feat: resource scan

internal/sql/repository/security/ImageScanHistoryRepository.go

@@ -18,20 +18,77 @@
 package security
 
 import (
+	serverBean "github.com/devtron-labs/devtron/pkg/server/bean"
 	"github.com/go-pg/pg"
 	"go.uber.org/zap"
 	"time"
 )
 
 type ImageScanExecutionHistory struct {
-	tableName     struct{}  `sql:"image_scan_execution_history" pg:",discard_unknown_columns"`
-	Id            int       `sql:"id,pk"`
-	Image         string    `sql:"image,notnull"`
-	ImageHash     string    `sql:"image_hash,notnull"`
-	ExecutionTime time.Time `sql:"execution_time"`
-	ExecutedBy    int       `sql:"executed_by,notnull"`
+	tableName                       struct{}      `sql:"image_scan_execution_history" pg:",discard_unknown_columns"`
+	Id                              int           `sql:"id,pk"`
+	Image                           string        `sql:"image,notnull"`
+	ImageHash                       string        `sql:"image_hash,notnull"` // TODO Migrate to request metadata
+	ExecutionTime                   time.Time     `sql:"execution_time"`
+	ExecutedBy                      int           `sql:"executed_by,notnull"`
+	SourceMetadataJson              string        `sql:"source_metadata_json"` // to have relevant info to process a scan for a given source type and subtype
+	SourceType                      SourceType    `sql:"source_type"`
+	SourceSubType                   SourceSubType `sql:"source_sub_type"`
+	ScanToolExecutionHistoryMapping *ScanToolExecutionHistoryMapping
 }
 
+func (ed *ExecutionData) IsBuiltImage() bool {
+	return ed.SourceType == SourceTypeImage && ed.SourceSubType == SourceSubTypeCi
+}
+
+func (ed *ExecutionData) IsManifestImage() bool {
+	return ed.SourceType == SourceTypeImage && ed.SourceSubType == SourceSubTypeManifest
+}
+
+func (ed *ExecutionData) IsManifest() bool {
+	return ed.SourceType == SourceTypeCode && ed.SourceSubType == SourceSubTypeManifest
+}
+
+func (ed *ExecutionData) IsCode() bool {
+	return ed.SourceType == SourceTypeCode && ed.SourceSubType == SourceSubTypeCi
+}
+
+func (ed *ExecutionData) ContainsType(typeToCheck ResourceScanType) bool {
+	for _, scanType := range ed.Types {
+		if scanType == int(typeToCheck) {
+			return true
+		}
+	}
+	return false
+}
+
+type ExecutionData struct {
+	Image         string
+	ScanDataJson  string
+	StartedOn     time.Time
+	ScanToolName  string
+	SourceType    SourceType
+	SourceSubType SourceSubType
+	Types         []int `sql:"types" pg:",array"`
+	Status        serverBean.ScanExecutionProcessState
+}
+
+// multiple history rows for one source event
+type SourceType int
+
+const (
+	SourceTypeImage SourceType = 1
+	SourceTypeCode  SourceType = 2
+	SourceTypeSbom  SourceType = 3 // can be used in future for direct sbom scanning
+)
+
+type SourceSubType int
+
+const (
+	SourceSubTypeCi       SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1)
+	SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2))
+)
+
 type ImageScanHistoryRepository interface {
 	Save(model *ImageScanExecutionHistory) error
 	FindAll() ([]*ImageScanExecutionHistory, error)

internal/sql/repository/security/ResourceScanExecutionRepository.go

@@ -0,0 +1,53 @@
+package security
+
+import (
+	"github.com/go-pg/pg"
+	"go.uber.org/zap"
+)
+
+type ResourceScanExecutionResult struct {
+	tableName                   struct{}           `sql:"resource_scan_execution_result" pg:",discard_unknown_columns"`
+	Id                          int                `sql:"id,pk"`
+	ImageScanExecutionHistoryId int                `sql:"image_scan_execution_history_id"`
+	ScanDataJson                string             `sql:"scan_data_json"`
+	Format                      ResourceScanFormat `sql:"format"`
+	Types                       []ResourceScanType `sql:"types"`
+	ScanToolId                  int                `sql:"scan_tool_id"`
+}
+
+type ResourceScanFormat int
+
+const (
+	CycloneDxSbom ResourceScanFormat = 1 // SBOM
+	TrivyJson                        = 2
+	Json                             = 3
+)
+
+type ResourceScanType int
+
+const (
+	Vulnerabilities ResourceScanType = 1
+	License                          = 2
+	Config                           = 3
+	Secrets                          = 4
+)
+
+type ResourceScanResultRepository interface {
+	SaveInBatch(tx *pg.Tx, models []*ResourceScanExecutionResult) error
+}
+
+type ResourceScanResultRepositoryImpl struct {
+	dbConnection *pg.DB
+	logger       *zap.SugaredLogger
+}
+
+func NewResourceScanResultRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *ResourceScanResultRepositoryImpl {
+	return &ResourceScanResultRepositoryImpl{
+		dbConnection: dbConnection,
+		logger:       logger,
+	}
+}
+
+func (impl ResourceScanResultRepositoryImpl) SaveInBatch(tx *pg.Tx, models []*ResourceScanExecutionResult) error {
+	return tx.Insert(&models)
+}

internal/sql/repository/security/ScanToolExecutionHistoryMapping.go

@@ -17,6 +17,7 @@ type ScanToolExecutionHistoryMapping struct {
 	ExecutionFinishTime         time.Time                            `sql:"execution_finish_time,notnull"`
 	State                       serverBean.ScanExecutionProcessState `sql:"state"`
 	TryCount                    int                                  `sql:"try_count"`
+	ErrorMessage                string                               `sql:"error_message"`
 	sql.AuditLog
 }
 

scripts/sql/242_resource_scan.down.sql

@@ -0,0 +1,5 @@
+ALTER TABLE public.scan_tool_execution_history_mapping DROP COLUMN IF EXISTS error_message;
+
+DELETE FROM public.scan_tool_step
+WHERE scan_tool_id = 3
+  AND index = 5;

scripts/sql/242_resource_scan.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.scan_tool_execution_history_mapping ADD COLUMN IF NOT EXISTS error_message  varchar NULL;
+INSERT INTO public.scan_tool_step(scan_tool_id, index, step_execution_type, step_execution_sync, retry_count, execute_step_on_fail, execute_step_on_pass, render_input_data_from_step, http_input_payload, http_method_type, http_req_headers, http_query_params, cli_command, cli_output_type, deleted, created_on, created_by, updated_on, updated_by) VALUES (3,5,'CLI',true,1,-1,-1,-1,null,null,null,null,'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}}', 'STATIC',false,now()::timestamp,'1',now()::timestamp,'1');

scripts/sql/243_code_image_scan.down.sql

@@ -0,0 +1,21 @@
+
+DROP TABLE IF EXISTS "public"."resource_scan_execution_result";
+
+DROP SEQUENCE IF EXISTS resource_scan_execution_result_id_seq;
+
+ALTER TABLE public.image_scan_execution_history DROP column IF EXISTS source_type;
+ALTER TABLE public.image_scan_execution_history DROP column IF EXISTS source_sub_type;
+ALTER TABLE public.image_scan_execution_history RENAME COLUMN source_metadata_json TO scan_event_json ;
+
+UPDATE scan_tool_step
+SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} --username {{.USERNAME}} --password {{.PASSWORD}}'
+WHERE scan_tool_id=3 and  index=1 and step_execution_type='CLI';
+UPDATE scan_tool_step
+SET cli_command = '(export AWS_ACCESS_KEY_ID={{.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY={{.AWS_SECRET_ACCESS_KEY}} AWS_DEFAULT_REGION={{.AWS_DEFAULT_REGION}}; trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}})'
+WHERE scan_tool_id=3 and index=2 and step_execution_type='CLI';
+UPDATE scan_tool_step
+SET cli_command = 'GOOGLE_APPLICATION_CREDENTIALS="{{.FILE_PATH}}/credentials.json" trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}}'
+WHERE scan_tool_id=3 and index=3 and step_execution_type='CLI';
+UPDATE scan_tool_step
+SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}}'
+WHERE scan_tool_id=3 and index=5 and step_execution_type='CLI';

scripts/sql/243_code_image_scan.up.sql

@@ -0,0 +1,116 @@
+
+CREATE SEQUENCE IF NOT EXISTS public.resource_scan_execution_result_id_seq;
+
+CREATE table if not exists public.resource_scan_execution_result (
+    id integer DEFAULT nextval('public.resource_scan_execution_result_id_seq'::regclass) NOT NULL,
+    image_scan_execution_history_id integer NOT NULL,
+    scan_data_json text,
+    format integer,
+    types integer[],
+    scan_tool_id int,
+    PRIMARY KEY ("id"),
+    CONSTRAINT image_scan_execution_history_id_fkey
+    FOREIGN KEY("image_scan_execution_history_id")
+    REFERENCES"public"."image_scan_execution_history" ("id")
+    );
+
+ALTER TABLE public.image_scan_execution_history ADD column IF NOT exists source_type integer NULL;
+ALTER TABLE public.image_scan_execution_history ADD column IF NOT exists source_sub_type integer NULL;
+ALTER TABLE public.image_scan_execution_history RENAME COLUMN scan_event_json TO source_metadata_json;
+
+
+UPDATE scan_tool_step
+SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} --username {{.USERNAME}} --password {{.PASSWORD}} {{.EXTRA_ARGS}}'
+WHERE scan_tool_id=3 and  index=1 and step_execution_type='CLI';
+UPDATE scan_tool_step
+SET cli_command = '(export AWS_ACCESS_KEY_ID={{.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY={{.AWS_SECRET_ACCESS_KEY}} AWS_DEFAULT_REGION={{.AWS_DEFAULT_REGION}}; trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} {{.EXTRA_ARGS}})'
+WHERE scan_tool_id=3 and index=2 and step_execution_type='CLI';
+UPDATE scan_tool_step
+SET cli_command = 'GOOGLE_APPLICATION_CREDENTIALS="{{.FILE_PATH}}/credentials.json" trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} {{.EXTRA_ARGS}}'
+WHERE scan_tool_id=3 and index=3 and step_execution_type='CLI';
+UPDATE scan_tool_step
+SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} {{.EXTRA_ARGS}}'
+WHERE scan_tool_id=3 and index=5 and step_execution_type='CLI';
+
+
+INSERT INTO plugin_metadata (id,name,description,type,icon,deleted,created_on,created_by,updated_on,updated_by)
+VALUES (nextval('id_seq_plugin_metadata'),'Vulnerabilty_Scanner v1.0.0' , 'Checks code vulnerability types in the Post-CI stage','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/devtron-logo-plugin.png',false,'now()',1,'now()',1);
+
+
+INSERT INTO plugin_stage_mapping (id,plugin_id,stage_type,created_on,created_by,updated_on,updated_by)VALUES (nextval('id_seq_plugin_stage_mapping'),
+                                                                                                              (SELECT id from plugin_metadata where name='Vulnerabilty_Scanner v1.0.0'),1,'now()',1,'now()',1);
+
+INSERT INTO "plugin_pipeline_script" ("id", "script","type","deleted","created_on", "created_by", "updated_on", "updated_by")
+VALUES ( nextval('id_seq_plugin_pipeline_script'),
+         E'#!/bin/bash
+
+json_data="$CI_CD_EVENT"
+base_url="$IMAGE_SCANNER_ENDPOINT"
+
+
+url="$base_url/scanner/image"
+
+ciProjectDetails=$(echo "$json_data" | jq -r \'.commonWorkflowRequest.ciProjectDetails\')
+ciWorkflowId=$(echo "$json_data" | jq -r \'.workflowId\')
+sourceType=2
+sourceSubType=1
+
+
+new_payload=$(cat <<EOF
+{
+  "ciProjectDetails": $ciProjectDetails,
+  "ciWorkflowId" : $ciWorkflowId,
+  "sourceType" : $sourceType,
+  "sourceSubType" : $sourceSubType
+
+}
+EOF
+)
+
+
+response=$(curl -s -X POST -H "Content-Type: application/json" -d "$new_payload" "$url")
+
+  export LOW=-1
+  export MEDIUM=-1
+  export HIGH=-1
+  export CRITICAL=-1
+  export UNKNOWN=-1
+
+
+if [[ $(echo "$response" | jq -r \'.status\') == "OK" ]]; then
+    # Extract severity values from the response JSON and replace null with zero
+    LOW=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.LOW // 0\')
+    MEDIUM=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.MEDIUM // 0\')
+             HIGH=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.HIGH  // 0\')
+             CRITICAL=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.CRITICAL // 0\')
+             UNKNOWN=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.UNKNOWN  // 0\')
+             else
+             echo "Response not OK: $response"
+             fi
+
+
+             echo "LOW = $LOW"
+             echo "MEDIUM = $MEDIUM"
+             echo "HIGH = $HIGH"
+             echo "CRITICAL = $CRITICAL"
+             echo "UNKNOWN = $UNKNOWN"',
+
+    'SHELL',
+    'f',
+    'now()',
+    1,
+    'now()',
+    1
+);
+
+
+
+INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='Vulnerabilty_Scanner v1.0.0'),'Step 1','Step 1 - Vulnerabilty_Scanner v1.0.0','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1);
+INSERT INTO plugin_step_variable (id,plugin_step_id,name,format,description,is_exposed,allow_empty_value,default_value,value,variable_type,value_type,previous_step_index,variable_step_index,variable_step_index_in_plugin,reference_variable_name,deleted,created_on,created_by,updated_on,updated_by)
+VALUES
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'LOW','NUMBER','Number of LOW vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'MEDIUM','NUMBER','Number of MEDIUM vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'HIGH','NUMBER','Number of HIGH vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'CRITICAL','NUMBER','Number of CRITICAL vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
+(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'UNKNOWN','NUMBER','Number of UNKNOWN vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1);
+