Skip to content

Commit 181cda7

Browse files
subhashish-devtronkomalreddy3
subhashish-devtron
authored and
komalreddy3
committed
feat: resource scan (#4977)
* cherry-pick * incorporating feedbacks
1 parent dd4fec8 commit 181cda7

File tree

7 files changed

+261
-6
lines changed

7 files changed

+261
-6
lines changed

‎internal/sql/repository/security/ImageScanHistoryRepository.go

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,77 @@
1818
package security
1919

2020
import (
21+
serverBean "github.com/devtron-labs/devtron/pkg/server/bean"
2122
"github.com/go-pg/pg"
2223
"go.uber.org/zap"
2324
"time"
2425
)
2526

2627
type ImageScanExecutionHistory struct {
27-
tableName struct{} `sql:"image_scan_execution_history" pg:",discard_unknown_columns"`
28-
Id int `sql:"id,pk"`
29-
Image string `sql:"image,notnull"`
30-
ImageHash string `sql:"image_hash,notnull"`
31-
ExecutionTime time.Time `sql:"execution_time"`
32-
ExecutedBy int `sql:"executed_by,notnull"`
28+
tableName struct{} `sql:"image_scan_execution_history" pg:",discard_unknown_columns"`
29+
Id int `sql:"id,pk"`
30+
Image string `sql:"image,notnull"`
31+
ImageHash string `sql:"image_hash,notnull"` // TODO Migrate to request metadata
32+
ExecutionTime time.Time `sql:"execution_time"`
33+
ExecutedBy int `sql:"executed_by,notnull"`
34+
SourceMetadataJson string `sql:"source_metadata_json"` // to have relevant info to process a scan for a given source type and subtype
35+
SourceType SourceType `sql:"source_type"`
36+
SourceSubType SourceSubType `sql:"source_sub_type"`
37+
ScanToolExecutionHistoryMapping *ScanToolExecutionHistoryMapping
3338
}
3439

40+
func (ed *ExecutionData) IsBuiltImage() bool {
41+
return ed.SourceType == SourceTypeImage && ed.SourceSubType == SourceSubTypeCi
42+
}
43+
44+
func (ed *ExecutionData) IsManifestImage() bool {
45+
return ed.SourceType == SourceTypeImage && ed.SourceSubType == SourceSubTypeManifest
46+
}
47+
48+
func (ed *ExecutionData) IsManifest() bool {
49+
return ed.SourceType == SourceTypeCode && ed.SourceSubType == SourceSubTypeManifest
50+
}
51+
52+
func (ed *ExecutionData) IsCode() bool {
53+
return ed.SourceType == SourceTypeCode && ed.SourceSubType == SourceSubTypeCi
54+
}
55+
56+
func (ed *ExecutionData) ContainsType(typeToCheck ResourceScanType) bool {
57+
for _, scanType := range ed.Types {
58+
if scanType == int(typeToCheck) {
59+
return true
60+
}
61+
}
62+
return false
63+
}
64+
65+
type ExecutionData struct {
66+
Image string
67+
ScanDataJson string
68+
StartedOn time.Time
69+
ScanToolName string
70+
SourceType SourceType
71+
SourceSubType SourceSubType
72+
Types []int `sql:"types" pg:",array"`
73+
Status serverBean.ScanExecutionProcessState
74+
}
75+
76+
// multiple history rows for one source event
77+
type SourceType int
78+
79+
const (
80+
SourceTypeImage SourceType = 1
81+
SourceTypeCode SourceType = 2
82+
SourceTypeSbom SourceType = 3 // can be used in future for direct sbom scanning
83+
)
84+
85+
type SourceSubType int
86+
87+
const (
88+
SourceSubTypeCi SourceSubType = 1 // relevant for ci code(2,1) or ci built image(1,1)
89+
SourceSubTypeManifest SourceSubType = 2 // relevant for devtron app deployment manifest/helm app manifest(2,2) or images retrieved from manifest(1,2))
90+
)
91+
3592
type ImageScanHistoryRepository interface {
3693
Save(model *ImageScanExecutionHistory) error
3794
FindAll() ([]*ImageScanExecutionHistory, error)

‎internal/sql/repository/security/ResourceScanExecutionRepository.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package security
2+
3+
import (
4+
"github.com/go-pg/pg"
5+
"go.uber.org/zap"
6+
)
7+
8+
type ResourceScanExecutionResult struct {
9+
tableName struct{} `sql:"resource_scan_execution_result" pg:",discard_unknown_columns"`
10+
Id int `sql:"id,pk"`
11+
ImageScanExecutionHistoryId int `sql:"image_scan_execution_history_id"`
12+
ScanDataJson string `sql:"scan_data_json"`
13+
Format ResourceScanFormat `sql:"format"`
14+
Types []ResourceScanType `sql:"types"`
15+
ScanToolId int `sql:"scan_tool_id"`
16+
}
17+
18+
type ResourceScanFormat int
19+
20+
const (
21+
CycloneDxSbom ResourceScanFormat = 1 // SBOM
22+
TrivyJson = 2
23+
Json = 3
24+
)
25+
26+
type ResourceScanType int
27+
28+
const (
29+
Vulnerabilities ResourceScanType = 1
30+
License = 2
31+
Config = 3
32+
Secrets = 4
33+
)
34+
35+
type ResourceScanResultRepository interface {
36+
SaveInBatch(tx *pg.Tx, models []*ResourceScanExecutionResult) error
37+
}
38+
39+
type ResourceScanResultRepositoryImpl struct {
40+
dbConnection *pg.DB
41+
logger *zap.SugaredLogger
42+
}
43+
44+
func NewResourceScanResultRepositoryImpl(dbConnection *pg.DB, logger *zap.SugaredLogger) *ResourceScanResultRepositoryImpl {
45+
return &ResourceScanResultRepositoryImpl{
46+
dbConnection: dbConnection,
47+
logger: logger,
48+
}
49+
}
50+
51+
func (impl ResourceScanResultRepositoryImpl) SaveInBatch(tx *pg.Tx, models []*ResourceScanExecutionResult) error {
52+
return tx.Insert(&models)
53+
}

‎internal/sql/repository/security/ScanToolExecutionHistoryMapping.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ type ScanToolExecutionHistoryMapping struct {
1717
ExecutionFinishTime time.Time `sql:"execution_finish_time,notnull"`
1818
State serverBean.ScanExecutionProcessState `sql:"state"`
1919
TryCount int `sql:"try_count"`
20+
ErrorMessage string `sql:"error_message"`
2021
sql.AuditLog
2122
}
2223

‎scripts/sql/242_resource_scan.down.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
ALTER TABLE public.scan_tool_execution_history_mapping DROP COLUMN IF EXISTS error_message;
2+
3+
DELETE FROM public.scan_tool_step
4+
WHERE scan_tool_id = 3
5+
AND index = 5;

‎scripts/sql/242_resource_scan.up.sql

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
ALTER TABLE public.scan_tool_execution_history_mapping ADD COLUMN IF NOT EXISTS error_message varchar NULL;
2+
INSERT INTO public.scan_tool_step(scan_tool_id, index, step_execution_type, step_execution_sync, retry_count, execute_step_on_fail, execute_step_on_pass, render_input_data_from_step, http_input_payload, http_method_type, http_req_headers, http_query_params, cli_command, cli_output_type, deleted, created_on, created_by, updated_on, updated_by) VALUES (3,5,'CLI',true,1,-1,-1,-1,null,null,null,null,'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}}', 'STATIC',false,now()::timestamp,'1',now()::timestamp,'1');

‎scripts/sql/243_code_image_scan.down.sql

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
2+
DROP TABLE IF EXISTS "public"."resource_scan_execution_result";
3+
4+
DROP SEQUENCE IF EXISTS resource_scan_execution_result_id_seq;
5+
6+
ALTER TABLE public.image_scan_execution_history DROP column IF EXISTS source_type;
7+
ALTER TABLE public.image_scan_execution_history DROP column IF EXISTS source_sub_type;
8+
ALTER TABLE public.image_scan_execution_history RENAME COLUMN source_metadata_json TO scan_event_json ;
9+
10+
UPDATE scan_tool_step
11+
SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} --username {{.USERNAME}} --password {{.PASSWORD}}'
12+
WHERE scan_tool_id=3 and index=1 and step_execution_type='CLI';
13+
UPDATE scan_tool_step
14+
SET cli_command = '(export AWS_ACCESS_KEY_ID={{.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY={{.AWS_SECRET_ACCESS_KEY}} AWS_DEFAULT_REGION={{.AWS_DEFAULT_REGION}}; trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}})'
15+
WHERE scan_tool_id=3 and index=2 and step_execution_type='CLI';
16+
UPDATE scan_tool_step
17+
SET cli_command = 'GOOGLE_APPLICATION_CREDENTIALS="{{.FILE_PATH}}/credentials.json" trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}}'
18+
WHERE scan_tool_id=3 and index=3 and step_execution_type='CLI';
19+
UPDATE scan_tool_step
20+
SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}}'
21+
WHERE scan_tool_id=3 and index=5 and step_execution_type='CLI';

‎scripts/sql/243_code_image_scan.up.sql

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
2+
CREATE SEQUENCE IF NOT EXISTS public.resource_scan_execution_result_id_seq;
3+
4+
CREATE table if not exists public.resource_scan_execution_result (
5+
id integer DEFAULT nextval('public.resource_scan_execution_result_id_seq'::regclass) NOT NULL,
6+
image_scan_execution_history_id integer NOT NULL,
7+
scan_data_json text,
8+
format integer,
9+
types integer[],
10+
scan_tool_id int,
11+
PRIMARY KEY ("id"),
12+
CONSTRAINT image_scan_execution_history_id_fkey
13+
FOREIGN KEY("image_scan_execution_history_id")
14+
REFERENCES"public"."image_scan_execution_history" ("id")
15+
);
16+
17+
ALTER TABLE public.image_scan_execution_history ADD column IF NOT exists source_type integer NULL;
18+
ALTER TABLE public.image_scan_execution_history ADD column IF NOT exists source_sub_type integer NULL;
19+
ALTER TABLE public.image_scan_execution_history RENAME COLUMN scan_event_json TO source_metadata_json;
20+
21+
22+
UPDATE scan_tool_step
23+
SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} --username {{.USERNAME}} --password {{.PASSWORD}} {{.EXTRA_ARGS}}'
24+
WHERE scan_tool_id=3 and index=1 and step_execution_type='CLI';
25+
UPDATE scan_tool_step
26+
SET cli_command = '(export AWS_ACCESS_KEY_ID={{.AWS_ACCESS_KEY_ID}} AWS_SECRET_ACCESS_KEY={{.AWS_SECRET_ACCESS_KEY}} AWS_DEFAULT_REGION={{.AWS_DEFAULT_REGION}}; trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} {{.EXTRA_ARGS}})'
27+
WHERE scan_tool_id=3 and index=2 and step_execution_type='CLI';
28+
UPDATE scan_tool_step
29+
SET cli_command = 'GOOGLE_APPLICATION_CREDENTIALS="{{.FILE_PATH}}/credentials.json" trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} {{.EXTRA_ARGS}}'
30+
WHERE scan_tool_id=3 and index=3 and step_execution_type='CLI';
31+
UPDATE scan_tool_step
32+
SET cli_command = 'trivy image -f json -o {{.OUTPUT_FILE_PATH}} --timeout {{.timeout}} {{.IMAGE_NAME}} {{.EXTRA_ARGS}}'
33+
WHERE scan_tool_id=3 and index=5 and step_execution_type='CLI';
34+
35+
36+
INSERT INTO plugin_metadata (id,name,description,type,icon,deleted,created_on,created_by,updated_on,updated_by)
37+
VALUES (nextval('id_seq_plugin_metadata'),'Vulnerabilty_Scanner v1.0.0' , 'Checks code vulnerability types in the Post-CI stage','PRESET','https://raw.githubusercontent.com/devtron-labs/devtron/main/assets/devtron-logo-plugin.png',false,'now()',1,'now()',1);
38+
39+
40+
INSERT INTO plugin_stage_mapping (id,plugin_id,stage_type,created_on,created_by,updated_on,updated_by)VALUES (nextval('id_seq_plugin_stage_mapping'),
41+
(SELECT id from plugin_metadata where name='Vulnerabilty_Scanner v1.0.0'),1,'now()',1,'now()',1);
42+
43+
INSERT INTO "plugin_pipeline_script" ("id", "script","type","deleted","created_on", "created_by", "updated_on", "updated_by")
44+
VALUES ( nextval('id_seq_plugin_pipeline_script'),
45+
E'#!/bin/bash
46+
47+
json_data="$CI_CD_EVENT"
48+
base_url="$IMAGE_SCANNER_ENDPOINT"
49+
50+
51+
url="$base_url/scanner/image"
52+
53+
ciProjectDetails=$(echo "$json_data" | jq -r \'.commonWorkflowRequest.ciProjectDetails\')
54+
ciWorkflowId=$(echo "$json_data" | jq -r \'.workflowId\')
55+
sourceType=2
56+
sourceSubType=1
57+
58+
59+
new_payload=$(cat <<EOF
60+
{
61+
"ciProjectDetails": $ciProjectDetails,
62+
"ciWorkflowId" : $ciWorkflowId,
63+
"sourceType" : $sourceType,
64+
"sourceSubType" : $sourceSubType
65+
66+
}
67+
EOF
68+
)
69+
70+
71+
response=$(curl -s -X POST -H "Content-Type: application/json" -d "$new_payload" "$url")
72+
73+
export LOW=-1
74+
export MEDIUM=-1
75+
export HIGH=-1
76+
export CRITICAL=-1
77+
export UNKNOWN=-1
78+
79+
80+
if [[ $(echo "$response" | jq -r \'.status\') == "OK" ]]; then
81+
# Extract severity values from the response JSON and replace null with zero
82+
LOW=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.LOW // 0\')
83+
MEDIUM=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.MEDIUM // 0\')
84+
HIGH=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.HIGH // 0\')
85+
CRITICAL=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.CRITICAL // 0\')
86+
UNKNOWN=$(echo "$response" | jq -r \'.result.codeScanResponse.misConfigurations.list[0].summary.severities.UNKNOWN // 0\')
87+
else
88+
echo "Response not OK: $response"
89+
fi
90+
91+
92+
echo "LOW = $LOW"
93+
echo "MEDIUM = $MEDIUM"
94+
echo "HIGH = $HIGH"
95+
echo "CRITICAL = $CRITICAL"
96+
echo "UNKNOWN = $UNKNOWN"',
97+
98+
'SHELL',
99+
'f',
100+
'now()',
101+
1,
102+
'now()',
103+
1
104+
);
105+
106+
107+
108+
INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES (nextval('id_seq_plugin_step'), (SELECT id FROM plugin_metadata WHERE name='Vulnerabilty_Scanner v1.0.0'),'Step 1','Step 1 - Vulnerabilty_Scanner v1.0.0','1','INLINE',(SELECT last_value FROM id_seq_plugin_pipeline_script),'f','now()', 1, 'now()', 1);
109+
INSERT INTO plugin_step_variable (id,plugin_step_id,name,format,description,is_exposed,allow_empty_value,default_value,value,variable_type,value_type,previous_step_index,variable_step_index,variable_step_index_in_plugin,reference_variable_name,deleted,created_on,created_by,updated_on,updated_by)
110+
VALUES
111+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'LOW','NUMBER','Number of LOW vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
112+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'MEDIUM','NUMBER','Number of MEDIUM vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
113+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'HIGH','NUMBER','Number of HIGH vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
114+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'CRITICAL','NUMBER','Number of CRITICAL vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1),
115+
(nextval('id_seq_plugin_step_variable'),(SELECT ps.id FROM plugin_metadata p inner JOIN plugin_step ps on ps.plugin_id=p.id WHERE p.name='Vulnerabilty_Scanner v1.0.0' and ps."index"=1 and ps.deleted=false),'UNKNOWN','NUMBER','Number of UNKNOWN vulnerability,','t','f',null,null,'OUTPUT','NEW',null,1,null,null,'f','now()',1,'now()',1);
116+

0 commit comments

Comments
 (0)