Skip to content

Commit dc4a9fb

Browse files
vikramdevtronvikramdevtron
authored
Merge pull request #51 from devtron-labs/release-candidate-v0.17.0
Release candidate v0.17.0
2 parents 187b78d + 6d42b81 commit dc4a9fb

File tree

8 files changed

+182
-49
lines changed

8 files changed

+182
-49
lines changed

‎internal/step-lib/util/common-util/util.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,12 @@ func ReadFile(fileName string) ([]byte, error) {
6969
func ParseJsonTemplate(inputTemplate string, data []byte) (string, error) {
7070
tmpl := template.Must(template.New("").Funcs(template.FuncMap{
7171
"add": func(a, b int) int { return a + b },
72+
"len": func(sliceIf interface{}) int {
73+
if slice, ok := sliceIf.([]any); ok {
74+
return len(slice)
75+
}
76+
return 0
77+
},
7278
}).Parse(inputTemplate))
7379
jsonMap := map[string]interface{}{}
7480
err := json.Unmarshal(data, &jsonMap)

‎pkg/security/ImageScanService.go‎

Lines changed: 42 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,7 @@ func (impl *ImageScanServiceImpl) RegisterScanExecutionHistoryAndState(scanEvent
274274
impl.Logger.Errorw("Failed to save executionHistory", "err", err, "model", executionHistoryModel)
275275
return nil, executionHistoryDirPath, err
276276
}
277+
277278
// creating folder for storing all details if not exist
278279
isExist, err := helper.DoesFileExist(bean.ScanOutputDirectory)
279280
if err != nil {
@@ -509,9 +510,9 @@ func (impl *ImageScanServiceImpl) ConvertEndStepOutputAndSaveVulnerabilities(ste
509510
for _, vul := range uniqueVulnerabilityMap {
510511
if val, ok := allSavedCvesMap[vul.Name]; ok {
511512
// updating cve here if vulnerability has a new severity
512-
vulnerabilitySeverity := bean.SeverityStringToEnum(bean.ConvertToLowerCase(vul.Severity))
513-
if vulnerabilitySeverity != val.Severity {
514-
val.UpdateNewSeverityInCveStore(vulnerabilitySeverity, userId)
513+
vulnerabilityStandardSeverity := bean.StandardSeverityStringToEnum(bean.ConvertToLowerCase(vul.Severity))
514+
if vulnerabilityStandardSeverity != val.GetSeverity() {
515+
val.UpdateNewSeverityInCveStore(vul.Severity, userId)
515516
cvesToUpdate = append(cvesToUpdate, val)
516517
}
517518
} else {
@@ -522,7 +523,7 @@ func (impl *ImageScanServiceImpl) ConvertEndStepOutputAndSaveVulnerabilities(ste
522523

523524
imageScanExecutionResults := make([]*repository.ImageScanExecutionResult, 0, len(vulnerabilities))
524525
for _, vul := range vulnerabilities {
525-
imageScanExecutionResult := createImageScanExecutionResultObject(executionHistoryId, vul.Name, vul.Package, vul.PackageVersion, vul.FixedInVersion, tool.Id)
526+
imageScanExecutionResult := createImageScanExecutionResultObject(executionHistoryId, vul.Name, vul.Package, vul.PackageVersion, vul.FixedInVersion, vul.Class, vul.Type, vul.TargetName, tool.Id)
526527
imageScanExecutionResults = append(imageScanExecutionResults, imageScanExecutionResult)
527528
}
528529
tx, err := impl.CveStoreRepository.GetConnection().Begin()
@@ -562,7 +563,7 @@ func (impl *ImageScanServiceImpl) ConvertEndStepOutputAndSaveVulnerabilities(ste
562563
}
563564

564565
func isV1Template(resultDescriptorTemplate string) bool {
565-
var mappings []bean.Mapping
566+
var mappings []map[string]interface{}
566567
err := json.Unmarshal([]byte(resultDescriptorTemplate), &mappings)
567568
return err != nil && isValidGoTemplate(resultDescriptorTemplate) //checking error too because our new result descriptor template can pass go templating too as it contains a simple json
568569
}
@@ -593,33 +594,46 @@ func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV1(stepOutput []byte,
593594

594595
func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV2(stepOutput []byte, resultDescriptorTemplate string) ([]*bean.ImageScanOutputObject, error) {
595596
var vulnerabilities []*bean.ImageScanOutputObject
596-
var mappings []bean.Mapping
597+
var mappings []map[string]interface{}
597598
err := json.Unmarshal([]byte(resultDescriptorTemplate), &mappings)
598599
if err != nil {
599600
impl.Logger.Errorw("error in un-marshaling result descriptor template", "err", err, "resultDescriptorTemplate", resultDescriptorTemplate)
600601
return nil, err
601602
}
602-
var processArray func(mapping bean.Mapping, value gjson.Result)
603-
processArray = func(mapping bean.Mapping, value gjson.Result) {
603+
var processArray func(mapping map[string]interface{}, value gjson.Result)
604+
processArray = func(mapping map[string]interface{}, value gjson.Result) {
605+
vulnerabilitiesPath := mapping[bean.MappingKeyPathToVulnerabilitiesArray].(string)
606+
vulnerabilityDataKeyPathsMap := mapping[bean.MappingKeyPathToVulnerabilityDataKeys].(map[string]interface{})
607+
resultDataKeyPathsMap := mapping[bean.MappingKeyPathToResultDataKeys].(map[string]interface{})
608+
604609
value.ForEach(func(_, nestedValue gjson.Result) bool {
605-
if nestedValue.IsArray() {
606-
// if the nested value is an array, recursively process it
607-
processArray(mapping, nestedValue)
608-
} else {
609-
vulnerability := &bean.ImageScanOutputObject{
610-
Name: nestedValue.Get(mapping[bean.MappingKeyName]).String(),
611-
Package: nestedValue.Get(mapping[bean.MappingKeyPackage]).String(),
612-
PackageVersion: nestedValue.Get(mapping[bean.MappingKeyPackageVersion]).String(),
613-
FixedInVersion: nestedValue.Get(mapping[bean.MappingKeyFixedInVersion]).String(),
614-
Severity: nestedValue.Get(mapping[bean.MappingKeySeverity]).String(),
610+
targetName, class, resType := "", "", ""
611+
if nestedValue.IsObject() {
612+
targetName, class, resType = nestedValue.Get(resultDataKeyPathsMap[bean.MappingTarget].(string)).String(), nestedValue.Get(resultDataKeyPathsMap[bean.MappingClass].(string)).String(), nestedValue.Get(resultDataKeyPathsMap[bean.MappingType].(string)).String()
613+
614+
if nestedValue.Get(vulnerabilitiesPath).IsArray() {
615+
nestedValue.Get(vulnerabilitiesPath).ForEach(func(_, vul gjson.Result) bool {
616+
vulnerability := &bean.ImageScanOutputObject{
617+
Name: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyName].(string)).String(),
618+
Package: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyPackage].(string)).String(),
619+
PackageVersion: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyPackageVersion].(string)).String(),
620+
FixedInVersion: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyFixedInVersion].(string)).String(),
621+
Severity: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeySeverity].(string)).String(),
622+
TargetName: targetName,
623+
Class: class,
624+
Type: resType,
625+
}
626+
vulnerabilities = append(vulnerabilities, vulnerability)
627+
return true
628+
})
615629
}
616-
vulnerabilities = append(vulnerabilities, vulnerability)
617630
}
618631
return true
619632
})
620633
}
634+
621635
for _, mapping := range mappings {
622-
result := gjson.Get(string(stepOutput), mapping[bean.MappingKeyPathToVulnerabilitiesArray])
636+
result := gjson.Get(string(stepOutput), mapping[bean.MappingKeyPathToResultsArray].(string))
623637
if !result.Exists() {
624638
continue
625639
}
@@ -734,13 +748,13 @@ func (impl *ImageScanServiceImpl) CreateScanExecutionRegistryForClairV4(vs []*cl
734748
cvesToBeSaved = append(cvesToBeSaved, cveStore)
735749
} else {
736750
// updating cve here if vulnerability has a new severity
737-
vulnerabilitySeverity := bean.SeverityStringToEnum(bean.ConvertToLowerCase(item.Severity))
738-
if vulnerabilitySeverity != cveStore.Severity {
739-
cveStore.UpdateNewSeverityInCveStore(vulnerabilitySeverity, userId)
751+
vulnerabilityStandardSeverity := bean.StandardSeverityStringToEnum(bean.ConvertToLowerCase(item.Severity))
752+
if vulnerabilityStandardSeverity != cveStore.GetSeverity() {
753+
cveStore.UpdateNewSeverityInCveStore(item.Severity, userId)
740754
cvesToUpdate = append(cvesToUpdate, cveStore)
741755
}
742756
}
743-
imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.Package.Name, item.Package.Version, item.FixedInVersion, toolId)
757+
imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.Package.Name, item.Package.Version, item.FixedInVersion, "", "", "", toolId)
744758
imageScanExecutionResultsToBeSaved = append(imageScanExecutionResultsToBeSaved, imageScanExecutionResult)
745759
}
746760
tx, err := impl.CveStoreRepository.GetConnection().Begin()
@@ -797,13 +811,13 @@ func (impl *ImageScanServiceImpl) CreateScanExecutionRegistryForClairV2(vs []*cl
797811
cvesToBeSaved = append(cvesToBeSaved, cveStore)
798812
} else {
799813
// updating cve here if vulnerability has a new severity
800-
vulnerabilitySeverity := bean.SeverityStringToEnum(bean.ConvertToLowerCase(item.Severity))
801-
if vulnerabilitySeverity != cveStore.Severity {
802-
cveStore.UpdateNewSeverityInCveStore(vulnerabilitySeverity, userId)
814+
vulnerabilityStandardSeverity := bean.StandardSeverityStringToEnum(bean.ConvertToLowerCase(item.Severity))
815+
if vulnerabilityStandardSeverity != cveStore.GetSeverity() {
816+
cveStore.UpdateNewSeverityInCveStore(item.Severity, userId)
803817
cvesToUpdate = append(cvesToUpdate, cveStore)
804818
}
805819
}
806-
imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.FeatureName, item.FeatureVersion, item.FixedBy, toolId)
820+
imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.FeatureName, item.FeatureVersion, item.FixedBy, "", "", "", toolId)
807821
imageScanExecutionResultsToBeSaved = append(imageScanExecutionResultsToBeSaved, imageScanExecutionResult)
808822
}
809823
tx, err := impl.CveStoreRepository.GetConnection().Begin()

‎pkg/security/helper.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,21 @@ func createCveStoreObject(name, version, fixedInVersion, severity string, userId
1313
}
1414
lowerCaseSeverity := bean.ConvertToLowerCase(severity)
1515
cveStore.Severity = bean.SeverityStringToEnum(lowerCaseSeverity)
16-
cveStore.StandardSeverity = bean.StandardSeverityStringToEnum(lowerCaseSeverity)
16+
cveStore.SetStandardSeverity(bean.StandardSeverityStringToEnum(lowerCaseSeverity))
1717
cveStore.CreateAuditLog(userId)
1818
return cveStore
1919
}
2020

21-
func createImageScanExecutionResultObject(executionHistoryId int, vulName, packageName, version, fixedInVersion string, toolId int) *repository.ImageScanExecutionResult {
21+
func createImageScanExecutionResultObject(executionHistoryId int, vulName, packageName, version, fixedInVersion, className, typeName, targetName string, toolId int) *repository.ImageScanExecutionResult {
2222
return &repository.ImageScanExecutionResult{
2323
ImageScanExecutionHistoryId: executionHistoryId,
2424
CveStoreName: vulName,
2525
Package: packageName,
2626
ScanToolId: toolId,
2727
Version: version,
2828
FixedVersion: fixedInVersion,
29+
Target: targetName,
30+
Type: typeName,
31+
Class: className,
2932
}
3033
}

‎pkg/security/imageScanService_test.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package security
2+
3+
import (
4+
"fmt"
5+
"github.com/devtron-labs/image-scanner/pkg/logger"
6+
"github.com/tidwall/gjson"
7+
"os"
8+
"testing"
9+
)
10+
11+
const jsonKeys = "[ \n { \n \"pathToResultArray\": \"Results\",\n \"pathToVulnerabilitiesArray\": \"Vulnerabilities\",\n \"vulnerabilityData\":{\n \t\t\"name\": \"VulnerabilityID\",\n \t\"package\": \"PkgName\", \n \t\"packageVersion\": \"InstalledVersion\",\n \t\"fixedInVersion\": \"FixedVersion\",\n \t\"severity\": \"Severity\"\n },\n \"resultData\":{\n \t\t\t\"target\":\"Target\",\n \t\"class\":\"Class\",\n \t\"type\":\"Type\" \n }\n }\n]"
12+
13+
func TestScan(t *testing.T) {
14+
bytes, err := os.ReadFile("testTrivyScanResultV2.json")
15+
if err != nil {
16+
t.Fail()
17+
}
18+
19+
logger, _ := logger.InitLogger()
20+
impl := ImageScanServiceImpl{Logger: logger}
21+
out, err := impl.getImageScanOutputObjectsV2(bytes, jsonKeys)
22+
fmt.Println(out, err)
23+
if err != nil || len(out) == 0 {
24+
t.Fail()
25+
}
26+
27+
cnt, err := countVuln(bytes)
28+
if err != nil {
29+
t.Fail()
30+
}
31+
32+
if cnt != len(out) {
33+
t.Fail()
34+
}
35+
}
36+
37+
func countVuln(jsonStr []byte) (int, error) {
38+
result := gjson.Get(string(jsonStr), "Results")
39+
if !result.Exists() {
40+
return 0, nil
41+
}
42+
43+
targetCountMap := make(map[string]int)
44+
45+
result.ForEach(func(_, nestedValue gjson.Result) bool {
46+
if nestedValue.IsObject() {
47+
count := 0
48+
targetName := nestedValue.Get("Target").String()
49+
if nestedValue.Get("Vulnerabilities").IsArray() {
50+
nestedValue.Get("Vulnerabilities").ForEach(func(_, vul gjson.Result) bool {
51+
count++
52+
return true
53+
})
54+
}
55+
targetCountMap[targetName] += count
56+
}
57+
return true
58+
})
59+
60+
count := 0
61+
for _, cnt := range targetCountMap {
62+
count += cnt
63+
}
64+
return count, nil
65+
}

‎pkg/sql/bean/bean.go‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ const (
5050
)
5151

5252
type ImageScanOutputObject struct {
53+
TargetName string `json:"targetName"`
54+
Class string `json:"class"`
55+
Type string `json:"type"`
5356
Name string `json:"name"`
5457
Package string `json:"package"`
5558
PackageVersion string `json:"packageVersion"`
@@ -61,12 +64,18 @@ type ImageScanOutputObject struct {
6164
type Mapping map[string]string
6265

6366
const (
64-
MappingKeyPathToVulnerabilitiesArray = "pathToVulnerabilitiesArray"
65-
MappingKeyName = "name"
66-
MappingKeyPackage = "package"
67-
MappingKeyPackageVersion = "packageVersion"
68-
MappingKeyFixedInVersion = "fixedInVersion"
69-
MappingKeySeverity = "severity"
67+
MappingKeyPathToResultDataKeys = "resultData"
68+
MappingKeyPathToVulnerabilityDataKeys = "vulnerabilityData"
69+
MappingKeyPathToResultsArray = "pathToResultArray"
70+
MappingKeyPathToVulnerabilitiesArray = "pathToVulnerabilitiesArray"
71+
MappingKeyName = "name"
72+
MappingKeyPackage = "package"
73+
MappingKeyPackageVersion = "packageVersion"
74+
MappingKeyFixedInVersion = "fixedInVersion"
75+
MappingKeySeverity = "severity"
76+
MappingTarget = "target"
77+
MappingType = "type"
78+
MappingClass = "class"
7079
)
7180

7281
type Severity int
@@ -78,6 +87,7 @@ const (
7887
LOW string = "low"
7988
MEDIUM string = "medium"
8089
MODERATE string = "moderate"
90+
UNKNOWN string = "unknown"
8191
)
8292

8393
const (
@@ -86,10 +96,11 @@ const (
8696
Critical
8797
High
8898
Safe
99+
Unknown
89100
)
90101

91102
func (sev Severity) String() string {
92-
return [...]string{"low", "medium", "critical", "high", "safe"}[sev]
103+
return [...]string{LOW, MEDIUM, CRITICAL, HIGH, SAFE, UNKNOWN}[sev]
93104
}
94105
func ConvertToLowerCase(input string) string {
95106
return strings.ToLower(input)
@@ -98,24 +109,29 @@ func ConvertToLowerCase(input string) string {
98109
func SeverityStringToEnum(severity string) Severity {
99110
if severity == LOW || severity == SAFE {
100111
return Low
101-
} else if severity == MEDIUM {
112+
} else if severity == MEDIUM || severity == MODERATE {
102113
return Medium
103114
} else if severity == HIGH || severity == CRITICAL {
104115
return Critical
116+
} else if severity == UNKNOWN {
117+
return Unknown
105118
}
106119
return Low
107120
}
121+
108122
func StandardSeverityStringToEnum(severity string) Severity {
109123
if severity == LOW {
110124
return Low
111-
} else if severity == MEDIUM {
125+
} else if severity == MEDIUM || severity == MODERATE {
112126
return Medium
113127
} else if severity == HIGH {
114128
return High
115129
} else if severity == CRITICAL {
116130
return Critical
117131
} else if severity == SAFE {
118132
return Safe
133+
} else if severity == UNKNOWN {
134+
return Unknown
119135
}
120136
return Low
121137
}

‎pkg/sql/repository/CveStoreRepository.go‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,30 +24,55 @@ import (
2424
)
2525

2626
type CveStore struct {
27-
tableName struct{} `sql:"cve_store" pg:",discard_unknown_columns"`
28-
Name string `sql:"name,pk"`
29-
Severity bean.Severity `sql:"severity,notnull"`
30-
Package string `sql:"package,notnull"` // deprecated, storing package data in image_scan_execution_result table
31-
Version string `sql:"version,notnull"`
32-
FixedVersion string `sql:"fixed_version,notnull"`
33-
StandardSeverity bean.Severity `sql:"standard_severity,notnull"`
27+
tableName struct{} `sql:"cve_store" pg:",discard_unknown_columns"`
28+
Name string `sql:"name,pk"`
29+
30+
// Deprecated: Severity, use StandardSeverity for all read purposes
31+
Severity bean.Severity `sql:"severity,notnull"`
32+
// Deprecated: Package
33+
Package string `sql:"package,notnull"` // deprecated, storing package data in image_scan_execution_result table
34+
// Deprecated: Version
35+
Version string `sql:"version,notnull"`
36+
// Deprecated: FixedVersion
37+
FixedVersion string `sql:"fixed_version,notnull"`
38+
39+
// StandardSeverity is the actual severity. use GetSeverity method to get severity of the vulnerability
40+
// earlier severity is maintained in Severity column by merging HIGH and CRITICAL severities.
41+
// later we introduced new column StandardSeverity to store raw severity, but didn't migrate the existing Severity data to StandardSeverity.
42+
// currently, we deprecated Severity.
43+
StandardSeverity *bean.Severity `sql:"standard_severity"`
3444
AuditLog
3545
}
3646

47+
// GetSeverity returns the actual severity of the vulnerability.
48+
func (cve *CveStore) GetSeverity() bean.Severity {
49+
if cve.StandardSeverity == nil {
50+
// we need this as there was a time when StandardSeverity didn't exist.
51+
// and migration of Severity data to StandardSeverity is not done.
52+
return cve.Severity
53+
}
54+
return *cve.StandardSeverity
55+
}
56+
3757
func (cve *CveStore) CreateAuditLog(userId int32) {
3858
cve.CreatedBy = userId
3959
cve.CreatedOn = time.Now()
4060
cve.UpdatedBy = userId
4161
cve.UpdatedOn = time.Now()
4262
}
4363

44-
func (cve *CveStore) UpdateNewSeverityInCveStore(severity bean.Severity, userId int32) {
45-
cve.Severity = severity
46-
cve.StandardSeverity = severity
64+
func (cve *CveStore) UpdateNewSeverityInCveStore(severity string, userId int32) {
65+
lowerCaseSeverity := bean.ConvertToLowerCase(severity)
66+
cve.Severity = bean.SeverityStringToEnum(lowerCaseSeverity)
67+
cve.SetStandardSeverity(bean.StandardSeverityStringToEnum(lowerCaseSeverity))
4768
cve.UpdatedOn = time.Now()
4869
cve.UpdatedBy = userId
4970
}
5071

72+
func (cve *CveStore) SetStandardSeverity(severity bean.Severity) {
73+
cve.StandardSeverity = &severity
74+
}
75+
5176
type CveStoreRepository interface {
5277
GetConnection() (dbConnection *pg.DB)
5378
Save(model *CveStore) error

0 commit comments

Comments
 (0)