feat: group vulnerabilities based on os/base image (#50)

* feat: using standard severity instead of severity

* fix: remove dml on cve_store and handle it in code handling this versioning

* fix: minor fix

* feat: group vulnerabilities based on os/base image

* fix: minor fix

* chore: move test file to tests folder

Author: gireesh-naidu

internal/step-lib/util/common-util/util.go

@@ -69,6 +69,12 @@ func ReadFile(fileName string) ([]byte, error) {
 func ParseJsonTemplate(inputTemplate string, data []byte) (string, error) {
 	tmpl := template.Must(template.New("").Funcs(template.FuncMap{
 		"add": func(a, b int) int { return a + b },
+		"len": func(sliceIf interface{}) int {
+			if slice, ok := sliceIf.([]any); ok {
+				return len(slice)
+			}
+			return 0
+		},
 	}).Parse(inputTemplate))
 	jsonMap := map[string]interface{}{}
 	err := json.Unmarshal(data, &jsonMap)

pkg/security/ImageScanService.go

@@ -274,6 +274,7 @@ func (impl *ImageScanServiceImpl) RegisterScanExecutionHistoryAndState(scanEvent
 		impl.Logger.Errorw("Failed to save executionHistory", "err", err, "model", executionHistoryModel)
 		return nil, executionHistoryDirPath, err
 	}
+
 	// creating folder for storing all details if not exist
 	isExist, err := helper.DoesFileExist(bean.ScanOutputDirectory)
 	if err != nil {
@@ -522,7 +523,7 @@ func (impl *ImageScanServiceImpl) ConvertEndStepOutputAndSaveVulnerabilities(ste
 
 	imageScanExecutionResults := make([]*repository.ImageScanExecutionResult, 0, len(vulnerabilities))
 	for _, vul := range vulnerabilities {
-		imageScanExecutionResult := createImageScanExecutionResultObject(executionHistoryId, vul.Name, vul.Package, vul.PackageVersion, vul.FixedInVersion, tool.Id)
+		imageScanExecutionResult := createImageScanExecutionResultObject(executionHistoryId, vul.Name, vul.Package, vul.PackageVersion, vul.FixedInVersion, vul.Class, vul.Type, vul.TargetName, tool.Id)
 		imageScanExecutionResults = append(imageScanExecutionResults, imageScanExecutionResult)
 	}
 	tx, err := impl.CveStoreRepository.GetConnection().Begin()
@@ -562,7 +563,7 @@ func (impl *ImageScanServiceImpl) ConvertEndStepOutputAndSaveVulnerabilities(ste
 }
 
 func isV1Template(resultDescriptorTemplate string) bool {
-	var mappings []bean.Mapping
+	var mappings []map[string]interface{}
 	err := json.Unmarshal([]byte(resultDescriptorTemplate), &mappings)
 	return err != nil && isValidGoTemplate(resultDescriptorTemplate) //checking error too because our new result descriptor template can pass go templating too as it contains a simple json
 }
@@ -593,33 +594,46 @@ func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV1(stepOutput []byte,
 
 func (impl *ImageScanServiceImpl) getImageScanOutputObjectsV2(stepOutput []byte, resultDescriptorTemplate string) ([]*bean.ImageScanOutputObject, error) {
 	var vulnerabilities []*bean.ImageScanOutputObject
-	var mappings []bean.Mapping
+	var mappings []map[string]interface{}
 	err := json.Unmarshal([]byte(resultDescriptorTemplate), &mappings)
 	if err != nil {
 		impl.Logger.Errorw("error in un-marshaling result descriptor template", "err", err, "resultDescriptorTemplate", resultDescriptorTemplate)
 		return nil, err
 	}
-	var processArray func(mapping bean.Mapping, value gjson.Result)
-	processArray = func(mapping bean.Mapping, value gjson.Result) {
+	var processArray func(mapping map[string]interface{}, value gjson.Result)
+	processArray = func(mapping map[string]interface{}, value gjson.Result) {
+		vulnerabilitiesPath := mapping[bean.MappingKeyPathToVulnerabilitiesArray].(string)
+		vulnerabilityDataKeyPathsMap := mapping[bean.MappingKeyPathToVulnerabilityDataKeys].(map[string]interface{})
+		resultDataKeyPathsMap := mapping[bean.MappingKeyPathToResultDataKeys].(map[string]interface{})
+
 		value.ForEach(func(_, nestedValue gjson.Result) bool {
-			if nestedValue.IsArray() {
-				// if the nested value is an array, recursively process it
-				processArray(mapping, nestedValue)
-			} else {
-				vulnerability := &bean.ImageScanOutputObject{
-					Name:           nestedValue.Get(mapping[bean.MappingKeyName]).String(),
-					Package:        nestedValue.Get(mapping[bean.MappingKeyPackage]).String(),
-					PackageVersion: nestedValue.Get(mapping[bean.MappingKeyPackageVersion]).String(),
-					FixedInVersion: nestedValue.Get(mapping[bean.MappingKeyFixedInVersion]).String(),
-					Severity:       nestedValue.Get(mapping[bean.MappingKeySeverity]).String(),
+			targetName, class, resType := "", "", ""
+			if nestedValue.IsObject() {
+				targetName, class, resType = nestedValue.Get(resultDataKeyPathsMap[bean.MappingTarget].(string)).String(), nestedValue.Get(resultDataKeyPathsMap[bean.MappingClass].(string)).String(), nestedValue.Get(resultDataKeyPathsMap[bean.MappingType].(string)).String()
+
+				if nestedValue.Get(vulnerabilitiesPath).IsArray() {
+					nestedValue.Get(vulnerabilitiesPath).ForEach(func(_, vul gjson.Result) bool {
+						vulnerability := &bean.ImageScanOutputObject{
+							Name:           vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyName].(string)).String(),
+							Package:        vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyPackage].(string)).String(),
+							PackageVersion: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyPackageVersion].(string)).String(),
+							FixedInVersion: vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeyFixedInVersion].(string)).String(),
+							Severity:       vul.Get(vulnerabilityDataKeyPathsMap[bean.MappingKeySeverity].(string)).String(),
+							TargetName:     targetName,
+							Class:          class,
+							Type:           resType,
+						}
+						vulnerabilities = append(vulnerabilities, vulnerability)
+						return true
+					})
 				}
-				vulnerabilities = append(vulnerabilities, vulnerability)
 			}
 			return true
 		})
 	}
+
 	for _, mapping := range mappings {
-		result := gjson.Get(string(stepOutput), mapping[bean.MappingKeyPathToVulnerabilitiesArray])
+		result := gjson.Get(string(stepOutput), mapping[bean.MappingKeyPathToResultsArray].(string))
 		if !result.Exists() {
 			continue
 		}
@@ -740,7 +754,7 @@ func (impl *ImageScanServiceImpl) CreateScanExecutionRegistryForClairV4(vs []*cl
 				cvesToUpdate = append(cvesToUpdate, cveStore)
 			}
 		}
-		imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.Package.Name, item.Package.Version, item.FixedInVersion, toolId)
+		imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.Package.Name, item.Package.Version, item.FixedInVersion, "", "", "", toolId)
 		imageScanExecutionResultsToBeSaved = append(imageScanExecutionResultsToBeSaved, imageScanExecutionResult)
 	}
 	tx, err := impl.CveStoreRepository.GetConnection().Begin()
@@ -803,7 +817,7 @@ func (impl *ImageScanServiceImpl) CreateScanExecutionRegistryForClairV2(vs []*cl
 				cvesToUpdate = append(cvesToUpdate, cveStore)
 			}
 		}
-		imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.FeatureName, item.FeatureVersion, item.FixedBy, toolId)
+		imageScanExecutionResult := createImageScanExecutionResultObject(executionHistory.Id, item.Name, item.FeatureName, item.FeatureVersion, item.FixedBy, "", "", "", toolId)
 		imageScanExecutionResultsToBeSaved = append(imageScanExecutionResultsToBeSaved, imageScanExecutionResult)
 	}
 	tx, err := impl.CveStoreRepository.GetConnection().Begin()

pkg/security/helper.go

@@ -18,13 +18,16 @@ func createCveStoreObject(name, version, fixedInVersion, severity string, userId
 	return cveStore
 }
 
-func createImageScanExecutionResultObject(executionHistoryId int, vulName, packageName, version, fixedInVersion string, toolId int) *repository.ImageScanExecutionResult {
+func createImageScanExecutionResultObject(executionHistoryId int, vulName, packageName, version, fixedInVersion, className, typeName, targetName string, toolId int) *repository.ImageScanExecutionResult {
 	return &repository.ImageScanExecutionResult{
 		ImageScanExecutionHistoryId: executionHistoryId,
 		CveStoreName:                vulName,
 		Package:                     packageName,
 		ScanToolId:                  toolId,
 		Version:                     version,
 		FixedVersion:                fixedInVersion,
+		Target:                      targetName,
+		Type:                        typeName,
+		Class:                       className,
 	}
 }

pkg/security/imageScanService_test.go

@@ -0,0 +1,65 @@
+package security
+
+import (
+	"fmt"
+	"github.com/devtron-labs/image-scanner/pkg/logger"
+	"github.com/tidwall/gjson"
+	"os"
+	"testing"
+)
+
+const jsonKeys = "[    \n      { \n        \"pathToResultArray\": \"Results\",\n        \"pathToVulnerabilitiesArray\": \"Vulnerabilities\",\n        \"vulnerabilityData\":{\n       \t\t\"name\": \"VulnerabilityID\",\n        \t\"package\": \"PkgName\", \n        \t\"packageVersion\": \"InstalledVersion\",\n        \t\"fixedInVersion\": \"FixedVersion\",\n        \t\"severity\": \"Severity\"\n        },\n        \"resultData\":{\n  \t\t\t\"target\":\"Target\",\n        \t\"class\":\"Class\",\n        \t\"type\":\"Type\"   \n       }\n      }\n]"
+
+func TestScan(t *testing.T) {
+	bytes, err := os.ReadFile("testTrivyScanResultV2.json")
+	if err != nil {
+		t.Fail()
+	}
+
+	logger, _ := logger.InitLogger()
+	impl := ImageScanServiceImpl{Logger: logger}
+	out, err := impl.getImageScanOutputObjectsV2(bytes, jsonKeys)
+	fmt.Println(out, err)
+	if err != nil || len(out) == 0 {
+		t.Fail()
+	}
+
+	cnt, err := countVuln(bytes)
+	if err != nil {
+		t.Fail()
+	}
+
+	if cnt != len(out) {
+		t.Fail()
+	}
+}
+
+func countVuln(jsonStr []byte) (int, error) {
+	result := gjson.Get(string(jsonStr), "Results")
+	if !result.Exists() {
+		return 0, nil
+	}
+
+	targetCountMap := make(map[string]int)
+
+	result.ForEach(func(_, nestedValue gjson.Result) bool {
+		if nestedValue.IsObject() {
+			count := 0
+			targetName := nestedValue.Get("Target").String()
+			if nestedValue.Get("Vulnerabilities").IsArray() {
+				nestedValue.Get("Vulnerabilities").ForEach(func(_, vul gjson.Result) bool {
+					count++
+					return true
+				})
+			}
+			targetCountMap[targetName] += count
+		}
+		return true
+	})
+
+	count := 0
+	for _, cnt := range targetCountMap {
+		count += cnt
+	}
+	return count, nil
+}

pkg/sql/bean/bean.go

@@ -50,6 +50,9 @@ const (
 )
 
 type ImageScanOutputObject struct {
+	TargetName     string `json:"targetName"`
+	Class          string `json:"class"`
+	Type           string `json:"type"`
 	Name           string `json:"name"`
 	Package        string `json:"package"`
 	PackageVersion string `json:"packageVersion"`
@@ -61,12 +64,18 @@ type ImageScanOutputObject struct {
 type Mapping map[string]string
 
 const (
-	MappingKeyPathToVulnerabilitiesArray = "pathToVulnerabilitiesArray"
-	MappingKeyName                       = "name"
-	MappingKeyPackage                    = "package"
-	MappingKeyPackageVersion             = "packageVersion"
-	MappingKeyFixedInVersion             = "fixedInVersion"
-	MappingKeySeverity                   = "severity"
+	MappingKeyPathToResultDataKeys        = "resultData"
+	MappingKeyPathToVulnerabilityDataKeys = "vulnerabilityData"
+	MappingKeyPathToResultsArray          = "pathToResultArray"
+	MappingKeyPathToVulnerabilitiesArray  = "pathToVulnerabilitiesArray"
+	MappingKeyName                        = "name"
+	MappingKeyPackage                     = "package"
+	MappingKeyPackageVersion              = "packageVersion"
+	MappingKeyFixedInVersion              = "fixedInVersion"
+	MappingKeySeverity                    = "severity"
+	MappingTarget                         = "target"
+	MappingType                           = "type"
+	MappingClass                          = "class"
 )
 
 type Severity int

pkg/sql/repository/ImageScanResultRepository.go

@@ -30,6 +30,9 @@ type ImageScanExecutionResult struct {
 	Package                     string   `sql:"package"`
 	Version                     string   `sql:"version"`
 	FixedVersion                string   `sql:"fixed_version"`
+	Target                      string   `sql:"target"`
+	Type                        string   `sql:"type"`
+	Class                       string   `sql:"class"`
 	CveStore                    CveStore
 	ImageScanExecutionHistory   ImageScanExecutionHistory
 }

tests/testTrivyScanResultV2.json

@@ -0,0 +1 @@
+{}