Merge branch 'main' of https://github.com/nexB/vulnerablecode into 1936-npm-importer-package-first

Author: michaelehab

vulnerabilities/importer.py

@@ -22,6 +22,7 @@
 from typing import Optional
 from typing import Set
 from typing import Tuple
+from typing import Union
 
 import pytz
 from dateutil import parser as dateparser
@@ -361,6 +362,7 @@ class AdvisoryData:
     weaknesses: List[int] = dataclasses.field(default_factory=list)
     severities: List[VulnerabilitySeverity] = dataclasses.field(default_factory=list)
     url: Optional[str] = None
+    original_advisory_text: Optional[str] = None
 
     def __post_init__(self):
         if self.date_published and not self.date_published.tzinfo:

vulnerabilities/importers/__init__.py

@@ -42,29 +42,41 @@
 from vulnerabilities.pipelines import pypa_importer
 from vulnerabilities.pipelines import pysec_importer
 from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2
+from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2
 from vulnerabilities.pipelines.v2_importers import (
     elixir_security_importer as elixir_security_importer_v2,
 )
-from vulnerabilities.pipelines.v2_importers import github_importer as github_importer_v2
+from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
+from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2
+from vulnerabilities.pipelines.v2_importers import mozilla_importer as mozilla_importer_v2
 from vulnerabilities.pipelines.v2_importers import npm_importer as npm_importer_v2
 from vulnerabilities.pipelines.v2_importers import nvd_importer as nvd_importer_v2
+from vulnerabilities.pipelines.v2_importers import oss_fuzz as oss_fuzz_v2
+from vulnerabilities.pipelines.v2_importers import postgresql_importer as postgresql_importer_v2
 from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2
 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2
 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2
+from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2
 from vulnerabilities.utils import create_registry
 
 IMPORTERS_REGISTRY = create_registry(
     [
         nvd_importer_v2.NVDImporterPipeline,
         elixir_security_importer_v2.ElixirSecurityImporterPipeline,
-        github_importer_v2.GitHubAPIImporterPipeline,
         npm_importer_v2.NpmImporterPipeline,
         vulnrichment_importer_v2.VulnrichImporterPipeline,
         apache_httpd_v2.ApacheHTTPDImporterPipeline,
         pypa_importer_v2.PyPaImporterPipeline,
         gitlab_importer_v2.GitLabImporterPipeline,
         pysec_importer_v2.PyPIImporterPipeline,
+        xen_importer_v2.XenImporterPipeline,
+        curl_importer_v2.CurlImporterPipeline,
+        oss_fuzz_v2.OSSFuzzImporterPipeline,
+        istio_importer_v2.IstioImporterPipeline,
+        postgresql_importer_v2.PostgreSQLImporterPipeline,
+        mozilla_importer_v2.MozillaImporterPipeline,
+        github_osv_importer_v2.GithubOSVImporterPipeline,
         nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,
         gitlab_importer.GitLabImporterPipeline,

vulnerabilities/importers/curl.py

@@ -97,7 +97,7 @@ def parse_advisory_data(raw_data) -> AdvisoryData:
         ...     ]
         ... }
         >>> parse_advisory_data(raw_data)
-        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], url='https://curl.se/docs/CVE-2024-2379.json')
+        AdvisoryData(advisory_id='', aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], references_v2=[], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], severities=[], url='https://curl.se/docs/CVE-2024-2379.json', original_advisory_text=None)
     """
 
     affected = get_item(raw_data, "affected")[0] if len(get_item(raw_data, "affected")) > 0 else []

vulnerabilities/importers/osv.py

@@ -7,6 +7,7 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 from typing import Iterable
 from typing import List
@@ -109,7 +110,7 @@ def parse_advisory_data(
 
 
 def parse_advisory_data_v2(
-    raw_data: dict, supported_ecosystems, advisory_url: str
+    raw_data: dict, supported_ecosystems, advisory_url: str, advisory_text: str
 ) -> Optional[AdvisoryData]:
     """
     Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and
@@ -173,6 +174,7 @@ def parse_advisory_data_v2(
         date_published=date_published,
         weaknesses=weaknesses,
         url=advisory_url,
+        original_advisory_text=advisory_text or json.dumps(raw_data, indent=2, ensure_ascii=False),
     )
 
 

vulnerabilities/improvers/__init__.py

@@ -10,7 +10,6 @@
 from vulnerabilities.improvers import valid_versions
 from vulnerabilities.improvers import vulnerability_status
 from vulnerabilities.pipelines import add_cvss31_to_CVEs
-from vulnerabilities.pipelines import collect_commits
 from vulnerabilities.pipelines import compute_advisory_todo
 from vulnerabilities.pipelines import compute_package_risk
 from vulnerabilities.pipelines import compute_package_version_rank
@@ -20,7 +19,6 @@
 from vulnerabilities.pipelines import flag_ghost_packages
 from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
 from vulnerabilities.pipelines import remove_duplicate_advisories
-from vulnerabilities.pipelines.v2_improvers import collect_commits as collect_commits_v2
 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2
 from vulnerabilities.pipelines.v2_improvers import (
     computer_package_version_rank as compute_version_rank_v2,
@@ -58,7 +56,6 @@
         enhance_with_exploitdb.ExploitDBImproverPipeline,
         compute_package_risk.ComputePackageRiskPipeline,
         compute_package_version_rank.ComputeVersionRankPipeline,
-        collect_commits.CollectFixCommitsPipeline,
         add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
         remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
         populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
@@ -68,7 +65,6 @@
         enhance_with_metasploit_v2.MetasploitImproverPipeline,
         compute_package_risk_v2.ComputePackageRiskPipeline,
         compute_version_rank_v2.ComputeVersionRankPipeline,
-        collect_commits_v2.CollectFixCommitsPipeline,
         compute_advisory_todo.ComputeToDo,
     ]
 )

vulnerabilities/migrations/0099_advisoryv2_original_advisory_text.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.22 on 2025-07-16 08:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0098_alter_advisory_options_alter_advisoryalias_options_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="original_advisory_text",
+            field=models.TextField(
+                blank=True,
+                help_text="Raw advisory data as collected from the upstream datasource.",
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -2744,6 +2744,12 @@ class AdvisoryV2(models.Model):
         blank=True, null=True, help_text="UTC Date on which the advisory was imported"
     )
 
+    original_advisory_text = models.TextField(
+        blank=True,
+        null=True,
+        help_text="Raw advisory data as collected from the upstream datasource.",
+    )
+
     affecting_packages = models.ManyToManyField(
         "PackageV2",
         related_name="affected_by_advisories",

vulnerabilities/pipelines/__init__.py

@@ -56,6 +56,7 @@ def __init__(
         run_instance: PipelineRun = None,
         selected_groups: List = None,
         selected_steps: List = None,
+        **kwargs,
     ):
         """Load the Pipeline class."""
         self.run = run_instance
@@ -68,6 +69,9 @@ def __init__(
         self.execution_log = []
         self.current_step = ""
 
+        # Optional args used as input in downstream pipeline steps
+        self.inputs = kwargs
+
     def append_to_log(self, message):
         if self.run and self.run.pipeline.live_logging:
             self.run.append_to_log(message)
@@ -303,13 +307,20 @@ def collect_and_store_advisories(self):
             if advisory is None:
                 self.log("Advisory is None, skipping")
                 continue
-            if _obj := insert_advisory_v2(
-                advisory=advisory,
-                pipeline_id=self.pipeline_id,
-                get_advisory_packages=self.get_advisory_packages,
-                logger=self.log,
-            ):
-                collected_advisory_count += 1
+            try:
+                if _obj := insert_advisory_v2(
+                    advisory=advisory,
+                    pipeline_id=self.pipeline_id,
+                    get_advisory_packages=self.get_advisory_packages,
+                    logger=self.log,
+                ):
+                    collected_advisory_count += 1
+            except Exception as e:
+                self.log(
+                    f"Failed to import advisory: {advisory!r} with error {e!r}:\n{traceback_format_exc()}",
+                    level=logging.ERROR,
+                )
+                continue
 
         self.log(f"Successfully collected {collected_advisory_count:,d} advisories")
 

vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py

@@ -7,13 +7,15 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+import json
 import logging
 import re
 import urllib.parse
 from typing import Iterable
 
 import requests
 from bs4 import BeautifulSoup
+from dateutil import parser as date_parser
 from packageurl import PackageURL
 from univers.version_constraint import VersionConstraint
 from univers.version_range import ApacheVersionRange
@@ -272,8 +274,11 @@ def to_advisory(self, data):
                     versions_data.append(version_data)
 
         fixed_versions = []
+        date_published = None
         for timeline_object in data.get("timeline") or []:
             timeline_value = timeline_object.get("value")
+            if timeline_value == "public":
+                date_published = timeline_object.get("time")
             if "release" in timeline_value:
                 split_timeline_value = timeline_value.split(" ")
                 if "never" in timeline_value:
@@ -307,6 +312,8 @@ def to_advisory(self, data):
             weaknesses=weaknesses,
             url=reference.url,
             severities=severities,
+            original_advisory_text=json.dumps(data, indent=2, ensure_ascii=False),
+            date_published=date_parser.parse(date_published) if date_published else None,
         )
 
     def to_version_ranges(self, versions_data, fixed_versions):