diff --git a/vulnerabilities/migrations/0115_impactedpackageaffecting_and_more.py b/vulnerabilities/migrations/0115_impactedpackageaffecting_and_more.py new file mode 100644 index 000000000..a0f35b624 --- /dev/null +++ b/vulnerabilities/migrations/0115_impactedpackageaffecting_and_more.py @@ -0,0 +1,107 @@ +# Generated by Django 5.2.11 on 2026-02-19 11:12 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0114_advisoryv2_related_advisory_severities"), + ] + + operations = [ + migrations.RemoveField( + model_name="impactedpackage", + name="affecting_packages", + ), + migrations.RemoveField( + model_name="impactedpackage", + name="fixed_by_packages", + ), + migrations.AlterField( + model_name="advisoryv2", + name="date_collected", + field=models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="UTC Date on which the advisory was collected", + ), + ), + migrations.CreateModel( + name="ImpactedPackageAffecting", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ( + "impacted_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="vulnerabilities.impactedpackage", + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.packagev2" + ), + ), + ], + options={ + "unique_together": {("impacted_package", "package")}, + }, + ), + migrations.AddField( + model_name="impactedpackage", + name="affecting_packages", + field=models.ManyToManyField( + help_text="Packages vulnerable to this impact.", + related_name="affected_in_impacts", + through="vulnerabilities.ImpactedPackageAffecting", + to="vulnerabilities.packagev2", + ), + ), + migrations.CreateModel( + name="ImpactedPackageFixedBy", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True, db_index=True)), + ( + "impacted_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="vulnerabilities.impactedpackage", + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.packagev2" + ), + ), + ], + options={ + "unique_together": {("impacted_package", "package")}, + }, + ), + migrations.AddField( + model_name="impactedpackage", + name="fixed_by_packages", + field=models.ManyToManyField( + help_text="Packages fixing the vulnerable packages in this impact.", + related_name="fixed_in_impacts", + through="vulnerabilities.ImpactedPackageFixedBy", + to="vulnerabilities.packagev2", + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index a83db0ad6..1981a2861 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2961,9 +2961,15 @@ class AdvisoryV2(models.Model): ) date_published = models.DateTimeField( - blank=True, null=True, help_text="UTC Date of publication of the advisory" + blank=True, + null=True, + help_text="UTC Date of publication of the advisory", + ) + date_collected = models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="UTC Date on which the advisory was collected", ) - date_collected = models.DateTimeField(help_text="UTC Date on which the advisory was collected") original_advisory_text = models.TextField( blank=True, @@ -3137,13 +3143,15 @@ class ImpactedPackage(models.Model): affecting_packages = models.ManyToManyField( "PackageV2", related_name="affected_in_impacts", + through="ImpactedPackageAffecting", help_text="Packages vulnerable to this impact.", ) fixed_by_packages = models.ManyToManyField( "PackageV2", related_name="fixed_in_impacts", - help_text="Packages vulnerable to this impact.", + through="ImpactedPackageFixedBy", + help_text="Packages fixing the vulnerable packages in this impact.", ) introduced_by_package_commit_patches = models.ManyToManyField( @@ -3491,6 +3499,44 @@ def current_version(self): return self.version_class(self.version) +class ImpactedPackageAffecting(models.Model): + impacted_package = models.ForeignKey( + ImpactedPackage, + on_delete=models.CASCADE, + ) + package = models.ForeignKey( + PackageV2, + on_delete=models.CASCADE, + ) + + created_at = models.DateTimeField( + auto_now_add=True, + db_index=True, + ) + + class Meta: + unique_together = ("impacted_package", "package") + + +class ImpactedPackageFixedBy(models.Model): + impacted_package = models.ForeignKey( + ImpactedPackage, + on_delete=models.CASCADE, + ) + package = models.ForeignKey( + PackageV2, + on_delete=models.CASCADE, + ) + + created_at = models.DateTimeField( + auto_now_add=True, + db_index=True, + ) + + class Meta: + unique_together = ("impacted_package", "package") + + class AdvisoryExploit(models.Model): """ A vulnerability exploit is code used to diff --git a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py index 27c6546ff..93eacaa1f 100644 --- a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py +++ b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py @@ -8,25 +8,32 @@ import itertools +import json import shutil +from datetime import datetime from operator import attrgetter from pathlib import Path -import saneyaml from aboutcode.pipeline import LoopProgress from django.conf import settings -from django.db.models import Prefetch +from django.utils import timezone from aboutcode.federated import DataFederation -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import ImpactedPackage -from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipes import export from vulnerabilities.pipes import federatedcode +from vulnerabilities.utils import load_json class FederatePackageVulnerabilities(VulnerableCodePipeline): - """Export package vulnerabilities and advisory to FederatedCode.""" + """ + Export package vulnerabilities and advisories to FederatedCode. + + - Export all packages and advisories to FederatedCode. + - On subsequent runs, export incremental updates. + - Remove `checkpoint.json` file from FederatedCode git repository to + force a full re-export of all packages and advisories. + """ pipeline_id = "federate_vulnerabilities_v2" @@ -37,8 +44,10 @@ def steps(cls): cls.create_federatedcode_working_dir, cls.fetch_federation_config, cls.clone_federation_repository, + cls.load_checkpoint, cls.publish_package_related_advisories, cls.publish_advisories, + cls.save_checkpoint, cls.delete_working_dir, ) @@ -64,39 +73,45 @@ def clone_federation_repository(self): clone_path=self.working_path / "advisories-data", logger=self.log, ) + self.repo_path = Path(self.repo.working_dir) + + def load_checkpoint(self): + checkpoint_file = self.repo_path / "checkpoint.json" + data = {} + self.start_time = str(timezone.now()) + self.checkpoint = None + if checkpoint_file.exists(): + data = load_json(checkpoint_file) + + if last_run := data.get("last_run"): + self.checkpoint = datetime.fromisoformat(last_run) def publish_package_related_advisories(self): """Publish package advisories relations to FederatedCode""" - repo_path = Path(self.repo.working_dir) commit_count = 1 - batch_size = 2000 + batch_size = 4000 chunk_size = 500 files_to_commit = set() - distinct_packages_count = ( - PackageV2.objects.values("type", "namespace", "name", "version") - .distinct("type", "namespace", "name", "version") - .count() - ) - package_qs = package_prefetched_qs() + packages_count, package_qs = export.package_prefetched_qs(self.checkpoint) grouped_packages = itertools.groupby( package_qs.iterator(chunk_size=chunk_size), key=attrgetter("type", "namespace", "name", "version"), ) - self.log(f"Exporting advisory relation for {distinct_packages_count} packages.") + self.log(f"Exporting advisory relation for {packages_count} packages.") progress = LoopProgress( - total_iterations=distinct_packages_count, + total_iterations=packages_count, progress_step=5, logger=self.log, ) for _, packages in progress.iter(grouped_packages): - purl, package_vulnerabilities = get_package_related_advisory(packages) + purl, package_vulnerabilities = export.get_package_related_advisory(packages) package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl) package_vulnerability_path = f"packages/{package_repo}/{datafile_path}" - write_file( - repo_path=repo_path, + export.write_file( + repo_path=self.repo_path, file_path=package_vulnerability_path, data=package_vulnerabilities, ) @@ -104,7 +119,10 @@ def publish_package_related_advisories(self): if len(files_to_commit) > batch_size: if federatedcode.commit_and_push_changes( - commit_message=self.commit_message("package advisory relations", commit_count), + commit_message=self.commit_message( + "Add new package advisory relations", + commit_count, + ), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, @@ -115,7 +133,7 @@ def publish_package_related_advisories(self): if files_to_commit: federatedcode.commit_and_push_changes( commit_message=self.commit_message( - "package advisory relations", + "Add new package advisory relations", commit_count, commit_count, ), @@ -124,16 +142,15 @@ def publish_package_related_advisories(self): logger=self.log, ) - self.log(f"Federated {distinct_packages_count} package advisories.") + self.log(f"Federated {packages_count} package advisories.") def publish_advisories(self): """Publish advisory to FederatedCode""" - repo_path = Path(self.repo.working_dir) commit_count = 1 - batch_size = 2000 + batch_size = 4000 chunk_size = 1000 files_to_commit = set() - advisory_qs = advisory_prefetched_qs() + advisory_qs = export.advisory_prefetched_qs(self.checkpoint) advisory_count = advisory_qs.count() self.log(f"Exporting {advisory_count} advisory.") @@ -143,10 +160,10 @@ def publish_advisories(self): logger=self.log, ) for advisory in progress.iter(advisory_qs.iterator(chunk_size=chunk_size)): - advisory_data = serialize_advisory(advisory) + advisory_data = export.serialize_advisory(advisory) adv_file = f"advisories/{advisory.avid}.yml" - write_file( - repo_path=repo_path, + export.write_file( + repo_path=self.repo_path, file_path=adv_file, data=advisory_data, ) @@ -154,7 +171,7 @@ def publish_advisories(self): if len(files_to_commit) > batch_size: if federatedcode.commit_and_push_changes( - commit_message=self.commit_message("advisories", commit_count), + commit_message=self.commit_message("Add new advisories", commit_count), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, @@ -165,7 +182,7 @@ def publish_advisories(self): if files_to_commit: federatedcode.commit_and_push_changes( commit_message=self.commit_message( - "advisories", + "Add new advisories", commit_count, commit_count, ), @@ -176,6 +193,19 @@ def publish_advisories(self): self.log(f"Successfully federated {advisory_count} advisories.") + def save_checkpoint(self): + checkpoint_file = self.repo_path / "checkpoint.json" + checkpoint = {"last_run": self.start_time} + with open(checkpoint_file, "w") as f: + json.dump(checkpoint, f, indent=2) + + federatedcode.commit_and_push_changes( + commit_message=self.commit_message("Update checkpoint", 1, 1), + repo=self.repo, + files_to_commit=[checkpoint_file], + logger=self.log, + ) + def delete_working_dir(self): """Remove temporary working dir.""" if hasattr(self, "working_path") and self.working_path: @@ -186,122 +216,13 @@ def on_failure(self): def commit_message( self, - item_type, + heading, commit_count, total_commit_count="many", ): """Commit message for pushing package vulnerability.""" return federatedcode.commit_message( - item_type=item_type, + heading=heading, commit_count=commit_count, total_commit_count=total_commit_count, ) - - -def package_prefetched_qs(): - return ( - PackageV2.objects.order_by("type", "namespace", "name", "version") - .only("package_url", "type", "namespace", "name", "version") - .prefetch_related( - Prefetch( - "affected_in_impacts", - queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( - Prefetch( - "advisory", - queryset=AdvisoryV2.objects.only("avid"), - ) - ), - ), - Prefetch( - "fixed_in_impacts", - queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( - Prefetch( - "advisory", - queryset=AdvisoryV2.objects.only("avid"), - ) - ), - ), - ) - ) - - -def get_package_related_advisory(packages): - package_vulnerabilities = [] - for package in packages: - affected_by_vulnerabilities = [ - impact.advisory.avid for impact in package.affected_in_impacts.all() - ] - fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()] - - package_vulnerability = { - "purl": package.package_url, - "affected_by_advisories": sorted(affected_by_vulnerabilities), - "fixing_advisories": sorted(fixing_vulnerabilities), - } - package_vulnerabilities.append(package_vulnerability) - - return package.package_url, package_vulnerabilities - - -def advisory_prefetched_qs(): - return AdvisoryV2.objects.prefetch_related( - "impacted_packages", - "aliases", - "references", - "severities", - "weaknesses", - ) - - -def serialize_severity(sev): - return { - "score": sev.value, - "scoring_system": sev.scoring_system, - "scoring_elements": sev.scoring_elements, - "published_at": str(sev.published_at), - "url": sev.url, - } - - -def serialize_references(reference): - return { - "url": reference.url, - "reference_type": reference.reference_type, - "reference_id": reference.reference_id, - } - - -def serialize_advisory(advisory): - """Return a plain data mapping serialized from advisory object.""" - aliases = sorted([a.alias for a in advisory.aliases.all()]) - severities = [serialize_severity(sev) for sev in advisory.severities.all()] - weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()] - references = [serialize_references(ref) for ref in advisory.references.all()] - impacts = [ - { - "purl": impact.base_purl, - "affected_versions": impact.affecting_vers, - "fixed_versions": impact.fixed_vers, - } - for impact in advisory.impacted_packages.all() - ] - - return { - "advisory_id": advisory.advisory_id, - "datasource_id": advisory.avid, - "datasource_url": advisory.url, - "aliases": aliases, - "summary": advisory.summary, - "impacted_packages": impacts, - "severities": severities, - "weaknesses": weaknesses, - "references": references, - } - - -def write_file(repo_path, file_path, data): - """Write ``data`` as YAML to ``repo_path``.""" - write_to = repo_path / file_path - write_to.parent.mkdir(parents=True, exist_ok=True) - with open(write_to, encoding="utf-8", mode="w") as f: - f.write(saneyaml.dump(data)) diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index 1d181ee2f..48d40e439 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -17,6 +17,7 @@ from univers.version_range import VersionRange from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import ImpactedPackageAffecting from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.fetchcode_utils import get_versions @@ -64,7 +65,7 @@ def unfurl_version_range(self): processed_affected_packages_count += bulk_create_with_m2m( purls=affected_purls, impact=impact, - relation=ImpactedPackage.affecting_packages.through, + relation=ImpactedPackageAffecting, logger=self.log, ) processed_impacted_packages_count += 1 @@ -118,7 +119,7 @@ def bulk_create_with_m2m(purls, impact, relation, logger): affected_packages_v2 = PackageV2.objects.bulk_get_or_create_from_purls(purls=purls) relations = [ - relation(impactedpackage=impact, packagev2=package) for package in affected_packages_v2 + relation(impacted_package=impact, package=package) for package in affected_packages_v2 ] try: diff --git a/vulnerabilities/pipes/__init__.py b/vulnerabilities/pipes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index 9d242c01f..96e1d4871 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -313,7 +313,6 @@ def insert_advisory_v2( "avid": f"{pipeline_id}/{advisory.advisory_id}", "summary": advisory.summary, "date_published": advisory.date_published, - "date_collected": datetime.now(timezone.utc), "original_advisory_text": advisory.original_advisory_text, "url": advisory.url, "precedence": precedence, diff --git a/vulnerabilities/pipes/export.py b/vulnerabilities/pipes/export.py new file mode 100644 index 000000000..8b77d53cb --- /dev/null +++ b/vulnerabilities/pipes/export.py @@ -0,0 +1,165 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import saneyaml +from django.db.models import Prefetch + +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import ImpactedPackageAffecting +from vulnerabilities.models import ImpactedPackageFixedBy +from vulnerabilities.models import PackageCommitPatch +from vulnerabilities.models import PackageV2 + + +def package_prefetched_qs(checkpoint): + count = None + qs = ( + PackageV2.objects.order_by("type", "namespace", "name", "version") + .only("package_url", "type", "namespace", "name", "version") + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( + Prefetch( + "advisory", + queryset=AdvisoryV2.objects.only("avid"), + ) + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.only("advisory_id").prefetch_related( + Prefetch( + "advisory", + queryset=AdvisoryV2.objects.only("avid"), + ) + ), + ), + ) + ) + + if checkpoint: + affected_package_ids_qs = ( + ImpactedPackageAffecting.objects.filter(created_at__gte=checkpoint) + .values_list("package_id", flat=True) + .distinct() + ) + fixing_package_ids_qs = ( + ImpactedPackageFixedBy.objects.filter(created_at__gte=checkpoint) + .values_list("package_id", flat=True) + .distinct() + ) + + updated_packages = affected_package_ids_qs.union(fixing_package_ids_qs) + count = updated_packages.count() + qs = qs.filter(id__in=updated_packages) + + count = qs.count() if not count else count + + return count, qs + + +def get_package_related_advisory(packages): + package_vulnerabilities = [] + for package in packages: + affected_by_vulnerabilities = [ + impact.advisory.avid for impact in package.affected_in_impacts.all() + ] + fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()] + + package_vulnerability = { + "purl": package.package_url, + "affected_by_advisories": sorted(affected_by_vulnerabilities), + "fixing_advisories": sorted(fixing_vulnerabilities), + } + package_vulnerabilities.append(package_vulnerability) + + return package.package_url, package_vulnerabilities + + +def advisory_prefetched_qs(checkpoint): + qs = AdvisoryV2.objects.order_by("date_collected").prefetch_related( + "impacted_packages", + "aliases", + "references", + "severities", + "weaknesses", + ) + + return qs.filter(date_collected__gte=checkpoint) if checkpoint else qs + + +def serialize_severity(sev): + return { + "score": sev.value, + "scoring_system": sev.scoring_system, + "scoring_elements": sev.scoring_elements, + "published_at": str(sev.published_at), + "url": sev.url, + } + + +def serialize_references(reference): + return { + "url": reference.url, + "reference_type": reference.reference_type, + "reference_id": reference.reference_id, + } + + +def serialize_commit_patches(patches): + return [ + { + "vcs_url": p.vcs_url, + "commit": p.commit_hash, + } + for p in patches.all() + ] + + +def serialize_advisory(advisory): + """Return a plain data mapping serialized from advisory object.""" + aliases = sorted([a.alias for a in advisory.aliases.all()]) + severities = [serialize_severity(sev) for sev in advisory.severities.all()] + weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()] + references = [serialize_references(ref) for ref in advisory.references.all()] + impacts = [ + { + "purl": impact.base_purl, + "affected_versions": impact.affecting_vers, + "fixed_versions": impact.fixed_vers, + "fixed_in_commits": serialize_commit_patches( + impact.fixed_by_package_commit_patches, + ), + "introduced_in_commits": serialize_commit_patches( + impact.introduced_by_package_commit_patches, + ), + } + for impact in advisory.impacted_packages.all() + ] + + return { + "advisory_id": advisory.advisory_id, + "datasource_id": advisory.avid, + "datasource_url": advisory.url, + "aliases": aliases, + "summary": advisory.summary, + "impacted_packages": impacts, + "severities": severities, + "weaknesses": weaknesses, + "references": references, + } + + +def write_file(repo_path, file_path, data): + """Write ``data`` as YAML to ``repo_path``.""" + write_to = repo_path / file_path + write_to.parent.mkdir(parents=True, exist_ok=True) + with open(write_to, encoding="utf-8", mode="w") as f: + f.write(saneyaml.dump(data)) diff --git a/vulnerabilities/pipes/federatedcode.py b/vulnerabilities/pipes/federatedcode.py index 560519c8d..3afda452e 100644 --- a/vulnerabilities/pipes/federatedcode.py +++ b/vulnerabilities/pipes/federatedcode.py @@ -157,7 +157,7 @@ def commit_changes(repo, files_to_commit, commit_message): ) -def commit_message(item_type, commit_count, total_commit_count): +def commit_message(heading, commit_count, total_commit_count): """Commit message for pushing Package vulnerability.""" from vulnerablecode import __version__ as VERSION @@ -167,7 +167,7 @@ def commit_message(item_type, commit_count, total_commit_count): tool_name = "pkg:github/aboutcode-org/vulnerablecode" return f"""\ - Add new {item_type} ({commit_count}/{total_commit_count}) + {heading} ({commit_count}/{total_commit_count}) Tool: {tool_name}@v{VERSION} diff --git a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py index 0d6fa32a5..a0bbbb772 100644 --- a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py +++ b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py @@ -21,6 +21,7 @@ from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.pipelines import insert_advisory_v2 from vulnerabilities.pipelines.exporters.federate_vulnerabilities import ( FederatePackageVulnerabilities, @@ -68,8 +69,13 @@ def setUp(self): package=PackageURL.from_string("pkg:npm/foobar"), affected_version_range=VersionRange.from_string("vers:npm/>=1.2.4"), fixed_version_range=VersionRange.from_string("vers:npm/2.0.0"), + fixed_by_commit_patches=[ + PackageCommitPatchData( + vcs_url="https://foobar.vcs/", + commit_hash="982f801f", + ) + ], introduced_by_commit_patches=[], - fixed_by_commit_patches=[], ), ], patches=[], @@ -99,8 +105,11 @@ def test_vulnerabilities_federation_v2(self, mock_check_fed, mock_commit, mock_c working_dir = Path(tempfile.mkdtemp()) pipeline = FederatePackageVulnerabilities() pipeline.repo = Repo.init(working_dir) + pipeline.repo_path = working_dir pipeline.log = self.logger.write - pipeline.execute() + exit_code, _ = pipeline.execute() + + self.assertEqual(exit_code, 0) result_advisories_yml = next(working_dir.rglob("1.2.4/advisories.yml")) result_advisory1_yml = next(working_dir.rglob("ADV-001.yml")) diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py b/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py index a175bbdb9..2319f27d0 100644 --- a/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_unfurl_version_range.py @@ -8,40 +8,59 @@ # +from datetime import datetime +from datetime import timedelta from unittest.mock import patch from django.test import TestCase +from packageurl import PackageURL +from univers.version_range import VersionRange +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines.v2_improvers.unfurl_version_range import UnfurlVersionRangePipeline +from vulnerabilities.pipes.advisory import insert_advisory_v2 class TestUnfurlVersionRangePipeline(TestCase): def setUp(self): - self.advisory1 = AdvisoryV2.objects.create( - datasource_id="ghsa", + advisory1 = AdvisoryDataV2( + summary="Test advisory", + aliases=["CVE-2025-0001"], + references=[], + severities=[], + weaknesses=[], + affected_packages=[ + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/>3.2.1|<4.0.0"), + fixed_version_range=VersionRange.from_string("vers:npm/4.0.0"), + introduced_by_commit_patches=[], + fixed_by_commit_patches=[], + ), + ], + patches=[], advisory_id="GHSA-1234", - avid="ghsa/GHSA-1234", - unique_content_id="f" * 64, + date_published=datetime.now() - timedelta(days=10), url="https://example.com/advisory", - date_collected="2025-07-01T00:00:00Z", ) - - self.impact1 = ImpactedPackage.objects.create( - advisory=self.advisory1, - base_purl="pkg:npm/foobar", - affecting_vers="vers:npm/>3.2.1|<4.0.0", - fixed_vers=None, + insert_advisory_v2( + advisory=advisory1, + pipeline_id="test_pipeline_v2", ) @patch("vulnerabilities.pipelines.v2_improvers.unfurl_version_range.get_purl_versions") def test_affecting_version_range_unfurl(self, mock_fetch): - self.assertEqual(0, PackageV2.objects.count()) + self.assertEqual(1, PackageV2.objects.count()) mock_fetch.return_value = {"3.4.1", "3.9.0", "2.1.0", "4.0.0", "4.1.0"} pipeline = UnfurlVersionRangePipeline() pipeline.execute() - self.assertEqual(2, PackageV2.objects.count()) - self.assertEqual(2, self.impact1.affecting_packages.count()) + advisory = AdvisoryV2.objects.first() + impact = advisory.impacted_packages.first() + + self.assertEqual(3, PackageV2.objects.count()) + self.assertEqual(1, impact.fixed_by_packages.count()) + self.assertEqual(2, impact.affecting_packages.count()) diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml index de4faff4f..57ea7e36c 100644 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-001-expected.yml @@ -8,6 +8,8 @@ impacted_packages: - purl: pkg:npm/foobar affected_versions: vers:npm/<=1.2.3 fixed_versions: vers:npm/1.2.4 + fixed_in_commits: [] + introduced_in_commits: [] severities: [] weaknesses: [] references: [] diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml index 1391dfa01..0b1861940 100644 --- a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-002-expected.yml @@ -8,6 +8,10 @@ impacted_packages: - purl: pkg:npm/foobar affected_versions: vers:npm/>=1.2.4 fixed_versions: vers:npm/2.0.0 + fixed_in_commits: + - vcs_url: https://foobar.vcs/ + commit: 982f801f + introduced_in_commits: [] severities: [] weaknesses: [] references: []