Sca fixed in improvements (#192)

* version needed for sbom-repo dependency

* exclude empty fixed in from vuln counters

* pkg-config, mysqlclient 2.2.4

* .

* dep versions

* fixes sbomrepor deps requirement

* dependencies

* .

* python 3.9 on integration test

Author: fpintoppb

.github/workflows/integration.yml

@@ -12,6 +12,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9' 
+      
       - name: Set-up environment
         run: pip install -r surface/requirements_test.txt
 

dev/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get update \
         libldap2-dev \
         libsasl2-dev \
         git \
+        pkg-config \
  && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /wheels

dev/Dockerfile-IN-A-BOX

@@ -7,6 +7,7 @@ RUN apt-get update \
         libldap2-dev \
         libsasl2-dev \
         git \
+        pkg-config \
  && rm -rf /var/lib/apt/lists/*
 
 RUN --mount=type=bind,target=/tmpapp \

pyproject.toml

@@ -38,4 +38,4 @@ select = [
 src = ['surface', 'e2e']
 
 [tool.ruff.isort]
-known-first-party = ["theme", "dkron", "django_restful_admin", "slackbot", "dbcleanup", "olympus", "notifications", "ppbenviron", "logbasecommand", "impersonate", "apitokens"]
+known-first-party = ["theme", "dkron", "django_restful_admin", "slackbot", "dbcleanup", "olympus", "notifications", "ppbenviron", "logbasecommand", "impersonate", "apitokens", "sbomrepo"]

surface/requirements.txt

@@ -1,5 +1,4 @@
 # Core Libraries
-
 Django==3.2.25
 django-admin-rangefilter==0.11.0
 django-after-response==0.2.2
@@ -27,7 +26,7 @@ django-impersonator==0.0.2
 django-apitokens==0.0.2
 django-sbomrepo==0.0.6
 
-mysqlclient==2.0.3
+mysqlclient==2.2.4
 tqdm==4.65.0  # for core_utils that is not really a app/package ..?
 django-database-locks==0.5  # distributed locks (on mysql)
 django-bulk-update-or-create==0.3.0  # for faster batch operations with update_or_create

surface/sca/admin.py

@@ -17,10 +17,10 @@
 from core_utils.admin_filters import DefaultFilterMixin
 from core_utils.utils import admin_reverse
 from dkron.utils import run_async
+from inventory.models import GitSource
 from sca import models
 from sca.utils import only_highest_version_dependencies
 from theme.filters import RelatedFieldAjaxListFilter
-from inventory.models import GitSource
 
 logger = logging.getLogger(__name__)
 
@@ -190,9 +190,9 @@ class Meta:
 
     def filter_fixed_in(self, queryset, name, value):
         if value == "true":
-            return queryset.exclude(Q(fixed_in="") | Q(fixed_in__isnull=True))
+            return queryset.exclude(fixed_in="")
         elif value == "false":
-            return queryset.filter(Q(fixed_in="") | Q(fixed_in__isnull=True))
+            return queryset.filter(fixed_in="")
         return queryset
 
 

surface/sca/migrations/0002_alter_scafinding_fixed_in.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-09-16 10:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sca', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='scafinding',
+            name='fixed_in',
+            field=models.TextField(default=''),
+        ),
+    ]

surface/sca/models.py

@@ -2,7 +2,6 @@
 from enum import Enum
 from typing import Union
 
-from bulk_update_or_create import BulkUpdateOrCreateQuerySet
 from django.db import models
 from django.db.models import Case, Count, Q, When
 
@@ -164,7 +163,7 @@ def get_dependencies(root_dependency: "SCADependency") -> list:
     def update_vulnerability_counters(self) -> "SCAFindingCounter":
         severity_counters = (
             SCAFinding.objects.filter(
-                (Q(fixed_in__isnull=False) | Q(finding_type=SCAFinding.FindingType.EOL)),
+                (Q(fixed_in__gt="") | Q(finding_type=SCAFinding.FindingType.EOL)),
                 dependency__purl__in=self.dependencies,
                 state__in=(SCAFinding.State.NEW, SCAFinding.State.OPEN),
             )
@@ -247,7 +246,7 @@ class FindingType(models.IntegerChoices):
     vuln_id = models.CharField(max_length=128)
     published = models.DateTimeField()
     aliases = models.TextField(default="")
-    fixed_in = models.TextField(default=None, null=True)
+    fixed_in = models.TextField(default="")
     cvss_vector = models.CharField(max_length=128, default="")
     ecosystem = models.CharField(max_length=20)
     finding_type = models.IntegerField(choices=FindingType.choices, default=FindingType.VULN)

surface/sca/tests/test_admin.py

@@ -73,7 +73,7 @@ def test_admin_changelist(self):
 
         # Assert Vulnerabilities Counters
         assert "1 Critical" in content
-        assert "4 High" in content
+        assert "3 High" in content
         assert "3 Medium" in content
         assert "0 Low" in content
         assert "0 End of Life" in content

surface/sca/tests/test_resync_sbom_repo.py

@@ -50,7 +50,7 @@ def test_resync_sbom_repo(self, now):
         assert SCAFindingCounter.objects.filter(dependency=main_dependency).exists()
         counter = SCAFindingCounter.objects.filter(dependency=main_dependency).first()
         assert counter.critical == 1
-        assert counter.high == 4
+        assert counter.high == 3
         assert counter.medium == 3
 
         # Asserts  main dependency has only one git source  "https://github.com/test/repo"