foundy_by: optimize for dedupe (#13888)

* dedupe: optimize found_by

* fix: update expected query counts in performance tests

* fix: update expected query counts in performance tests

Author: valentijnscholten

dojo/finding/deduplication.py

@@ -147,7 +147,14 @@ def set_duplicate(new_finding, existing_finding):
     for find in new_finding.original_finding.all():
         new_finding.original_finding.remove(find)
         set_duplicate(find, existing_finding)
-    existing_finding.found_by.add(new_finding.test.test_type)
+    # Only add test type to found_by if it is not already present.
+    # This is efficient because `found_by` is prefetched for candidates via `build_dedupe_scope_queryset()`.
+    test_type = getattr(getattr(new_finding, "test", None), "test_type", None)
+    if test_type is not None and test_type not in existing_finding.found_by.all():
+        existing_finding.found_by.add(test_type)
+
+    # existing_finding.found_by.add(new_finding.test.test_type)
+
     logger.debug("saving new finding: %d", new_finding.id)
     super(Finding, new_finding).save()
     logger.debug("saving existing finding: %d", existing_finding.id)
@@ -239,7 +246,7 @@ def build_dedupe_scope_queryset(test):
     return (
         Finding.objects.filter(scope_q)
         .select_related("test", "test__engagement", "test__test_type")
-        .prefetch_related("endpoints")
+        .prefetch_related("endpoints", "found_by")
     )
 
 

dojo/models.py

@@ -2766,6 +2766,8 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru
         logger.debug("Start saving finding of id " + str(self.id) + " dedupe_option:" + str(dedupe_option) + " (self.pk is %s)", "None" if self.pk is None else "not None")
         from dojo.finding import helper as finding_helper  # noqa: PLC0415 circular import
 
+        is_new_finding = self.pk is None
+
         # if not isinstance(self.date, (datetime, date)):
         #     raise ValidationError(_("The 'date' field must be a valid date or datetime object."))
 
@@ -2809,7 +2811,7 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru
 
         self.set_hash_code(dedupe_option)
 
-        if self.pk is None:
+        if is_new_finding:
             # We enter here during the first call from serializers.py
             from dojo.utils import apply_cwe_to_template  # noqa: PLC0415 circular import
             # No need to use the returned variable since `self` Is updated in memory
@@ -2838,7 +2840,9 @@ def save(self, dedupe_option=True, rules_option=True, product_grading_option=Tru
         logger.debug("Saving finding of id " + str(self.id) + " dedupe_option:" + str(dedupe_option) + " (self.pk is %s)", "None" if self.pk is None else "not None")
         super().save(*args, **kwargs)
 
-        self.found_by.add(self.test.test_type)
+        # Only add to found_by for newly-created findings (avoid doing this on every update)
+        if is_new_finding:
+            self.found_by.add(self.test.test_type)
 
         # only perform post processing (in celery task) if needed. this check avoids submitting 1000s of tasks to celery that will do nothing
         system_settings = System_Settings.objects.get()

unittests/test_importers_performance.py

@@ -217,11 +217,11 @@ def test_import_reimport_reimport_performance_no_async(self):
         testuser.usercontactinfo.block_execution = True
         testuser.usercontactinfo.save()
         self._import_reimport_performance(
-            expected_num_queries1=345,
+            expected_num_queries1=346,
             expected_num_async_tasks1=6,
-            expected_num_queries2=293,
+            expected_num_queries2=294,
             expected_num_async_tasks2=17,
-            expected_num_queries3=180,
+            expected_num_queries3=181,
             expected_num_async_tasks3=16,
         )
 
@@ -239,11 +239,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self):
         testuser.usercontactinfo.save()
 
         self._import_reimport_performance(
-            expected_num_queries1=311,
+            expected_num_queries1=312,
             expected_num_async_tasks1=6,
-            expected_num_queries2=286,
+            expected_num_queries2=287,
             expected_num_async_tasks2=17,
-            expected_num_queries3=175,
+            expected_num_queries3=176,
             expected_num_async_tasks3=16,
         )
 
@@ -265,11 +265,11 @@ def test_import_reimport_reimport_performance_no_async_with_product_grading(self
         self.system_settings(enable_product_grade=True)
 
         self._import_reimport_performance(
-            expected_num_queries1=347,
+            expected_num_queries1=348,
             expected_num_async_tasks1=8,
-            expected_num_queries2=295,
+            expected_num_queries2=296,
             expected_num_async_tasks2=19,
-            expected_num_queries3=182,
+            expected_num_queries3=183,
             expected_num_async_tasks3=18,
         )
 
@@ -288,11 +288,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr
         self.system_settings(enable_product_grade=True)
 
         self._import_reimport_performance(
-            expected_num_queries1=313,
+            expected_num_queries1=314,
             expected_num_async_tasks1=8,
-            expected_num_queries2=288,
+            expected_num_queries2=289,
             expected_num_async_tasks2=19,
-            expected_num_queries3=177,
+            expected_num_queries3=178,
             expected_num_async_tasks3=18,
         )
 
@@ -449,9 +449,9 @@ def test_deduplication_performance_no_async(self):
         testuser.usercontactinfo.save()
 
         self._deduplication_performance(
-            expected_num_queries1=316,
+            expected_num_queries1=317,
             expected_num_async_tasks1=7,
-            expected_num_queries2=287,
+            expected_num_queries2=282,
             expected_num_async_tasks2=7,
         )
 
@@ -469,8 +469,8 @@ def test_deduplication_performance_pghistory_no_async(self):
         testuser.usercontactinfo.save()
 
         self._deduplication_performance(
-            expected_num_queries1=280,
+            expected_num_queries1=281,
             expected_num_async_tasks1=7,
-            expected_num_queries2=250,
+            expected_num_queries2=245,
             expected_num_async_tasks2=7,
         )