Skip to content

Commit 508905e

Browse files
valentijnscholtenvalentijnscholten
authored
deduplication logic: add cross scanner unique_id tests and fix bug (#13499)
* deduplication logic: add cross scanner unique_id tests * unique_id_from_tool dedupe: fix cross parser logic * update finding ids in tests * notifications test: replace hardcoded ids with references * fix merge artifacts
1 parent fb96105 commit 508905e

4 files changed

Lines changed: 461 additions & 37 deletions

File tree

‎dojo/fixtures/dojo_testdata.json‎

Lines changed: 320 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3071,5 +3071,325 @@
30713071
"note": null,
30723072
"owner": 6
30733073
}
3074+
},
3075+
{
3076+
"model": "dojo.test_type",
3077+
"pk": 1000,
3078+
"fields": {
3079+
"name": "SonarQube Scan detailed",
3080+
"static_tool": false,
3081+
"dynamic_tool": false,
3082+
"active": true,
3083+
"dynamically_generated": false
3084+
}
3085+
},
3086+
{
3087+
"model": "dojo.test",
3088+
"pk": 90,
3089+
"fields": {
3090+
"engagement": 5,
3091+
"lead": [
3092+
"admin"
3093+
],
3094+
"test_type": 1000,
3095+
"scan_type": "SonarQube Scan detailed",
3096+
"title": null,
3097+
"description": null,
3098+
"target_start": "2025-10-22T08:29:41.333Z",
3099+
"target_end": "2025-10-22T08:29:41.333Z",
3100+
"percent_complete": 100,
3101+
"environment": 1,
3102+
"updated": "2025-10-22T08:29:41.590Z",
3103+
"created": "2025-10-22T08:29:41.343Z",
3104+
"version": "",
3105+
"build_id": "",
3106+
"commit_hash": "",
3107+
"branch_tag": "",
3108+
"api_scan_configuration": null,
3109+
"notes": [],
3110+
"files": [],
3111+
"tags": [],
3112+
"inherited_tags": []
3113+
}
3114+
},
3115+
{
3116+
"model": "dojo.finding",
3117+
"pk": 232,
3118+
"fields": {
3119+
"title": "Disabling CSRF Protections Is Security-Sensitive",
3120+
"date": "2025-10-22",
3121+
"sla_start_date": null,
3122+
"sla_expiration_date": "2025-11-21",
3123+
"cwe": 352,
3124+
"cve": null,
3125+
"epss_score": null,
3126+
"epss_percentile": null,
3127+
"known_exploited": false,
3128+
"ransomware_used": false,
3129+
"kev_date": null,
3130+
"cvssv3": null,
3131+
"cvssv3_score": null,
3132+
"cvssv4": null,
3133+
"cvssv4_score": null,
3134+
"url": null,
3135+
"severity": "High",
3136+
"description": "A cross-site request forgery (CSRF) attack occurs when a trusted user of a web application can be forced, by an attacker, to perform sensitive\nactions that he didn’t intend, such as updating his profile or sending a message, more generally anything that can change the state of the\napplication.\nThe attacker can trick the user/victim to click on a link, corresponding to the privileged action, or to visit a malicious web site that embeds a\nhidden web request and as web browsers automatically include cookies, the actions can be authenticated and sensitive.\n**Ask Yourself Whether**\n\n The web application uses cookies to authenticate users. \n There exist sensitive operations in the web application that can be performed when the user is authenticated. \n The state / resources of the web application can be modified by doing HTTP POST or HTTP DELETE requests for example. \n\nThere is a risk if you answered yes to any of those questions.\n**Recommended Secure Coding Practices**\n\n Protection against CSRF attacks is strongly recommended:\n \n to be activated by default for all unsafe HTTP\n methods. \n implemented, for example, with an unguessable CSRF token \n \n Of course all sensitive operations should not be performed with safe HTTP methods like GET which are designed to be\n used only for information retrieval. \n\n**Sensitive Code Example**\nFor a Django application, the code is sensitive when,\n\n django.middleware.csrf.CsrfViewMiddleware is not used in the Django settings: \n\n\nMIDDLEWARE = [\n 'django.middleware.security.SecurityMiddleware',\n 'django.contrib.sessions.middleware.SessionMiddleware',\n 'django.middleware.common.CommonMiddleware',\n 'django.contrib.auth.middleware.AuthenticationMiddleware',\n 'django.contrib.messages.middleware.MessageMiddleware',\n 'django.middleware.clickjacking.XFrameOptionsMiddleware',\n] # Sensitive: django.middleware.csrf.CsrfViewMiddleware is missing\n\n\n the CSRF protection is disabled on a view: \n\n\n@csrf_exempt # Sensitive\ndef example(request):\n return HttpResponse(\"default\")\n\nFor a Flask application, the code is sensitive when,\n\n the WTF_CSRF_ENABLED setting is set to false: \n\n\napp = Flask(__name__)\napp.config['WTF_CSRF_ENABLED'] = False # Sensitive\n\n\n the application doesn’t use the CSRFProtect module: \n\n\napp = Flask(__name__) # Sensitive: CSRFProtect is missing\n\n@app.route('/')\ndef hello_world():\n return 'Hello, World!'\n\n\n the CSRF protection is disabled on a view: \n\n\napp = Flask(__name__)\ncsrf = CSRFProtect()\ncsrf.init_app(app)\n\n@app.route('/example/', methods=['POST'])\n@csrf.exempt # Sensitive\ndef example():\n return 'example '\n\n\n the CSRF protection is disabled on a form: \n\n\nclass unprotectedForm(FlaskForm):\n class Meta:\n csrf = False # Sensitive\n\n name = TextField('name')\n submit = SubmitField('submit')\n\n**Compliant Solution**\nFor a Django application,\n\n it is recommended to protect all the views with django.middleware.csrf.CsrfViewMiddleware: \n\n\nMIDDLEWARE = [\n 'django.middleware.security.SecurityMiddleware',\n 'django.contrib.sessions.middleware.SessionMiddleware',\n 'django.middleware.common.CommonMiddleware',\n 'django.middleware.csrf.CsrfViewMiddleware', # Compliant\n 'django.contrib.auth.middleware.AuthenticationMiddleware',\n 'django.contrib.messages.middleware.MessageMiddleware',\n 'django.middleware.clickjacking.XFrameOptionsMiddleware',\n]\n\n\n and to not disable the CSRF protection on specific views: \n\n\ndef example(request): # Compliant\n return HttpResponse(\"default\")\n\nFor a Flask application,\n\n the CSRFProtect module should be used (and not disabled further with WTF_CSRF_ENABLED set to false):\n \n\n\napp = Flask(__name__)\ncsrf = CSRFProtect()\ncsrf.init_app(app) # Compliant\n\n\n and it is recommended to not disable the CSRF protection on specific views or forms: \n\n\n@app.route('/example/', methods=['POST']) # Compliant\ndef example():\n return 'example '\n\nclass unprotectedForm(FlaskForm):\n class Meta:\n csrf = True # Compliant\n\n name = TextField('name')\n submit = SubmitField('submit')",
3137+
"mitigation": "Make sure disabling CSRF protection is safe here.",
3138+
"fix_available": null,
3139+
"impact": "No impact provided",
3140+
"steps_to_reproduce": null,
3141+
"severity_justification": null,
3142+
"references": "python:S4502\nunsafe HTTP\n methods\nsafe HTTP\nDjango\nDjango settings\nFlask\nDjango\nFlask\nOWASP Top 10 2021 Category A1\nMITRE, CWE-352\nOWASP Top 10 2017 Category A6\nOWASP: Cross-Site Request Forgery\nSANS Top 25",
3143+
"test": 90,
3144+
"active": true,
3145+
"verified": true,
3146+
"false_p": false,
3147+
"duplicate": false,
3148+
"duplicate_finding": null,
3149+
"out_of_scope": false,
3150+
"risk_accepted": false,
3151+
"under_review": false,
3152+
"last_status_update": "2025-10-22T08:29:41.361Z",
3153+
"review_requested_by": null,
3154+
"under_defect_review": false,
3155+
"defect_review_requested_by": null,
3156+
"is_mitigated": false,
3157+
"thread_id": 0,
3158+
"mitigated": null,
3159+
"mitigated_by": null,
3160+
"reporter": [
3161+
"admin"
3162+
],
3163+
"numerical_severity": "S1",
3164+
"last_reviewed": "2025-10-22T08:29:41.336Z",
3165+
"last_reviewed_by": [
3166+
"admin"
3167+
],
3168+
"param": null,
3169+
"payload": null,
3170+
"hash_code": "ed09b1b5980bd7b9d67b58ba3a3200b788b567a8b3359b5b66d861112b025b9e",
3171+
"line": 8,
3172+
"file_path": "vulnerable-flask-app.py",
3173+
"component_name": null,
3174+
"component_version": null,
3175+
"static_finding": true,
3176+
"dynamic_finding": false,
3177+
"created": "2025-10-22T08:29:41.361Z",
3178+
"scanner_confidence": null,
3179+
"sonarqube_issue": null,
3180+
"unique_id_from_tool": "AYvNd32RyD1npIoQXyT1",
3181+
"vuln_id_from_tool": null,
3182+
"sast_source_object": null,
3183+
"sast_sink_object": null,
3184+
"sast_source_line": null,
3185+
"sast_source_file_path": null,
3186+
"nb_occurences": null,
3187+
"publish_date": null,
3188+
"service": null,
3189+
"planned_remediation_date": null,
3190+
"planned_remediation_version": null,
3191+
"effort_for_fixing": null,
3192+
"reviewers": [],
3193+
"notes": [],
3194+
"files": [],
3195+
"found_by": [
3196+
1000
3197+
],
3198+
"tags": [],
3199+
"inherited_tags": []
3200+
}
3201+
},
3202+
{
3203+
"model": "dojo.test_import",
3204+
"pk": 6829,
3205+
"fields": {
3206+
"created": "2025-10-22T08:29:41.453Z",
3207+
"modified": "2025-10-22T08:29:41.453Z",
3208+
"test": 90,
3209+
"import_settings": {
3210+
"tags": [],
3211+
"active": true,
3212+
"verified": true,
3213+
"push_to_jira": false,
3214+
"minimum_severity": "Info",
3215+
"close_old_findings": false
3216+
},
3217+
"type": "import",
3218+
"version": "",
3219+
"build_id": "",
3220+
"commit_hash": "",
3221+
"branch_tag": ""
3222+
}
3223+
},
3224+
{
3225+
"model": "dojo.test_import_finding_action",
3226+
"pk": 80213,
3227+
"fields": {
3228+
"created": "2025-10-22T08:29:41.458Z",
3229+
"modified": "2025-10-22T08:29:41.458Z",
3230+
"test_import": 6829,
3231+
"finding": 232,
3232+
"action": "N"
3233+
}
3234+
},
3235+
{
3236+
"model": "dojo.test_type",
3237+
"pk": 1001,
3238+
"fields": {
3239+
"name": "HackerOne Cases",
3240+
"static_tool": false,
3241+
"dynamic_tool": false,
3242+
"active": true,
3243+
"dynamically_generated": false
3244+
}
3245+
},
3246+
{
3247+
"model": "dojo.test",
3248+
"pk": 91,
3249+
"fields": {
3250+
"engagement": 5,
3251+
"lead": [
3252+
"admin"
3253+
],
3254+
"test_type": 1001,
3255+
"scan_type": "HackerOne Cases",
3256+
"title": null,
3257+
"description": null,
3258+
"target_start": "2025-10-22T08:52:53.734Z",
3259+
"target_end": "2025-10-22T08:52:53.734Z",
3260+
"percent_complete": 100,
3261+
"environment": 1,
3262+
"updated": "2025-10-22T08:52:53.859Z",
3263+
"created": "2025-10-22T08:52:53.737Z",
3264+
"version": "",
3265+
"build_id": "",
3266+
"commit_hash": "",
3267+
"branch_tag": "",
3268+
"api_scan_configuration": null,
3269+
"notes": [],
3270+
"files": [],
3271+
"tags": [],
3272+
"inherited_tags": []
3273+
}
3274+
},
3275+
{
3276+
"model": "dojo.finding",
3277+
"pk": 233,
3278+
"fields": {
3279+
"title": "Sensitive Account Balance Information Exposure via Example's DaviPlata Payment Link Integration",
3280+
"date": "2025-10-22",
3281+
"sla_start_date": null,
3282+
"sla_expiration_date": "2026-01-20",
3283+
"cwe": 0,
3284+
"cve": null,
3285+
"epss_score": null,
3286+
"epss_percentile": null,
3287+
"known_exploited": false,
3288+
"ransomware_used": false,
3289+
"kev_date": null,
3290+
"cvssv3": null,
3291+
"cvssv3_score": null,
3292+
"cvssv4": null,
3293+
"cvssv4_score": null,
3294+
"url": null,
3295+
"severity": "Medium",
3296+
"description": "**ID**: 2501687\n**Weakness Category**: Information Disclosure\n**Substate**: triaged\n**Reporter**: reporter\n**Assigned To**: Group example.co Team\n**Public**: no\n**Awarded On**: 2024-08-28 19:40:24 UTC\n**Bounty Price**: 400.0\n**First Response On**: 2024-05-14 22:14:16 UTC\n**Structured Scope**: 1489537348\n",
3297+
"mitigation": null,
3298+
"fix_available": null,
3299+
"impact": null,
3300+
"steps_to_reproduce": null,
3301+
"severity_justification": null,
3302+
"references": null,
3303+
"test": 91,
3304+
"active": true,
3305+
"verified": true,
3306+
"false_p": false,
3307+
"duplicate": false,
3308+
"duplicate_finding": null,
3309+
"out_of_scope": false,
3310+
"risk_accepted": false,
3311+
"under_review": false,
3312+
"last_status_update": "2025-10-22T08:52:53.755Z",
3313+
"review_requested_by": null,
3314+
"under_defect_review": false,
3315+
"defect_review_requested_by": null,
3316+
"is_mitigated": false,
3317+
"thread_id": 0,
3318+
"mitigated": null,
3319+
"mitigated_by": null,
3320+
"reporter": [
3321+
"admin"
3322+
],
3323+
"numerical_severity": "S2",
3324+
"last_reviewed": "2025-10-22T08:52:53.735Z",
3325+
"last_reviewed_by": [
3326+
"admin"
3327+
],
3328+
"param": null,
3329+
"payload": null,
3330+
"hash_code": "684facb6f2fd8faa50a28637d4f7fc1ba9ad3d3a932d39960e99e3c10aec3495",
3331+
"line": null,
3332+
"file_path": null,
3333+
"component_name": null,
3334+
"component_version": null,
3335+
"static_finding": false,
3336+
"dynamic_finding": true,
3337+
"created": "2025-10-22T08:52:53.755Z",
3338+
"scanner_confidence": null,
3339+
"sonarqube_issue": null,
3340+
"unique_id_from_tool": null,
3341+
"vuln_id_from_tool": null,
3342+
"sast_source_object": null,
3343+
"sast_sink_object": null,
3344+
"sast_source_line": null,
3345+
"sast_source_file_path": null,
3346+
"nb_occurences": null,
3347+
"publish_date": null,
3348+
"service": null,
3349+
"planned_remediation_date": null,
3350+
"planned_remediation_version": null,
3351+
"effort_for_fixing": null,
3352+
"reviewers": [],
3353+
"notes": [],
3354+
"files": [],
3355+
"found_by": [
3356+
1001
3357+
],
3358+
"tags": [],
3359+
"inherited_tags": []
3360+
}
3361+
},
3362+
{
3363+
"model": "dojo.test_import",
3364+
"pk": 6830,
3365+
"fields": {
3366+
"created": "2025-10-22T08:52:53.797Z",
3367+
"modified": "2025-10-22T08:52:53.797Z",
3368+
"test": 91,
3369+
"import_settings": {
3370+
"tags": [],
3371+
"active": true,
3372+
"verified": true,
3373+
"push_to_jira": false,
3374+
"minimum_severity": "Info",
3375+
"close_old_findings": false
3376+
},
3377+
"type": "import",
3378+
"version": "",
3379+
"build_id": "",
3380+
"commit_hash": "",
3381+
"branch_tag": ""
3382+
}
3383+
},
3384+
{
3385+
"model": "dojo.test_import_finding_action",
3386+
"pk": 80214,
3387+
"fields": {
3388+
"created": "2025-10-22T08:52:53.798Z",
3389+
"modified": "2025-10-22T08:52:53.798Z",
3390+
"test_import": 6830,
3391+
"finding": 233,
3392+
"action": "N"
3393+
}
30743394
}
30753395
]

‎dojo/utils.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,8 @@ def deduplicate_unique_id_from_tool(new_finding):
429429
if new_finding.test.engagement.deduplication_on_engagement:
430430
existing_findings = Finding.objects.filter(
431431
test__engagement=new_finding.test.engagement,
432+
# the unique_id_from_tool is unique for a given tool: do not compare with other tools
433+
test__test_type=new_finding.test.test_type,
432434
unique_id_from_tool=new_finding.unique_id_from_tool).exclude(
433435
id=new_finding.id).exclude(
434436
unique_id_from_tool=None).exclude(

0 commit comments

Comments
 (0)