remove duplicate urls in vulncheck-kev.external_references (#114)

* remove duplicate urls in vulncheck-kev.external_references #113

* fix test

* bump

Author: fqrious

arango_cve_processor/managers/cve_kev_vulncheck.py

@@ -1,7 +1,7 @@
 import logging
 import os
 from typing import Any
-from urllib.parse import urlparse
+from urllib.parse import urlparse, parse_qsl, urlencode, urlunparse
 import uuid
 import requests
 from tqdm import tqdm
@@ -18,7 +18,8 @@ class VulnCheckKevManager(CISAKevManager, relationship_note="cve-vulncheck-kev")
     """
     content_fmt = "Vulncheck KEV: {cve_id}"
     CHUNK_SIZE = 1500
-    UPLOAD_CHUNK_SIZE = 2500
+    UPLOAD_CHUNK_SIZE = 500
+    UPDATE_CHUNK_SIZE = 500
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -61,14 +62,27 @@ def get_all_kevs(self):
             if meta["last_item"] >= meta["total_documents"]:
                 break
 
+    @staticmethod
+    def sanitize_url(url):
+        """ this function removes #frgments and day=/date= queries from the link"""
+        parsed = urlparse(url)
+        qs = parse_qsl(parsed.query, keep_blank_values=True)
+        filtered = [(k, v) for (k, v) in qs if k.lower() not in ("day", "date")]
+        new_query = urlencode(filtered, doseq=True)
+        new_parsed = parsed._replace(query=new_query, fragment="")
+        return urlunparse(new_parsed)
+
     def get_additional_refs(self, kev_obj):
-        for reported in kev_obj["vulncheck_reported_exploitation"]:
+        refs = {}
+        for reported in sorted(kev_obj["vulncheck_reported_exploitation"], key=lambda x: x["date_added"]):
+            ref_url = self.sanitize_url(reported["url"])
             ref = dict(
-                url=reported["url"],
+                url=ref_url,
                 description=f"Added on: {reported['date_added']}",
                 source_name=urlparse(reported["url"]).hostname,
             )
-            yield ref
+            refs[ref_url] = ref
+        return reversed(refs.values()) #return descending
 
     def get_dates(self, cve):
         kev_obj = cve['kev']

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arango_cve_processor"
-version = "1.4.5"
+version = "1.4.6"
 authors = [
   { name = "dogesec" }
 ]

tests/unit/managers/test_cve_kev_vulncheck.py

@@ -42,6 +42,10 @@ def test_relate_single(vulncheck_kev_manager, patched_retriever):
                 }
             ],
             "vulncheck_reported_exploitation": [
+                {
+                    "url": "https://dashboard.shadowserver.org/statistics/honeypot/vulnerability/map/?day=2025-11-29&host_type=src&vulnerability=cve-2024-53704",
+                    "date_added": "2025-11-29T00:00:00Z",
+                },
                 {
                     "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
                     "date_added": "2025-02-18T00:00:00Z",
@@ -97,16 +101,16 @@ def test_relate_single(vulncheck_kev_manager, patched_retriever):
                     "description": "Apply mitigations per vendor instructions or discontinue use of the product if mitigations are unavailable.",
                 },
                 {"source_name": "action_due", "description": "2025-03-11T00:00:00Z"},
+                {
+                    "url": "https://dashboard.shadowserver.org/statistics/honeypot/vulnerability/map/?host_type=src&vulnerability=cve-2024-53704",
+                    "description": "Added on: 2025-11-29T00:00:00Z",
+                    "source_name": "dashboard.shadowserver.org",
+                },
                 {
                     "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
                     "description": "Added on: 2025-02-18T00:00:00Z",
                     "source_name": "www.cisa.gov",
                 },
-                {
-                    "url": "https://dashboard.shadowserver.org/statistics/honeypot/vulnerability/map/?day=2025-02-24&host_type=src&vulnerability=cve-2024-53704",
-                    "description": "Added on: 2025-02-24T00:00:00Z",
-                    "source_name": "dashboard.shadowserver.org",
-                },
                 {
                     "source_name": "cwe",
                     "url": "http://cwe.mitre.org/data/definitions/94.html",