Update CVE EPSS (#111)

* add top-level arango properties for muchdogesec/vulmatch#304

* update tests

Author: fqrious

arango_cve_processor/managers/base_manager.py

@@ -26,6 +26,7 @@ class STIXRelationManager:
     MIN_DATE_STR = "1970-01-01"
     BATCH_SIZE = 1000
     CHUNK_SIZE = BATCH_SIZE
+    UPDATE_CHUNK_SIZE = 5000
     DESCRIPTION = "please set"
 
     def __init_subclass__(cls, /, relationship_note, register=True) -> None:
@@ -105,9 +106,9 @@ def upload_vertex_data(self, objects):
 
             logging.info("updating %d existing reports", len(self.update_objects))
             for batch in chunked_tqdm(
-                self.update_objects, n=10_000, description="update existing objects"
+                self.update_objects, n=self.UPDATE_CHUNK_SIZE, description="update existing objects"
             ):
-                self.arango.db.collection(self.vertex_collection).update_many(batch)
+                self.arango.db.collection(self.vertex_collection).update_many(batch, silent=True, check_rev=False)
 
         self.update_objects.clear()
         logging.info("uploading %d vertices", len(objects))

arango_cve_processor/managers/cve_epss.py

@@ -19,7 +19,7 @@ class _CveEpssWorker(STIXRelationManager, relationship_note="cve-epss", register
     default_objects = [
         "https://raw.githubusercontent.com/muchdogesec/stix2extensions/refs/heads/main/automodel_generated/extension-definitions/properties/report-epss-scoring.json"
     ]
-    CHUNK_SIZE = 20_000
+    CHUNK_SIZE = 10_000
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
@@ -101,6 +101,8 @@ def relate_single(self, cve_object):
                             "x_epss": all_epss,
                             "modified": latest_epss["date"] + "T00:00:00.000Z",
                             "_arango_cve_processor_note": self.relationship_note,
+                            "_epss_score": latest_epss["epss"],
+                            "_epss_percentile": latest_epss["percentile"],
                         },
                         self.make_opencti_properties(cve_object["_key"], latest_epss),
                     ]

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "arango_cve_processor"
-version = "1.4.3"
+version = "1.4.4"
 authors = [
   { name = "dogesec" }
 ]

tests/unit/managers/test_cve_epss.py

@@ -66,6 +66,8 @@ def test_epss_backfill(acp_processor):
             "_record_md5_hash": "dd9295679421ae55cd4ab7a20532a509",
             "_is_latest": True,
             "_taxii": {"last": True, "first": True, "visible": True},
+            "_epss_score": 0.00043,
+            "_epss_percentile": 0.11665,
         },
         {
             "type": "report",
@@ -104,6 +106,8 @@ def test_epss_backfill(acp_processor):
             "_record_md5_hash": "f25e43d30c9d44a2e73a698d12dc015d",
             "_is_latest": True,
             "_taxii": {"last": True, "first": True, "visible": True},
+            "_epss_score": 0.00043,
+            "_epss_percentile": 0.11665,
         },
         {
             "type": "report",
@@ -142,6 +146,8 @@ def test_epss_backfill(acp_processor):
             "_record_md5_hash": "19c7516d89633f14acb35319480dcd3c",
             "_is_latest": True,
             "_taxii": {"last": True, "first": True, "visible": True},
+            "_epss_score": 0.00087,
+            "_epss_percentile": 0.39353,
         },
         {
             "type": "report",
@@ -180,6 +186,8 @@ def test_epss_backfill(acp_processor):
             "_record_md5_hash": "daf44f8b6498071b6c6cd7d213fbecd8",
             "_is_latest": True,
             "_taxii": {"last": True, "first": True, "visible": True},
+            "_epss_score": 0.00054,
+            "_epss_percentile": 0.25386,
         },
     ]