diff --git a/CHANGELOG.md b/CHANGELOG.md index f3fe0a75f..46432c60e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Development ### Data Format - - Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`, `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following @@ -38,6 +37,8 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; ``` +- added `severity` field to help with triaging received events (PR#2575 by Kamil Mańkowski). + To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. ### Bots #### Collectors diff --git a/NEWS.md b/NEWS.md index 4c948919e..27e3b20ac 100644 --- a/NEWS.md +++ b/NEWS.md @@ -27,6 +27,7 @@ ALTER TABLE events ADD "product.name" text; ALTER TABLE events ADD "product.vendor" text; ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; +ALTER TABLE events ADD severity varchar(10); ``` ### Configuration diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index b9e57d6e2..e25a93872 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -253,6 +253,12 @@ "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.", "type": "URL" }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString" + }, "source.abuse_contact": { "description": "Abuse contact for source address. A comma separated list.", "type": "LowercaseString" diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 3da3e95fc..208684c03 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -42,8 +42,8 @@ 'v322_url_replacement', 'v322_removed_feeds_and_bots', 'v340_deprecations', - 'v341_blueliv_removal', - 'v342_new_fields' + 'v350_blueliv_removal', + 'v350_new_fields', ] @@ -976,7 +976,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization -def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): +def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ Remove blueliv collector and parser """ @@ -999,7 +999,7 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): return message, configuration, harmonization -def v342_new_fields(configuration, harmonization, dry_run, **kwargs): +def v350_new_fields(configuration, harmonization, dry_run, **kwargs): """ Add new fields to IntelMQ Data Format """ @@ -1011,6 +1011,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): resource_filename("intelmq", "etc/harmonization.conf") ) for field in [ + "severity", "product.full_name", "product.name", "product.vendor", @@ -1056,8 +1057,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), (v341_blueliv_removal, )), - ((3, 4, 2), (v342_new_fields, )), + ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 19faecb85..7e09d549d 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -57,6 +57,7 @@ CREATE TABLE events ( "raw" text, "rtir_id" integer, "screenshot_url" text, + "severity" varchar(10), "source.abuse_contact" text, "source.account" text, "source.allocated" timestamp with time zone, diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 6021a0fc0..047ecb4d4 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -616,7 +616,7 @@ "module": "intelmq.bots.collectors.twitter.collector", }, } -V341_BLUELIV_REMOVAL = { +V350_BLUELIV_REMOVAL = { "global": {}, "blueliv-collector": { "module": "intelmq.bots.collectors.blueliv.collector_crimeserver" @@ -865,16 +865,16 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) - def test_v341_blueliv_removal(self): - """ Test v341_blueliv_removal deprecation warning """ - result = upgrades.v341_blueliv_removal(V341_BLUELIV_REMOVAL, {}, False) + def test_v350_blueliv_removal(self): + """ Test v350_blueliv_removal deprecation warning """ + result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False) self.assertIn('blueliv-collector', result[0]) self.assertIn('blueliv-parser', result[0]) - self.assertEqual(V341_BLUELIV_REMOVAL, result[1]) + self.assertEqual(V350_BLUELIV_REMOVAL, result[1]) - def test_v342_new_fields(self): + def test_v350_new_fields(self): """ Test adding new harmonisation fields """ - result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) + result = upgrades.v350_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) @@ -882,6 +882,9 @@ def test_v342_new_fields(self): self.assertIn("product.vendor", result[2]["event"]) self.assertIn("product.version", result[2]["event"]) self.assertIn("product.vulnerabilities", result[2]["event"]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("severity", result[2]["event"]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name,