diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 11fa5126a..89d931a4e 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -19,10 +19,12 @@ from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline from vulnerabilities.pipelines import remove_duplicate_advisories from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees +from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import ( computer_package_version_rank as compute_version_rank_v2, ) +from vulnerabilities.pipelines.v2_improvers import detection_rules from vulnerabilities.pipelines.v2_improvers import enhance_with_exploitdb as exploitdb_v2 from vulnerabilities.pipelines.v2_improvers import enhance_with_kev as enhance_with_kev_v2 from vulnerabilities.pipelines.v2_improvers import ( @@ -72,5 +74,6 @@ collect_ssvc_trees.CollectSSVCPipeline, relate_severities.RelateSeveritiesPipeline, group_advisories_for_packages.GroupAdvisoriesForPackages, + detection_rules.DetectionRulesPipeline, ] ) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 45d8acf55..99d93e764 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3740,3 +3740,43 @@ class GroupedAdvisory(NamedTuple): weighted_severity: Optional[float] exploitability: Optional[float] risk_score: Optional[float] + + +class DetectionRuleTypes(models.TextChoices): + """Defines the supported formats for security detection rules.""" + + YARA = "yara", "Yara" + YARA_X = "yara-x", "Yara-X" + SIGMA = "sigma", "Sigma" + CLAMAV = "clamav", "ClamAV" + SURICATA = "suricata", "Suricata" + + +class DetectionRule(models.Model): + """ + A Detection Rule is code used to identify malicious activity or security threats. + """ + + rule_type = models.CharField( + max_length=50, + choices=DetectionRuleTypes.choices, + help_text="The type of the detection rule content (e.g., YARA, Sigma).", + ) + + source_url = models.URLField( + max_length=1024, help_text="URL to the original source or reference for this rule." + ) + + rule_metadata = models.JSONField( + null=True, + blank=True, + help_text="Additional structured data such as tags, or author information.", + ) + + rule_text = models.TextField(help_text="The content of the detection signature.") + + related_advisories = models.ManyToManyField( + AdvisoryV2, + related_name="detection_rules", + help_text="Advisories associated with this DetectionRule.", + ) diff --git a/vulnerabilities/pipelines/v2_improvers/detection_rules.py b/vulnerabilities/pipelines/v2_improvers/detection_rules.py new file mode 100644 index 000000000..e506eb9e2 --- /dev/null +++ b/vulnerabilities/pipelines/v2_improvers/detection_rules.py @@ -0,0 +1,102 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +from pathlib import Path + +from fetchcode.vcs import fetch_via_vcs + +from vulnerabilities.models import AdvisoryAlias +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import DetectionRule +from vulnerabilities.models import DetectionRuleTypes +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class DetectionRulesPipeline(VulnerableCodePipeline): + """ + Pipeline to collect vulnerability scanner rules (Sigma, YARA, Suricata, ClamAV entries) + """ + + pipeline_id = "detection_rules" + license_url = "https://github.com/ziadhany/detection-rules-collector/blob/master/LICENSE" + precedence = 200 + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_detection_rules, + cls.clean_downloads, + ) + + def clone(self): + self.repo_url = "git+https://github.com/aboutcode-data/detection-rules-collector" + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) + return sum(1 for _ in root.rglob("*.json")) + + def collect_detection_rules(self): + base_path = Path(self.vcs_response.dest_dir) / "data" + rule_type_mapping = { + DetectionRuleTypes.YARA: "yara/**/*.json", + DetectionRuleTypes.SURICATA: "suricata/**/*.json", + DetectionRuleTypes.SIGMA: "sigma/**/*.json", + DetectionRuleTypes.CLAMAV: "clamav/**/*.json", + } + + for rule_type, glob_pattern in rule_type_mapping.items(): + for file_path in base_path.glob(glob_pattern): + with open(file_path, "r") as f: + try: + json_data = json.load(f) + except json.JSONDecodeError: + self.log(f"Failed to parse JSON in {file_path}") + continue + + source_url = json_data.get("source_url") + for rule in json_data.get("rules", []): + advisories = set() + for vulnerability_id in rule.get("vulnerabilities", []): + try: + if alias := AdvisoryAlias.objects.get(alias=vulnerability_id): + for adv in alias.advisories.all(): + advisories.add(adv) + else: + advs = AdvisoryV2.objects.filter( + advisory_id=vulnerability_id + ).latest_per_avid() + for adv in advs: + advisories.add(adv) + except AdvisoryAlias.DoesNotExist: + self.log(f"No advisory found for aliases {vulnerability_id}") + + raw_text = rule.get("rule_text") + detection_rule, _ = DetectionRule.objects.get_or_create( + rule_text=raw_text, + rule_type=rule_type, + defaults={ + "source_url": source_url, + }, + ) + if advisories: + detection_rule.related_advisories.add(*advisories) + + def clean_downloads(self): + """Cleanup any temporary repository data.""" + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + """Ensure cleanup is always performed on failure.""" + self.clean_downloads() diff --git a/vulnerabilities/tests/pipelines/v2_improvers/test_detection_rules.py b/vulnerabilities/tests/pipelines/v2_improvers/test_detection_rules.py new file mode 100644 index 000000000..f1706262c --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_improvers/test_detection_rules.py @@ -0,0 +1,46 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from datetime import datetime +from unittest.mock import Mock +from unittest.mock import patch + +import pytest + +from vulnerabilities.models import AdvisoryAlias +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import DetectionRule +from vulnerabilities.pipelines.v2_improvers.detection_rules import DetectionRulesPipeline + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../../test_data", "detection_rules") + + +@pytest.mark.django_db +@patch("vulnerabilities.pipelines.v2_improvers.detection_rules.fetch_via_vcs") +def test_detection_rules_improver(mock_fetch_via_vcs): + mock_vcs_response = Mock() + mock_vcs_response.dest_dir = TEST_DATA + mock_fetch_via_vcs.return_value = mock_vcs_response + + adv1 = AdvisoryV2.objects.create( + advisory_id="VCIO-123-2002", + datasource_id="ds", + avid="ds/VCIO-123-2002", + unique_content_id="i3giu", + url="https://test.com", + date_collected=datetime.now(), + ) + alias = AdvisoryAlias.objects.create(alias="CVE-2007-4387") + adv1.aliases.add(alias) + + improver = DetectionRulesPipeline() + improver.execute() + assert DetectionRule.objects.count() > 0 diff --git a/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_hdb.json b/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_hdb.json new file mode 100644 index 000000000..e02e590c7 --- /dev/null +++ b/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_hdb.json @@ -0,0 +1,32 @@ +{ + "source_url": "https://database.clamav.net/main.cvd?api-version=1", + "source_filename": "main.hdb", + "rules": [ + { + "rule_metadata": { + "name": "Eicar-Test-Signature", + "line_num": 1 + }, + "rule_text": "44d88612fea8a8f36de82e1278abb02f:68:Eicar-Test-Signature", + "vulnerabilities": [] + }, + { + "rule_metadata": { + "name": "Win.Trojan.Yat-2", + "line_num": 3 + }, + "rule_text": "de3430cd6a3e24bfb9f78743a25f7c96:1098752:Win.Trojan.Yat-2", + "vulnerabilities": [] + }, + { + "rule_metadata": { + "name": "Java.Exploit.CVE_2012_5076-1", + "line_num": 10079 + }, + "rule_text": "a9b65b78619002a1b30ceee2d85fa770:205:Java.Exploit.CVE_2012_5076-1", + "vulnerabilities": [ + "CVE-2012-5076" + ] + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ldb.json b/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ldb.json new file mode 100644 index 000000000..9f4160799 --- /dev/null +++ b/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ldb.json @@ -0,0 +1,32 @@ +{ + "source_url": "https://database.clamav.net/main.cvd?api-version=1", + "source_filename": "main.ldb", + "rules": [ + { + "rule_metadata": { + "name": "Win.Exploit.CVE_2016_7185-1", + "line_num": 1 + }, + "rule_text": "Win.Exploit.CVE_2016_7185-1;Engine:51-255,Target:1;(0&1&2&3);44616e6765726f757347657448616e646c65;5361666548616e646c655a65726f4f724d696e75734f6e654973496e76616c6964;52656c6561736548616e646c65;5c004400650076006900630065005c0044006600730043006c00690065006e007400", + "vulnerabilities": [ + "CVE-2016-7185" + ] + }, + { + "rule_metadata": { + "name": "Doc.Trojan.Agent-1383193", + "line_num": 2 + }, + "rule_text": "Doc.Trojan.Agent-1383193;Engine:53-255,Target:2;0&1&2&3&4;57683370314d4c73576c69454b30626476376d707563704156724856585141694f30383755365a48556f;507a33593934674e796c784e724d5937706a3068;586c49766b65446349324259514d5169556b764d436165345144415452746d3842;434c70577561534d6f4c4845437a4172754d4d6466484b3334444e78;4256504271623368394c6e6c", + "vulnerabilities": [] + }, + { + "rule_metadata": { + "name": "Doc.Trojan.Agent-1383194", + "line_num": 3 + }, + "rule_text": "Doc.Trojan.Agent-1383194;Engine:53-255,Target:2;0&1>50;28373835362920417320427974652c20;3d2059656172284e6f77292027", + "vulnerabilities": [] + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ndb.json b/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ndb.json new file mode 100644 index 000000000..3d2afe2db --- /dev/null +++ b/vulnerabilities/tests/test_data/detection_rules/data/clamav/main_ndb.json @@ -0,0 +1,32 @@ +{ + "source_url": "https://database.clamav.net/main.cvd?api-version=1", + "source_filename": "main.ndb", + "rules": [ + { + "rule_metadata": { + "name": "Legacy.Trojan.Agent-1", + "line_num": 1 + }, + "rule_text": "Legacy.Trojan.Agent-1:0:*:dd6d70241f674d8fc13e1eb3af731a7b5c43173c1cdd75722fa556c373b65c5275d513147b070077757064080386898ae75c6fb7f717b562ef636f6d6d613f2e0e202f6336c5eed52064f120228e2f6d27c101", + "vulnerabilities": [] + }, + { + "rule_metadata": { + "name": "Win.Trojan.Hotkey-1", + "line_num": 2 + }, + "rule_text": "Win.Trojan.Hotkey-1:0:*:c01640006a3cffb684000000ff159cef420089869800000089be940000008bc75f5ec20400565733ff8bf1397c240c741fff762089be8c000000ff1560ef42", + "vulnerabilities": [] + }, + { + "rule_metadata": { + "name": "Win.Exploit.CVE_2001_0500-1", + "line_num": 31344 + }, + "rule_text": "Win.Exploit.CVE_2001_0500-1:0:*:7961686f6f3a20607065726c202d6520277072696e7420225c783930227831313830302760245348454c4c434f44453d3230", + "vulnerabilities": [ + "CVE-2001-0500" + ] + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/detection_rules/data/sigma/web_sql_injection_in_access_logs.json b/vulnerabilities/tests/test_data/detection_rules/data/sigma/web_sql_injection_in_access_logs.json new file mode 100644 index 000000000..158a5e673 --- /dev/null +++ b/vulnerabilities/tests/test_data/detection_rules/data/sigma/web_sql_injection_in_access_logs.json @@ -0,0 +1,16 @@ +{ + "source_url": "https://github.com/SigmaHQ/sigma/blob/master/rules/web/webserver_generic/web_sql_injection_in_access_logs.yml", + "rules": [ + { + "rule_metadata": { + "status": "test", + "author": "Saw Win Naung, Nasreddine Bencherchali (Nextron Systems), Thurein Oo (Yoma Bank)", + "date": "2020-02-22", + "title": "SQL Injection Strings In URI", + "id": "5513deaf-f49a-46c2-a6c8-3f111b5cb453" + }, + "rule_text": "title: SQL Injection Strings In URI\nid: 5513deaf-f49a-46c2-a6c8-3f111b5cb453\nstatus: test\ndescription: Detects potential SQL injection attempts via GET requests in access logs.\nreferences:\n- https://www.acunetix.com/blog/articles/exploiting-sql-injection-example/\n- https://www.acunetix.com/blog/articles/using-logs-to-investigate-a-web-application-attack/\n- https://brightsec.com/blog/sql-injection-payloads/\n- https://github.com/payloadbox/sql-injection-payload-list\n- https://book.hacktricks.xyz/pentesting-web/sql-injection/mysql-injection\nauthor: Saw Win Naung, Nasreddine Bencherchali (Nextron Systems), Thurein Oo (Yoma\n Bank)\ndate: 2020-02-22\nmodified: 2023-09-04\ntags:\n- attack.initial-access\n- attack.t1190\nlogsource:\n category: webserver\ndetection:\n selection:\n cs-method: GET\n keywords:\n - '@@version'\n - '%271%27%3D%271'\n - '=select '\n - =select(\n - =select%20\n - concat_ws(\n - CONCAT(0x\n - from mysql.innodb_table_stats\n - from%20mysql.innodb_table_stats\n - group_concat(\n - information_schema.tables\n - json_arrayagg(\n - or 1=1#\n - or%201=1#\n - 'order by '\n - order%20by%20\n - 'select * '\n - select database()\n - select version()\n - select%20*%20\n - select%20database()\n - select%20version()\n - select%28sleep%2810%29\n - SELECTCHAR(\n - table_schema\n - UNION ALL SELECT\n - UNION SELECT\n - UNION%20ALL%20SELECT\n - UNION%20SELECT\n - '''1''=''1'\n filter_main_status:\n sc-status: 404\n condition: selection and keywords and not 1 of filter_main_*\nfalsepositives:\n- Java scripts and CSS Files\n- User searches in search boxes of the respective website\n- Internal vulnerability scanners can cause some serious FPs when used, if you experience\n a lot of FPs due to this think of adding more filters such as \"User Agent\" strings\n and more response codes\nlevel: high\n", + "vulnerabilities": [] + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/detection_rules/data/suricata/dhcp-events.json b/vulnerabilities/tests/test_data/detection_rules/data/suricata/dhcp-events.json new file mode 100644 index 000000000..5639c71f5 --- /dev/null +++ b/vulnerabilities/tests/test_data/detection_rules/data/suricata/dhcp-events.json @@ -0,0 +1,25 @@ +{ + "source_url": "https://github.com/OISF/suricata/blob/master/rules/dhcp-events.rules", + "rules": [ + { + "rule_metadata": { + "name": "SURICATA DHCP malformed options", + "version": 1, + "id": 2227000, + "enabled": true + }, + "rule_text": "alert dhcp any any -> any any (msg:\"SURICATA DHCP malformed options\"; app-layer-event:dhcp.malformed_options; classtype:protocol-command-decode; sid:2227000; rev:1;)", + "vulnerabilities": [] + }, + { + "rule_metadata": { + "name": "SURICATA DHCP truncated options", + "version": 1, + "id": 2227001, + "enabled": true + }, + "rule_text": "alert dhcp any any -> any any (msg:\"SURICATA DHCP truncated options\"; app-layer-event:dhcp.truncated_options; classtype:protocol-command-decode; sid:2227001; rev:1;)", + "vulnerabilities": [] + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/detection_rules/data/yara/Linux_Backdoor_Bash.json b/vulnerabilities/tests/test_data/detection_rules/data/yara/Linux_Backdoor_Bash.json new file mode 100644 index 000000000..eb8027415 --- /dev/null +++ b/vulnerabilities/tests/test_data/detection_rules/data/yara/Linux_Backdoor_Bash.json @@ -0,0 +1,25 @@ +{ + "source_url": "https://github.com/elastic/protections-artifacts/blob/master/yara/rules/Linux_Backdoor_Bash.yar", + "rules": [ + { + "rule_metadata": { + "name": "Linux_Backdoor_Bash_e427876d", + "tags": [], + "author": "Elastic Security", + "id": "e427876d-c7c5-447a-ad6d-5cbc12d9dacf", + "fingerprint": "6cc13bb2591d896affc58f4a22b3463a72f6c9d896594fe1714b825e064b0956", + "creation_date": "2021-01-12", + "last_modified": "2021-09-16", + "threat_name": "Linux.Backdoor.Bash", + "reference_sample": "07db41a4ddaac802b04df5e5bbae0881fead30cb8f6fa53a8a2e1edf14f2d36b", + "severity": 100, + "arch_context": "x86", + "scan_context": "file, memory", + "license": "Elastic License v2", + "os": "linux" + }, + "rule_text": "rule Linux_Backdoor_Bash_e427876d\n{\n\tmeta:\n\t\tauthor = \"Elastic Security\"\n\t\tid = \"e427876d-c7c5-447a-ad6d-5cbc12d9dacf\"\n\t\tfingerprint = \"6cc13bb2591d896affc58f4a22b3463a72f6c9d896594fe1714b825e064b0956\"\n\t\tcreation_date = \"2021-01-12\"\n\t\tlast_modified = \"2021-09-16\"\n\t\tthreat_name = \"Linux.Backdoor.Bash\"\n\t\treference_sample = \"07db41a4ddaac802b04df5e5bbae0881fead30cb8f6fa53a8a2e1edf14f2d36b\"\n\t\tseverity = 100\n\t\tarch_context = \"x86\"\n\t\tscan_context = \"file, memory\"\n\t\tlicense = \"Elastic License v2\"\n\t\tos = \"linux\"\n\n\tstrings:\n\t\t$a = { 67 65 44 6F 6B 4B 47 6C 6B 49 43 31 31 4B 54 6F 67 4C 32 56 }\n\n\tcondition:\n\t\tall of them\n}\n", + "vulnerabilities": [] + } + ] +} \ No newline at end of file