From 55f89ceedbfb151e6203882bffab9d5ee17d9c34 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 01/11] Add OWASP Top 10 and API importer support --- .../tests/owasp_api_top10_2023_parser_test.py | 43 ++++++++++ .../tests/owasp_top10_2025_parser_test.py | 80 +++++++++++++++++++ .../data/owasp_api_top10_2023.json | 62 ++++++++++++++ .../data/owasp_top10_2025.json | 62 ++++++++++++++ .../parsers/owasp_api_top10_2023.py | 47 +++++++++++ .../parsers/owasp_top10_2025.py | 47 +++++++++++ cre.py | 30 +++++++ 7 files changed, 371 insertions(+) create mode 100644 application/tests/owasp_api_top10_2023_parser_test.py create mode 100644 application/tests/owasp_top10_2025_parser_test.py create mode 100644 application/utils/external_project_parsers/data/owasp_api_top10_2023.json create mode 100644 application/utils/external_project_parsers/data/owasp_top10_2025.json create mode 100644 application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py create mode 100644 application/utils/external_project_parsers/parsers/owasp_top10_2025.py diff --git a/application/tests/owasp_api_top10_2023_parser_test.py b/application/tests/owasp_api_top10_2023_parser_test.py new file mode 100644 index 000000000..806d11bed --- /dev/null +++ b/application/tests/owasp_api_top10_2023_parser_test.py @@ -0,0 +1,43 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_api_top10_2023 + + +class TestOwaspApiTop10_2023Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("304-667", "Protect API against unauthorized access/modification (IDOR)"), + ("724-770", "Technical application access control"), + ("715-223", "Ensure trusted origin of third party resources"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_api_top10_2023.OwaspApiTop10_2023().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP API Security Top 10 2023"] + self.assertEqual(10, len(entries)) + self.assertEqual("API1", entries[0].sectionID) + self.assertEqual("Broken Object Level Authorization", entries[0].section) + self.assertEqual( + ["304-667", "724-770"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("API10", entries[-1].sectionID) + self.assertEqual(["715-223"], [l.document.id for l in entries[-1].links]) diff --git a/application/tests/owasp_top10_2025_parser_test.py b/application/tests/owasp_top10_2025_parser_test.py new file mode 100644 index 000000000..de4f86a9f --- /dev/null +++ b/application/tests/owasp_top10_2025_parser_test.py @@ -0,0 +1,80 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_top10_2025 + + +class TestOwaspTop10_2025Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + self.collection.add_cre( + defs.CRE(id="177-260", name="Session management", description="") + ) + self.collection.add_cre( + defs.CRE( + id="117-371", + name="Use a centralized access control mechanism", + description="", + ) + ) + self.collection.add_cre( + defs.CRE( + id="724-770", + name="Technical application access control", + description="", + ) + ) + self.collection.add_cre( + defs.CRE( + id="031-447", name="Whitelist all external (HTTP) input", description="" + ) + ) + self.collection.add_cre( + defs.CRE( + id="064-808", name="Encode output context-specifically", description="" + ) + ) + self.collection.add_cre( + defs.CRE(id="760-764", name="Injection protection", description="") + ) + self.collection.add_cre( + defs.CRE(id="513-183", name="Error handling", description="") + ) + + result = owasp_top10_2025.OwaspTop10_2025().parse( + self.collection, + prompt_client.PromptHandler(database=self.collection), + ) + + entries = result.results["OWASP Top 10 2025"] + self.assertEqual(10, len(entries)) + self.assertEqual("A01", entries[0].sectionID) + self.assertEqual("Broken Access Control", entries[0].section) + self.assertEqual( + "https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + entries[0].hyperlink, + ) + self.assertEqual( + ["117-371", "177-260", "724-770"], + [link.document.id for link in entries[0].links], + ) + self.assertEqual( + ["031-447", "064-808", "760-764"], + [link.document.id for link in entries[4].links], + ) + self.assertEqual("A10", entries[-1].sectionID) + self.assertEqual(["513-183"], [link.document.id for link in entries[-1].links]) diff --git a/application/utils/external_project_parsers/data/owasp_api_top10_2023.json b/application/utils/external_project_parsers/data/owasp_api_top10_2023.json new file mode 100644 index 000000000..7a8df0ed0 --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_api_top10_2023.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "API1", + "section": "Broken Object Level Authorization", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa1-broken-object-level-authorization/", + "cre_ids": ["304-667", "724-770"] + }, + { + "section_id": "API2", + "section": "Broken Authentication", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa2-broken-authentication/", + "cre_ids": ["177-260", "586-842", "633-428"] + }, + { + "section_id": "API3", + "section": "Broken Object Property Level Authorization", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa3-broken-object-property-level-authorization/", + "cre_ids": ["538-770", "724-770", "128-128"] + }, + { + "section_id": "API4", + "section": "Unrestricted Resource Consumption", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa4-unrestricted-resource-consumption/", + "cre_ids": ["623-550"] + }, + { + "section_id": "API5", + "section": "Broken Function Level Authorization", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa5-broken-function-level-authorization/", + "cre_ids": ["650-560", "724-770"] + }, + { + "section_id": "API6", + "section": "Unrestricted Access to Sensitive Business Flows", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa6-unrestricted-access-to-sensitive-business-flows/", + "cre_ids": ["534-605", "630-573"] + }, + { + "section_id": "API7", + "section": "Server Side Request Forgery", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa7-server-side-request-forgery/", + "cre_ids": ["028-728", "657-084"] + }, + { + "section_id": "API8", + "section": "Security Misconfiguration", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa8-security-misconfiguration/", + "cre_ids": ["486-813"] + }, + { + "section_id": "API9", + "section": "Improper Inventory Management", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa9-improper-inventory-management/", + "cre_ids": ["162-655", "863-521"] + }, + { + "section_id": "API10", + "section": "Unsafe Consumption of APIs", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xaa-unsafe-consumption-of-apis/", + "cre_ids": ["715-223"] + } +] diff --git a/application/utils/external_project_parsers/data/owasp_top10_2025.json b/application/utils/external_project_parsers/data/owasp_top10_2025.json new file mode 100644 index 000000000..7e19d1a4e --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_top10_2025.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "A01", + "section": "Broken Access Control", + "hyperlink": "https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + "cre_ids": ["117-371", "177-260", "724-770"] + }, + { + "section_id": "A02", + "section": "Security Misconfiguration", + "hyperlink": "https://owasp.org/Top10/2025/A02_2025-Security_Misconfiguration/", + "cre_ids": ["486-813"] + }, + { + "section_id": "A03", + "section": "Software Supply Chain Failures", + "hyperlink": "https://owasp.org/Top10/2025/A03_2025-Software_Supply_Chain_Failures/", + "cre_ids": ["613-286", "613-287", "715-223", "863-521"] + }, + { + "section_id": "A04", + "section": "Cryptographic Failures", + "hyperlink": "https://owasp.org/Top10/2025/A04_2025-Cryptographic_Failures/", + "cre_ids": ["170-772", "227-045"] + }, + { + "section_id": "A05", + "section": "Injection", + "hyperlink": "https://owasp.org/Top10/2025/A05_2025-Injection/", + "cre_ids": ["031-447", "064-808", "760-764"] + }, + { + "section_id": "A06", + "section": "Insecure Design", + "hyperlink": "https://owasp.org/Top10/2025/A06_2025-Insecure_Design/", + "cre_ids": ["126-668", "155-155"] + }, + { + "section_id": "A07", + "section": "Authentication Failures", + "hyperlink": "https://owasp.org/Top10/2025/A07_2025-Authentication_Failures/", + "cre_ids": ["002-630", "177-260", "586-842", "633-428"] + }, + { + "section_id": "A08", + "section": "Software or Data Integrity Failures", + "hyperlink": "https://owasp.org/Top10/2025/A08_2025-Software_or_Data_Integrity_Failures/", + "cre_ids": ["613-287", "836-068"] + }, + { + "section_id": "A09", + "section": "Security Logging and Alerting Failures", + "hyperlink": "https://owasp.org/Top10/2025/A09_2025-Security_Logging_and_Alerting_Failures/", + "cre_ids": ["067-050", "148-420", "402-706", "843-841"] + }, + { + "section_id": "A10", + "section": "Mishandling of Exceptional Conditions", + "hyperlink": "https://owasp.org/Top10/2025/A10_2025-Mishandling_of_Exceptional_Conditions/", + "cre_ids": ["513-183"] + } +] diff --git a/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py b/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py new file mode 100644 index 000000000..08157a1e9 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspApiTop10_2023(ParserInterface): + name = "OWASP API Security Top 10 2023" + data_file = ( + Path(__file__).resolve().parent.parent / "data" / "owasp_api_top10_2023.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/application/utils/external_project_parsers/parsers/owasp_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_top10_2025.py new file mode 100644 index 000000000..070f869af --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_top10_2025.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspTop10_2025(ParserInterface): + name = "OWASP Top 10 2025" + data_file = ( + Path(__file__).resolve().parent.parent / "data" / "owasp_top10_2025.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/cre.py b/cre.py index 99735890f..1ffc24951 100644 --- a/cre.py +++ b/cre.py @@ -167,6 +167,36 @@ def main() -> None: action="store_true", help="import owasp secure headers", ) + parser.add_argument( + "--owasp_top10_2025_in", + action="store_true", + help="import OWASP Top 10 2025", + ) + parser.add_argument( + "--owasp_api_top10_2023_in", + action="store_true", + help="import OWASP API Security Top 10 2023", + ) + parser.add_argument( + "--owasp_kubernetes_top10_2022_in", + action="store_true", + help="import OWASP Kubernetes Top Ten 2022", + ) + parser.add_argument( + "--owasp_kubernetes_top10_2025_in", + action="store_true", + help="import OWASP Kubernetes Top Ten 2025 draft", + ) + parser.add_argument( + "--owasp_llm_top10_2025_in", + action="store_true", + help="import OWASP Top 10 for LLM and Gen AI Apps 2025", + ) + parser.add_argument( + "--owasp_aisvs_in", + action="store_true", + help="import OWASP AI Security Verification Standard (AISVS)", + ) parser.add_argument( "--pci_dss_3_2_in", action="store_true", From 5560673ce5678ccae199e04e80239eb36a15308c Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 16:13:04 +0530 Subject: [PATCH 02/11] Fix cheat sheet parser test expectations on importer branches --- application/tests/cheatsheets_parser_test.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/application/tests/cheatsheets_parser_test.py b/application/tests/cheatsheets_parser_test.py index 1a3ba4bf0..e2c0910d6 100644 --- a/application/tests/cheatsheets_parser_test.py +++ b/application/tests/cheatsheets_parser_test.py @@ -34,7 +34,13 @@ class Repo: repo.working_dir = loc cre = defs.CRE(name="blah", id="223-780") self.collection.add_cre(cre) - with open(os.path.join(os.path.join(loc, "cheatsheets"), "cs.md"), "w") as mdf: + with open( + os.path.join( + os.path.join(loc, "cheatsheets"), + "Secrets_Management_Cheat_Sheet.md", + ), + "w", + ) as mdf: mdf.write(cs) mock_clone.return_value = repo entries = cheatsheets_parser.Cheatsheets().parse( @@ -45,22 +51,26 @@ class Repo: # verify the external tagging convention, not just enum wiring. expected = defs.Standard( name="OWASP Cheat Sheets", - hyperlink="https://github.com/foo/bar/tree/master/cs.md", + hyperlink="https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html", section="Secrets Management Cheat Sheet", - links=[defs.Link(document=cre, ltype=defs.LinkTypes.LinkedTo)], + links=[ + defs.Link( + document=cre, ltype=defs.LinkTypes.AutomaticallyLinkedTo + ) + ], tags=[ "family:guidance", "subtype:cheatsheet", - "source:owasp_cheatsheets", "audience:developer", "maturity:stable", + "source:owasp_cheatsheets", ], ) self.maxDiff = None for name, nodes in entries.results.items(): self.assertEqual(name, parser.name) self.assertEqual(len(nodes), 1) - self.assertCountEqual(expected.todict(), nodes[0].todict()) + self.assertEqual(expected.todict(), nodes[0].todict()) cheatsheets_md = """ # Secrets Management Cheat Sheet From c27cc7ae4512aa2fa7208d37888c54d28f404d5f Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 16:14:21 +0530 Subject: [PATCH 03/11] Use official OWASP cheat sheet URLs in importer branches --- .../external_project_parsers/parsers/cheatsheets_parser.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py index e695414d8..e234dadda 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py +++ b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py @@ -15,6 +15,7 @@ class Cheatsheets(ParserInterface): name = "OWASP Cheat Sheets" + cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets" def cheatsheet( self, section: str, hyperlink: str, tags: List[str] @@ -33,6 +34,10 @@ def cheatsheet( hyperlink=hyperlink, ) + def official_cheatsheet_url(self, markdown_filename: str) -> str: + html_name = os.path.splitext(markdown_filename)[0] + ".html" + return f"{self.cheatsheetseries_base_url}/{html_name}" + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): c_repo = "https://github.com/OWASP/CheatSheetSeries.git" cheatsheets_path = "cheatsheets/" @@ -65,7 +70,7 @@ def register_cheatsheets( name = title.group("title") cre_id = cre.group("cre") cres = cache.get_CREs(external_id=cre_id) - hyperlink = f"{repo_path.replace('.git','')}/tree/master/{cheatsheets_path}{mdfile}" + hyperlink = self.official_cheatsheet_url(mdfile) cs = self.cheatsheet(section=name, hyperlink=hyperlink, tags=[]) for cre in cres: cs.add_link( From a11c5afe31be2e6538a69370a102650f6909a565 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 04/11] Add OWASP AI resource importer support --- application/tests/owasp_aisvs_parser_test.py | 62 +++++++++++++ .../tests/owasp_llm_top10_2025_parser_test.py | 45 ++++++++++ .../data/owasp_aisvs_1_0.json | 86 +++++++++++++++++++ .../data/owasp_llm_top10_2025.json | 62 +++++++++++++ .../parsers/owasp_aisvs.py | 45 ++++++++++ .../parsers/owasp_llm_top10_2025.py | 47 ++++++++++ 6 files changed, 347 insertions(+) create mode 100644 application/tests/owasp_aisvs_parser_test.py create mode 100644 application/tests/owasp_llm_top10_2025_parser_test.py create mode 100644 application/utils/external_project_parsers/data/owasp_aisvs_1_0.json create mode 100644 application/utils/external_project_parsers/data/owasp_llm_top10_2025.json create mode 100644 application/utils/external_project_parsers/parsers/owasp_aisvs.py create mode 100644 application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py diff --git a/application/tests/owasp_aisvs_parser_test.py b/application/tests/owasp_aisvs_parser_test.py new file mode 100644 index 000000000..461b2d68d --- /dev/null +++ b/application/tests/owasp_aisvs_parser_test.py @@ -0,0 +1,62 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_aisvs + + +class TestOwaspAisvsParser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("227-045", "Identify sensitive data and subject it to a policy"), + ( + "307-507", + "Allow only trusted sources both build time and runtime; therefore perform integrity checks on all resources and code", + ), + ( + "162-655", + "Documentation of all components' business or security function", + ), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_aisvs.OwaspAisvs().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP AI Security Verification Standard (AISVS)"] + self.assertEqual(14, len(entries)) + self.assertEqual("AISVS1", entries[0].sectionID) + self.assertEqual( + "Training Data Governance & Bias Management", entries[0].section + ) + self.assertEqual( + "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C01-Training-Data-Governance.md", + entries[0].hyperlink, + ) + self.assertEqual( + ["227-045", "307-507"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("AISVS14", entries[-1].sectionID) + self.assertEqual( + "Human Oversight, Accountability & Governance", entries[-1].section + ) + self.assertEqual( + "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C14-Human-Oversight.md", + entries[-1].hyperlink, + ) + self.assertEqual(["162-655"], [l.document.id for l in entries[-1].links]) diff --git a/application/tests/owasp_llm_top10_2025_parser_test.py b/application/tests/owasp_llm_top10_2025_parser_test.py new file mode 100644 index 000000000..75b282c34 --- /dev/null +++ b/application/tests/owasp_llm_top10_2025_parser_test.py @@ -0,0 +1,45 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_llm_top10_2025 + + +class TestOwaspLlmTop10_2025Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("161-451", "Output encoding and injection prevention"), + ("064-808", "Encode output context-specifically"), + ("760-764", "Injection protection"), + ("623-550", "Denial Of Service protection"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_llm_top10_2025.OwaspLlmTop10_2025().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP Top 10 for LLM and Gen AI Apps 2025"] + self.assertEqual(10, len(entries)) + self.assertEqual("LLM01", entries[0].sectionID) + self.assertEqual("Prompt Injection", entries[0].section) + self.assertEqual( + ["161-451", "760-764"], [l.document.id for l in entries[0].links] + ) + self.assertEqual(["064-808"], [l.document.id for l in entries[4].links]) + self.assertEqual("LLM10", entries[-1].sectionID) + self.assertEqual(["623-550"], [l.document.id for l in entries[-1].links]) diff --git a/application/utils/external_project_parsers/data/owasp_aisvs_1_0.json b/application/utils/external_project_parsers/data/owasp_aisvs_1_0.json new file mode 100644 index 000000000..c4880546f --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_aisvs_1_0.json @@ -0,0 +1,86 @@ +[ + { + "section_id": "AISVS1", + "section": "Training Data Governance & Bias Management", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C01-Training-Data-Governance.md", + "cre_ids": ["227-045", "307-507"] + }, + { + "section_id": "AISVS2", + "section": "User Input Validation", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C02-User-Input-Validation.md", + "cre_ids": ["031-447", "760-764"] + }, + { + "section_id": "AISVS3", + "section": "Model Lifecycle Management & Change Control", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C03-Model-Lifecycle-Management.md", + "cre_ids": ["148-853", "613-285"] + }, + { + "section_id": "AISVS4", + "section": "Infrastructure, Configuration & Deployment Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C04-Infrastructure.md", + "cre_ids": ["233-748", "486-813"] + }, + { + "section_id": "AISVS5", + "section": "Access Control & Identity for AI Components & Users", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C05-Access-Control-and-Identity.md", + "cre_ids": ["633-428", "724-770"] + }, + { + "section_id": "AISVS6", + "section": "Supply Chain Security for Models, Frameworks & Data", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C06-Supply-Chain.md", + "cre_ids": ["613-285", "613-287", "863-521"] + }, + { + "section_id": "AISVS7", + "section": "Model Behavior, Output Control & Safety Assurance", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C07-Model-Behavior.md", + "cre_ids": ["064-808", "141-555"] + }, + { + "section_id": "AISVS8", + "section": "Memory, Embeddings & Vector Database Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C08-Memory-Embeddings-and-Vector-Database.md", + "cre_ids": ["126-668", "538-770"] + }, + { + "section_id": "AISVS9", + "section": "Autonomous Orchestration & Agentic Action Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C09-Orchestration-and-Agentic-Action.md", + "cre_ids": ["117-371", "650-560"] + }, + { + "section_id": "AISVS10", + "section": "Model Context Protocol (MCP) Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C10-MCP-Security.md", + "cre_ids": ["307-507", "715-223"] + }, + { + "section_id": "AISVS11", + "section": "Adversarial Robustness & Privacy Defense", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C11-Adversarial-Robustness.md", + "cre_ids": ["141-555", "623-550"] + }, + { + "section_id": "AISVS12", + "section": "Privacy Protection & Personal Data Management", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C12-Privacy.md", + "cre_ids": ["126-668", "227-045", "482-866"] + }, + { + "section_id": "AISVS13", + "section": "Monitoring, Logging & Anomaly Detection", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C13-Monitoring-and-Logging.md", + "cre_ids": ["058-083", "148-420", "402-706", "843-841"] + }, + { + "section_id": "AISVS14", + "section": "Human Oversight, Accountability & Governance", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C14-Human-Oversight.md", + "cre_ids": ["162-655", "766-162"] + } +] diff --git a/application/utils/external_project_parsers/data/owasp_llm_top10_2025.json b/application/utils/external_project_parsers/data/owasp_llm_top10_2025.json new file mode 100644 index 000000000..b761d5e09 --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_llm_top10_2025.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "LLM01", + "section": "Prompt Injection", + "hyperlink": "https://genai.owasp.org/llmrisk/llm01-prompt-injection/", + "cre_ids": ["161-451", "760-764"] + }, + { + "section_id": "LLM02", + "section": "Sensitive Information Disclosure", + "hyperlink": "https://genai.owasp.org/llmrisk/llm022025-sensitive-information-disclosure/", + "cre_ids": ["126-668", "227-045"] + }, + { + "section_id": "LLM03", + "section": "Supply Chain", + "hyperlink": "https://genai.owasp.org/llmrisk/llm032025-supply-chain/", + "cre_ids": ["613-285", "613-287"] + }, + { + "section_id": "LLM04", + "section": "Data and Model Poisoning", + "hyperlink": "https://genai.owasp.org/llmrisk/llm042025-data-and-model-poisoning/", + "cre_ids": ["307-507", "613-287"] + }, + { + "section_id": "LLM05", + "section": "Improper Output Handling", + "hyperlink": "https://genai.owasp.org/llmrisk/llm052025-improper-output-handling/", + "cre_ids": ["064-808"] + }, + { + "section_id": "LLM06", + "section": "Excessive Agency", + "hyperlink": "https://genai.owasp.org/llmrisk/llm062025-excessive-agency/", + "cre_ids": ["117-371", "650-560"] + }, + { + "section_id": "LLM07", + "section": "System Prompt Leakage", + "hyperlink": "https://genai.owasp.org/llmrisk/llm072025-system-prompt-leakage/", + "cre_ids": ["126-668", "227-045"] + }, + { + "section_id": "LLM08", + "section": "Vector and Embedding Weaknesses", + "hyperlink": "https://genai.owasp.org/llmrisk/llm082025-vector-and-embedding-weaknesses/", + "cre_ids": ["126-668", "538-770"] + }, + { + "section_id": "LLM09", + "section": "Misinformation", + "hyperlink": "https://genai.owasp.org/llmrisk/llm092025-misinformation/", + "cre_ids": ["141-555"] + }, + { + "section_id": "LLM10", + "section": "Unbounded Consumption", + "hyperlink": "https://genai.owasp.org/llmrisk/llm102025-unbounded-consumption/", + "cre_ids": ["267-031", "623-550"] + } +] diff --git a/application/utils/external_project_parsers/parsers/owasp_aisvs.py b/application/utils/external_project_parsers/parsers/owasp_aisvs.py new file mode 100644 index 000000000..cec4abad9 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_aisvs.py @@ -0,0 +1,45 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspAisvs(ParserInterface): + name = "OWASP AI Security Verification Standard (AISVS)" + data_file = Path(__file__).resolve().parent.parent / "data" / "owasp_aisvs_1_0.json" + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py new file mode 100644 index 000000000..3971b9e6b --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspLlmTop10_2025(ParserInterface): + name = "OWASP Top 10 for LLM and Gen AI Apps 2025" + data_file = ( + Path(__file__).resolve().parent.parent / "data" / "owasp_llm_top10_2025.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) From c9b49f87572e6de5b0cd65678b8261d1f61137ff Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 05/11] Add OWASP Kubernetes importer support --- ...owasp_kubernetes_top10_2022_parser_test.py | 45 ++++++++ ...owasp_kubernetes_top10_2025_parser_test.py | 102 ++++++++++++++++++ .../data/owasp_kubernetes_top10_2022.json | 62 +++++++++++ .../data/owasp_kubernetes_top10_2025.json | 72 +++++++++++++ .../parsers/owasp_kubernetes_top10_2022.py | 49 +++++++++ .../parsers/owasp_kubernetes_top10_2025.py | 78 ++++++++++++++ 6 files changed, 408 insertions(+) create mode 100644 application/tests/owasp_kubernetes_top10_2022_parser_test.py create mode 100644 application/tests/owasp_kubernetes_top10_2025_parser_test.py create mode 100644 application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json create mode 100644 application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json create mode 100644 application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py create mode 100644 application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py diff --git a/application/tests/owasp_kubernetes_top10_2022_parser_test.py b/application/tests/owasp_kubernetes_top10_2022_parser_test.py new file mode 100644 index 000000000..30b0922c9 --- /dev/null +++ b/application/tests/owasp_kubernetes_top10_2022_parser_test.py @@ -0,0 +1,45 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2022, +) + + +class TestOwaspKubernetesTop10_2022Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("233-748", "Configuration hardening"), + ("486-813", "Configuration"), + ("053-751", "Force build pipeline to check outdated/insecure components"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_kubernetes_top10_2022.OwaspKubernetesTop10_2022().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP Kubernetes Top Ten 2022"] + self.assertEqual(10, len(entries)) + self.assertEqual("K01", entries[0].sectionID) + self.assertEqual("Insecure Workload Configurations", entries[0].section) + self.assertEqual( + ["233-748", "486-813"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("K10", entries[-1].sectionID) + self.assertEqual(["053-751"], [l.document.id for l in entries[-1].links]) diff --git a/application/tests/owasp_kubernetes_top10_2025_parser_test.py b/application/tests/owasp_kubernetes_top10_2025_parser_test.py new file mode 100644 index 000000000..6f444c9a9 --- /dev/null +++ b/application/tests/owasp_kubernetes_top10_2025_parser_test.py @@ -0,0 +1,102 @@ +import unittest +import tempfile +from pathlib import Path + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2025, +) + + +class TestOwaspKubernetesTop10_2025Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("233-748", "Configuration hardening"), + ("486-813", "Configuration"), + ("148-420", "Log integrity"), + ("402-706", "Log relevant"), + ("843-841", "Log discretely"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP Kubernetes Top Ten 2025 (Draft)"] + self.assertEqual(10, len(entries)) + self.assertEqual("K01", entries[0].sectionID) + self.assertEqual("Insecure Workload Configurations", entries[0].section) + self.assertEqual( + ["233-748", "486-813"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("K10", entries[-1].sectionID) + self.assertEqual( + ["148-420", "402-706", "843-841"], + [l.document.id for l in entries[-1].links], + ) + + def test_parse_falls_back_to_2022_mapping_when_2025_links_missing(self) -> None: + self.collection.add_cre( + defs.CRE(id="148-420", name="Log integrity", description="") + ) + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + current_file = tmp_path / "k8s_2025.json" + fallback_file = tmp_path / "k8s_2022.json" + current_file.write_text( + """ +[ + { + "section_id": "K10", + "section": "Inadequate Logging And Monitoring", + "hyperlink": "https://example.com/k10", + "cre_ids": ["999-999"], + "fallback_section_ids": ["K05"] + } +] + """.strip(), + encoding="utf-8", + ) + fallback_file.write_text( + """ +[ + { + "section_id": "K05", + "section": "Inadequate Logging and Monitoring", + "hyperlink": "https://example.com/k05", + "cre_ids": ["148-420"] + } +] + """.strip(), + encoding="utf-8", + ) + + parser = owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025() + parser.data_file = current_file + parser.fallback_data_file = fallback_file + + result = parser.parse( + self.collection, + prompt_client.PromptHandler(database=self.collection), + ) + + entries = result.results["OWASP Kubernetes Top Ten 2025 (Draft)"] + self.assertEqual(1, len(entries)) + self.assertEqual(["148-420"], [link.document.id for link in entries[0].links]) diff --git a/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json new file mode 100644 index 000000000..c4eb3d6fd --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "K01", + "section": "Insecure Workload Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K01-insecure-workload-configurations", + "cre_ids": ["233-748", "486-813"] + }, + { + "section_id": "K02", + "section": "Supply Chain Vulnerabilities", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K02-supply-chain-vulnerabilities", + "cre_ids": ["613-285", "613-287"] + }, + { + "section_id": "K03", + "section": "Overly Permissive RBAC Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K03-overly-permissive-rbac-configurations", + "cre_ids": ["128-128", "724-770"] + }, + { + "section_id": "K04", + "section": "Lack of Centralized Policy Enforcement", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K04-lack-of-centralized-policy-enforcement", + "cre_ids": ["117-371"] + }, + { + "section_id": "K05", + "section": "Inadequate Logging and Monitoring", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K05-inadequate-logging-and-monitoring", + "cre_ids": ["058-083", "148-420", "402-706", "843-841"] + }, + { + "section_id": "K06", + "section": "Broken Authentication Mechanisms", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K06-broken-authentication-mechanisms", + "cre_ids": ["177-260", "586-842", "633-428"] + }, + { + "section_id": "K07", + "section": "Missing Network Segmentation Controls", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K07-missing-network-segmentation-controls", + "cre_ids": ["132-146", "467-784", "515-021"] + }, + { + "section_id": "K08", + "section": "Secrets Management Failures", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K08-secrets-management-failures", + "cre_ids": ["340-375", "774-888", "813-610"] + }, + { + "section_id": "K09", + "section": "Misconfigured Cluster Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K09-misconfigured-cluster-components", + "cre_ids": ["233-748", "486-813"] + }, + { + "section_id": "K10", + "section": "Outdated and Vulnerable Kubernetes Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K10-outdated-and-vulnerable-kubernetes-components", + "cre_ids": ["053-751", "715-334", "863-521"] + } +] diff --git a/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json new file mode 100644 index 000000000..c55afb059 --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json @@ -0,0 +1,72 @@ +[ + { + "section_id": "K01", + "section": "Insecure Workload Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["233-748", "486-813"], + "fallback_section_ids": ["K01"] + }, + { + "section_id": "K02", + "section": "Overly Permissive Authorization Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["128-128", "724-770"], + "fallback_section_ids": ["K03"] + }, + { + "section_id": "K03", + "section": "Secrets Management Failures", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["340-375", "774-888", "813-610"], + "fallback_section_ids": ["K08"] + }, + { + "section_id": "K04", + "section": "Lack Of Cluster Level Policy Enforcement", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["117-371"], + "fallback_section_ids": ["K04"] + }, + { + "section_id": "K05", + "section": "Missing Network Segmentation Controls", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["132-146", "467-784", "515-021"], + "fallback_section_ids": ["K07"] + }, + { + "section_id": "K06", + "section": "Overly Exposed Kubernetes Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["152-725", "640-364"], + "fallback_section_ids": ["K09"] + }, + { + "section_id": "K07", + "section": "Misconfigured And Vulnerable Cluster Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["053-751", "233-748", "486-813", "715-334"], + "fallback_section_ids": ["K09", "K10"] + }, + { + "section_id": "K08", + "section": "Cluster To Cloud Lateral Movement", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["132-146", "640-364", "724-770"], + "fallback_section_ids": ["K03", "K07"] + }, + { + "section_id": "K09", + "section": "Broken Authentication Mechanisms", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["177-260", "586-842", "633-428"], + "fallback_section_ids": ["K06"] + }, + { + "section_id": "K10", + "section": "Inadequate Logging And Monitoring", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["058-083", "148-420", "402-706", "843-841"], + "fallback_section_ids": ["K05"] + } +] diff --git a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py new file mode 100644 index 000000000..9d3822ab7 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspKubernetesTop10_2022(ParserInterface): + name = "OWASP Kubernetes Top Ten 2022" + data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_kubernetes_top10_2022.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py new file mode 100644 index 000000000..31deed8da --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py @@ -0,0 +1,78 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspKubernetesTop10_2025(ParserInterface): + name = "OWASP Kubernetes Top Ten 2025 (Draft)" + data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_kubernetes_top10_2025.json" + ) + fallback_data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_kubernetes_top10_2022.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + with self.fallback_data_file.open("r", encoding="utf-8") as handle: + fallback_entries = { + entry["section_id"]: entry for entry in json.load(handle) + } + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + linked_cre_ids = [] + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + linked_cre_ids.append(cre_id) + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + if not linked_cre_ids: + for section_id in entry.get("fallback_section_ids", []): + fallback_entry = fallback_entries.get(section_id) + if not fallback_entry: + continue + for cre_id in fallback_entry.get("cre_ids", []): + if cre_id in linked_cre_ids: + continue + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + linked_cre_ids.append(cre_id) + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) From 8df31e94e23c42f48f8db277b3da6e16880e91b0 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 06/11] Normalize OWASP cheat sheet references --- application/tests/cheatsheets_parser_test.py | 33 +++++++++- .../data/owasp_cheatsheets_supplement.json | 47 ++++++++++++++ .../parsers/cheatsheets_parser.py | 62 +++++++++++++++++-- 3 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json diff --git a/application/tests/cheatsheets_parser_test.py b/application/tests/cheatsheets_parser_test.py index e2c0910d6..fb2a9c277 100644 --- a/application/tests/cheatsheets_parser_test.py +++ b/application/tests/cheatsheets_parser_test.py @@ -69,8 +69,37 @@ class Repo: self.maxDiff = None for name, nodes in entries.results.items(): self.assertEqual(name, parser.name) - self.assertEqual(len(nodes), 1) - self.assertEqual(expected.todict(), nodes[0].todict()) + sections = {node.section for node in nodes} + self.assertIn("Secrets Management Cheat Sheet", sections) + secret_entry = [ + node + for node in nodes + if node.section == "Secrets Management Cheat Sheet" + ][0] + self.assertEqual(expected.todict(), secret_entry.todict()) + + def test_register_supplemental_cheatsheets(self) -> None: + for cre_id, name in [ + ("118-110", "API/web services"), + ("724-770", "Technical application access control"), + ("623-550", "Denial Of Service protection"), + ]: + self.collection.add_cre(defs.CRE(name=name, id=cre_id)) + + entries = cheatsheets_parser.Cheatsheets().register_supplemental_cheatsheets( + cache=self.collection + ) + rest = [ + entry for entry in entries if entry.section == "REST Security Cheat Sheet" + ][0] + self.assertEqual( + "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html", + rest.hyperlink, + ) + self.assertEqual( + ["118-110", "724-770", "623-550"], + [link.document.id for link in rest.links], + ) cheatsheets_md = """ # Secrets Management Cheat Sheet diff --git a/application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json b/application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json new file mode 100644 index 000000000..4e06bee8c --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json @@ -0,0 +1,47 @@ +[ + { + "section": "Authorization Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Authorization_Cheat_Sheet.html", + "cre_ids": ["128-128", "117-371"] + }, + { + "section": "REST Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html", + "cre_ids": ["118-110", "724-770", "623-550"] + }, + { + "section": "Server Side Request Forgery Prevention Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html", + "cre_ids": ["028-728", "657-084"] + }, + { + "section": "Docker Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Docker_Security_Cheat_Sheet.html", + "cre_ids": ["233-748", "486-813"] + }, + { + "section": "Kubernetes Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Kubernetes_Security_Cheat_Sheet.html", + "cre_ids": ["467-784", "233-748", "486-813"] + }, + { + "section": "Secure Cloud Architecture Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_Cloud_Architecture_Cheat_Sheet.html", + "cre_ids": ["155-155", "467-784"] + }, + { + "section": "LLM Prompt Injection Prevention Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/LLM_Prompt_Injection_Prevention_Cheat_Sheet.html", + "cre_ids": ["161-451", "760-764"] + }, + { + "section": "AI Agent Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/AI_Agent_Security_Cheat_Sheet.html", + "cre_ids": ["117-371", "650-560", "126-668"] + }, + { + "section": "Secure AI Model Ops Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_AI_Model_Ops_Cheat_Sheet.html", + "cre_ids": ["148-853", "613-285", "613-287"] + } +] diff --git a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py index e234dadda..02003b7bd 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py +++ b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py @@ -6,6 +6,9 @@ import os import re from application.utils.external_project_parsers import base_parser_defs +import json +from pathlib import Path +import logging from application.utils.external_project_parsers.base_parser_defs import ( ParserInterface, ParseResult, @@ -16,6 +19,12 @@ class Cheatsheets(ParserInterface): name = "OWASP Cheat Sheets" cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets" + supplement_data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_cheatsheets_supplement.json" + ) + logger = logging.getLogger(__name__) def cheatsheet( self, section: str, hyperlink: str, tags: List[str] @@ -41,10 +50,22 @@ def official_cheatsheet_url(self, markdown_filename: str) -> str: def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): c_repo = "https://github.com/OWASP/CheatSheetSeries.git" cheatsheets_path = "cheatsheets/" - repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True) - cheatsheets = self.register_cheatsheets( - repo=repo, cache=cache, cheatsheets_path=cheatsheets_path, repo_path=c_repo - ) + cheatsheets = [] + try: + repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True) + cheatsheets = self.register_cheatsheets( + repo=repo, + cache=cache, + cheatsheets_path=cheatsheets_path, + repo_path=c_repo, + ) + except Exception as exc: + self.logger.warning( + "Unable to clone OWASP CheatSheetSeries, continuing with supplemental cheat sheets only: %s", + exc, + ) + cheatsheets.extend(self.register_supplemental_cheatsheets(cache=cache)) + cheatsheets = self.deduplicate_entries(cheatsheets) results = {self.name: cheatsheets} base_parser_defs.validate_classification_tags(results) return ParseResult(results=results) @@ -80,3 +101,36 @@ def register_cheatsheets( ) standard_entries.append(cs) return standard_entries + + def register_supplemental_cheatsheets(self, cache: db.Node_collection): + with self.supplement_data_file.open("r", encoding="utf-8") as handle: + supplement_entries = json.load(handle) + + standard_entries = [] + for entry in supplement_entries: + cs = self.cheatsheet( + section=entry["section"], + hyperlink=entry["hyperlink"], + tags=[], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + for cre in cres: + try: + cs.add_link( + defs.Link( + document=cre.shallow_copy(), + ltype=defs.LinkTypes.AutomaticallyLinkedTo, + ) + ) + except Exception: + continue + if cs.links: + standard_entries.append(cs) + return standard_entries + + def deduplicate_entries(self, entries: List[defs.Standard]) -> List[defs.Standard]: + deduped = {} + for entry in entries: + deduped[(entry.section, entry.hyperlink)] = entry + return list(deduped.values()) From e4dd489203175fb6a7a5d09d93d3aa557c802c19 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 07/11] Add refresh scripts for OWASP resources --- scripts/update-cheatsheets.sh | 55 +++++++++ scripts/update-owasp-top10-2025-mappings.sh | 38 ++++++ scripts/update-owasp-top10-standards.sh | 129 ++++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100755 scripts/update-cheatsheets.sh create mode 100755 scripts/update-owasp-top10-2025-mappings.sh create mode 100644 scripts/update-owasp-top10-standards.sh diff --git a/scripts/update-cheatsheets.sh b/scripts/update-cheatsheets.sh new file mode 100755 index 000000000..48d5eccc5 --- /dev/null +++ b/scripts/update-cheatsheets.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DB_PATH="${1:-$ROOT_DIR/standards_cache.sqlite}" +VENV_DIR="$ROOT_DIR/venv" + +if [[ ! -d "$VENV_DIR" ]]; then + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import flask" >/dev/null 2>&1; then + pip install -r "$ROOT_DIR/requirements.txt" +fi + +CRE_NO_CALCULATE_GAP_ANALYSIS=1 \ +CRE_NO_GEN_EMBEDDINGS=1 \ +python "$ROOT_DIR/cre.py" --cheatsheets_in --cache_file "$DB_PATH" + +python - "$DB_PATH" <<'PY' +import os +import sqlite3 +import sys + +db_path = sys.argv[1] +conn = sqlite3.connect(db_path) +cur = conn.cursor() + +github_prefix = "https://github.com/OWASP/CheatSheetSeries/tree/master/cheatsheets/" +official_prefix = "https://cheatsheetseries.owasp.org/cheatsheets/" + +rows = cur.execute( + """ + select id, link + from node + where name = 'OWASP Cheat Sheets' + and link like ? + """, + (f"{github_prefix}%",), +).fetchall() + +for node_id, link in rows: + filename = os.path.basename(link) + html_name = os.path.splitext(filename)[0] + ".html" + cur.execute( + "update node set link = ? where id = ?", + (f"{official_prefix}{html_name}", node_id), + ) + +conn.commit() +conn.close() +print(f"Normalized {len(rows)} OWASP Cheat Sheet links") +PY diff --git a/scripts/update-owasp-top10-2025-mappings.sh b/scripts/update-owasp-top10-2025-mappings.sh new file mode 100755 index 000000000..04258646b --- /dev/null +++ b/scripts/update-owasp-top10-2025-mappings.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$ROOT_DIR/venv" +CACHE_FILE="${1:-$ROOT_DIR/standards_cache.sqlite}" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +BACKUP_FILE="${CACHE_FILE}.bak.${TIMESTAMP}" + +if [[ ! -d "$VENV_DIR" ]]; then + echo "Creating virtual environment in $VENV_DIR" + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import requests" >/dev/null 2>&1; then + echo "Installing Python dependencies" + pip install -r "$ROOT_DIR/requirements.txt" +fi + +if [[ -f "$CACHE_FILE" ]]; then + cp "$CACHE_FILE" "$BACKUP_FILE" + echo "Backed up database to $BACKUP_FILE" +fi + +export CRE_NO_NEO4J="${CRE_NO_NEO4J:-1}" +export CRE_NO_GEN_EMBEDDINGS="${CRE_NO_GEN_EMBEDDINGS:-1}" +export CRE_UPSTREAM_MAX_ATTEMPTS="${CRE_UPSTREAM_MAX_ATTEMPTS:-6}" +export CRE_UPSTREAM_RETRY_BACKOFF_SECONDS="${CRE_UPSTREAM_RETRY_BACKOFF_SECONDS:-2}" +export CRE_UPSTREAM_TIMEOUT_SECONDS="${CRE_UPSTREAM_TIMEOUT_SECONDS:-30}" + +echo "Refreshing official OpenCRE upstream data in $CACHE_FILE" +python "$ROOT_DIR/cre.py" --upstream_sync --cache_file "$CACHE_FILE" + +echo "Reapplying OWASP Top 10 2025 CRE mappings" +exec python "$ROOT_DIR/cre.py" --owasp_top10_2025_in --cache_file "$CACHE_FILE" diff --git a/scripts/update-owasp-top10-standards.sh b/scripts/update-owasp-top10-standards.sh new file mode 100644 index 000000000..a795cf872 --- /dev/null +++ b/scripts/update-owasp-top10-standards.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$ROOT_DIR/venv" +CACHE_FILE="${1:-$ROOT_DIR/standards_cache.sqlite}" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +BACKUP_FILE="${CACHE_FILE}.bak.${TIMESTAMP}" + +if [[ ! -d "$VENV_DIR" ]]; then + echo "Creating virtual environment in $VENV_DIR" + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import requests" >/dev/null 2>&1; then + echo "Installing Python dependencies" + pip install -r "$ROOT_DIR/requirements.txt" +fi + +if [[ -f "$CACHE_FILE" ]]; then + cp "$CACHE_FILE" "$BACKUP_FILE" + echo "Backed up database to $BACKUP_FILE" +fi + +export CRE_NO_NEO4J="${CRE_NO_NEO4J:-1}" +export CRE_NO_GEN_EMBEDDINGS="${CRE_NO_GEN_EMBEDDINGS:-1}" +export CRE_UPSTREAM_MAX_ATTEMPTS="${CRE_UPSTREAM_MAX_ATTEMPTS:-6}" +export CRE_UPSTREAM_RETRY_BACKOFF_SECONDS="${CRE_UPSTREAM_RETRY_BACKOFF_SECONDS:-2}" +export CRE_UPSTREAM_TIMEOUT_SECONDS="${CRE_UPSTREAM_TIMEOUT_SECONDS:-30}" + +echo "Refreshing official OpenCRE upstream data in $CACHE_FILE" +python "$ROOT_DIR/cre.py" --upstream_sync --cache_file "$CACHE_FILE" + +echo "Reapplying OWASP Top 10 standards and CRE mappings" +python "$ROOT_DIR/cre.py" \ + --owasp_top10_2025_in \ + --owasp_api_top10_2023_in \ + --owasp_kubernetes_top10_2025_in \ + --owasp_llm_top10_2025_in \ + --owasp_aisvs_in \ + --cache_file "$CACHE_FILE" + +echo "Selecting preferred Kubernetes Top Ten version" +if python - <<'PY' "$CACHE_FILE" +import sqlite3 +import sys + +cache_file = sys.argv[1] +name_2025 = "OWASP Kubernetes Top Ten 2025 (Draft)" +name_2022 = "OWASP Kubernetes Top Ten 2022" + +conn = sqlite3.connect(cache_file) +cur = conn.cursor() + +linked_2025 = cur.execute( + """ + select count(*) + from node n + join cre_node_links l on l.node = n.id + where n.name = ? + """, + (name_2025,), +).fetchone()[0] + +if linked_2025 > 0: + cur.execute("delete from node where name = ?", (name_2022,)) + print(f"Using {name_2025}; removed {name_2022}") +else: + raise SystemExit(f"{name_2025} not linked") + +conn.commit() +conn.close() +PY +then + : +else + echo "OWASP Kubernetes Top Ten 2025 (Draft) is unavailable or unmapped, importing 2022" + python "$ROOT_DIR/cre.py" \ + --owasp_kubernetes_top10_2022_in \ + --cache_file "$CACHE_FILE" +fi + +echo "Pruning OWASP Top 10 entries that still have no CRE links" +python - <<'PY' "$CACHE_FILE" +import sqlite3 +import sys + +cache_file = sys.argv[1] +standard_names = ( + "OWASP Top 10 2025", + "OWASP API Security Top 10 2023", + "OWASP Kubernetes Top Ten 2025 (Draft)", + "OWASP Top 10 for LLM and Gen AI Apps 2025", + "OWASP AI Security Verification Standard (AISVS)", +) + +conn = sqlite3.connect(cache_file) +cur = conn.cursor() + +has_2022 = cur.execute( + "select 1 from node where name = 'OWASP Kubernetes Top Ten 2022' limit 1" +).fetchone() +if has_2022: + standard_names = standard_names + ("OWASP Kubernetes Top Ten 2022",) + +rows = list( + cur.execute( + f""" + select n.id, n.name, coalesce(n.section_id, ''), coalesce(n.section, '') + from node n + left join cre_node_links l on l.node = n.id + where n.name in ({','.join('?' for _ in standard_names)}) + group by n.id + having count(l.cre) = 0 + """, + standard_names, + ) +) + +for node_id, name, section_id, section in rows: + cur.execute("delete from node where id = ?", (node_id,)) + print(f"Removed unmapped entry: {name} {section_id} {section}".strip()) + +conn.commit() +conn.close() +PY From 6b800fc7e8b1596d51301694cd8608157e1fcf36 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 08/11] Retry transient failures during upstream sync --- application/cmd/cre_main.py | 115 ++++++++++++++++++++++++----- application/tests/cre_main_test.py | 107 +++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 18 deletions(-) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index 5369617fb..28dc6411e 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -36,6 +36,51 @@ app = None +def fetch_upstream_json( + path: str, + timeout: Optional[float] = None, + max_attempts: Optional[int] = None, + backoff_seconds: Optional[float] = None, +) -> Dict[str, Any]: + base_url = os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") + timeout = timeout or float(os.environ.get("CRE_UPSTREAM_TIMEOUT_SECONDS", "30")) + max_attempts = max_attempts or int(os.environ.get("CRE_UPSTREAM_MAX_ATTEMPTS", "4")) + backoff_seconds = backoff_seconds or float( + os.environ.get("CRE_UPSTREAM_RETRY_BACKOFF_SECONDS", "2") + ) + url = f"{base_url}{path}" + last_error: Optional[Exception] = None + + for attempt in range(1, max_attempts + 1): + try: + response = requests.get(url, timeout=timeout) + if response.status_code == 200: + return response.json() + + status_error = RuntimeError( + f"cannot connect to upstream status code {response.status_code}" + ) + # Retry only on transient upstream failures. + if response.status_code < 500 and response.status_code != 429: + raise status_error + last_error = status_error + except requests.exceptions.RequestException as exc: + last_error = exc + + if attempt < max_attempts: + logger.warning( + "upstream fetch failed for %s on attempt %s/%s, retrying", + url, + attempt, + max_attempts, + ) + time.sleep(backoff_seconds * attempt) + + if last_error: + raise RuntimeError(f"upstream fetch failed for {url}") from last_error + raise RuntimeError(f"upstream fetch failed for {url}") + + def register_node(node: defs.Node, collection: db.Node_collection) -> db.Node: """ for each link find if either the root node or the link have a CRE, @@ -353,6 +398,8 @@ def register_standard( ): if os.environ.get("CRE_NO_GEN_EMBEDDINGS") == "1": generate_embeddings = False + if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"): + calculate_gap_analysis = False if not standard_entries: logger.warning("register_standard() called with no standard_entries") @@ -589,15 +636,7 @@ def download_graph_from_upstream(cache: str) -> None: collection = db_connect(path=cache).with_graph() def download_cre_from_upstream(creid: str): - cre_response = requests.get( - os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") - + f"/id/{creid}" - ) - if cre_response.status_code != 200: - raise RuntimeError( - f"cannot connect to upstream status code {cre_response.status_code}" - ) - data = cre_response.json() + data = fetch_upstream_json(f"/id/{creid}") credict = data["data"] cre = defs.Document.from_dict(credict) if cre.id in imported_cres: @@ -609,15 +648,7 @@ def download_cre_from_upstream(creid: str): if link.document.doctype == defs.Credoctypes.CRE: download_cre_from_upstream(link.document.id) - root_cres_response = requests.get( - os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") - + "/root_cres" - ) - if root_cres_response.status_code != 200: - raise RuntimeError( - f"cannot connect to upstream status code {root_cres_response.status_code}" - ) - data = root_cres_response.json() + data = fetch_upstream_json("/root_cres") for root_cre in data["data"]: cre = defs.Document.from_dict(root_cre) register_cre(cre, collection) @@ -880,6 +911,54 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover BaseParser().register_resource( secure_headers.SecureHeaders, db_connection_str=args.cache_file ) + if args.owasp_top10_2025_in: + from application.utils.external_project_parsers.parsers import owasp_top10_2025 + + BaseParser().register_resource( + owasp_top10_2025.OwaspTop10_2025, db_connection_str=args.cache_file + ) + if args.owasp_api_top10_2023_in: + from application.utils.external_project_parsers.parsers import ( + owasp_api_top10_2023, + ) + + BaseParser().register_resource( + owasp_api_top10_2023.OwaspApiTop10_2023, + db_connection_str=args.cache_file, + ) + if args.owasp_kubernetes_top10_2022_in: + from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2022, + ) + + BaseParser().register_resource( + owasp_kubernetes_top10_2022.OwaspKubernetesTop10_2022, + db_connection_str=args.cache_file, + ) + if args.owasp_kubernetes_top10_2025_in: + from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2025, + ) + + BaseParser().register_resource( + owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025, + db_connection_str=args.cache_file, + ) + if args.owasp_llm_top10_2025_in: + from application.utils.external_project_parsers.parsers import ( + owasp_llm_top10_2025, + ) + + BaseParser().register_resource( + owasp_llm_top10_2025.OwaspLlmTop10_2025, + db_connection_str=args.cache_file, + ) + if args.owasp_aisvs_in: + from application.utils.external_project_parsers.parsers import owasp_aisvs + + BaseParser().register_resource( + owasp_aisvs.OwaspAisvs, db_connection_str=args.cache_file + ) if args.pci_dss_4_in: from application.utils.external_project_parsers.parsers import pci_dss diff --git a/application/tests/cre_main_test.py b/application/tests/cre_main_test.py index 097b0b6d9..68d721ab1 100644 --- a/application/tests/cre_main_test.py +++ b/application/tests/cre_main_test.py @@ -7,6 +7,8 @@ from unittest import mock from unittest.mock import Mock, patch from rq import Queue, job +import requests +from rq import Queue, job from application.utils import redis from application.prompt_client import prompt_client as prompt_client from application.tests.utils import data_gen @@ -470,6 +472,111 @@ def test_register_cre(self) -> None: ], ) + @patch("application.cmd.cre_main.time.sleep") + @patch("application.cmd.cre_main.requests.get") + def test_fetch_upstream_json_retries_transient_failures( + self, mock_get, mock_sleep + ) -> None: + transient_error = requests.exceptions.ConnectionError("reset by peer") + success_response = Mock() + success_response.status_code = 200 + success_response.json.return_value = {"data": []} + mock_get.side_effect = [transient_error, success_response] + + data = main.fetch_upstream_json("/root_cres") + + self.assertEqual(data, {"data": []}) + self.assertEqual(mock_get.call_count, 2) + mock_sleep.assert_called_once() + + def test_parse_file(self) -> None: + file: List[Dict[str, Any]] = [ + { + "description": "Verify that approved cryptographic algorithms are used in the generation, seeding, and verification.", + "doctype": defs.Credoctypes.CRE, + "id": "157-573", + "links": [ + { + "type": defs.LinkTypes.LinkedTo, + "document": { + "doctype": defs.Credoctypes.Standard, + "name": "TOP10", + "section": "https://owasp.org/www-project-top-ten/2017/A5_2017-Broken_Access_Control", + }, + }, + { + "type": defs.LinkTypes.LinkedTo, + "document": { + "doctype": defs.Credoctypes.Standard, + "name": "ISO 25010", + "section": "Secure data storage", + }, + }, + ], + "name": "CREDENTIALS_MANAGEMENT_CRYPTOGRAPHIC_DIRECTIVES", + }, + { + "description": "Desc", + "doctype": defs.Credoctypes.CRE, + "id": "141-141", + "name": "name", + }, + ] + expected = [ + defs.CRE( + doctype=defs.Credoctypes.CRE, + id="157-573", + description="Verify that approved cryptographic algorithms are used in the generation, seeding, and verification.", + name="CREDENTIALS_MANAGEMENT_CRYPTOGRAPHIC_DIRECTIVES", + links=[ + defs.Link( + document=defs.Standard( + doctype=defs.Credoctypes.Standard, + name="TOP10", + section="https://owasp.org/www-project-top-ten/2017/A5_2017-Broken_Access_Control", + ), + ltype=defs.LinkTypes.LinkedTo, + ), + defs.Link( + document=defs.Standard( + doctype=defs.Credoctypes.Standard, + name="ISO 25010", + section="Secure data storage", + ), + ltype=defs.LinkTypes.LinkedTo, + ), + ], + ), + defs.CRE(id="141-141", description="Desc", name="name"), + ] + with self.assertLogs("application.cmd.cre_main", level=logging.FATAL) as logs: + # negative test first parse_file accepts a list of objects + result = main.parse_file( + filename="tests", + yamldocs=[ + "no", + "valid", + "objects", + "here", + { + "1": 2, + }, + ], + scollection=self.collection, + ) + + self.assertEqual(result, None) + self.assertIn( + "CRITICAL:application.cmd.cre_main:Malformed file tests, skipping", + logs.output, + ) + + self.maxDiff = None + + res = main.parse_file( + filename="tests", yamldocs=file, scollection=self.collection + ) + self.assertCountEqual(res, expected) @patch.object(main, "db_connect") @patch.object(Queue, "enqueue_call") @patch.object(redis, "wait_for_jobs") From 49d6ad5e29f7d20acb04cc91701dedd7adcf3723 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 09/11] Add local helper scripts for standards development --- scripts/run-local.sh | 26 ++++++++++++++++++ scripts/show-db-stats.sh | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100755 scripts/run-local.sh create mode 100755 scripts/show-db-stats.sh diff --git a/scripts/run-local.sh b/scripts/run-local.sh new file mode 100755 index 000000000..94631cbe9 --- /dev/null +++ b/scripts/run-local.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$ROOT_DIR/venv" + +if [[ ! -d "$VENV_DIR" ]]; then + echo "Creating virtual environment in $VENV_DIR" + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import flask" >/dev/null 2>&1; then + echo "Installing Python dependencies" + pip install -r "$ROOT_DIR/requirements.txt" +fi + +export NO_LOGIN="${NO_LOGIN:-1}" +export INSECURE_REQUESTS="${INSECURE_REQUESTS:-1}" +export FLASK_APP="$ROOT_DIR/cre.py" +export FLASK_CONFIG="${FLASK_CONFIG:-development}" + +echo "Starting OpenCRE on http://127.0.0.1:5000" +exec flask run --host 127.0.0.1 --port 5000 diff --git a/scripts/show-db-stats.sh b/scripts/show-db-stats.sh new file mode 100755 index 000000000..626e30dda --- /dev/null +++ b/scripts/show-db-stats.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DB_PATH="${1:-$ROOT_DIR/standards_cache.sqlite}" + +if [[ ! -f "$DB_PATH" ]]; then + echo "Database not found: $DB_PATH" >&2 + exit 1 +fi + +echo "Database: $DB_PATH" +du -h "$DB_PATH" + +"$ROOT_DIR/venv/bin/python" - "$DB_PATH" <<'PY' +import os +import sqlite3 +import sys + +db_path = sys.argv[1] +conn = sqlite3.connect(db_path) +cur = conn.cursor() + +print(f"size_bytes {os.path.getsize(db_path)}") + +tables = [ + "node", + "cre", + "cre_links", + "cre_node_links", + "embeddings", +] + +for table in tables: + try: + count = cur.execute(f"select count(*) from {table}").fetchone()[0] + print(f"{table}_count {count}") + except sqlite3.Error as exc: + print(f"{table}_count unavailable ({exc})") + +try: + standards = cur.execute( + """ + select name, count(*) + from node + where name is not null + group by name + order by count(*) desc, name asc + limit 15 + """ + ).fetchall() + print("top_standards") + for name, count in standards: + print(f"{name}\t{count}") +except sqlite3.Error as exc: + print(f"top_standards unavailable ({exc})") + +conn.close() +PY From c9ebac08037a4026c9610d69d40e0c1d6750825c Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:46:38 +0530 Subject: [PATCH 10/11] Add OWASP comparison and fallback map analysis support --- application/tests/web_main_test.py | 304 ++++++++++++++++++++ application/web/web_main.py | 432 +++++++++++++++++++++++++---- 2 files changed, 677 insertions(+), 59 deletions(-) diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 4b21571d7..d880812f4 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -519,6 +519,63 @@ def test_find_root_cres(self) -> None: self.assertEqual(json.loads(response.data.decode()), expected) self.assertEqual(200, response.status_code) + def test_find_root_cres_featured_standards(self) -> None: + collection = db.Node_collection().with_graph() + + root_cre = defs.CRE(id="111-115", description="CA", name="CA", tags=["ta"]) + collection.add_cre(root_cre) + + featured_nodes = [ + defs.Standard( + name="OWASP Top 10 for LLM and Gen AI Apps 2025", + sectionID="LLM01", + section="Prompt Injection", + ), + defs.Standard( + name="OWASP AI Security Verification Standard (AISVS)", + sectionID="AISVS1", + section="Training Data Governance & Bias Management", + ), + defs.Standard( + name="OWASP API Security Top 10 2023", + sectionID="API1", + section="Broken Object Level Authorization", + ), + defs.Standard( + name="Cloud Controls Matrix", + sectionID="AIS-01", + section="Application and Interface Security", + ), + defs.Standard( + name="OWASP Kubernetes Top Ten 2025 (Draft)", + sectionID="K01", + section="Insecure Workload Configurations", + ), + ] + for node in featured_nodes: + collection.add_node(node) + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/root_cres", + headers={"Content-Type": "application/json"}, + ) + + self.assertEqual(200, response.status_code) + payload = response.get_json() + self.assertIn("data", payload) + self.assertIn("featured_standards", payload) + self.assertCountEqual( + ["AI", "API", "Cloud"], payload["featured_standards"].keys() + ) + self.assertEqual(2, len(payload["featured_standards"]["AI"])) + self.assertEqual(1, len(payload["featured_standards"]["API"])) + self.assertEqual(2, len(payload["featured_standards"]["Cloud"])) + self.assertEqual( + "OWASP Top 10 for LLM and Gen AI Apps 2025", + payload["featured_standards"]["AI"][0]["name"], + ) + def test_smartlink(self) -> None: self.maxDiff = None collection = db.Node_collection().with_graph() @@ -624,6 +681,253 @@ def test_gap_analysis_from_cache_full_response( self.assertEqual(200, response.status_code) self.assertEqual(expected, json.loads(response.data)) + @patch.object(redis, "from_url") + @patch.object(db, "Node_collection") + def test_gap_analysis_adds_owasp_top10_comparison_section( + self, db_mock, redis_conn_mock + ) -> None: + expected = {"result": "hello"} + top10_2021 = [ + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A01", + section="Broken Access Controls", + hyperlink="https://owasp.org/Top10/A01_2021-Broken_Access_Control/", + ), + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A02", + section="Cryptographic Failures", + hyperlink="https://owasp.org/Top10/A02_2021-Cryptographic_Failures/", + ), + ] + top10_2025 = [ + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A01", + section="Broken Access Control", + hyperlink="https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + ), + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A02", + section="Security Misconfiguration", + hyperlink="https://owasp.org/Top10/2025/A02_2025-Security_Misconfiguration/", + ), + ] + redis_conn_mock.return_value.exists.return_value = True + redis_conn_mock.return_value.get.return_value = json.dumps(expected) + db_mock.return_value.get_gap_analysis_result.return_value = json.dumps(expected) + db_mock.return_value.gap_analysis_exists.return_value = True + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Top 10 2021": + return top10_2021 + if name == "OWASP Top 10 2025": + return top10_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Top%2010%202025", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual("hello", payload["result"]) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + ["OWASP Top 10 2021", "OWASP Top 10 2025"], + payload["owasp_top10_comparison"]["standards"], + ) + self.assertEqual( + "Broken Access Controls", + payload["owasp_top10_comparison"]["items"][0]["top10_2021"]["section"], + ) + self.assertEqual( + "Broken Access Control", + payload["owasp_top10_comparison"]["items"][0]["top10_2025"]["section"], + ) + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_owasp_comparison_when_schedule_fails( + self, db_mock, schedule_mock + ) -> None: + top10_2021 = [ + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A01", + section="Broken Access Controls", + hyperlink="https://owasp.org/Top10/A01_2021-Broken_Access_Control/", + ) + ] + top10_2025 = [ + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A01", + section="Broken Access Control", + hyperlink="https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + ) + ] + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Top 10 2021": + return top10_2021 + if name == "OWASP Top 10 2025": + return top10_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + schedule_mock.side_effect = RuntimeError("redis unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Top%2010%202025", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + "Broken Access Controls", + payload["owasp_top10_comparison"]["items"][0]["top10_2021"]["section"], + ) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_upstream_result_when_schedule_fails( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + expected = {"result": {"A01": {"start": "x"}}} + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.return_value = expected + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + self.assertEqual(200, response.status_code) + self.assertEqual(expected, json.loads(response.data)) + upstream_fetch_mock.assert_called_once_with( + "/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Cheat%20Sheets", + timeout=5.0, + max_attempts=1, + backoff_seconds=0.5, + ) + db_mock.return_value.add_gap_analysis_result.assert_called_once_with( + cache_key="OWASP Top 10 2021 >> OWASP Cheat Sheets", + ga_object=json.dumps(expected), + ) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_direct_cre_overlap_when_backends_fail( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + shared_cre = defs.CRE(id="170-772", name="Cryptography", description="") + base = defs.Standard( + name="OWASP Top 10 2025", + sectionID="A04", + section="Cryptographic Failures", + ) + base.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + compare = defs.Standard( + name="OWASP Web Security Testing Guide (WSTG)", + section="WSTG-CRYP-04", + ) + compare.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [base] + if name == "OWASP Top 10 2025" + else [compare] if name == "OWASP Web Security Testing Guide (WSTG)" else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202025&standard=OWASP%20Web%20Security%20Testing%20Guide%20(WSTG)", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertIn(base.id, payload["result"]) + self.assertIn(compare.id, payload["result"][base.id]["paths"]) + db_mock.return_value.add_gap_analysis_result.assert_called_once() + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_normalizes_owasp_top_2025_alias( + self, db_mock, schedule_mock + ) -> None: + top10_2021 = [ + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A01", + section="Broken Access Controls", + hyperlink="https://owasp.org/Top10/A01_2021-Broken_Access_Control/", + ) + ] + top10_2025 = [ + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A01", + section="Broken Access Control", + hyperlink="https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + ) + ] + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Top 10 2021": + return top10_2021 + if name == "OWASP Top 10 2025": + return top10_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + schedule_mock.side_effect = RuntimeError("redis unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Top%202025", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + "Broken Access Control", + payload["owasp_top10_comparison"]["items"][0]["top10_2025"]["section"], + ) + schedule_mock.assert_called_once_with( + ["OWASP Top 10 2021", "OWASP Top 10 2025"], db_mock.return_value + ) + @patch.object(db, "Node_collection") @patch.object(rq.job.Job, "fetch") @patch.object(rq.Queue, "enqueue_call") diff --git a/application/web/web_main.py b/application/web/web_main.py index a499818cd..9063445c7 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -47,6 +47,13 @@ ITEMS_PER_PAGE = 20 +OWASP_TOP10_2025_DATA_FILE = ( + pathlib.Path(__file__).resolve().parent.parent + / "utils" + / "external_project_parsers" + / "data" + / "owasp_top10_2025.json" +) app = Blueprint( "web", @@ -60,6 +67,32 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +STANDARD_NAME_ALIASES = { + "owasp top 2025": "OWASP Top 10 2025", +} + +ROOT_CRES_FEATURED_STANDARDS = { + "AI": [ + "OWASP Top 10 for LLM and Gen AI Apps 2025", + "OWASP AI Security Verification Standard (AISVS)", + ], + "API": [ + "OWASP API Security Top 10 2023", + ], + "Cloud": [ + "Cloud Controls Matrix", + "OWASP Kubernetes Top Ten 2025 (Draft)", + ], +} +ROOT_CRES_FEATURED_STANDARD_LIMIT = 2 +LLM_TOP10_STANDARD_NAME = "OWASP Top 10 for LLM and Gen AI Apps 2025" +AISVS_STANDARD_NAME = "OWASP AI Security Verification Standard (AISVS)" +API_TOP10_STANDARD_NAME = "OWASP API Security Top 10 2023" +CCM_STANDARD_NAME = "Cloud Controls Matrix" +KUBERNETES_TOP10_2025_STANDARD_NAME = "OWASP Kubernetes Top Ten 2025 (Draft)" +KUBERNETES_TOP10_2022_STANDARD_NAME = "OWASP Kubernetes Top Ten 2022" +OWASP_CHEATSHEETS_STANDARD_NAME = "OWASP Cheat Sheets" + def _ga_timeout_seconds() -> int: raw = str(gap_analysis.GAP_ANALYSIS_TIMEOUT or "129600s") @@ -88,6 +121,244 @@ class SupportedFormats(Enum): YAML = "yaml" OSCAL = "oscal" +def _normalize_source_name(source: Any) -> str | None: + """ + Normalize integration source names so analytics aggregation stays stable. + """ + if source is None: + return None + + normalized_source = str(source).strip().lower() + if not normalized_source: + return None + + normalized_source = normalized_source.replace(" ", "-") + normalized_source = re.sub(r"[^a-z0-9._:-]", "-", normalized_source) + normalized_source = re.sub(r"-{2,}", "-", normalized_source).strip("-") + if not normalized_source: + return None + + return normalized_source[:64] + + +def _load_owasp_top10_2025_entries() -> list[dict[str, str]]: + with OWASP_TOP10_2025_DATA_FILE.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _normalize_standard_name(standard: str) -> str: + normalized_standard = str(standard).strip() + return STANDARD_NAME_ALIASES.get(normalized_standard.lower(), normalized_standard) + + +def _build_owasp_top10_comparison( + standards: list[str], collection: db.Node_collection +) -> dict[str, Any] | None: + requested = {str(standard).strip().lower() for standard in standards} + if requested != {"owasp top 10 2021", "owasp top 10 2025"}: + return None + + top10_2021 = sorted( + collection.get_nodes(name="OWASP Top 10 2021"), + key=lambda node: node.sectionID or "", + ) + if not top10_2021: + return None + + top10_2025_nodes = sorted( + collection.get_nodes(name="OWASP Top 10 2025"), + key=lambda node: node.sectionID or "", + ) + if top10_2025_nodes: + top10_2025 = [ + { + "section_id": node.sectionID or "", + "section": node.section or "", + "hyperlink": node.hyperlink or "", + } + for node in top10_2025_nodes + ] + else: + top10_2025 = _load_owasp_top10_2025_entries() + + top10_2021_by_rank = { + node.sectionID + or "": { + "section_id": node.sectionID or "", + "section": node.section or "", + "hyperlink": node.hyperlink or "", + } + for node in top10_2021 + } + top10_2025_by_rank = { + entry["section_id"]: entry for entry in top10_2025 if entry.get("section_id") + } + ranks = sorted(set(top10_2021_by_rank.keys()) | set(top10_2025_by_rank.keys())) + comparison = [] + for rank in ranks: + item_2021 = top10_2021_by_rank.get(rank) + item_2025 = top10_2025_by_rank.get(rank) + comparison.append( + { + "rank": rank, + "top10_2021": item_2021, + "top10_2025": item_2025, + "changed": (item_2021 or {}).get("section") + != (item_2025 or {}).get("section"), + } + ) + + return { + "standards": ["OWASP Top 10 2021", "OWASP Top 10 2025"], + "items": comparison, + } + + +def _build_root_cres_featured_standards( + collection: db.Node_collection, +) -> dict[str, list[dict[str, Any]]]: + featured: dict[str, list[dict[str, Any]]] = {} + + for category, standard_names in ROOT_CRES_FEATURED_STANDARDS.items(): + category_entries: list[dict[str, Any]] = [] + for standard_name in standard_names[:ROOT_CRES_FEATURED_STANDARD_LIMIT]: + nodes = sorted( + collection.get_nodes(name=standard_name), + key=lambda node: ( + node.sectionID or "", + node.section or "", + node.id or "", + ), + ) + if not nodes: + continue + category_entries.append(nodes[0].todict()) + + if category_entries: + featured[category] = category_entries + + return featured + + +def _fetch_upstream_map_analysis( + standards: list[str], + standards_hash: str, + collection: db.Node_collection, +) -> dict[str, Any] | None: + if len(standards) < 2: + return None + + encoded_standards = "&".join( + f"standard={urllib.parse.quote(str(standard), safe='')}" + for standard in standards + ) + timeout = float(os.environ.get("CRE_WEB_UPSTREAM_TIMEOUT_SECONDS", "5")) + max_attempts = int(os.environ.get("CRE_WEB_UPSTREAM_MAX_ATTEMPTS", "1")) + backoff_seconds = float( + os.environ.get("CRE_WEB_UPSTREAM_RETRY_BACKOFF_SECONDS", "0.5") + ) + try: + result = cre_main.fetch_upstream_json( + f"/map_analysis?{encoded_standards}", + timeout=timeout, + max_attempts=max_attempts, + backoff_seconds=backoff_seconds, + ) + except Exception as exc: + logger.warning( + "Unable to fetch upstream map analysis for %s: %s", + standards_hash, + exc, + ) + return None + + if not isinstance(result, dict): + return None + + if result.get("result"): + try: + collection.add_gap_analysis_result( + cache_key=standards_hash, ga_object=flask_json.dumps(result) + ) + except Exception as exc: + logger.warning( + "Unable to cache upstream map analysis for %s: %s", + standards_hash, + exc, + ) + return result + + +def _build_direct_cre_overlap_map_analysis( + standards: list[str], + standards_hash: str, + collection: db.Node_collection, +) -> dict[str, Any] | None: + if len(standards) < 2: + return None + + base_nodes = collection.get_nodes(name=standards[0]) + compare_nodes = collection.get_nodes(name=standards[1]) + if not base_nodes or not compare_nodes: + return None + + if not base_nodes or not compare_nodes: + return None + + compare_nodes_by_cre: dict[str, list[defs.Standard]] = {} + for compare_node in compare_nodes: + for link in compare_node.links: + if link.document.doctype != defs.Credoctypes.CRE: + continue + compare_nodes_by_cre.setdefault(link.document.id, []).append(compare_node) + + grouped_paths: dict[str, dict[str, Any]] = {} + for base_node in base_nodes: + shared_paths: dict[str, Any] = {} + for link in base_node.links: + if link.document.doctype != defs.Credoctypes.CRE: + continue + for compare_node in compare_nodes_by_cre.get(link.document.id, []): + shared_paths.setdefault( + compare_node.id, + { + "end": compare_node.shallow_copy(), + "path": [ + { + "start": base_node.shallow_copy(), + "end": link.document.shallow_copy(), + "relationship": "LINKED_TO", + "score": 0, + }, + { + "start": link.document.shallow_copy(), + "end": compare_node.shallow_copy(), + "relationship": "LINKED_TO", + "score": 0, + }, + ], + "score": 0, + }, + ) + + grouped_paths[base_node.id] = { + "start": base_node.shallow_copy(), + "paths": shared_paths, + "extra": 0, + } + + result = {"result": grouped_paths} + try: + collection.add_gap_analysis_result( + cache_key=standards_hash, ga_object=flask_json.dumps(result) + ) + except Exception as exc: + logger.warning( + "Unable to cache direct CRE-overlap map analysis for %s: %s", + standards_hash, + exc, + ) + return result def extend_cre_with_tag_links( cre: defs.CRE, collection: db.Node_collection @@ -298,73 +569,113 @@ def find_document_by_tag() -> Any: @app.route("/rest/v1/map_analysis", methods=["GET"]) def map_analysis() -> Any: - standards = request.args.getlist("standard") + standards = [_normalize_standard_name(s) for s in request.args.getlist("standard")] if posthog: posthog.capture(f"map_analysis", f"standards:{standards}") database = db.Node_collection() - if len(standards) < 2: - abort(400, "Please provide two standards") - standards = standards[:2] - cache_key = gap_analysis.make_resources_key(standards) - cached = database.get_gap_analysis_result(cache_key=cache_key) - if cached: - parsed = json.loads(cached) - if parsed.get("result"): - return jsonify({"result": parsed.get("result")}) - if os.environ.get("HEROKU"): - abort(404, "No such Cache") - - db_url = os.environ.get("CRE_CACHE_FILE") or os.environ.get("PROD_DATABASE_URL") - if not db_url: - # Derive from current SQLAlchemy bind when not explicitly set. - db_url = str(getattr(getattr(database.session, "bind", None), "url", "")) - try: - conn = redis.connect() - ga_queue_name = os.environ.get("CRE_GA_QUEUE_NAME", "ga") - q = Queue(name=ga_queue_name, connection=conn) - inflight_key = f"ga:inflight:{cache_key}" - inflight_job_id_raw = conn.get(inflight_key) - inflight_job_id = ( - inflight_job_id_raw.decode("utf-8") - if isinstance(inflight_job_id_raw, bytes) - else str(inflight_job_id_raw) if inflight_job_id_raw else "" + standards_hash = gap_analysis.make_resources_key(standards) + owasp_top10_comparison = _build_owasp_top10_comparison(standards, database) + + # First, check if we have cached results in the database + if database.gap_analysis_exists(standards_hash): + gap_analysis_result = database.get_gap_analysis_result(standards_hash) + if gap_analysis_result: + result = flask_json.loads(gap_analysis_result) + if owasp_top10_comparison: + result["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(result) + + # On Heroku (read-only), check if standards exist before attempting Redis/queue operations + is_heroku = os.environ.get("DYNO") is not None + if is_heroku: + # Check if all requested standards exist + try: + existing_standards = database.standards() + if isinstance(existing_standards, (list, tuple, set)): + existing_lower = {str(s).lower() for s in existing_standards} + missing = [s for s in standards if str(s).lower() not in existing_lower] + if missing: + logger.info( + f"On Heroku: gap analysis request {standards_hash} references " + f"standards that do not exist: {', '.join(missing)}, returning 404" + ) + abort( + 404, f"One or more standards do not exist: {', '.join(missing)}" + ) + except Exception as exc: + # If we can't verify standards, log but don't fail (defensive) + logger.warning(f"Could not verify standards existence on Heroku: {exc}") + + # If calculations are disabled, return 404 + if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"): + logger.info( + f"Gap analysis calculations are disabled by CRE_NO_CALCULATE_GAP_ANALYSIS; " + f"refusing to schedule new job for {standards_hash}" ) - if inflight_job_id: - try: - inflight_job = job.Job.fetch(id=inflight_job_id, connection=conn) - if inflight_job.get_status() in ( - job.JobStatus.QUEUED, - job.JobStatus.STARTED, - ): - return jsonify({"job_id": inflight_job_id}) - except exceptions.NoSuchJobError: - conn.delete(inflight_key) - - j = q.enqueue_call( - description=f"{standards[0]}->{standards[1]}", - func=cre_main.run_gap_pair_job, - kwargs={ - "importing_name": standards[0], - "peer_name": standards[1], - "db_connection_str": db_url, - }, - timeout=gap_analysis.GAP_ANALYSIS_TIMEOUT, + upstream_gap_analysis = _fetch_upstream_map_analysis( + standards, standards_hash, database + ) + if upstream_gap_analysis: + if owasp_top10_comparison: + upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(upstream_gap_analysis) + direct_gap_analysis = _build_direct_cre_overlap_map_analysis( + standards, standards_hash, database ) - conn.set(inflight_key, str(j.id)) - conn.expire(inflight_key, _ga_timeout_seconds()) - return jsonify({"job_id": str(j.id)}) + if direct_gap_analysis: + if owasp_top10_comparison: + direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(direct_gap_analysis) + if owasp_top10_comparison: + return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) + abort(404, "Gap analysis calculations are disabled") + + # Now call schedule() which will handle Redis/queue operations + try: + gap_analysis_dict = gap_analysis.schedule(standards, database) except Exception as exc: - logger.warning( - "Redis/RQ unavailable in map_analysis for %s; using synchronous fallback: %s", - cache_key, - exc, + logger.error(f"Gap analysis scheduling failed for {standards_hash}: {exc}") + upstream_gap_analysis = _fetch_upstream_map_analysis( + standards, standards_hash, database ) - try: - return jsonify(_compute_ga_without_redis(database, standards)) - except Exception as fallback_exc: - logger.exception("Synchronous GA fallback failed for %s", cache_key) - abort(503, f"Gap analysis unavailable: {fallback_exc}") + if upstream_gap_analysis: + if owasp_top10_comparison: + upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(upstream_gap_analysis) + direct_gap_analysis = _build_direct_cre_overlap_map_analysis( + standards, standards_hash, database + ) + if direct_gap_analysis: + if owasp_top10_comparison: + direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(direct_gap_analysis) + if owasp_top10_comparison: + return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) + raise + if owasp_top10_comparison: + gap_analysis_dict["owasp_top10_comparison"] = owasp_top10_comparison + if "result" in gap_analysis_dict: + return jsonify(gap_analysis_dict) + if gap_analysis_dict.get("error"): + upstream_gap_analysis = _fetch_upstream_map_analysis( + standards, standards_hash, database + ) + if upstream_gap_analysis: + if owasp_top10_comparison: + upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(upstream_gap_analysis) + direct_gap_analysis = _build_direct_cre_overlap_map_analysis( + standards, standards_hash, database + ) + if direct_gap_analysis: + if owasp_top10_comparison: + direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(direct_gap_analysis) + if owasp_top10_comparison: + return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) + abort(404) + return jsonify({"job_id": gap_analysis_dict.get("job_id")}) @app.route("/rest/v1/map_analysis_weak_links", methods=["GET"]) @@ -549,6 +860,9 @@ def find_root_cres() -> Any: if documents: res = [doc.todict() for doc in documents] result = {"data": res} + featured_standards = _build_root_cres_featured_standards(database) + if featured_standards: + result["featured_standards"] = featured_standards # if opt_osib: # result["osib"] = odefs.cre2osib(documents).todict() if opt_format == SupportedFormats.Markdown.value: From 646cf9cb3ce95599681e8a5357cad0679d2082f1 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Sat, 28 Mar 2026 21:08:59 +0530 Subject: [PATCH 11/11] Merge OpenCRE direct map analysis behavior into issue #471 core review --- application/tests/web_main_test.py | 190 +++++++++++++++++++++++++++++ application/web/web_main.py | 120 +++++++++++++++++- 2 files changed, 309 insertions(+), 1 deletion(-) diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index d880812f4..2efd71758 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -7,6 +7,7 @@ import json import unittest import tempfile +from types import SimpleNamespace from unittest.mock import patch, Mock import redis @@ -877,6 +878,195 @@ def test_gap_analysis_returns_direct_cre_overlap_when_backends_fail( self.assertIn(compare.id, payload["result"][base.id]["paths"]) db_mock.return_value.add_gap_analysis_result.assert_called_once() + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_supports_opencre_as_standard( + self, db_mock, schedule_mock + ) -> None: + compare = defs.Standard( + name="OWASP Web Security Testing Guide (WSTG)", + section="WSTG-CRYP-04", + ) + opencre = defs.CRE(id="170-772", name="Cryptography", description="") + opencre.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=compare.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [compare] if name == "OWASP Web Security Testing Guide (WSTG)" else [] + ) + db_mock.return_value.session.query.return_value.all.return_value = [ + SimpleNamespace(id="cre-internal-1") + ] + db_mock.return_value.get_CREs.return_value = [opencre] + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OpenCRE&standard=OWASP%20Web%20Security%20Testing%20Guide%20(WSTG)", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertIn(opencre.id, payload["result"]) + self.assertEqual(1, len(payload["result"][opencre.id]["paths"])) + path = next(iter(payload["result"][opencre.id]["paths"].values())) + self.assertEqual(compare.id, path["end"]["id"]) + schedule_mock.assert_not_called() + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_only_direct_opencre_mappings( + self, db_mock, schedule_mock + ) -> None: + compare = defs.Standard( + name="CWE", + sectionID="1004", + section="Sensitive Cookie Without 'HttpOnly' Flag", + ) + direct_cre = defs.CRE( + id="804-220", + name="Set httponly attribute for cookie-based session tokens", + description="", + ) + direct_cre.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=compare.shallow_copy()) + ) + auto_linked_cres = [] + for i, cre_id in enumerate( + [ + "117-371", + "166-151", + "284-521", + "368-633", + "612-252", + "664-080", + "801-310", + ], + start=1, + ): + cre = defs.CRE( + id=cre_id, + name=f"Automatically mapped CRE {i}", + description="", + ) + cre.add_link( + defs.Link( + ltype=defs.LinkTypes.AutomaticallyLinkedTo, + document=compare.shallow_copy(), + ) + ) + auto_linked_cres.append(cre) + + opencre_documents = [direct_cre] + auto_linked_cres + internal_ids = [ + SimpleNamespace(id=f"cre-internal-{i}") + for i in range(len(opencre_documents)) + ] + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [compare] if name == "CWE" else [] + ) + db_mock.return_value.session.query.return_value.all.return_value = internal_ids + db_mock.return_value.get_CREs.side_effect = lambda internal_id=None, **kwargs: [ + next( + cre + for index, cre in enumerate(opencre_documents) + if internal_id == f"cre-internal-{index}" + ) + ] + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=CWE&standard=OpenCRE", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertEqual([compare.id], list(payload["result"].keys())) + self.assertEqual(1, len(payload["result"][compare.id]["paths"])) + path = next(iter(payload["result"][compare.id]["paths"].values())) + self.assertEqual(compare.id, payload["result"][compare.id]["start"]["id"]) + self.assertEqual(direct_cre.id, path["end"]["id"]) + self.assertEqual(direct_cre.name, path["end"]["name"]) + self.assertEqual("", path["path"][0]["start"]["id"]) + self.assertEqual(direct_cre.id, path["path"][0]["end"]["id"]) + schedule_mock.assert_not_called() + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_only_direct_opencre_mappings_when_opencre_is_left( + self, db_mock, schedule_mock + ) -> None: + compare = defs.Standard( + name="CWE", + sectionID="1004", + section="Sensitive Cookie Without 'HttpOnly' Flag", + ) + direct_cre = defs.CRE( + id="804-220", + name="Set httponly attribute for cookie-based session tokens", + description="", + ) + direct_cre.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=compare.shallow_copy()) + ) + indirect_cre = defs.CRE( + id="117-371", + name="Use a centralized access control mechanism", + description="", + ) + indirect_cre.add_link( + defs.Link( + ltype=defs.LinkTypes.AutomaticallyLinkedTo, + document=compare.shallow_copy(), + ) + ) + + opencre_documents = [direct_cre, indirect_cre] + internal_ids = [ + SimpleNamespace(id=f"cre-internal-{i}") + for i in range(len(opencre_documents)) + ] + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [compare] if name == "CWE" else [] + ) + db_mock.return_value.session.query.return_value.all.return_value = internal_ids + db_mock.return_value.get_CREs.side_effect = lambda internal_id=None, **kwargs: [ + next( + cre + for index, cre in enumerate(opencre_documents) + if internal_id == f"cre-internal-{index}" + ) + ] + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OpenCRE&standard=CWE", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual([direct_cre.id], list(payload["result"].keys())) + self.assertEqual(1, len(payload["result"][direct_cre.id]["paths"])) + path = next(iter(payload["result"][direct_cre.id]["paths"].values())) + self.assertEqual(direct_cre.id, payload["result"][direct_cre.id]["start"]["id"]) + self.assertEqual(compare.id, path["end"]["id"]) + self.assertEqual(direct_cre.id, path["path"][0]["start"]["id"]) + self.assertEqual(compare.id, path["path"][0]["end"]["id"]) + schedule_mock.assert_not_called() + @patch.object(web_main.gap_analysis, "schedule") @patch.object(db, "Node_collection") def test_gap_analysis_normalizes_owasp_top_2025_alias( diff --git a/application/web/web_main.py b/application/web/web_main.py index 9063445c7..8b6dd823c 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -85,6 +85,7 @@ ], } ROOT_CRES_FEATURED_STANDARD_LIMIT = 2 +OPENCRE_STANDARD_NAME = "OpenCRE" LLM_TOP10_STANDARD_NAME = "OWASP Top 10 for LLM and Gen AI Apps 2025" AISVS_STANDARD_NAME = "OWASP AI Security Verification Standard (AISVS)" API_TOP10_STANDARD_NAME = "OWASP API Security Top 10 2023" @@ -289,6 +290,114 @@ def _fetch_upstream_map_analysis( return result +def _get_opencre_documents(collection: db.Node_collection) -> list[defs.CRE]: + return [ + collection.get_CREs(internal_id=cre.id)[0] + for cre in collection.session.query(db.CRE).all() + ] + + +def _get_opencre_map_analysis_documents( + standard: str, collection: db.Node_collection +) -> list[defs.Document]: + if standard == OPENCRE_STANDARD_NAME: + return _get_opencre_documents(collection) + return collection.get_nodes(name=standard) + + +def _build_opencre_direct_link_path( + start_document: defs.Document, end_document: defs.Document +) -> dict[str, Any]: + segment_start = start_document.shallow_copy() + # Keep the direct-link response compatible with the current popup renderer. + if segment_start.doctype != defs.Credoctypes.CRE.value: + segment_start.id = "" + return { + "end": end_document.shallow_copy(), + "path": [ + { + "start": segment_start, + "end": end_document.shallow_copy(), + "relationship": "LINKED_TO", + "score": 0, + } + ], + "score": 0, + } + + +def _make_opencre_direct_link_path_key(end_document: defs.Document) -> str: + return end_document.id + + +def _add_opencre_direct_link_result( + grouped_paths: dict[str, dict[str, Any]], + start_document: defs.Document, + end_document: defs.Document, +) -> None: + shared_paths = grouped_paths.setdefault( + start_document.id, + { + "start": start_document.shallow_copy(), + "paths": {}, + "extra": 0, + }, + )["paths"] + shared_paths.setdefault( + _make_opencre_direct_link_path_key(end_document), + _build_opencre_direct_link_path(start_document, end_document), + ) + + +def _build_opencre_direct_map_analysis( + standards: list[str], + standards_hash: str, + collection: db.Node_collection, +) -> dict[str, Any] | None: + if len(standards) < 2: + return None + + base_standard = standards[0] + compare_standard = standards[1] + base_nodes = _get_opencre_map_analysis_documents(base_standard, collection) + compare_nodes = _get_opencre_map_analysis_documents(compare_standard, collection) + if not base_nodes or not compare_nodes: + return None + + base_is_opencre = base_standard == OPENCRE_STANDARD_NAME + opencre_nodes = base_nodes if base_is_opencre else compare_nodes + standard_nodes = compare_nodes if base_is_opencre else base_nodes + + standard_nodes_by_id = { + standard_node.id: standard_node for standard_node in standard_nodes + } + direct_pairs: list[tuple[defs.CRE, defs.Document]] = [] + for opencre_node in opencre_nodes: + for link in opencre_node.links: + if link.ltype != defs.LinkTypes.LinkedTo: + continue + standard_node = standard_nodes_by_id.get(link.document.id) + if not standard_node: + continue + direct_pairs.append((opencre_node, standard_node)) + + grouped_paths: dict[str, dict[str, Any]] = {} + for opencre_node, standard_node in direct_pairs: + if base_is_opencre: + _add_opencre_direct_link_result(grouped_paths, opencre_node, standard_node) + else: + _add_opencre_direct_link_result(grouped_paths, standard_node, opencre_node) + + if not grouped_paths: + return None + + result = {"result": grouped_paths} + collection.add_gap_analysis_result( + cache_key=standards_hash, ga_object=flask_json.dumps(result) + ) + return result + + def _build_direct_cre_overlap_map_analysis( standards: list[str], standards_hash: str, @@ -575,6 +684,13 @@ def map_analysis() -> Any: database = db.Node_collection() standards_hash = gap_analysis.make_resources_key(standards) + if OPENCRE_STANDARD_NAME in standards: + direct_gap_analysis = _build_opencre_direct_map_analysis( + standards, standards_hash, database + ) + if direct_gap_analysis: + return jsonify(direct_gap_analysis) + abort(404, "No direct overlap found for requested standards") owasp_top10_comparison = _build_owasp_top10_comparison(standards, database) # First, check if we have cached results in the database @@ -771,7 +887,9 @@ def standards() -> Any: posthog.capture(f"standards", "") database = db.Node_collection() - standards = database.standards() + standards = list(database.standards()) + if OPENCRE_STANDARD_NAME not in standards: + standards.append(OPENCRE_STANDARD_NAME) return standards