From 55f89ceedbfb151e6203882bffab9d5ee17d9c34 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 01/20] Add OWASP Top 10 and API importer support --- .../tests/owasp_api_top10_2023_parser_test.py | 43 ++++++++++ .../tests/owasp_top10_2025_parser_test.py | 80 +++++++++++++++++++ .../data/owasp_api_top10_2023.json | 62 ++++++++++++++ .../data/owasp_top10_2025.json | 62 ++++++++++++++ .../parsers/owasp_api_top10_2023.py | 47 +++++++++++ .../parsers/owasp_top10_2025.py | 47 +++++++++++ cre.py | 30 +++++++ 7 files changed, 371 insertions(+) create mode 100644 application/tests/owasp_api_top10_2023_parser_test.py create mode 100644 application/tests/owasp_top10_2025_parser_test.py create mode 100644 application/utils/external_project_parsers/data/owasp_api_top10_2023.json create mode 100644 application/utils/external_project_parsers/data/owasp_top10_2025.json create mode 100644 application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py create mode 100644 application/utils/external_project_parsers/parsers/owasp_top10_2025.py diff --git a/application/tests/owasp_api_top10_2023_parser_test.py b/application/tests/owasp_api_top10_2023_parser_test.py new file mode 100644 index 000000000..806d11bed --- /dev/null +++ b/application/tests/owasp_api_top10_2023_parser_test.py @@ -0,0 +1,43 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_api_top10_2023 + + +class TestOwaspApiTop10_2023Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("304-667", "Protect API against unauthorized access/modification (IDOR)"), + ("724-770", "Technical application access control"), + ("715-223", "Ensure trusted origin of third party resources"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_api_top10_2023.OwaspApiTop10_2023().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP API Security Top 10 2023"] + self.assertEqual(10, len(entries)) + self.assertEqual("API1", entries[0].sectionID) + self.assertEqual("Broken Object Level Authorization", entries[0].section) + self.assertEqual( + ["304-667", "724-770"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("API10", entries[-1].sectionID) + self.assertEqual(["715-223"], [l.document.id for l in entries[-1].links]) diff --git a/application/tests/owasp_top10_2025_parser_test.py b/application/tests/owasp_top10_2025_parser_test.py new file mode 100644 index 000000000..de4f86a9f --- /dev/null +++ b/application/tests/owasp_top10_2025_parser_test.py @@ -0,0 +1,80 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_top10_2025 + + +class TestOwaspTop10_2025Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + self.collection.add_cre( + defs.CRE(id="177-260", name="Session management", description="") + ) + self.collection.add_cre( + defs.CRE( + id="117-371", + name="Use a centralized access control mechanism", + description="", + ) + ) + self.collection.add_cre( + defs.CRE( + id="724-770", + name="Technical application access control", + description="", + ) + ) + self.collection.add_cre( + defs.CRE( + id="031-447", name="Whitelist all external (HTTP) input", description="" + ) + ) + self.collection.add_cre( + defs.CRE( + id="064-808", name="Encode output context-specifically", description="" + ) + ) + self.collection.add_cre( + defs.CRE(id="760-764", name="Injection protection", description="") + ) + self.collection.add_cre( + defs.CRE(id="513-183", name="Error handling", description="") + ) + + result = owasp_top10_2025.OwaspTop10_2025().parse( + self.collection, + prompt_client.PromptHandler(database=self.collection), + ) + + entries = result.results["OWASP Top 10 2025"] + self.assertEqual(10, len(entries)) + self.assertEqual("A01", entries[0].sectionID) + self.assertEqual("Broken Access Control", entries[0].section) + self.assertEqual( + "https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + entries[0].hyperlink, + ) + self.assertEqual( + ["117-371", "177-260", "724-770"], + [link.document.id for link in entries[0].links], + ) + self.assertEqual( + ["031-447", "064-808", "760-764"], + [link.document.id for link in entries[4].links], + ) + self.assertEqual("A10", entries[-1].sectionID) + self.assertEqual(["513-183"], [link.document.id for link in entries[-1].links]) diff --git a/application/utils/external_project_parsers/data/owasp_api_top10_2023.json b/application/utils/external_project_parsers/data/owasp_api_top10_2023.json new file mode 100644 index 000000000..7a8df0ed0 --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_api_top10_2023.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "API1", + "section": "Broken Object Level Authorization", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa1-broken-object-level-authorization/", + "cre_ids": ["304-667", "724-770"] + }, + { + "section_id": "API2", + "section": "Broken Authentication", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa2-broken-authentication/", + "cre_ids": ["177-260", "586-842", "633-428"] + }, + { + "section_id": "API3", + "section": "Broken Object Property Level Authorization", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa3-broken-object-property-level-authorization/", + "cre_ids": ["538-770", "724-770", "128-128"] + }, + { + "section_id": "API4", + "section": "Unrestricted Resource Consumption", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa4-unrestricted-resource-consumption/", + "cre_ids": ["623-550"] + }, + { + "section_id": "API5", + "section": "Broken Function Level Authorization", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa5-broken-function-level-authorization/", + "cre_ids": ["650-560", "724-770"] + }, + { + "section_id": "API6", + "section": "Unrestricted Access to Sensitive Business Flows", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa6-unrestricted-access-to-sensitive-business-flows/", + "cre_ids": ["534-605", "630-573"] + }, + { + "section_id": "API7", + "section": "Server Side Request Forgery", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa7-server-side-request-forgery/", + "cre_ids": ["028-728", "657-084"] + }, + { + "section_id": "API8", + "section": "Security Misconfiguration", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa8-security-misconfiguration/", + "cre_ids": ["486-813"] + }, + { + "section_id": "API9", + "section": "Improper Inventory Management", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xa9-improper-inventory-management/", + "cre_ids": ["162-655", "863-521"] + }, + { + "section_id": "API10", + "section": "Unsafe Consumption of APIs", + "hyperlink": "https://owasp.org/API-Security/editions/2023/en/0xaa-unsafe-consumption-of-apis/", + "cre_ids": ["715-223"] + } +] diff --git a/application/utils/external_project_parsers/data/owasp_top10_2025.json b/application/utils/external_project_parsers/data/owasp_top10_2025.json new file mode 100644 index 000000000..7e19d1a4e --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_top10_2025.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "A01", + "section": "Broken Access Control", + "hyperlink": "https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + "cre_ids": ["117-371", "177-260", "724-770"] + }, + { + "section_id": "A02", + "section": "Security Misconfiguration", + "hyperlink": "https://owasp.org/Top10/2025/A02_2025-Security_Misconfiguration/", + "cre_ids": ["486-813"] + }, + { + "section_id": "A03", + "section": "Software Supply Chain Failures", + "hyperlink": "https://owasp.org/Top10/2025/A03_2025-Software_Supply_Chain_Failures/", + "cre_ids": ["613-286", "613-287", "715-223", "863-521"] + }, + { + "section_id": "A04", + "section": "Cryptographic Failures", + "hyperlink": "https://owasp.org/Top10/2025/A04_2025-Cryptographic_Failures/", + "cre_ids": ["170-772", "227-045"] + }, + { + "section_id": "A05", + "section": "Injection", + "hyperlink": "https://owasp.org/Top10/2025/A05_2025-Injection/", + "cre_ids": ["031-447", "064-808", "760-764"] + }, + { + "section_id": "A06", + "section": "Insecure Design", + "hyperlink": "https://owasp.org/Top10/2025/A06_2025-Insecure_Design/", + "cre_ids": ["126-668", "155-155"] + }, + { + "section_id": "A07", + "section": "Authentication Failures", + "hyperlink": "https://owasp.org/Top10/2025/A07_2025-Authentication_Failures/", + "cre_ids": ["002-630", "177-260", "586-842", "633-428"] + }, + { + "section_id": "A08", + "section": "Software or Data Integrity Failures", + "hyperlink": "https://owasp.org/Top10/2025/A08_2025-Software_or_Data_Integrity_Failures/", + "cre_ids": ["613-287", "836-068"] + }, + { + "section_id": "A09", + "section": "Security Logging and Alerting Failures", + "hyperlink": "https://owasp.org/Top10/2025/A09_2025-Security_Logging_and_Alerting_Failures/", + "cre_ids": ["067-050", "148-420", "402-706", "843-841"] + }, + { + "section_id": "A10", + "section": "Mishandling of Exceptional Conditions", + "hyperlink": "https://owasp.org/Top10/2025/A10_2025-Mishandling_of_Exceptional_Conditions/", + "cre_ids": ["513-183"] + } +] diff --git a/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py b/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py new file mode 100644 index 000000000..08157a1e9 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspApiTop10_2023(ParserInterface): + name = "OWASP API Security Top 10 2023" + data_file = ( + Path(__file__).resolve().parent.parent / "data" / "owasp_api_top10_2023.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/application/utils/external_project_parsers/parsers/owasp_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_top10_2025.py new file mode 100644 index 000000000..070f869af --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_top10_2025.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspTop10_2025(ParserInterface): + name = "OWASP Top 10 2025" + data_file = ( + Path(__file__).resolve().parent.parent / "data" / "owasp_top10_2025.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/cre.py b/cre.py index 99735890f..1ffc24951 100644 --- a/cre.py +++ b/cre.py @@ -167,6 +167,36 @@ def main() -> None: action="store_true", help="import owasp secure headers", ) + parser.add_argument( + "--owasp_top10_2025_in", + action="store_true", + help="import OWASP Top 10 2025", + ) + parser.add_argument( + "--owasp_api_top10_2023_in", + action="store_true", + help="import OWASP API Security Top 10 2023", + ) + parser.add_argument( + "--owasp_kubernetes_top10_2022_in", + action="store_true", + help="import OWASP Kubernetes Top Ten 2022", + ) + parser.add_argument( + "--owasp_kubernetes_top10_2025_in", + action="store_true", + help="import OWASP Kubernetes Top Ten 2025 draft", + ) + parser.add_argument( + "--owasp_llm_top10_2025_in", + action="store_true", + help="import OWASP Top 10 for LLM and Gen AI Apps 2025", + ) + parser.add_argument( + "--owasp_aisvs_in", + action="store_true", + help="import OWASP AI Security Verification Standard (AISVS)", + ) parser.add_argument( "--pci_dss_3_2_in", action="store_true", From 5560673ce5678ccae199e04e80239eb36a15308c Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 16:13:04 +0530 Subject: [PATCH 02/20] Fix cheat sheet parser test expectations on importer branches --- application/tests/cheatsheets_parser_test.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/application/tests/cheatsheets_parser_test.py b/application/tests/cheatsheets_parser_test.py index 1a3ba4bf0..e2c0910d6 100644 --- a/application/tests/cheatsheets_parser_test.py +++ b/application/tests/cheatsheets_parser_test.py @@ -34,7 +34,13 @@ class Repo: repo.working_dir = loc cre = defs.CRE(name="blah", id="223-780") self.collection.add_cre(cre) - with open(os.path.join(os.path.join(loc, "cheatsheets"), "cs.md"), "w") as mdf: + with open( + os.path.join( + os.path.join(loc, "cheatsheets"), + "Secrets_Management_Cheat_Sheet.md", + ), + "w", + ) as mdf: mdf.write(cs) mock_clone.return_value = repo entries = cheatsheets_parser.Cheatsheets().parse( @@ -45,22 +51,26 @@ class Repo: # verify the external tagging convention, not just enum wiring. expected = defs.Standard( name="OWASP Cheat Sheets", - hyperlink="https://github.com/foo/bar/tree/master/cs.md", + hyperlink="https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html", section="Secrets Management Cheat Sheet", - links=[defs.Link(document=cre, ltype=defs.LinkTypes.LinkedTo)], + links=[ + defs.Link( + document=cre, ltype=defs.LinkTypes.AutomaticallyLinkedTo + ) + ], tags=[ "family:guidance", "subtype:cheatsheet", - "source:owasp_cheatsheets", "audience:developer", "maturity:stable", + "source:owasp_cheatsheets", ], ) self.maxDiff = None for name, nodes in entries.results.items(): self.assertEqual(name, parser.name) self.assertEqual(len(nodes), 1) - self.assertCountEqual(expected.todict(), nodes[0].todict()) + self.assertEqual(expected.todict(), nodes[0].todict()) cheatsheets_md = """ # Secrets Management Cheat Sheet From c27cc7ae4512aa2fa7208d37888c54d28f404d5f Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 16:14:21 +0530 Subject: [PATCH 03/20] Use official OWASP cheat sheet URLs in importer branches --- .../external_project_parsers/parsers/cheatsheets_parser.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py index e695414d8..e234dadda 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py +++ b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py @@ -15,6 +15,7 @@ class Cheatsheets(ParserInterface): name = "OWASP Cheat Sheets" + cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets" def cheatsheet( self, section: str, hyperlink: str, tags: List[str] @@ -33,6 +34,10 @@ def cheatsheet( hyperlink=hyperlink, ) + def official_cheatsheet_url(self, markdown_filename: str) -> str: + html_name = os.path.splitext(markdown_filename)[0] + ".html" + return f"{self.cheatsheetseries_base_url}/{html_name}" + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): c_repo = "https://github.com/OWASP/CheatSheetSeries.git" cheatsheets_path = "cheatsheets/" @@ -65,7 +70,7 @@ def register_cheatsheets( name = title.group("title") cre_id = cre.group("cre") cres = cache.get_CREs(external_id=cre_id) - hyperlink = f"{repo_path.replace('.git','')}/tree/master/{cheatsheets_path}{mdfile}" + hyperlink = self.official_cheatsheet_url(mdfile) cs = self.cheatsheet(section=name, hyperlink=hyperlink, tags=[]) for cre in cres: cs.add_link( From a11c5afe31be2e6538a69370a102650f6909a565 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 04/20] Add OWASP AI resource importer support --- application/tests/owasp_aisvs_parser_test.py | 62 +++++++++++++ .../tests/owasp_llm_top10_2025_parser_test.py | 45 ++++++++++ .../data/owasp_aisvs_1_0.json | 86 +++++++++++++++++++ .../data/owasp_llm_top10_2025.json | 62 +++++++++++++ .../parsers/owasp_aisvs.py | 45 ++++++++++ .../parsers/owasp_llm_top10_2025.py | 47 ++++++++++ 6 files changed, 347 insertions(+) create mode 100644 application/tests/owasp_aisvs_parser_test.py create mode 100644 application/tests/owasp_llm_top10_2025_parser_test.py create mode 100644 application/utils/external_project_parsers/data/owasp_aisvs_1_0.json create mode 100644 application/utils/external_project_parsers/data/owasp_llm_top10_2025.json create mode 100644 application/utils/external_project_parsers/parsers/owasp_aisvs.py create mode 100644 application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py diff --git a/application/tests/owasp_aisvs_parser_test.py b/application/tests/owasp_aisvs_parser_test.py new file mode 100644 index 000000000..461b2d68d --- /dev/null +++ b/application/tests/owasp_aisvs_parser_test.py @@ -0,0 +1,62 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_aisvs + + +class TestOwaspAisvsParser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("227-045", "Identify sensitive data and subject it to a policy"), + ( + "307-507", + "Allow only trusted sources both build time and runtime; therefore perform integrity checks on all resources and code", + ), + ( + "162-655", + "Documentation of all components' business or security function", + ), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_aisvs.OwaspAisvs().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP AI Security Verification Standard (AISVS)"] + self.assertEqual(14, len(entries)) + self.assertEqual("AISVS1", entries[0].sectionID) + self.assertEqual( + "Training Data Governance & Bias Management", entries[0].section + ) + self.assertEqual( + "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C01-Training-Data-Governance.md", + entries[0].hyperlink, + ) + self.assertEqual( + ["227-045", "307-507"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("AISVS14", entries[-1].sectionID) + self.assertEqual( + "Human Oversight, Accountability & Governance", entries[-1].section + ) + self.assertEqual( + "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C14-Human-Oversight.md", + entries[-1].hyperlink, + ) + self.assertEqual(["162-655"], [l.document.id for l in entries[-1].links]) diff --git a/application/tests/owasp_llm_top10_2025_parser_test.py b/application/tests/owasp_llm_top10_2025_parser_test.py new file mode 100644 index 000000000..75b282c34 --- /dev/null +++ b/application/tests/owasp_llm_top10_2025_parser_test.py @@ -0,0 +1,45 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import owasp_llm_top10_2025 + + +class TestOwaspLlmTop10_2025Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("161-451", "Output encoding and injection prevention"), + ("064-808", "Encode output context-specifically"), + ("760-764", "Injection protection"), + ("623-550", "Denial Of Service protection"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_llm_top10_2025.OwaspLlmTop10_2025().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP Top 10 for LLM and Gen AI Apps 2025"] + self.assertEqual(10, len(entries)) + self.assertEqual("LLM01", entries[0].sectionID) + self.assertEqual("Prompt Injection", entries[0].section) + self.assertEqual( + ["161-451", "760-764"], [l.document.id for l in entries[0].links] + ) + self.assertEqual(["064-808"], [l.document.id for l in entries[4].links]) + self.assertEqual("LLM10", entries[-1].sectionID) + self.assertEqual(["623-550"], [l.document.id for l in entries[-1].links]) diff --git a/application/utils/external_project_parsers/data/owasp_aisvs_1_0.json b/application/utils/external_project_parsers/data/owasp_aisvs_1_0.json new file mode 100644 index 000000000..c4880546f --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_aisvs_1_0.json @@ -0,0 +1,86 @@ +[ + { + "section_id": "AISVS1", + "section": "Training Data Governance & Bias Management", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C01-Training-Data-Governance.md", + "cre_ids": ["227-045", "307-507"] + }, + { + "section_id": "AISVS2", + "section": "User Input Validation", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C02-User-Input-Validation.md", + "cre_ids": ["031-447", "760-764"] + }, + { + "section_id": "AISVS3", + "section": "Model Lifecycle Management & Change Control", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C03-Model-Lifecycle-Management.md", + "cre_ids": ["148-853", "613-285"] + }, + { + "section_id": "AISVS4", + "section": "Infrastructure, Configuration & Deployment Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C04-Infrastructure.md", + "cre_ids": ["233-748", "486-813"] + }, + { + "section_id": "AISVS5", + "section": "Access Control & Identity for AI Components & Users", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C05-Access-Control-and-Identity.md", + "cre_ids": ["633-428", "724-770"] + }, + { + "section_id": "AISVS6", + "section": "Supply Chain Security for Models, Frameworks & Data", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C06-Supply-Chain.md", + "cre_ids": ["613-285", "613-287", "863-521"] + }, + { + "section_id": "AISVS7", + "section": "Model Behavior, Output Control & Safety Assurance", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C07-Model-Behavior.md", + "cre_ids": ["064-808", "141-555"] + }, + { + "section_id": "AISVS8", + "section": "Memory, Embeddings & Vector Database Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C08-Memory-Embeddings-and-Vector-Database.md", + "cre_ids": ["126-668", "538-770"] + }, + { + "section_id": "AISVS9", + "section": "Autonomous Orchestration & Agentic Action Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C09-Orchestration-and-Agentic-Action.md", + "cre_ids": ["117-371", "650-560"] + }, + { + "section_id": "AISVS10", + "section": "Model Context Protocol (MCP) Security", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C10-MCP-Security.md", + "cre_ids": ["307-507", "715-223"] + }, + { + "section_id": "AISVS11", + "section": "Adversarial Robustness & Privacy Defense", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C11-Adversarial-Robustness.md", + "cre_ids": ["141-555", "623-550"] + }, + { + "section_id": "AISVS12", + "section": "Privacy Protection & Personal Data Management", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C12-Privacy.md", + "cre_ids": ["126-668", "227-045", "482-866"] + }, + { + "section_id": "AISVS13", + "section": "Monitoring, Logging & Anomaly Detection", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C13-Monitoring-and-Logging.md", + "cre_ids": ["058-083", "148-420", "402-706", "843-841"] + }, + { + "section_id": "AISVS14", + "section": "Human Oversight, Accountability & Governance", + "hyperlink": "https://github.com/OWASP/AISVS/tree/main/1.0/en/0x10-C14-Human-Oversight.md", + "cre_ids": ["162-655", "766-162"] + } +] diff --git a/application/utils/external_project_parsers/data/owasp_llm_top10_2025.json b/application/utils/external_project_parsers/data/owasp_llm_top10_2025.json new file mode 100644 index 000000000..b761d5e09 --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_llm_top10_2025.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "LLM01", + "section": "Prompt Injection", + "hyperlink": "https://genai.owasp.org/llmrisk/llm01-prompt-injection/", + "cre_ids": ["161-451", "760-764"] + }, + { + "section_id": "LLM02", + "section": "Sensitive Information Disclosure", + "hyperlink": "https://genai.owasp.org/llmrisk/llm022025-sensitive-information-disclosure/", + "cre_ids": ["126-668", "227-045"] + }, + { + "section_id": "LLM03", + "section": "Supply Chain", + "hyperlink": "https://genai.owasp.org/llmrisk/llm032025-supply-chain/", + "cre_ids": ["613-285", "613-287"] + }, + { + "section_id": "LLM04", + "section": "Data and Model Poisoning", + "hyperlink": "https://genai.owasp.org/llmrisk/llm042025-data-and-model-poisoning/", + "cre_ids": ["307-507", "613-287"] + }, + { + "section_id": "LLM05", + "section": "Improper Output Handling", + "hyperlink": "https://genai.owasp.org/llmrisk/llm052025-improper-output-handling/", + "cre_ids": ["064-808"] + }, + { + "section_id": "LLM06", + "section": "Excessive Agency", + "hyperlink": "https://genai.owasp.org/llmrisk/llm062025-excessive-agency/", + "cre_ids": ["117-371", "650-560"] + }, + { + "section_id": "LLM07", + "section": "System Prompt Leakage", + "hyperlink": "https://genai.owasp.org/llmrisk/llm072025-system-prompt-leakage/", + "cre_ids": ["126-668", "227-045"] + }, + { + "section_id": "LLM08", + "section": "Vector and Embedding Weaknesses", + "hyperlink": "https://genai.owasp.org/llmrisk/llm082025-vector-and-embedding-weaknesses/", + "cre_ids": ["126-668", "538-770"] + }, + { + "section_id": "LLM09", + "section": "Misinformation", + "hyperlink": "https://genai.owasp.org/llmrisk/llm092025-misinformation/", + "cre_ids": ["141-555"] + }, + { + "section_id": "LLM10", + "section": "Unbounded Consumption", + "hyperlink": "https://genai.owasp.org/llmrisk/llm102025-unbounded-consumption/", + "cre_ids": ["267-031", "623-550"] + } +] diff --git a/application/utils/external_project_parsers/parsers/owasp_aisvs.py b/application/utils/external_project_parsers/parsers/owasp_aisvs.py new file mode 100644 index 000000000..cec4abad9 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_aisvs.py @@ -0,0 +1,45 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspAisvs(ParserInterface): + name = "OWASP AI Security Verification Standard (AISVS)" + data_file = Path(__file__).resolve().parent.parent / "data" / "owasp_aisvs_1_0.json" + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py new file mode 100644 index 000000000..3971b9e6b --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspLlmTop10_2025(ParserInterface): + name = "OWASP Top 10 for LLM and Gen AI Apps 2025" + data_file = ( + Path(__file__).resolve().parent.parent / "data" / "owasp_llm_top10_2025.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) From c9b49f87572e6de5b0cd65678b8261d1f61137ff Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 05/20] Add OWASP Kubernetes importer support --- ...owasp_kubernetes_top10_2022_parser_test.py | 45 ++++++++ ...owasp_kubernetes_top10_2025_parser_test.py | 102 ++++++++++++++++++ .../data/owasp_kubernetes_top10_2022.json | 62 +++++++++++ .../data/owasp_kubernetes_top10_2025.json | 72 +++++++++++++ .../parsers/owasp_kubernetes_top10_2022.py | 49 +++++++++ .../parsers/owasp_kubernetes_top10_2025.py | 78 ++++++++++++++ 6 files changed, 408 insertions(+) create mode 100644 application/tests/owasp_kubernetes_top10_2022_parser_test.py create mode 100644 application/tests/owasp_kubernetes_top10_2025_parser_test.py create mode 100644 application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json create mode 100644 application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json create mode 100644 application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py create mode 100644 application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py diff --git a/application/tests/owasp_kubernetes_top10_2022_parser_test.py b/application/tests/owasp_kubernetes_top10_2022_parser_test.py new file mode 100644 index 000000000..30b0922c9 --- /dev/null +++ b/application/tests/owasp_kubernetes_top10_2022_parser_test.py @@ -0,0 +1,45 @@ +import unittest + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2022, +) + + +class TestOwaspKubernetesTop10_2022Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("233-748", "Configuration hardening"), + ("486-813", "Configuration"), + ("053-751", "Force build pipeline to check outdated/insecure components"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_kubernetes_top10_2022.OwaspKubernetesTop10_2022().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP Kubernetes Top Ten 2022"] + self.assertEqual(10, len(entries)) + self.assertEqual("K01", entries[0].sectionID) + self.assertEqual("Insecure Workload Configurations", entries[0].section) + self.assertEqual( + ["233-748", "486-813"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("K10", entries[-1].sectionID) + self.assertEqual(["053-751"], [l.document.id for l in entries[-1].links]) diff --git a/application/tests/owasp_kubernetes_top10_2025_parser_test.py b/application/tests/owasp_kubernetes_top10_2025_parser_test.py new file mode 100644 index 000000000..6f444c9a9 --- /dev/null +++ b/application/tests/owasp_kubernetes_top10_2025_parser_test.py @@ -0,0 +1,102 @@ +import unittest +import tempfile +from pathlib import Path + +from application import create_app, sqla # type: ignore +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2025, +) + + +class TestOwaspKubernetesTop10_2025Parser(unittest.TestCase): + def tearDown(self) -> None: + sqla.session.remove() + sqla.drop_all() + self.app_context.pop() + + def setUp(self) -> None: + self.app = create_app(mode="test") + self.app_context = self.app.app_context() + self.app_context.push() + sqla.create_all() + self.collection = db.Node_collection() + + def test_parse(self) -> None: + for cre_id, name in [ + ("233-748", "Configuration hardening"), + ("486-813", "Configuration"), + ("148-420", "Log integrity"), + ("402-706", "Log relevant"), + ("843-841", "Log discretely"), + ]: + self.collection.add_cre(defs.CRE(id=cre_id, name=name, description="")) + + result = owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025().parse( + self.collection, prompt_client.PromptHandler(database=self.collection) + ) + + entries = result.results["OWASP Kubernetes Top Ten 2025 (Draft)"] + self.assertEqual(10, len(entries)) + self.assertEqual("K01", entries[0].sectionID) + self.assertEqual("Insecure Workload Configurations", entries[0].section) + self.assertEqual( + ["233-748", "486-813"], [l.document.id for l in entries[0].links] + ) + self.assertEqual("K10", entries[-1].sectionID) + self.assertEqual( + ["148-420", "402-706", "843-841"], + [l.document.id for l in entries[-1].links], + ) + + def test_parse_falls_back_to_2022_mapping_when_2025_links_missing(self) -> None: + self.collection.add_cre( + defs.CRE(id="148-420", name="Log integrity", description="") + ) + + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + current_file = tmp_path / "k8s_2025.json" + fallback_file = tmp_path / "k8s_2022.json" + current_file.write_text( + """ +[ + { + "section_id": "K10", + "section": "Inadequate Logging And Monitoring", + "hyperlink": "https://example.com/k10", + "cre_ids": ["999-999"], + "fallback_section_ids": ["K05"] + } +] + """.strip(), + encoding="utf-8", + ) + fallback_file.write_text( + """ +[ + { + "section_id": "K05", + "section": "Inadequate Logging and Monitoring", + "hyperlink": "https://example.com/k05", + "cre_ids": ["148-420"] + } +] + """.strip(), + encoding="utf-8", + ) + + parser = owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025() + parser.data_file = current_file + parser.fallback_data_file = fallback_file + + result = parser.parse( + self.collection, + prompt_client.PromptHandler(database=self.collection), + ) + + entries = result.results["OWASP Kubernetes Top Ten 2025 (Draft)"] + self.assertEqual(1, len(entries)) + self.assertEqual(["148-420"], [link.document.id for link in entries[0].links]) diff --git a/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json new file mode 100644 index 000000000..c4eb3d6fd --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2022.json @@ -0,0 +1,62 @@ +[ + { + "section_id": "K01", + "section": "Insecure Workload Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K01-insecure-workload-configurations", + "cre_ids": ["233-748", "486-813"] + }, + { + "section_id": "K02", + "section": "Supply Chain Vulnerabilities", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K02-supply-chain-vulnerabilities", + "cre_ids": ["613-285", "613-287"] + }, + { + "section_id": "K03", + "section": "Overly Permissive RBAC Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K03-overly-permissive-rbac-configurations", + "cre_ids": ["128-128", "724-770"] + }, + { + "section_id": "K04", + "section": "Lack of Centralized Policy Enforcement", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K04-lack-of-centralized-policy-enforcement", + "cre_ids": ["117-371"] + }, + { + "section_id": "K05", + "section": "Inadequate Logging and Monitoring", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K05-inadequate-logging-and-monitoring", + "cre_ids": ["058-083", "148-420", "402-706", "843-841"] + }, + { + "section_id": "K06", + "section": "Broken Authentication Mechanisms", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K06-broken-authentication-mechanisms", + "cre_ids": ["177-260", "586-842", "633-428"] + }, + { + "section_id": "K07", + "section": "Missing Network Segmentation Controls", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K07-missing-network-segmentation-controls", + "cre_ids": ["132-146", "467-784", "515-021"] + }, + { + "section_id": "K08", + "section": "Secrets Management Failures", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K08-secrets-management-failures", + "cre_ids": ["340-375", "774-888", "813-610"] + }, + { + "section_id": "K09", + "section": "Misconfigured Cluster Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K09-misconfigured-cluster-components", + "cre_ids": ["233-748", "486-813"] + }, + { + "section_id": "K10", + "section": "Outdated and Vulnerable Kubernetes Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K10-outdated-and-vulnerable-kubernetes-components", + "cre_ids": ["053-751", "715-334", "863-521"] + } +] diff --git a/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json new file mode 100644 index 000000000..c55afb059 --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json @@ -0,0 +1,72 @@ +[ + { + "section_id": "K01", + "section": "Insecure Workload Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["233-748", "486-813"], + "fallback_section_ids": ["K01"] + }, + { + "section_id": "K02", + "section": "Overly Permissive Authorization Configurations", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["128-128", "724-770"], + "fallback_section_ids": ["K03"] + }, + { + "section_id": "K03", + "section": "Secrets Management Failures", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["340-375", "774-888", "813-610"], + "fallback_section_ids": ["K08"] + }, + { + "section_id": "K04", + "section": "Lack Of Cluster Level Policy Enforcement", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["117-371"], + "fallback_section_ids": ["K04"] + }, + { + "section_id": "K05", + "section": "Missing Network Segmentation Controls", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["132-146", "467-784", "515-021"], + "fallback_section_ids": ["K07"] + }, + { + "section_id": "K06", + "section": "Overly Exposed Kubernetes Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["152-725", "640-364"], + "fallback_section_ids": ["K09"] + }, + { + "section_id": "K07", + "section": "Misconfigured And Vulnerable Cluster Components", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["053-751", "233-748", "486-813", "715-334"], + "fallback_section_ids": ["K09", "K10"] + }, + { + "section_id": "K08", + "section": "Cluster To Cloud Lateral Movement", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["132-146", "640-364", "724-770"], + "fallback_section_ids": ["K03", "K07"] + }, + { + "section_id": "K09", + "section": "Broken Authentication Mechanisms", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["177-260", "586-842", "633-428"], + "fallback_section_ids": ["K06"] + }, + { + "section_id": "K10", + "section": "Inadequate Logging And Monitoring", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "cre_ids": ["058-083", "148-420", "402-706", "843-841"], + "fallback_section_ids": ["K05"] + } +] diff --git a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py new file mode 100644 index 000000000..9d3822ab7 --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspKubernetesTop10_2022(ParserInterface): + name = "OWASP Kubernetes Top Ten 2022" + data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_kubernetes_top10_2022.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) diff --git a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py new file mode 100644 index 000000000..31deed8da --- /dev/null +++ b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py @@ -0,0 +1,78 @@ +import json +from pathlib import Path + +from application.database import db +from application.defs import cre_defs as defs +from application.prompt_client import prompt_client +from application.utils.external_project_parsers.base_parser_defs import ( + ParseResult, + ParserInterface, +) + + +class OwaspKubernetesTop10_2025(ParserInterface): + name = "OWASP Kubernetes Top Ten 2025 (Draft)" + data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_kubernetes_top10_2025.json" + ) + fallback_data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_kubernetes_top10_2022.json" + ) + + def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): + with self.data_file.open("r", encoding="utf-8") as handle: + raw_entries = json.load(handle) + with self.fallback_data_file.open("r", encoding="utf-8") as handle: + fallback_entries = { + entry["section_id"]: entry for entry in json.load(handle) + } + + entries = [] + for entry in raw_entries: + standard = defs.Standard( + name=self.name, + sectionID=entry["section_id"], + section=entry["section"], + hyperlink=entry["hyperlink"], + ) + linked_cre_ids = [] + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + linked_cre_ids.append(cre_id) + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + if not linked_cre_ids: + for section_id in entry.get("fallback_section_ids", []): + fallback_entry = fallback_entries.get(section_id) + if not fallback_entry: + continue + for cre_id in fallback_entry.get("cre_ids", []): + if cre_id in linked_cre_ids: + continue + cres = cache.get_CREs(external_id=cre_id) + if not cres: + continue + linked_cre_ids.append(cre_id) + standard.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, + document=cres[0].shallow_copy(), + ) + ) + entries.append(standard) + + return ParseResult( + results={self.name: entries}, + calculate_gap_analysis=False, + calculate_embeddings=False, + ) From 8df31e94e23c42f48f8db277b3da6e16880e91b0 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 06/20] Normalize OWASP cheat sheet references --- application/tests/cheatsheets_parser_test.py | 33 +++++++++- .../data/owasp_cheatsheets_supplement.json | 47 ++++++++++++++ .../parsers/cheatsheets_parser.py | 62 +++++++++++++++++-- 3 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json diff --git a/application/tests/cheatsheets_parser_test.py b/application/tests/cheatsheets_parser_test.py index e2c0910d6..fb2a9c277 100644 --- a/application/tests/cheatsheets_parser_test.py +++ b/application/tests/cheatsheets_parser_test.py @@ -69,8 +69,37 @@ class Repo: self.maxDiff = None for name, nodes in entries.results.items(): self.assertEqual(name, parser.name) - self.assertEqual(len(nodes), 1) - self.assertEqual(expected.todict(), nodes[0].todict()) + sections = {node.section for node in nodes} + self.assertIn("Secrets Management Cheat Sheet", sections) + secret_entry = [ + node + for node in nodes + if node.section == "Secrets Management Cheat Sheet" + ][0] + self.assertEqual(expected.todict(), secret_entry.todict()) + + def test_register_supplemental_cheatsheets(self) -> None: + for cre_id, name in [ + ("118-110", "API/web services"), + ("724-770", "Technical application access control"), + ("623-550", "Denial Of Service protection"), + ]: + self.collection.add_cre(defs.CRE(name=name, id=cre_id)) + + entries = cheatsheets_parser.Cheatsheets().register_supplemental_cheatsheets( + cache=self.collection + ) + rest = [ + entry for entry in entries if entry.section == "REST Security Cheat Sheet" + ][0] + self.assertEqual( + "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html", + rest.hyperlink, + ) + self.assertEqual( + ["118-110", "724-770", "623-550"], + [link.document.id for link in rest.links], + ) cheatsheets_md = """ # Secrets Management Cheat Sheet diff --git a/application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json b/application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json new file mode 100644 index 000000000..4e06bee8c --- /dev/null +++ b/application/utils/external_project_parsers/data/owasp_cheatsheets_supplement.json @@ -0,0 +1,47 @@ +[ + { + "section": "Authorization Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Authorization_Cheat_Sheet.html", + "cre_ids": ["128-128", "117-371"] + }, + { + "section": "REST Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/REST_Security_Cheat_Sheet.html", + "cre_ids": ["118-110", "724-770", "623-550"] + }, + { + "section": "Server Side Request Forgery Prevention Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Server_Side_Request_Forgery_Prevention_Cheat_Sheet.html", + "cre_ids": ["028-728", "657-084"] + }, + { + "section": "Docker Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Docker_Security_Cheat_Sheet.html", + "cre_ids": ["233-748", "486-813"] + }, + { + "section": "Kubernetes Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Kubernetes_Security_Cheat_Sheet.html", + "cre_ids": ["467-784", "233-748", "486-813"] + }, + { + "section": "Secure Cloud Architecture Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_Cloud_Architecture_Cheat_Sheet.html", + "cre_ids": ["155-155", "467-784"] + }, + { + "section": "LLM Prompt Injection Prevention Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/LLM_Prompt_Injection_Prevention_Cheat_Sheet.html", + "cre_ids": ["161-451", "760-764"] + }, + { + "section": "AI Agent Security Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/AI_Agent_Security_Cheat_Sheet.html", + "cre_ids": ["117-371", "650-560", "126-668"] + }, + { + "section": "Secure AI Model Ops Cheat Sheet", + "hyperlink": "https://cheatsheetseries.owasp.org/cheatsheets/Secure_AI_Model_Ops_Cheat_Sheet.html", + "cre_ids": ["148-853", "613-285", "613-287"] + } +] diff --git a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py index e234dadda..02003b7bd 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py +++ b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py @@ -6,6 +6,9 @@ import os import re from application.utils.external_project_parsers import base_parser_defs +import json +from pathlib import Path +import logging from application.utils.external_project_parsers.base_parser_defs import ( ParserInterface, ParseResult, @@ -16,6 +19,12 @@ class Cheatsheets(ParserInterface): name = "OWASP Cheat Sheets" cheatsheetseries_base_url = "https://cheatsheetseries.owasp.org/cheatsheets" + supplement_data_file = ( + Path(__file__).resolve().parent.parent + / "data" + / "owasp_cheatsheets_supplement.json" + ) + logger = logging.getLogger(__name__) def cheatsheet( self, section: str, hyperlink: str, tags: List[str] @@ -41,10 +50,22 @@ def official_cheatsheet_url(self, markdown_filename: str) -> str: def parse(self, cache: db.Node_collection, ph: prompt_client.PromptHandler): c_repo = "https://github.com/OWASP/CheatSheetSeries.git" cheatsheets_path = "cheatsheets/" - repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True) - cheatsheets = self.register_cheatsheets( - repo=repo, cache=cache, cheatsheets_path=cheatsheets_path, repo_path=c_repo - ) + cheatsheets = [] + try: + repo = git.clone(c_repo, sparse_paths=["cheatsheets"], sparse_cone=True) + cheatsheets = self.register_cheatsheets( + repo=repo, + cache=cache, + cheatsheets_path=cheatsheets_path, + repo_path=c_repo, + ) + except Exception as exc: + self.logger.warning( + "Unable to clone OWASP CheatSheetSeries, continuing with supplemental cheat sheets only: %s", + exc, + ) + cheatsheets.extend(self.register_supplemental_cheatsheets(cache=cache)) + cheatsheets = self.deduplicate_entries(cheatsheets) results = {self.name: cheatsheets} base_parser_defs.validate_classification_tags(results) return ParseResult(results=results) @@ -80,3 +101,36 @@ def register_cheatsheets( ) standard_entries.append(cs) return standard_entries + + def register_supplemental_cheatsheets(self, cache: db.Node_collection): + with self.supplement_data_file.open("r", encoding="utf-8") as handle: + supplement_entries = json.load(handle) + + standard_entries = [] + for entry in supplement_entries: + cs = self.cheatsheet( + section=entry["section"], + hyperlink=entry["hyperlink"], + tags=[], + ) + for cre_id in entry.get("cre_ids", []): + cres = cache.get_CREs(external_id=cre_id) + for cre in cres: + try: + cs.add_link( + defs.Link( + document=cre.shallow_copy(), + ltype=defs.LinkTypes.AutomaticallyLinkedTo, + ) + ) + except Exception: + continue + if cs.links: + standard_entries.append(cs) + return standard_entries + + def deduplicate_entries(self, entries: List[defs.Standard]) -> List[defs.Standard]: + deduped = {} + for entry in entries: + deduped[(entry.section, entry.hyperlink)] = entry + return list(deduped.values()) From e4dd489203175fb6a7a5d09d93d3aa557c802c19 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 07/20] Add refresh scripts for OWASP resources --- scripts/update-cheatsheets.sh | 55 +++++++++ scripts/update-owasp-top10-2025-mappings.sh | 38 ++++++ scripts/update-owasp-top10-standards.sh | 129 ++++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100755 scripts/update-cheatsheets.sh create mode 100755 scripts/update-owasp-top10-2025-mappings.sh create mode 100644 scripts/update-owasp-top10-standards.sh diff --git a/scripts/update-cheatsheets.sh b/scripts/update-cheatsheets.sh new file mode 100755 index 000000000..48d5eccc5 --- /dev/null +++ b/scripts/update-cheatsheets.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DB_PATH="${1:-$ROOT_DIR/standards_cache.sqlite}" +VENV_DIR="$ROOT_DIR/venv" + +if [[ ! -d "$VENV_DIR" ]]; then + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import flask" >/dev/null 2>&1; then + pip install -r "$ROOT_DIR/requirements.txt" +fi + +CRE_NO_CALCULATE_GAP_ANALYSIS=1 \ +CRE_NO_GEN_EMBEDDINGS=1 \ +python "$ROOT_DIR/cre.py" --cheatsheets_in --cache_file "$DB_PATH" + +python - "$DB_PATH" <<'PY' +import os +import sqlite3 +import sys + +db_path = sys.argv[1] +conn = sqlite3.connect(db_path) +cur = conn.cursor() + +github_prefix = "https://github.com/OWASP/CheatSheetSeries/tree/master/cheatsheets/" +official_prefix = "https://cheatsheetseries.owasp.org/cheatsheets/" + +rows = cur.execute( + """ + select id, link + from node + where name = 'OWASP Cheat Sheets' + and link like ? + """, + (f"{github_prefix}%",), +).fetchall() + +for node_id, link in rows: + filename = os.path.basename(link) + html_name = os.path.splitext(filename)[0] + ".html" + cur.execute( + "update node set link = ? where id = ?", + (f"{official_prefix}{html_name}", node_id), + ) + +conn.commit() +conn.close() +print(f"Normalized {len(rows)} OWASP Cheat Sheet links") +PY diff --git a/scripts/update-owasp-top10-2025-mappings.sh b/scripts/update-owasp-top10-2025-mappings.sh new file mode 100755 index 000000000..04258646b --- /dev/null +++ b/scripts/update-owasp-top10-2025-mappings.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$ROOT_DIR/venv" +CACHE_FILE="${1:-$ROOT_DIR/standards_cache.sqlite}" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +BACKUP_FILE="${CACHE_FILE}.bak.${TIMESTAMP}" + +if [[ ! -d "$VENV_DIR" ]]; then + echo "Creating virtual environment in $VENV_DIR" + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import requests" >/dev/null 2>&1; then + echo "Installing Python dependencies" + pip install -r "$ROOT_DIR/requirements.txt" +fi + +if [[ -f "$CACHE_FILE" ]]; then + cp "$CACHE_FILE" "$BACKUP_FILE" + echo "Backed up database to $BACKUP_FILE" +fi + +export CRE_NO_NEO4J="${CRE_NO_NEO4J:-1}" +export CRE_NO_GEN_EMBEDDINGS="${CRE_NO_GEN_EMBEDDINGS:-1}" +export CRE_UPSTREAM_MAX_ATTEMPTS="${CRE_UPSTREAM_MAX_ATTEMPTS:-6}" +export CRE_UPSTREAM_RETRY_BACKOFF_SECONDS="${CRE_UPSTREAM_RETRY_BACKOFF_SECONDS:-2}" +export CRE_UPSTREAM_TIMEOUT_SECONDS="${CRE_UPSTREAM_TIMEOUT_SECONDS:-30}" + +echo "Refreshing official OpenCRE upstream data in $CACHE_FILE" +python "$ROOT_DIR/cre.py" --upstream_sync --cache_file "$CACHE_FILE" + +echo "Reapplying OWASP Top 10 2025 CRE mappings" +exec python "$ROOT_DIR/cre.py" --owasp_top10_2025_in --cache_file "$CACHE_FILE" diff --git a/scripts/update-owasp-top10-standards.sh b/scripts/update-owasp-top10-standards.sh new file mode 100644 index 000000000..a795cf872 --- /dev/null +++ b/scripts/update-owasp-top10-standards.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$ROOT_DIR/venv" +CACHE_FILE="${1:-$ROOT_DIR/standards_cache.sqlite}" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" +BACKUP_FILE="${CACHE_FILE}.bak.${TIMESTAMP}" + +if [[ ! -d "$VENV_DIR" ]]; then + echo "Creating virtual environment in $VENV_DIR" + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import requests" >/dev/null 2>&1; then + echo "Installing Python dependencies" + pip install -r "$ROOT_DIR/requirements.txt" +fi + +if [[ -f "$CACHE_FILE" ]]; then + cp "$CACHE_FILE" "$BACKUP_FILE" + echo "Backed up database to $BACKUP_FILE" +fi + +export CRE_NO_NEO4J="${CRE_NO_NEO4J:-1}" +export CRE_NO_GEN_EMBEDDINGS="${CRE_NO_GEN_EMBEDDINGS:-1}" +export CRE_UPSTREAM_MAX_ATTEMPTS="${CRE_UPSTREAM_MAX_ATTEMPTS:-6}" +export CRE_UPSTREAM_RETRY_BACKOFF_SECONDS="${CRE_UPSTREAM_RETRY_BACKOFF_SECONDS:-2}" +export CRE_UPSTREAM_TIMEOUT_SECONDS="${CRE_UPSTREAM_TIMEOUT_SECONDS:-30}" + +echo "Refreshing official OpenCRE upstream data in $CACHE_FILE" +python "$ROOT_DIR/cre.py" --upstream_sync --cache_file "$CACHE_FILE" + +echo "Reapplying OWASP Top 10 standards and CRE mappings" +python "$ROOT_DIR/cre.py" \ + --owasp_top10_2025_in \ + --owasp_api_top10_2023_in \ + --owasp_kubernetes_top10_2025_in \ + --owasp_llm_top10_2025_in \ + --owasp_aisvs_in \ + --cache_file "$CACHE_FILE" + +echo "Selecting preferred Kubernetes Top Ten version" +if python - <<'PY' "$CACHE_FILE" +import sqlite3 +import sys + +cache_file = sys.argv[1] +name_2025 = "OWASP Kubernetes Top Ten 2025 (Draft)" +name_2022 = "OWASP Kubernetes Top Ten 2022" + +conn = sqlite3.connect(cache_file) +cur = conn.cursor() + +linked_2025 = cur.execute( + """ + select count(*) + from node n + join cre_node_links l on l.node = n.id + where n.name = ? + """, + (name_2025,), +).fetchone()[0] + +if linked_2025 > 0: + cur.execute("delete from node where name = ?", (name_2022,)) + print(f"Using {name_2025}; removed {name_2022}") +else: + raise SystemExit(f"{name_2025} not linked") + +conn.commit() +conn.close() +PY +then + : +else + echo "OWASP Kubernetes Top Ten 2025 (Draft) is unavailable or unmapped, importing 2022" + python "$ROOT_DIR/cre.py" \ + --owasp_kubernetes_top10_2022_in \ + --cache_file "$CACHE_FILE" +fi + +echo "Pruning OWASP Top 10 entries that still have no CRE links" +python - <<'PY' "$CACHE_FILE" +import sqlite3 +import sys + +cache_file = sys.argv[1] +standard_names = ( + "OWASP Top 10 2025", + "OWASP API Security Top 10 2023", + "OWASP Kubernetes Top Ten 2025 (Draft)", + "OWASP Top 10 for LLM and Gen AI Apps 2025", + "OWASP AI Security Verification Standard (AISVS)", +) + +conn = sqlite3.connect(cache_file) +cur = conn.cursor() + +has_2022 = cur.execute( + "select 1 from node where name = 'OWASP Kubernetes Top Ten 2022' limit 1" +).fetchone() +if has_2022: + standard_names = standard_names + ("OWASP Kubernetes Top Ten 2022",) + +rows = list( + cur.execute( + f""" + select n.id, n.name, coalesce(n.section_id, ''), coalesce(n.section, '') + from node n + left join cre_node_links l on l.node = n.id + where n.name in ({','.join('?' for _ in standard_names)}) + group by n.id + having count(l.cre) = 0 + """, + standard_names, + ) +) + +for node_id, name, section_id, section in rows: + cur.execute("delete from node where id = ?", (node_id,)) + print(f"Removed unmapped entry: {name} {section_id} {section}".strip()) + +conn.commit() +conn.close() +PY From 6b800fc7e8b1596d51301694cd8608157e1fcf36 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 08/20] Retry transient failures during upstream sync --- application/cmd/cre_main.py | 115 ++++++++++++++++++++++++----- application/tests/cre_main_test.py | 107 +++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 18 deletions(-) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index 5369617fb..28dc6411e 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -36,6 +36,51 @@ app = None +def fetch_upstream_json( + path: str, + timeout: Optional[float] = None, + max_attempts: Optional[int] = None, + backoff_seconds: Optional[float] = None, +) -> Dict[str, Any]: + base_url = os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") + timeout = timeout or float(os.environ.get("CRE_UPSTREAM_TIMEOUT_SECONDS", "30")) + max_attempts = max_attempts or int(os.environ.get("CRE_UPSTREAM_MAX_ATTEMPTS", "4")) + backoff_seconds = backoff_seconds or float( + os.environ.get("CRE_UPSTREAM_RETRY_BACKOFF_SECONDS", "2") + ) + url = f"{base_url}{path}" + last_error: Optional[Exception] = None + + for attempt in range(1, max_attempts + 1): + try: + response = requests.get(url, timeout=timeout) + if response.status_code == 200: + return response.json() + + status_error = RuntimeError( + f"cannot connect to upstream status code {response.status_code}" + ) + # Retry only on transient upstream failures. + if response.status_code < 500 and response.status_code != 429: + raise status_error + last_error = status_error + except requests.exceptions.RequestException as exc: + last_error = exc + + if attempt < max_attempts: + logger.warning( + "upstream fetch failed for %s on attempt %s/%s, retrying", + url, + attempt, + max_attempts, + ) + time.sleep(backoff_seconds * attempt) + + if last_error: + raise RuntimeError(f"upstream fetch failed for {url}") from last_error + raise RuntimeError(f"upstream fetch failed for {url}") + + def register_node(node: defs.Node, collection: db.Node_collection) -> db.Node: """ for each link find if either the root node or the link have a CRE, @@ -353,6 +398,8 @@ def register_standard( ): if os.environ.get("CRE_NO_GEN_EMBEDDINGS") == "1": generate_embeddings = False + if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"): + calculate_gap_analysis = False if not standard_entries: logger.warning("register_standard() called with no standard_entries") @@ -589,15 +636,7 @@ def download_graph_from_upstream(cache: str) -> None: collection = db_connect(path=cache).with_graph() def download_cre_from_upstream(creid: str): - cre_response = requests.get( - os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") - + f"/id/{creid}" - ) - if cre_response.status_code != 200: - raise RuntimeError( - f"cannot connect to upstream status code {cre_response.status_code}" - ) - data = cre_response.json() + data = fetch_upstream_json(f"/id/{creid}") credict = data["data"] cre = defs.Document.from_dict(credict) if cre.id in imported_cres: @@ -609,15 +648,7 @@ def download_cre_from_upstream(creid: str): if link.document.doctype == defs.Credoctypes.CRE: download_cre_from_upstream(link.document.id) - root_cres_response = requests.get( - os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") - + "/root_cres" - ) - if root_cres_response.status_code != 200: - raise RuntimeError( - f"cannot connect to upstream status code {root_cres_response.status_code}" - ) - data = root_cres_response.json() + data = fetch_upstream_json("/root_cres") for root_cre in data["data"]: cre = defs.Document.from_dict(root_cre) register_cre(cre, collection) @@ -880,6 +911,54 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover BaseParser().register_resource( secure_headers.SecureHeaders, db_connection_str=args.cache_file ) + if args.owasp_top10_2025_in: + from application.utils.external_project_parsers.parsers import owasp_top10_2025 + + BaseParser().register_resource( + owasp_top10_2025.OwaspTop10_2025, db_connection_str=args.cache_file + ) + if args.owasp_api_top10_2023_in: + from application.utils.external_project_parsers.parsers import ( + owasp_api_top10_2023, + ) + + BaseParser().register_resource( + owasp_api_top10_2023.OwaspApiTop10_2023, + db_connection_str=args.cache_file, + ) + if args.owasp_kubernetes_top10_2022_in: + from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2022, + ) + + BaseParser().register_resource( + owasp_kubernetes_top10_2022.OwaspKubernetesTop10_2022, + db_connection_str=args.cache_file, + ) + if args.owasp_kubernetes_top10_2025_in: + from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2025, + ) + + BaseParser().register_resource( + owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025, + db_connection_str=args.cache_file, + ) + if args.owasp_llm_top10_2025_in: + from application.utils.external_project_parsers.parsers import ( + owasp_llm_top10_2025, + ) + + BaseParser().register_resource( + owasp_llm_top10_2025.OwaspLlmTop10_2025, + db_connection_str=args.cache_file, + ) + if args.owasp_aisvs_in: + from application.utils.external_project_parsers.parsers import owasp_aisvs + + BaseParser().register_resource( + owasp_aisvs.OwaspAisvs, db_connection_str=args.cache_file + ) if args.pci_dss_4_in: from application.utils.external_project_parsers.parsers import pci_dss diff --git a/application/tests/cre_main_test.py b/application/tests/cre_main_test.py index 097b0b6d9..68d721ab1 100644 --- a/application/tests/cre_main_test.py +++ b/application/tests/cre_main_test.py @@ -7,6 +7,8 @@ from unittest import mock from unittest.mock import Mock, patch from rq import Queue, job +import requests +from rq import Queue, job from application.utils import redis from application.prompt_client import prompt_client as prompt_client from application.tests.utils import data_gen @@ -470,6 +472,111 @@ def test_register_cre(self) -> None: ], ) + @patch("application.cmd.cre_main.time.sleep") + @patch("application.cmd.cre_main.requests.get") + def test_fetch_upstream_json_retries_transient_failures( + self, mock_get, mock_sleep + ) -> None: + transient_error = requests.exceptions.ConnectionError("reset by peer") + success_response = Mock() + success_response.status_code = 200 + success_response.json.return_value = {"data": []} + mock_get.side_effect = [transient_error, success_response] + + data = main.fetch_upstream_json("/root_cres") + + self.assertEqual(data, {"data": []}) + self.assertEqual(mock_get.call_count, 2) + mock_sleep.assert_called_once() + + def test_parse_file(self) -> None: + file: List[Dict[str, Any]] = [ + { + "description": "Verify that approved cryptographic algorithms are used in the generation, seeding, and verification.", + "doctype": defs.Credoctypes.CRE, + "id": "157-573", + "links": [ + { + "type": defs.LinkTypes.LinkedTo, + "document": { + "doctype": defs.Credoctypes.Standard, + "name": "TOP10", + "section": "https://owasp.org/www-project-top-ten/2017/A5_2017-Broken_Access_Control", + }, + }, + { + "type": defs.LinkTypes.LinkedTo, + "document": { + "doctype": defs.Credoctypes.Standard, + "name": "ISO 25010", + "section": "Secure data storage", + }, + }, + ], + "name": "CREDENTIALS_MANAGEMENT_CRYPTOGRAPHIC_DIRECTIVES", + }, + { + "description": "Desc", + "doctype": defs.Credoctypes.CRE, + "id": "141-141", + "name": "name", + }, + ] + expected = [ + defs.CRE( + doctype=defs.Credoctypes.CRE, + id="157-573", + description="Verify that approved cryptographic algorithms are used in the generation, seeding, and verification.", + name="CREDENTIALS_MANAGEMENT_CRYPTOGRAPHIC_DIRECTIVES", + links=[ + defs.Link( + document=defs.Standard( + doctype=defs.Credoctypes.Standard, + name="TOP10", + section="https://owasp.org/www-project-top-ten/2017/A5_2017-Broken_Access_Control", + ), + ltype=defs.LinkTypes.LinkedTo, + ), + defs.Link( + document=defs.Standard( + doctype=defs.Credoctypes.Standard, + name="ISO 25010", + section="Secure data storage", + ), + ltype=defs.LinkTypes.LinkedTo, + ), + ], + ), + defs.CRE(id="141-141", description="Desc", name="name"), + ] + with self.assertLogs("application.cmd.cre_main", level=logging.FATAL) as logs: + # negative test first parse_file accepts a list of objects + result = main.parse_file( + filename="tests", + yamldocs=[ + "no", + "valid", + "objects", + "here", + { + "1": 2, + }, + ], + scollection=self.collection, + ) + + self.assertEqual(result, None) + self.assertIn( + "CRITICAL:application.cmd.cre_main:Malformed file tests, skipping", + logs.output, + ) + + self.maxDiff = None + + res = main.parse_file( + filename="tests", yamldocs=file, scollection=self.collection + ) + self.assertCountEqual(res, expected) @patch.object(main, "db_connect") @patch.object(Queue, "enqueue_call") @patch.object(redis, "wait_for_jobs") From 49d6ad5e29f7d20acb04cc91701dedd7adcf3723 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 09/20] Add local helper scripts for standards development --- scripts/run-local.sh | 26 ++++++++++++++++++ scripts/show-db-stats.sh | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100755 scripts/run-local.sh create mode 100755 scripts/show-db-stats.sh diff --git a/scripts/run-local.sh b/scripts/run-local.sh new file mode 100755 index 000000000..94631cbe9 --- /dev/null +++ b/scripts/run-local.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$ROOT_DIR/venv" + +if [[ ! -d "$VENV_DIR" ]]; then + echo "Creating virtual environment in $VENV_DIR" + python3 -m venv "$VENV_DIR" +fi + +source "$VENV_DIR/bin/activate" + +if ! python -c "import flask" >/dev/null 2>&1; then + echo "Installing Python dependencies" + pip install -r "$ROOT_DIR/requirements.txt" +fi + +export NO_LOGIN="${NO_LOGIN:-1}" +export INSECURE_REQUESTS="${INSECURE_REQUESTS:-1}" +export FLASK_APP="$ROOT_DIR/cre.py" +export FLASK_CONFIG="${FLASK_CONFIG:-development}" + +echo "Starting OpenCRE on http://127.0.0.1:5000" +exec flask run --host 127.0.0.1 --port 5000 diff --git a/scripts/show-db-stats.sh b/scripts/show-db-stats.sh new file mode 100755 index 000000000..626e30dda --- /dev/null +++ b/scripts/show-db-stats.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DB_PATH="${1:-$ROOT_DIR/standards_cache.sqlite}" + +if [[ ! -f "$DB_PATH" ]]; then + echo "Database not found: $DB_PATH" >&2 + exit 1 +fi + +echo "Database: $DB_PATH" +du -h "$DB_PATH" + +"$ROOT_DIR/venv/bin/python" - "$DB_PATH" <<'PY' +import os +import sqlite3 +import sys + +db_path = sys.argv[1] +conn = sqlite3.connect(db_path) +cur = conn.cursor() + +print(f"size_bytes {os.path.getsize(db_path)}") + +tables = [ + "node", + "cre", + "cre_links", + "cre_node_links", + "embeddings", +] + +for table in tables: + try: + count = cur.execute(f"select count(*) from {table}").fetchone()[0] + print(f"{table}_count {count}") + except sqlite3.Error as exc: + print(f"{table}_count unavailable ({exc})") + +try: + standards = cur.execute( + """ + select name, count(*) + from node + where name is not null + group by name + order by count(*) desc, name asc + limit 15 + """ + ).fetchall() + print("top_standards") + for name, count in standards: + print(f"{name}\t{count}") +except sqlite3.Error as exc: + print(f"top_standards unavailable ({exc})") + +conn.close() +PY From c9ebac08037a4026c9610d69d40e0c1d6750825c Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:46:38 +0530 Subject: [PATCH 10/20] Add OWASP comparison and fallback map analysis support --- application/tests/web_main_test.py | 304 ++++++++++++++++++++ application/web/web_main.py | 432 +++++++++++++++++++++++++---- 2 files changed, 677 insertions(+), 59 deletions(-) diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 4b21571d7..d880812f4 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -519,6 +519,63 @@ def test_find_root_cres(self) -> None: self.assertEqual(json.loads(response.data.decode()), expected) self.assertEqual(200, response.status_code) + def test_find_root_cres_featured_standards(self) -> None: + collection = db.Node_collection().with_graph() + + root_cre = defs.CRE(id="111-115", description="CA", name="CA", tags=["ta"]) + collection.add_cre(root_cre) + + featured_nodes = [ + defs.Standard( + name="OWASP Top 10 for LLM and Gen AI Apps 2025", + sectionID="LLM01", + section="Prompt Injection", + ), + defs.Standard( + name="OWASP AI Security Verification Standard (AISVS)", + sectionID="AISVS1", + section="Training Data Governance & Bias Management", + ), + defs.Standard( + name="OWASP API Security Top 10 2023", + sectionID="API1", + section="Broken Object Level Authorization", + ), + defs.Standard( + name="Cloud Controls Matrix", + sectionID="AIS-01", + section="Application and Interface Security", + ), + defs.Standard( + name="OWASP Kubernetes Top Ten 2025 (Draft)", + sectionID="K01", + section="Insecure Workload Configurations", + ), + ] + for node in featured_nodes: + collection.add_node(node) + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/root_cres", + headers={"Content-Type": "application/json"}, + ) + + self.assertEqual(200, response.status_code) + payload = response.get_json() + self.assertIn("data", payload) + self.assertIn("featured_standards", payload) + self.assertCountEqual( + ["AI", "API", "Cloud"], payload["featured_standards"].keys() + ) + self.assertEqual(2, len(payload["featured_standards"]["AI"])) + self.assertEqual(1, len(payload["featured_standards"]["API"])) + self.assertEqual(2, len(payload["featured_standards"]["Cloud"])) + self.assertEqual( + "OWASP Top 10 for LLM and Gen AI Apps 2025", + payload["featured_standards"]["AI"][0]["name"], + ) + def test_smartlink(self) -> None: self.maxDiff = None collection = db.Node_collection().with_graph() @@ -624,6 +681,253 @@ def test_gap_analysis_from_cache_full_response( self.assertEqual(200, response.status_code) self.assertEqual(expected, json.loads(response.data)) + @patch.object(redis, "from_url") + @patch.object(db, "Node_collection") + def test_gap_analysis_adds_owasp_top10_comparison_section( + self, db_mock, redis_conn_mock + ) -> None: + expected = {"result": "hello"} + top10_2021 = [ + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A01", + section="Broken Access Controls", + hyperlink="https://owasp.org/Top10/A01_2021-Broken_Access_Control/", + ), + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A02", + section="Cryptographic Failures", + hyperlink="https://owasp.org/Top10/A02_2021-Cryptographic_Failures/", + ), + ] + top10_2025 = [ + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A01", + section="Broken Access Control", + hyperlink="https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + ), + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A02", + section="Security Misconfiguration", + hyperlink="https://owasp.org/Top10/2025/A02_2025-Security_Misconfiguration/", + ), + ] + redis_conn_mock.return_value.exists.return_value = True + redis_conn_mock.return_value.get.return_value = json.dumps(expected) + db_mock.return_value.get_gap_analysis_result.return_value = json.dumps(expected) + db_mock.return_value.gap_analysis_exists.return_value = True + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Top 10 2021": + return top10_2021 + if name == "OWASP Top 10 2025": + return top10_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Top%2010%202025", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual("hello", payload["result"]) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + ["OWASP Top 10 2021", "OWASP Top 10 2025"], + payload["owasp_top10_comparison"]["standards"], + ) + self.assertEqual( + "Broken Access Controls", + payload["owasp_top10_comparison"]["items"][0]["top10_2021"]["section"], + ) + self.assertEqual( + "Broken Access Control", + payload["owasp_top10_comparison"]["items"][0]["top10_2025"]["section"], + ) + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_owasp_comparison_when_schedule_fails( + self, db_mock, schedule_mock + ) -> None: + top10_2021 = [ + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A01", + section="Broken Access Controls", + hyperlink="https://owasp.org/Top10/A01_2021-Broken_Access_Control/", + ) + ] + top10_2025 = [ + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A01", + section="Broken Access Control", + hyperlink="https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + ) + ] + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Top 10 2021": + return top10_2021 + if name == "OWASP Top 10 2025": + return top10_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + schedule_mock.side_effect = RuntimeError("redis unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Top%2010%202025", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + "Broken Access Controls", + payload["owasp_top10_comparison"]["items"][0]["top10_2021"]["section"], + ) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_upstream_result_when_schedule_fails( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + expected = {"result": {"A01": {"start": "x"}}} + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.return_value = expected + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + self.assertEqual(200, response.status_code) + self.assertEqual(expected, json.loads(response.data)) + upstream_fetch_mock.assert_called_once_with( + "/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Cheat%20Sheets", + timeout=5.0, + max_attempts=1, + backoff_seconds=0.5, + ) + db_mock.return_value.add_gap_analysis_result.assert_called_once_with( + cache_key="OWASP Top 10 2021 >> OWASP Cheat Sheets", + ga_object=json.dumps(expected), + ) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_direct_cre_overlap_when_backends_fail( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + shared_cre = defs.CRE(id="170-772", name="Cryptography", description="") + base = defs.Standard( + name="OWASP Top 10 2025", + sectionID="A04", + section="Cryptographic Failures", + ) + base.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + compare = defs.Standard( + name="OWASP Web Security Testing Guide (WSTG)", + section="WSTG-CRYP-04", + ) + compare.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [base] + if name == "OWASP Top 10 2025" + else [compare] if name == "OWASP Web Security Testing Guide (WSTG)" else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202025&standard=OWASP%20Web%20Security%20Testing%20Guide%20(WSTG)", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertIn(base.id, payload["result"]) + self.assertIn(compare.id, payload["result"][base.id]["paths"]) + db_mock.return_value.add_gap_analysis_result.assert_called_once() + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_normalizes_owasp_top_2025_alias( + self, db_mock, schedule_mock + ) -> None: + top10_2021 = [ + defs.Standard( + name="OWASP Top 10 2021", + sectionID="A01", + section="Broken Access Controls", + hyperlink="https://owasp.org/Top10/A01_2021-Broken_Access_Control/", + ) + ] + top10_2025 = [ + defs.Standard( + name="OWASP Top 10 2025", + sectionID="A01", + section="Broken Access Control", + hyperlink="https://owasp.org/Top10/2025/A01_2025-Broken_Access_Control/", + ) + ] + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Top 10 2021": + return top10_2021 + if name == "OWASP Top 10 2025": + return top10_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + schedule_mock.side_effect = RuntimeError("redis unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%202021&standard=OWASP%20Top%202025", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + "Broken Access Control", + payload["owasp_top10_comparison"]["items"][0]["top10_2025"]["section"], + ) + schedule_mock.assert_called_once_with( + ["OWASP Top 10 2021", "OWASP Top 10 2025"], db_mock.return_value + ) + @patch.object(db, "Node_collection") @patch.object(rq.job.Job, "fetch") @patch.object(rq.Queue, "enqueue_call") diff --git a/application/web/web_main.py b/application/web/web_main.py index a499818cd..9063445c7 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -47,6 +47,13 @@ ITEMS_PER_PAGE = 20 +OWASP_TOP10_2025_DATA_FILE = ( + pathlib.Path(__file__).resolve().parent.parent + / "utils" + / "external_project_parsers" + / "data" + / "owasp_top10_2025.json" +) app = Blueprint( "web", @@ -60,6 +67,32 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +STANDARD_NAME_ALIASES = { + "owasp top 2025": "OWASP Top 10 2025", +} + +ROOT_CRES_FEATURED_STANDARDS = { + "AI": [ + "OWASP Top 10 for LLM and Gen AI Apps 2025", + "OWASP AI Security Verification Standard (AISVS)", + ], + "API": [ + "OWASP API Security Top 10 2023", + ], + "Cloud": [ + "Cloud Controls Matrix", + "OWASP Kubernetes Top Ten 2025 (Draft)", + ], +} +ROOT_CRES_FEATURED_STANDARD_LIMIT = 2 +LLM_TOP10_STANDARD_NAME = "OWASP Top 10 for LLM and Gen AI Apps 2025" +AISVS_STANDARD_NAME = "OWASP AI Security Verification Standard (AISVS)" +API_TOP10_STANDARD_NAME = "OWASP API Security Top 10 2023" +CCM_STANDARD_NAME = "Cloud Controls Matrix" +KUBERNETES_TOP10_2025_STANDARD_NAME = "OWASP Kubernetes Top Ten 2025 (Draft)" +KUBERNETES_TOP10_2022_STANDARD_NAME = "OWASP Kubernetes Top Ten 2022" +OWASP_CHEATSHEETS_STANDARD_NAME = "OWASP Cheat Sheets" + def _ga_timeout_seconds() -> int: raw = str(gap_analysis.GAP_ANALYSIS_TIMEOUT or "129600s") @@ -88,6 +121,244 @@ class SupportedFormats(Enum): YAML = "yaml" OSCAL = "oscal" +def _normalize_source_name(source: Any) -> str | None: + """ + Normalize integration source names so analytics aggregation stays stable. + """ + if source is None: + return None + + normalized_source = str(source).strip().lower() + if not normalized_source: + return None + + normalized_source = normalized_source.replace(" ", "-") + normalized_source = re.sub(r"[^a-z0-9._:-]", "-", normalized_source) + normalized_source = re.sub(r"-{2,}", "-", normalized_source).strip("-") + if not normalized_source: + return None + + return normalized_source[:64] + + +def _load_owasp_top10_2025_entries() -> list[dict[str, str]]: + with OWASP_TOP10_2025_DATA_FILE.open("r", encoding="utf-8") as handle: + return json.load(handle) + + +def _normalize_standard_name(standard: str) -> str: + normalized_standard = str(standard).strip() + return STANDARD_NAME_ALIASES.get(normalized_standard.lower(), normalized_standard) + + +def _build_owasp_top10_comparison( + standards: list[str], collection: db.Node_collection +) -> dict[str, Any] | None: + requested = {str(standard).strip().lower() for standard in standards} + if requested != {"owasp top 10 2021", "owasp top 10 2025"}: + return None + + top10_2021 = sorted( + collection.get_nodes(name="OWASP Top 10 2021"), + key=lambda node: node.sectionID or "", + ) + if not top10_2021: + return None + + top10_2025_nodes = sorted( + collection.get_nodes(name="OWASP Top 10 2025"), + key=lambda node: node.sectionID or "", + ) + if top10_2025_nodes: + top10_2025 = [ + { + "section_id": node.sectionID or "", + "section": node.section or "", + "hyperlink": node.hyperlink or "", + } + for node in top10_2025_nodes + ] + else: + top10_2025 = _load_owasp_top10_2025_entries() + + top10_2021_by_rank = { + node.sectionID + or "": { + "section_id": node.sectionID or "", + "section": node.section or "", + "hyperlink": node.hyperlink or "", + } + for node in top10_2021 + } + top10_2025_by_rank = { + entry["section_id"]: entry for entry in top10_2025 if entry.get("section_id") + } + ranks = sorted(set(top10_2021_by_rank.keys()) | set(top10_2025_by_rank.keys())) + comparison = [] + for rank in ranks: + item_2021 = top10_2021_by_rank.get(rank) + item_2025 = top10_2025_by_rank.get(rank) + comparison.append( + { + "rank": rank, + "top10_2021": item_2021, + "top10_2025": item_2025, + "changed": (item_2021 or {}).get("section") + != (item_2025 or {}).get("section"), + } + ) + + return { + "standards": ["OWASP Top 10 2021", "OWASP Top 10 2025"], + "items": comparison, + } + + +def _build_root_cres_featured_standards( + collection: db.Node_collection, +) -> dict[str, list[dict[str, Any]]]: + featured: dict[str, list[dict[str, Any]]] = {} + + for category, standard_names in ROOT_CRES_FEATURED_STANDARDS.items(): + category_entries: list[dict[str, Any]] = [] + for standard_name in standard_names[:ROOT_CRES_FEATURED_STANDARD_LIMIT]: + nodes = sorted( + collection.get_nodes(name=standard_name), + key=lambda node: ( + node.sectionID or "", + node.section or "", + node.id or "", + ), + ) + if not nodes: + continue + category_entries.append(nodes[0].todict()) + + if category_entries: + featured[category] = category_entries + + return featured + + +def _fetch_upstream_map_analysis( + standards: list[str], + standards_hash: str, + collection: db.Node_collection, +) -> dict[str, Any] | None: + if len(standards) < 2: + return None + + encoded_standards = "&".join( + f"standard={urllib.parse.quote(str(standard), safe='')}" + for standard in standards + ) + timeout = float(os.environ.get("CRE_WEB_UPSTREAM_TIMEOUT_SECONDS", "5")) + max_attempts = int(os.environ.get("CRE_WEB_UPSTREAM_MAX_ATTEMPTS", "1")) + backoff_seconds = float( + os.environ.get("CRE_WEB_UPSTREAM_RETRY_BACKOFF_SECONDS", "0.5") + ) + try: + result = cre_main.fetch_upstream_json( + f"/map_analysis?{encoded_standards}", + timeout=timeout, + max_attempts=max_attempts, + backoff_seconds=backoff_seconds, + ) + except Exception as exc: + logger.warning( + "Unable to fetch upstream map analysis for %s: %s", + standards_hash, + exc, + ) + return None + + if not isinstance(result, dict): + return None + + if result.get("result"): + try: + collection.add_gap_analysis_result( + cache_key=standards_hash, ga_object=flask_json.dumps(result) + ) + except Exception as exc: + logger.warning( + "Unable to cache upstream map analysis for %s: %s", + standards_hash, + exc, + ) + return result + + +def _build_direct_cre_overlap_map_analysis( + standards: list[str], + standards_hash: str, + collection: db.Node_collection, +) -> dict[str, Any] | None: + if len(standards) < 2: + return None + + base_nodes = collection.get_nodes(name=standards[0]) + compare_nodes = collection.get_nodes(name=standards[1]) + if not base_nodes or not compare_nodes: + return None + + if not base_nodes or not compare_nodes: + return None + + compare_nodes_by_cre: dict[str, list[defs.Standard]] = {} + for compare_node in compare_nodes: + for link in compare_node.links: + if link.document.doctype != defs.Credoctypes.CRE: + continue + compare_nodes_by_cre.setdefault(link.document.id, []).append(compare_node) + + grouped_paths: dict[str, dict[str, Any]] = {} + for base_node in base_nodes: + shared_paths: dict[str, Any] = {} + for link in base_node.links: + if link.document.doctype != defs.Credoctypes.CRE: + continue + for compare_node in compare_nodes_by_cre.get(link.document.id, []): + shared_paths.setdefault( + compare_node.id, + { + "end": compare_node.shallow_copy(), + "path": [ + { + "start": base_node.shallow_copy(), + "end": link.document.shallow_copy(), + "relationship": "LINKED_TO", + "score": 0, + }, + { + "start": link.document.shallow_copy(), + "end": compare_node.shallow_copy(), + "relationship": "LINKED_TO", + "score": 0, + }, + ], + "score": 0, + }, + ) + + grouped_paths[base_node.id] = { + "start": base_node.shallow_copy(), + "paths": shared_paths, + "extra": 0, + } + + result = {"result": grouped_paths} + try: + collection.add_gap_analysis_result( + cache_key=standards_hash, ga_object=flask_json.dumps(result) + ) + except Exception as exc: + logger.warning( + "Unable to cache direct CRE-overlap map analysis for %s: %s", + standards_hash, + exc, + ) + return result def extend_cre_with_tag_links( cre: defs.CRE, collection: db.Node_collection @@ -298,73 +569,113 @@ def find_document_by_tag() -> Any: @app.route("/rest/v1/map_analysis", methods=["GET"]) def map_analysis() -> Any: - standards = request.args.getlist("standard") + standards = [_normalize_standard_name(s) for s in request.args.getlist("standard")] if posthog: posthog.capture(f"map_analysis", f"standards:{standards}") database = db.Node_collection() - if len(standards) < 2: - abort(400, "Please provide two standards") - standards = standards[:2] - cache_key = gap_analysis.make_resources_key(standards) - cached = database.get_gap_analysis_result(cache_key=cache_key) - if cached: - parsed = json.loads(cached) - if parsed.get("result"): - return jsonify({"result": parsed.get("result")}) - if os.environ.get("HEROKU"): - abort(404, "No such Cache") - - db_url = os.environ.get("CRE_CACHE_FILE") or os.environ.get("PROD_DATABASE_URL") - if not db_url: - # Derive from current SQLAlchemy bind when not explicitly set. - db_url = str(getattr(getattr(database.session, "bind", None), "url", "")) - try: - conn = redis.connect() - ga_queue_name = os.environ.get("CRE_GA_QUEUE_NAME", "ga") - q = Queue(name=ga_queue_name, connection=conn) - inflight_key = f"ga:inflight:{cache_key}" - inflight_job_id_raw = conn.get(inflight_key) - inflight_job_id = ( - inflight_job_id_raw.decode("utf-8") - if isinstance(inflight_job_id_raw, bytes) - else str(inflight_job_id_raw) if inflight_job_id_raw else "" + standards_hash = gap_analysis.make_resources_key(standards) + owasp_top10_comparison = _build_owasp_top10_comparison(standards, database) + + # First, check if we have cached results in the database + if database.gap_analysis_exists(standards_hash): + gap_analysis_result = database.get_gap_analysis_result(standards_hash) + if gap_analysis_result: + result = flask_json.loads(gap_analysis_result) + if owasp_top10_comparison: + result["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(result) + + # On Heroku (read-only), check if standards exist before attempting Redis/queue operations + is_heroku = os.environ.get("DYNO") is not None + if is_heroku: + # Check if all requested standards exist + try: + existing_standards = database.standards() + if isinstance(existing_standards, (list, tuple, set)): + existing_lower = {str(s).lower() for s in existing_standards} + missing = [s for s in standards if str(s).lower() not in existing_lower] + if missing: + logger.info( + f"On Heroku: gap analysis request {standards_hash} references " + f"standards that do not exist: {', '.join(missing)}, returning 404" + ) + abort( + 404, f"One or more standards do not exist: {', '.join(missing)}" + ) + except Exception as exc: + # If we can't verify standards, log but don't fail (defensive) + logger.warning(f"Could not verify standards existence on Heroku: {exc}") + + # If calculations are disabled, return 404 + if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"): + logger.info( + f"Gap analysis calculations are disabled by CRE_NO_CALCULATE_GAP_ANALYSIS; " + f"refusing to schedule new job for {standards_hash}" ) - if inflight_job_id: - try: - inflight_job = job.Job.fetch(id=inflight_job_id, connection=conn) - if inflight_job.get_status() in ( - job.JobStatus.QUEUED, - job.JobStatus.STARTED, - ): - return jsonify({"job_id": inflight_job_id}) - except exceptions.NoSuchJobError: - conn.delete(inflight_key) - - j = q.enqueue_call( - description=f"{standards[0]}->{standards[1]}", - func=cre_main.run_gap_pair_job, - kwargs={ - "importing_name": standards[0], - "peer_name": standards[1], - "db_connection_str": db_url, - }, - timeout=gap_analysis.GAP_ANALYSIS_TIMEOUT, + upstream_gap_analysis = _fetch_upstream_map_analysis( + standards, standards_hash, database + ) + if upstream_gap_analysis: + if owasp_top10_comparison: + upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(upstream_gap_analysis) + direct_gap_analysis = _build_direct_cre_overlap_map_analysis( + standards, standards_hash, database ) - conn.set(inflight_key, str(j.id)) - conn.expire(inflight_key, _ga_timeout_seconds()) - return jsonify({"job_id": str(j.id)}) + if direct_gap_analysis: + if owasp_top10_comparison: + direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(direct_gap_analysis) + if owasp_top10_comparison: + return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) + abort(404, "Gap analysis calculations are disabled") + + # Now call schedule() which will handle Redis/queue operations + try: + gap_analysis_dict = gap_analysis.schedule(standards, database) except Exception as exc: - logger.warning( - "Redis/RQ unavailable in map_analysis for %s; using synchronous fallback: %s", - cache_key, - exc, + logger.error(f"Gap analysis scheduling failed for {standards_hash}: {exc}") + upstream_gap_analysis = _fetch_upstream_map_analysis( + standards, standards_hash, database ) - try: - return jsonify(_compute_ga_without_redis(database, standards)) - except Exception as fallback_exc: - logger.exception("Synchronous GA fallback failed for %s", cache_key) - abort(503, f"Gap analysis unavailable: {fallback_exc}") + if upstream_gap_analysis: + if owasp_top10_comparison: + upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(upstream_gap_analysis) + direct_gap_analysis = _build_direct_cre_overlap_map_analysis( + standards, standards_hash, database + ) + if direct_gap_analysis: + if owasp_top10_comparison: + direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(direct_gap_analysis) + if owasp_top10_comparison: + return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) + raise + if owasp_top10_comparison: + gap_analysis_dict["owasp_top10_comparison"] = owasp_top10_comparison + if "result" in gap_analysis_dict: + return jsonify(gap_analysis_dict) + if gap_analysis_dict.get("error"): + upstream_gap_analysis = _fetch_upstream_map_analysis( + standards, standards_hash, database + ) + if upstream_gap_analysis: + if owasp_top10_comparison: + upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(upstream_gap_analysis) + direct_gap_analysis = _build_direct_cre_overlap_map_analysis( + standards, standards_hash, database + ) + if direct_gap_analysis: + if owasp_top10_comparison: + direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + return jsonify(direct_gap_analysis) + if owasp_top10_comparison: + return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) + abort(404) + return jsonify({"job_id": gap_analysis_dict.get("job_id")}) @app.route("/rest/v1/map_analysis_weak_links", methods=["GET"]) @@ -549,6 +860,9 @@ def find_root_cres() -> Any: if documents: res = [doc.todict() for doc in documents] result = {"data": res} + featured_standards = _build_root_cres_featured_standards(database) + if featured_standards: + result["featured_standards"] = featured_standards # if opt_osib: # result["osib"] = odefs.cre2osib(documents).todict() if opt_format == SupportedFormats.Markdown.value: From 646cf9cb3ce95599681e8a5357cad0679d2082f1 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Sat, 28 Mar 2026 21:08:59 +0530 Subject: [PATCH 11/20] Merge OpenCRE direct map analysis behavior into issue #471 core review --- application/tests/web_main_test.py | 190 +++++++++++++++++++++++++++++ application/web/web_main.py | 120 +++++++++++++++++- 2 files changed, 309 insertions(+), 1 deletion(-) diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index d880812f4..2efd71758 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -7,6 +7,7 @@ import json import unittest import tempfile +from types import SimpleNamespace from unittest.mock import patch, Mock import redis @@ -877,6 +878,195 @@ def test_gap_analysis_returns_direct_cre_overlap_when_backends_fail( self.assertIn(compare.id, payload["result"][base.id]["paths"]) db_mock.return_value.add_gap_analysis_result.assert_called_once() + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_supports_opencre_as_standard( + self, db_mock, schedule_mock + ) -> None: + compare = defs.Standard( + name="OWASP Web Security Testing Guide (WSTG)", + section="WSTG-CRYP-04", + ) + opencre = defs.CRE(id="170-772", name="Cryptography", description="") + opencre.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=compare.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [compare] if name == "OWASP Web Security Testing Guide (WSTG)" else [] + ) + db_mock.return_value.session.query.return_value.all.return_value = [ + SimpleNamespace(id="cre-internal-1") + ] + db_mock.return_value.get_CREs.return_value = [opencre] + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OpenCRE&standard=OWASP%20Web%20Security%20Testing%20Guide%20(WSTG)", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertIn(opencre.id, payload["result"]) + self.assertEqual(1, len(payload["result"][opencre.id]["paths"])) + path = next(iter(payload["result"][opencre.id]["paths"].values())) + self.assertEqual(compare.id, path["end"]["id"]) + schedule_mock.assert_not_called() + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_only_direct_opencre_mappings( + self, db_mock, schedule_mock + ) -> None: + compare = defs.Standard( + name="CWE", + sectionID="1004", + section="Sensitive Cookie Without 'HttpOnly' Flag", + ) + direct_cre = defs.CRE( + id="804-220", + name="Set httponly attribute for cookie-based session tokens", + description="", + ) + direct_cre.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=compare.shallow_copy()) + ) + auto_linked_cres = [] + for i, cre_id in enumerate( + [ + "117-371", + "166-151", + "284-521", + "368-633", + "612-252", + "664-080", + "801-310", + ], + start=1, + ): + cre = defs.CRE( + id=cre_id, + name=f"Automatically mapped CRE {i}", + description="", + ) + cre.add_link( + defs.Link( + ltype=defs.LinkTypes.AutomaticallyLinkedTo, + document=compare.shallow_copy(), + ) + ) + auto_linked_cres.append(cre) + + opencre_documents = [direct_cre] + auto_linked_cres + internal_ids = [ + SimpleNamespace(id=f"cre-internal-{i}") + for i in range(len(opencre_documents)) + ] + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [compare] if name == "CWE" else [] + ) + db_mock.return_value.session.query.return_value.all.return_value = internal_ids + db_mock.return_value.get_CREs.side_effect = lambda internal_id=None, **kwargs: [ + next( + cre + for index, cre in enumerate(opencre_documents) + if internal_id == f"cre-internal-{index}" + ) + ] + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=CWE&standard=OpenCRE", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertEqual([compare.id], list(payload["result"].keys())) + self.assertEqual(1, len(payload["result"][compare.id]["paths"])) + path = next(iter(payload["result"][compare.id]["paths"].values())) + self.assertEqual(compare.id, payload["result"][compare.id]["start"]["id"]) + self.assertEqual(direct_cre.id, path["end"]["id"]) + self.assertEqual(direct_cre.name, path["end"]["name"]) + self.assertEqual("", path["path"][0]["start"]["id"]) + self.assertEqual(direct_cre.id, path["path"][0]["end"]["id"]) + schedule_mock.assert_not_called() + + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_only_direct_opencre_mappings_when_opencre_is_left( + self, db_mock, schedule_mock + ) -> None: + compare = defs.Standard( + name="CWE", + sectionID="1004", + section="Sensitive Cookie Without 'HttpOnly' Flag", + ) + direct_cre = defs.CRE( + id="804-220", + name="Set httponly attribute for cookie-based session tokens", + description="", + ) + direct_cre.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=compare.shallow_copy()) + ) + indirect_cre = defs.CRE( + id="117-371", + name="Use a centralized access control mechanism", + description="", + ) + indirect_cre.add_link( + defs.Link( + ltype=defs.LinkTypes.AutomaticallyLinkedTo, + document=compare.shallow_copy(), + ) + ) + + opencre_documents = [direct_cre, indirect_cre] + internal_ids = [ + SimpleNamespace(id=f"cre-internal-{i}") + for i in range(len(opencre_documents)) + ] + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [compare] if name == "CWE" else [] + ) + db_mock.return_value.session.query.return_value.all.return_value = internal_ids + db_mock.return_value.get_CREs.side_effect = lambda internal_id=None, **kwargs: [ + next( + cre + for index, cre in enumerate(opencre_documents) + if internal_id == f"cre-internal-{index}" + ) + ] + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OpenCRE&standard=CWE", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual([direct_cre.id], list(payload["result"].keys())) + self.assertEqual(1, len(payload["result"][direct_cre.id]["paths"])) + path = next(iter(payload["result"][direct_cre.id]["paths"].values())) + self.assertEqual(direct_cre.id, payload["result"][direct_cre.id]["start"]["id"]) + self.assertEqual(compare.id, path["end"]["id"]) + self.assertEqual(direct_cre.id, path["path"][0]["start"]["id"]) + self.assertEqual(compare.id, path["path"][0]["end"]["id"]) + schedule_mock.assert_not_called() + @patch.object(web_main.gap_analysis, "schedule") @patch.object(db, "Node_collection") def test_gap_analysis_normalizes_owasp_top_2025_alias( diff --git a/application/web/web_main.py b/application/web/web_main.py index 9063445c7..8b6dd823c 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -85,6 +85,7 @@ ], } ROOT_CRES_FEATURED_STANDARD_LIMIT = 2 +OPENCRE_STANDARD_NAME = "OpenCRE" LLM_TOP10_STANDARD_NAME = "OWASP Top 10 for LLM and Gen AI Apps 2025" AISVS_STANDARD_NAME = "OWASP AI Security Verification Standard (AISVS)" API_TOP10_STANDARD_NAME = "OWASP API Security Top 10 2023" @@ -289,6 +290,114 @@ def _fetch_upstream_map_analysis( return result +def _get_opencre_documents(collection: db.Node_collection) -> list[defs.CRE]: + return [ + collection.get_CREs(internal_id=cre.id)[0] + for cre in collection.session.query(db.CRE).all() + ] + + +def _get_opencre_map_analysis_documents( + standard: str, collection: db.Node_collection +) -> list[defs.Document]: + if standard == OPENCRE_STANDARD_NAME: + return _get_opencre_documents(collection) + return collection.get_nodes(name=standard) + + +def _build_opencre_direct_link_path( + start_document: defs.Document, end_document: defs.Document +) -> dict[str, Any]: + segment_start = start_document.shallow_copy() + # Keep the direct-link response compatible with the current popup renderer. + if segment_start.doctype != defs.Credoctypes.CRE.value: + segment_start.id = "" + return { + "end": end_document.shallow_copy(), + "path": [ + { + "start": segment_start, + "end": end_document.shallow_copy(), + "relationship": "LINKED_TO", + "score": 0, + } + ], + "score": 0, + } + + +def _make_opencre_direct_link_path_key(end_document: defs.Document) -> str: + return end_document.id + + +def _add_opencre_direct_link_result( + grouped_paths: dict[str, dict[str, Any]], + start_document: defs.Document, + end_document: defs.Document, +) -> None: + shared_paths = grouped_paths.setdefault( + start_document.id, + { + "start": start_document.shallow_copy(), + "paths": {}, + "extra": 0, + }, + )["paths"] + shared_paths.setdefault( + _make_opencre_direct_link_path_key(end_document), + _build_opencre_direct_link_path(start_document, end_document), + ) + + +def _build_opencre_direct_map_analysis( + standards: list[str], + standards_hash: str, + collection: db.Node_collection, +) -> dict[str, Any] | None: + if len(standards) < 2: + return None + + base_standard = standards[0] + compare_standard = standards[1] + base_nodes = _get_opencre_map_analysis_documents(base_standard, collection) + compare_nodes = _get_opencre_map_analysis_documents(compare_standard, collection) + if not base_nodes or not compare_nodes: + return None + + base_is_opencre = base_standard == OPENCRE_STANDARD_NAME + opencre_nodes = base_nodes if base_is_opencre else compare_nodes + standard_nodes = compare_nodes if base_is_opencre else base_nodes + + standard_nodes_by_id = { + standard_node.id: standard_node for standard_node in standard_nodes + } + direct_pairs: list[tuple[defs.CRE, defs.Document]] = [] + for opencre_node in opencre_nodes: + for link in opencre_node.links: + if link.ltype != defs.LinkTypes.LinkedTo: + continue + standard_node = standard_nodes_by_id.get(link.document.id) + if not standard_node: + continue + direct_pairs.append((opencre_node, standard_node)) + + grouped_paths: dict[str, dict[str, Any]] = {} + for opencre_node, standard_node in direct_pairs: + if base_is_opencre: + _add_opencre_direct_link_result(grouped_paths, opencre_node, standard_node) + else: + _add_opencre_direct_link_result(grouped_paths, standard_node, opencre_node) + + if not grouped_paths: + return None + + result = {"result": grouped_paths} + collection.add_gap_analysis_result( + cache_key=standards_hash, ga_object=flask_json.dumps(result) + ) + return result + + def _build_direct_cre_overlap_map_analysis( standards: list[str], standards_hash: str, @@ -575,6 +684,13 @@ def map_analysis() -> Any: database = db.Node_collection() standards_hash = gap_analysis.make_resources_key(standards) + if OPENCRE_STANDARD_NAME in standards: + direct_gap_analysis = _build_opencre_direct_map_analysis( + standards, standards_hash, database + ) + if direct_gap_analysis: + return jsonify(direct_gap_analysis) + abort(404, "No direct overlap found for requested standards") owasp_top10_comparison = _build_owasp_top10_comparison(standards, database) # First, check if we have cached results in the database @@ -771,7 +887,9 @@ def standards() -> Any: posthog.capture(f"standards", "") database = db.Node_collection() - standards = database.standards() + standards = list(database.standards()) + if OPENCRE_STANDARD_NAME not in standards: + standards.append(OPENCRE_STANDARD_NAME) return standards From 0da3813007f4e3eec58c40ef7b6782305db81362 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:47:44 +0530 Subject: [PATCH 12/20] Add specialized OWASP cheat sheet map analysis behavior --- application/tests/web_main_test.py | 256 +++++++++++++++++++++++++++++ application/web/web_main.py | 170 ++++++++++++++++++- 2 files changed, 425 insertions(+), 1 deletion(-) diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 2efd71758..763e32ae9 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -34,6 +34,32 @@ def get_status(self): return rq.job.JobStatus.STARTED +class MockFinishedJobResult: + class Type: + SUCCESSFUL = "successful" + FAILED = "failed" + + type = Type.SUCCESSFUL + + @property + def return_value(self): + return [ + [ + "OWASP Top 10 for LLM and Gen AI Apps 2025", + "OWASP Cheat Sheets", + ], + {"status": "done"}, + ] + + +class MockFinishedJob: + def get_status(self): + return rq.job.JobStatus.FINISHED + + def latest_result(self): + return MockFinishedJobResult() + + class TestMain(unittest.TestCase): def tearDown(self) -> None: sqla.session.remove() @@ -878,6 +904,180 @@ def test_gap_analysis_returns_direct_cre_overlap_when_backends_fail( self.assertIn(compare.id, payload["result"][base.id]["paths"]) db_mock.return_value.add_gap_analysis_result.assert_called_once() + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_filters_generic_cheatsheets_for_llm_top10( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + broad_cre = defs.CRE(id="064-808", name="Output encoding", description="") + llm_specific_cre = defs.CRE( + id="161-451", name="Prompt boundary protection", description="" + ) + + llm = defs.Standard( + name="OWASP Top 10 for LLM and Gen AI Apps 2025", + sectionID="LLM05", + section="Improper Output Handling", + ) + llm.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=broad_cre.shallow_copy()) + ) + llm.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, document=llm_specific_cre.shallow_copy() + ) + ) + + generic_cheatsheet = defs.Standard( + name="OWASP Cheat Sheets", + section="Cross Site Scripting Prevention Cheat Sheet", + ) + generic_cheatsheet.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=broad_cre.shallow_copy()) + ) + + llm_cheatsheet = defs.Standard( + name="OWASP Cheat Sheets", + section="LLM Prompt Injection Prevention Cheat Sheet", + ) + llm_cheatsheet.add_link( + defs.Link( + ltype=defs.LinkTypes.LinkedTo, document=llm_specific_cre.shallow_copy() + ) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [llm] + if name == "OWASP Top 10 for LLM and Gen AI Apps 2025" + else ( + [generic_cheatsheet, llm_cheatsheet] + if name == "OWASP Cheat Sheets" + else [] + ) + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%20for%20LLM%20and%20Gen%20AI%20Apps%202025&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertEqual( + "AI / LLM Cheat Sheets", + payload["specialized_cheatsheet_section"]["category"], + ) + self.assertIn(llm.id, payload["result"]) + self.assertNotIn(llm_cheatsheet.id, payload["result"][llm.id]["paths"]) + self.assertNotIn(generic_cheatsheet.id, payload["result"][llm.id]["paths"]) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_keeps_prompt_injection_cheatsheet_for_llm01( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + prompt_cre = defs.CRE( + id="161-451", name="Prompt boundary protection", description="" + ) + + llm = defs.Standard( + name="OWASP Top 10 for LLM and Gen AI Apps 2025", + sectionID="LLM01", + section="Prompt Injection", + ) + llm.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=prompt_cre.shallow_copy()) + ) + + llm_cheatsheet = defs.Standard( + name="OWASP Cheat Sheets", + section="LLM Prompt Injection Prevention Cheat Sheet", + ) + llm_cheatsheet.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=prompt_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [llm] + if name == "OWASP Top 10 for LLM and Gen AI Apps 2025" + else [llm_cheatsheet] + if name == "OWASP Cheat Sheets" + else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Top%2010%20for%20LLM%20and%20Gen%20AI%20Apps%202025&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn(llm.id, payload["result"]) + self.assertIn(llm_cheatsheet.id, payload["result"][llm.id]["paths"]) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_adds_specialized_section_for_api_cheatsheets( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + authz_cre = defs.CRE( + id="117-371", name="Centralized access control", description="" + ) + base = defs.Standard( + name="OWASP API Security Top 10 2023", + sectionID="API1", + section="Broken Object Level Authorization", + ) + base.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=authz_cre.shallow_copy()) + ) + + compare = defs.Standard( + name="OWASP Cheat Sheets", + section="Authorization Cheat Sheet", + ) + compare.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=authz_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [base] + if name == "OWASP API Security Top 10 2023" + else [compare] if name == "OWASP Cheat Sheets" else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20API%20Security%20Top%2010%202023&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual( + "API Cheat Sheets", + payload["specialized_cheatsheet_section"]["category"], + ) + self.assertIn(base.id, payload["specialized_cheatsheet_section"]["result"]) + @patch.object(web_main.gap_analysis, "schedule") @patch.object(db, "Node_collection") def test_gap_analysis_supports_opencre_as_standard( @@ -1222,6 +1422,62 @@ def test_gap_analysis_heroku_cache_miss_returns_404( self.assertEqual(404, response.status_code) redis_conn_mock.assert_not_called() + @patch.object(web_main.redis, "connect") + @patch.object(rq.job.Job, "fetch") + @patch.object(db, "Node_collection") + def test_ma_job_results_adds_specialized_cheatsheet_section( + self, db_mock, fetch_mock, redis_connect_mock + ) -> None: + shared_cre = defs.CRE(id="161-451", name="Prompt boundary protection", description="") + llm = defs.Standard( + name="OWASP Top 10 for LLM and Gen AI Apps 2025", + sectionID="LLM01", + section="Prompt Injection", + ) + llm.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + cheatsheet = defs.Standard( + name="OWASP Cheat Sheets", + section="LLM Prompt Injection Prevention Cheat Sheet", + ) + cheatsheet.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + + cached_gap = { + "result": { + llm.id: { + "start": llm.shallow_copy().todict(), + "paths": { + cheatsheet.id: { + "end": cheatsheet.shallow_copy().todict(), + "path": [], + "score": 0, + } + }, + "extra": 0, + } + } + } + + fetch_mock.return_value = MockFinishedJob() + redis_connect_mock.return_value.exists.return_value = True + db_mock.return_value.get_gap_analysis_result.return_value = json.dumps(cached_gap) + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/ma_job_results?id=ABC", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual( + "AI / LLM Cheat Sheets", + payload["specialized_cheatsheet_section"]["category"], + ) + @patch.object(redis, "from_url") @patch.object(db, "Node_collection") def test_standards_from_db(self, node_mock, redis_conn_mock) -> None: diff --git a/application/web/web_main.py b/application/web/web_main.py index 8b6dd823c..b610d8b28 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -93,6 +93,49 @@ KUBERNETES_TOP10_2025_STANDARD_NAME = "OWASP Kubernetes Top Ten 2025 (Draft)" KUBERNETES_TOP10_2022_STANDARD_NAME = "OWASP Kubernetes Top Ten 2022" OWASP_CHEATSHEETS_STANDARD_NAME = "OWASP Cheat Sheets" +SPECIALIZED_CHEATSHEET_GROUPS = { + "AI / LLM Cheat Sheets": { + "standards": {LLM_TOP10_STANDARD_NAME, AISVS_STANDARD_NAME}, + "sections": { + "LLM Prompt Injection Prevention Cheat Sheet", + "AI Agent Security Cheat Sheet", + "Secure AI Model Ops Cheat Sheet", + }, + }, + "API Cheat Sheets": { + "standards": {API_TOP10_STANDARD_NAME}, + "sections": { + "REST Security Cheat Sheet", + "Authorization Cheat Sheet", + "Server Side Request Forgery Prevention Cheat Sheet", + "Web Service Security Cheat Sheet", + }, + }, + "Cloud Cheat Sheets": { + "standards": { + CCM_STANDARD_NAME, + KUBERNETES_TOP10_2025_STANDARD_NAME, + KUBERNETES_TOP10_2022_STANDARD_NAME, + }, + "sections": { + "Docker Security Cheat Sheet", + "Kubernetes Security Cheat Sheet", + "Secure Cloud Architecture Cheat Sheet", + }, + }, +} +AI_LLM_SECTION_CHEATSHEET_MATCHES = { + "LLM01": {"LLM Prompt Injection Prevention Cheat Sheet"}, + "LLM02": {"AI Agent Security Cheat Sheet"}, + "LLM03": {"Secure AI Model Ops Cheat Sheet"}, + "LLM04": {"Secure AI Model Ops Cheat Sheet"}, + "LLM05": set(), + "LLM06": {"AI Agent Security Cheat Sheet"}, + "LLM07": {"AI Agent Security Cheat Sheet"}, + "LLM08": {"AI Agent Security Cheat Sheet"}, + "LLM09": set(), + "LLM10": set(), +} def _ga_timeout_seconds() -> int: @@ -411,6 +454,18 @@ def _build_direct_cre_overlap_map_analysis( if not base_nodes or not compare_nodes: return None + specialized_group = _get_specialized_cheatsheet_group(standards) + if specialized_group: + allowed_sections = SPECIALIZED_CHEATSHEET_GROUPS[specialized_group]["sections"] + if standards[1] == OWASP_CHEATSHEETS_STANDARD_NAME: + compare_nodes = [ + node for node in compare_nodes if node.section in allowed_sections + ] + elif standards[0] == OWASP_CHEATSHEETS_STANDARD_NAME: + base_nodes = [ + node for node in base_nodes if node.section in allowed_sections + ] + if not base_nodes or not compare_nodes: return None @@ -428,6 +483,10 @@ def _build_direct_cre_overlap_map_analysis( if link.document.doctype != defs.Credoctypes.CRE: continue for compare_node in compare_nodes_by_cre.get(link.document.id, []): + if not _is_allowed_specialized_cheatsheet_match( + specialized_group, base_node, compare_node + ): + continue shared_paths.setdefault( compare_node.id, { @@ -469,6 +528,52 @@ def _build_direct_cre_overlap_map_analysis( ) return result +def _get_specialized_cheatsheet_group(standards: list[str]) -> str | None: + if OWASP_CHEATSHEETS_STANDARD_NAME not in standards: + return None + + for category, config in SPECIALIZED_CHEATSHEET_GROUPS.items(): + if any(standard in config["standards"] for standard in standards): + return category + return None + + +def _build_specialized_cheatsheet_section( + standards: list[str], gap_analysis_result: dict[str, Any] | None +) -> dict[str, Any] | None: + category = _get_specialized_cheatsheet_group(standards) + if not category or not gap_analysis_result: + return None + + result = gap_analysis_result.get("result") + if not isinstance(result, dict): + return None + + return { + "category": category, + "standards": standards, + "result": result, + } + + +def _is_allowed_specialized_cheatsheet_match( + category: str | None, base_node: defs.Standard, compare_node: defs.Standard +) -> bool: + if category != "AI / LLM Cheat Sheets": + return True + + if base_node.name == LLM_TOP10_STANDARD_NAME and compare_node.name == OWASP_CHEATSHEETS_STANDARD_NAME: + allowed = AI_LLM_SECTION_CHEATSHEET_MATCHES.get(base_node.sectionID or "", set()) + return compare_node.section in allowed + + if ( + base_node.name == OWASP_CHEATSHEETS_STANDARD_NAME + and compare_node.name == LLM_TOP10_STANDARD_NAME + ): + allowed = AI_LLM_SECTION_CHEATSHEET_MATCHES.get(compare_node.sectionID or "", set()) + return base_node.section in allowed + + return True def extend_cre_with_tag_links( cre: defs.CRE, collection: db.Node_collection ) -> defs.CRE: @@ -698,8 +803,15 @@ def map_analysis() -> Any: gap_analysis_result = database.get_gap_analysis_result(standards_hash) if gap_analysis_result: result = flask_json.loads(gap_analysis_result) + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, result + ) if owasp_top10_comparison: result["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + result["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(result) # On Heroku (read-only), check if standards exist before attempting Redis/queue operations @@ -733,15 +845,29 @@ def map_analysis() -> Any: standards, standards_hash, database ) if upstream_gap_analysis: + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, upstream_gap_analysis + ) if owasp_top10_comparison: upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + upstream_gap_analysis["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(upstream_gap_analysis) direct_gap_analysis = _build_direct_cre_overlap_map_analysis( standards, standards_hash, database ) if direct_gap_analysis: + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, direct_gap_analysis + ) if owasp_top10_comparison: direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + direct_gap_analysis["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(direct_gap_analysis) if owasp_top10_comparison: return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) @@ -756,21 +882,42 @@ def map_analysis() -> Any: standards, standards_hash, database ) if upstream_gap_analysis: + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, upstream_gap_analysis + ) if owasp_top10_comparison: upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + upstream_gap_analysis["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(upstream_gap_analysis) direct_gap_analysis = _build_direct_cre_overlap_map_analysis( standards, standards_hash, database ) if direct_gap_analysis: + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, direct_gap_analysis + ) if owasp_top10_comparison: direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + direct_gap_analysis["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(direct_gap_analysis) if owasp_top10_comparison: return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) raise if owasp_top10_comparison: gap_analysis_dict["owasp_top10_comparison"] = owasp_top10_comparison + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, gap_analysis_dict + ) + if specialized_cheatsheet_section: + gap_analysis_dict["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) if "result" in gap_analysis_dict: return jsonify(gap_analysis_dict) if gap_analysis_dict.get("error"): @@ -778,15 +925,29 @@ def map_analysis() -> Any: standards, standards_hash, database ) if upstream_gap_analysis: + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, upstream_gap_analysis + ) if owasp_top10_comparison: upstream_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + upstream_gap_analysis["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(upstream_gap_analysis) direct_gap_analysis = _build_direct_cre_overlap_map_analysis( standards, standards_hash, database ) if direct_gap_analysis: + specialized_cheatsheet_section = _build_specialized_cheatsheet_section( + standards, direct_gap_analysis + ) if owasp_top10_comparison: direct_gap_analysis["owasp_top10_comparison"] = owasp_top10_comparison + if specialized_cheatsheet_section: + direct_gap_analysis["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) return jsonify(direct_gap_analysis) if owasp_top10_comparison: return jsonify({"owasp_top10_comparison": owasp_top10_comparison}) @@ -865,7 +1026,14 @@ def fetch_job() -> Any: if ga: # logger.__delattr__("and results in cache") ga = flask_json.loads(ga) - if ga.get("result"): + specialized_cheatsheet_section = ( + _build_specialized_cheatsheet_section(standards, ga) + ) + if specialized_cheatsheet_section: + ga["specialized_cheatsheet_section"] = ( + specialized_cheatsheet_section + ) + if "result" in ga: return jsonify(ga) else: logger.error( From b8ee4e0233f8837c729e089be2ea8fc6f7d71a55 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Tue, 24 Mar 2026 23:04:55 +0530 Subject: [PATCH 13/20] Surface OWASP standards in explorer and analysis UI --- .../frontend/src/pages/Explorer/explorer.scss | 63 +++++-- .../frontend/src/pages/Explorer/explorer.tsx | 40 +++- .../src/pages/GapAnalysis/GapAnalysis.scss | 17 ++ .../src/pages/GapAnalysis/GapAnalysis.tsx | 178 +++++++++++++++++- .../frontend/src/providers/DataProvider.tsx | 55 +++--- application/frontend/src/types.ts | 24 +++ 6 files changed, 317 insertions(+), 60 deletions(-) diff --git a/application/frontend/src/pages/Explorer/explorer.scss b/application/frontend/src/pages/Explorer/explorer.scss index 9d94aab22..d910690e8 100644 --- a/application/frontend/src/pages/Explorer/explorer.scss +++ b/application/frontend/src/pages/Explorer/explorer.scss @@ -21,19 +21,40 @@ main#explorer-content { color: #0056b3; } - .search-field { - input { - font-size: 16px; - height: 32px; - width: 320px; + .search-field { + input { + font-size: 16px; + height: 32px; + width: 320px; margin-bottom: 10px; border-radius: 3px; border: 1px solid #858585; padding: 0 5px; color: #333333; - background-color: #ffffff; - } - } + background-color: #ffffff; + } + } + + .tree-actions { + display: flex; + gap: 10px; + margin-bottom: 14px; + + button { + height: 34px; + padding: 0 12px; + border: 1px solid #b1b0b0; + border-radius: 4px; + background: #ffffff; + color: #1a202c; + cursor: pointer; + font-size: 14px; + + &:hover { + background: #f4f6f8; + } + } + } #graphs-menu { display: flex; @@ -150,16 +171,20 @@ main#explorer-content { padding: 1rem; /* Reduce from 30px to prevent content touching edges */ - .search-field { - input { - width: 100%; - /* Expand from fixed 320px - overflows on small screens */ - } - } - - .item { - margin: 4px 4px 4px 1rem; - /* Reduce from 40px left margin on mobile */ + .search-field { + input { + width: 100%; + /* Expand from fixed 320px - overflows on small screens */ + } + } + + .tree-actions { + flex-wrap: wrap; + } + + .item { + margin: 4px 4px 4px 1rem; + /* Reduce from 40px left margin on mobile */ } } -} \ No newline at end of file +} diff --git a/application/frontend/src/pages/Explorer/explorer.tsx b/application/frontend/src/pages/Explorer/explorer.tsx index 64bee6566..a54e484c2 100644 --- a/application/frontend/src/pages/Explorer/explorer.tsx +++ b/application/frontend/src/pages/Explorer/explorer.tsx @@ -65,6 +65,32 @@ export const Explorer = () => { } }; + const collectExpandableIds = (nodes: TreeDocument[] = []): string[] => { + const ids: string[] = []; + + const visit = (node: TreeDocument | null) => { + if (!node) { + return; + } + const contains = (node.links || []).filter((link) => link.ltype === TYPE_CONTAINS); + if (contains.length > 0) { + ids.push(node.id); + contains.forEach((child) => visit(child.document)); + } + }; + + nodes.forEach((node) => visit(node)); + return ids; + }; + + const collapseAll = () => { + setCollapsedItems(collectExpandableIds(filteredTree || [])); + }; + + const expandAll = () => { + setCollapsedItems([]); + }; + useEffect(() => { if (dataTree.length) { const treeCopy = structuredClone(dataTree); @@ -136,11 +162,7 @@ export const Explorer = () => {

Open CRE Explorer

A visual explorer of Open Common Requirement Enumerations (CREs). Originally created by:{' '} - + Zeljko Obrenovic . @@ -151,6 +173,14 @@ export const Explorer = () => {

+
+ + +

Explore visually:

    diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.scss b/application/frontend/src/pages/GapAnalysis/GapAnalysis.scss index feccf0672..8f4c573b0 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.scss +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.scss @@ -5,6 +5,23 @@ main#gap-analysis { span.name { padding: 0 10px; } + + .specialized-cheatsheets { + margin-top: 1.5rem; + + h2 { + margin-bottom: 0.4rem; + } + + p { + margin-bottom: 0.9rem; + color: #4a5568; + } + + table { + border-top: 3px solid #2b6cb0; + } + } } @media (min-width: 0px) and (max-width: 500px) { diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 1a0a72c38..7b70cd497 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -8,7 +8,7 @@ import { Button, Dropdown, DropdownItemProps, Icon, Popup, Table } from 'semanti import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator'; import { GA_STRONG_UPPER_LIMIT } from '../../const'; import { useEnvironment } from '../../hooks'; -import { GapAnalysisPathStart } from '../../types'; +import { GapAnalysisPathStart, OwaspTop10Comparison, SpecializedCheatsheetSection } from '../../types'; import { getDocumentDisplayName } from '../../utils'; import { getInternalUrl } from '../../utils/document'; @@ -41,6 +41,21 @@ function useQuery() { return React.useMemo(() => new URLSearchParams(search), [search]); } +const getInitialStandardsFromQuery = (searchParams: URLSearchParams) => { + const standardParams = searchParams.getAll('standard').filter(Boolean); + if (standardParams.length >= 2) { + return { + base: standardParams[0], + compare: standardParams[1], + }; + } + + return { + base: searchParams.get('base') ?? '', + compare: searchParams.get('compare') ?? '', + }; +}; + const GetStrength = (score) => { if (score == 0) return 'Direct'; if (score <= GA_STRONG_UPPER_LIMIT) return 'Strong'; @@ -55,7 +70,38 @@ const GetStrengthColor = (score) => { return 'Orange'; }; -const GetResultLine = (path, gapAnalysis, key) => { +const CHEATSHEET_CATEGORY_LABELS = { + 'LLM Prompt Injection Prevention Cheat Sheet': 'AI', + 'AI Agent Security Cheat Sheet': 'AI', + 'Secure AI Model Ops Cheat Sheet': 'AI', + 'REST Security Cheat Sheet': 'API', + 'Authorization Cheat Sheet': 'API', + 'Server Side Request Forgery Prevention Cheat Sheet': 'API', + 'Web Service Security Cheat Sheet': 'API', + 'Docker Security Cheat Sheet': 'Cloud', + 'Kubernetes Security Cheat Sheet': 'Cloud', + 'Secure Cloud Architecture Cheat Sheet': 'Cloud', +}; + +const formatCheatsheetLabel = (document, specializedCategory?: string) => { + if (document?.name !== 'OWASP Cheat Sheets') { + return getDocumentDisplayName(document, true); + } + + const category = + CHEATSHEET_CATEGORY_LABELS[document.section] ?? + (specializedCategory?.includes('AI') + ? 'AI' + : specializedCategory?.includes('API') + ? 'API' + : specializedCategory?.includes('Cloud') + ? 'Cloud' + : 'Web'); + + return `${category} Cheat Sheet: ${document.section}`; +}; + +const GetResultLine = (path, gapAnalysis, key, specializedCategory?: string) => { let segmentID = gapAnalysis[key].start.id; return (
    @@ -66,7 +112,7 @@ const GetResultLine = (path, gapAnalysis, key) => { style={{ textAlign: 'center' }} hoverable position="right center" - trigger={{getDocumentDisplayName(path.end, true)} } + trigger={{formatCheatsheetLabel(path.end, specializedCategory)} } > {getDocumentDisplayName(gapAnalysis[key].start, true)} @@ -114,15 +160,17 @@ const GetResultLine = (path, gapAnalysis, key) => { export const GapAnalysis = () => { const standardOptionsDefault = [{ key: '', text: '', value: undefined }]; const searchParams = useQuery(); + const initialStandards = getInitialStandardsFromQuery(searchParams); const [standardOptions, setStandardOptions] = useState( standardOptionsDefault ); - const [BaseStandard, setBaseStandard] = useState(searchParams.get('base') ?? ''); - const [CompareStandard, setCompareStandard] = useState( - searchParams.get('compare') ?? '' - ); + const [BaseStandard, setBaseStandard] = useState(initialStandards.base); + const [CompareStandard, setCompareStandard] = useState(initialStandards.compare); const [gaJob, setgaJob] = useState(''); const [gapAnalysis, setGapAnalysis] = useState>(); + const [owaspComparison, setOwaspComparison] = useState(); + const [specializedCheatsheetSection, setSpecializedCheatsheetSection] = + useState(); const [loadingStandards, setLoadingStandards] = useState(false); const [loadingGA, setLoadingGA] = useState(false); const [error, setError] = useState(null); @@ -158,6 +206,7 @@ export const GapAnalysis = () => { if (result.data.result) { setLoadingGA(false); setGapAnalysis(result.data.result); + setSpecializedCheatsheetSection(result.data.specialized_cheatsheet_section); setgaJob(''); } }; @@ -197,6 +246,13 @@ export const GapAnalysis = () => { if (result.data.result) { setLoadingGA(false); setGapAnalysis(result.data.result); + setOwaspComparison(result.data.owasp_top10_comparison); + setSpecializedCheatsheetSection(result.data.specialized_cheatsheet_section); + } else if (result.data.owasp_top10_comparison) { + setLoadingGA(false); + setGapAnalysis(undefined); + setOwaspComparison(result.data.owasp_top10_comparison); + setSpecializedCheatsheetSection(result.data.specialized_cheatsheet_section); } else if (result.data.job_id) { setgaJob(result.data.job_id); } @@ -204,6 +260,8 @@ export const GapAnalysis = () => { if (!BaseStandard || !CompareStandard || BaseStandard === CompareStandard) return; setGapAnalysis(undefined); + setOwaspComparison(undefined); + setSpecializedCheatsheetSection(undefined); setLoadingGA(true); fetchData().catch((e) => { setLoadingGA(false); @@ -282,7 +340,7 @@ export const GapAnalysis = () => { - {gapAnalysis && ( + {gapAnalysis && !specializedCheatsheetSection && ( <> {Object.keys(gapAnalysis) .sort((a, b) => @@ -326,6 +384,110 @@ export const GapAnalysis = () => { )} + {specializedCheatsheetSection && ( +
    +

    {specializedCheatsheetSection.category}

    +

    + Showing only the specialized OWASP Cheat Sheets for this comparison so the results stay focused. +

    + + + {Object.keys(specializedCheatsheetSection.result) + .sort((a, b) => + getDocumentDisplayName( + specializedCheatsheetSection.result[a].start, + true + ).localeCompare( + getDocumentDisplayName( + specializedCheatsheetSection.result[b].start, + true + ) + ) + ) + .map((key) => ( + + + +

    + {getDocumentDisplayName(specializedCheatsheetSection.result[key].start, true)} +

    +     +     + + {Object.values(specializedCheatsheetSection.result[key].paths) + .sort((a, b) => a.score - b.score) + .map((path) => + GetResultLine( + path, + specializedCheatsheetSection.result, + key, + specializedCheatsheetSection.category + ) + )} + {specializedCheatsheetSection.result[key].weakLinks && + Object.values(specializedCheatsheetSection.result[key].weakLinks) + .sort((a, b) => a.score - b.score) + .map((path) => + GetResultLine( + path, + specializedCheatsheetSection.result, + key, + specializedCheatsheetSection.category + ) + )} + {Object.keys(specializedCheatsheetSection.result[key].paths).length === 0 && + specializedCheatsheetSection.result[key].extra === 0 && No links Found} + +     + ))} +     +
    +
    + )} + {owaspComparison && ( + + + + Rank + OWASP Top 10 2021 + OWASP Top 10 2025 + Changed + + + + {owaspComparison.items.map((item) => ( + + + {item.rank} + + + {item.top10_2021?.hyperlink ? ( + + {item.top10_2021.section} + + ) : ( + item.top10_2021?.section ?? Not mapped + )} + + + {item.top10_2025?.hyperlink ? ( + + {item.top10_2025.section} + + ) : ( + item.top10_2025?.section ?? Not mapped + )} + + {item.changed ? 'Yes' : 'No'} + + ))} + +
    + )} ); }; diff --git a/application/frontend/src/providers/DataProvider.tsx b/application/frontend/src/providers/DataProvider.tsx index 37962c014..aaa0cb738 100644 --- a/application/frontend/src/providers/DataProvider.tsx +++ b/application/frontend/src/providers/DataProvider.tsx @@ -9,8 +9,9 @@ import { Document, TreeDocument } from '../types'; import { getDbObject, setDbObject } from '../utils'; // Assumes utils/index.tsx is the entry point import { getInternalUrl, getTopicDisplayName } from '../utils/document'; -const DATA_STORE_KEY = 'data-store', - DATA_TREE_KEY = 'record-tree'; +const CACHE_VERSION = 'v2'; +const DATA_STORE_KEY = `data-store-${CACHE_VERSION}`, + DATA_TREE_KEY = `record-tree-${CACHE_VERSION}`; type DataContextValues = { dataLoading: boolean; @@ -84,7 +85,7 @@ export const DataProvider = ({ children }: { children: React.ReactNode }) => { const getTreeQuery = useQuery( 'root_cres', async () => { - if (!dataTree.length && Object.keys(dataStore).length) { + if (Object.keys(dataStore).length) { try { const result = await axios.get(`${apiUrl}/root_cres`); const treeData = result.data.data.map((x) => buildTree(x)); @@ -108,32 +109,30 @@ export const DataProvider = ({ children }: { children: React.ReactNode }) => { const getStoreQuery = useQuery( 'all_cres', async () => { - if (!Object.keys(dataStore).length) { - try { - const result = await axios.get(`${apiUrl}/all_cres?page=1&per_page=1000`); - let data = result.data.data; - let store = {}; - - if (data.length) { - data.forEach((x) => { - store[getStoreKey(x)] = { - links: x.links, - displayName: getTopicDisplayName(x), - url: getInternalUrl(x), - ...x, - }; - }); - - // CHANGE 5: Save to IndexedDB (async) instead of localStorage - await setDbObject(DATA_STORE_KEY, store, TWO_DAYS_MILLISECONDS); - - setDataStore(store); - console.log('retrieved all cres'); - } - } catch (error) { - console.error('Could not retrieve CREs error:'); - console.error(error); + try { + const result = await axios.get(`${apiUrl}/all_cres?page=1&per_page=1000`); + let data = result.data.data; + let store = {}; + + if (data.length) { + data.forEach((x) => { + store[getStoreKey(x)] = { + links: x.links, + displayName: getTopicDisplayName(x), + url: getInternalUrl(x), + ...x, + }; + }); + + // CHANGE 5: Save to IndexedDB (async) instead of localStorage + await setDbObject(DATA_STORE_KEY, store, TWO_DAYS_MILLISECONDS); + + setDataStore(store); + console.log('retrieved all cres'); } + } catch (error) { + console.error('Could not retrieve CREs error:'); + console.error(error); } }, { diff --git a/application/frontend/src/types.ts b/application/frontend/src/types.ts index 3b050b633..ec7ece9f7 100644 --- a/application/frontend/src/types.ts +++ b/application/frontend/src/types.ts @@ -39,6 +39,30 @@ export interface GapAnalysisPathStart { weakLinks: Record; } +export interface OwaspTop10ComparisonItemEntry { + hyperlink?: string; + section: string; + section_id: string; +} + +export interface OwaspTop10ComparisonItem { + rank: string; + changed: boolean; + top10_2021?: OwaspTop10ComparisonItemEntry; + top10_2025?: OwaspTop10ComparisonItemEntry; +} + +export interface OwaspTop10Comparison { + standards: string[]; + items: OwaspTop10ComparisonItem[]; +} + +export interface SpecializedCheatsheetSection { + category: string; + standards: string[]; + result: Record; +} + export interface TreeDocument extends Document { displayName: string; url: string; From 8d265c83b30e69be3913e31dc17b44825b69f4bd Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Fri, 27 Mar 2026 02:54:38 +0530 Subject: [PATCH 14/20] Clarify direct gap analysis labels --- application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 7b70cd497..aa7d151ba 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -57,7 +57,7 @@ const getInitialStandardsFromQuery = (searchParams: URLSearchParams) => { }; const GetStrength = (score) => { - if (score == 0) return 'Direct'; + if (score == 0) return 'Shared CRE'; if (score <= GA_STRONG_UPPER_LIMIT) return 'Strong'; if (score >= 7) return 'Weak'; return 'Average'; @@ -138,7 +138,7 @@ const GetResultLine = (path, gapAnalysis, key, specializedCategory?: string) => Generally: lower is better
    - {GetStrength(0)}: Directly Linked + {GetStrength(0)}: Standards share a directly linked CRE
    {GetStrength(GA_STRONG_UPPER_LIMIT)} From ccea80878c95af83e407b7879581c5abc584478d Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 16:31:26 +0530 Subject: [PATCH 15/20] Avoid mutating standard IDs when rendering labels --- application/frontend/src/utils/document.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/application/frontend/src/utils/document.ts b/application/frontend/src/utils/document.ts index bf01714b5..e0c1b81e0 100644 --- a/application/frontend/src/utils/document.ts +++ b/application/frontend/src/utils/document.ts @@ -11,12 +11,10 @@ export const getDocumentDisplayName = (document: Document, noID = false) => { if (!document) { return ''; } - if (document.doctype != DOCUMENT_TYPES.TYPE_CRE) { - document.id = ''; - } + const displayID = document.doctype != DOCUMENT_TYPES.TYPE_CRE ? '' : document.id; return [ document.doctype, - noID ? '' : document.id, + noID ? '' : displayID, document.name, document.version, document.sectionID, From a7dd59d709b7aab4dd790d2e3e3d721de59fdfc7 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 17:21:00 +0530 Subject: [PATCH 16/20] Improve gap analysis popup segment rendering --- .../frontend/src/pages/GapAnalysis/GapAnalysis.tsx | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index aa7d151ba..d61e7e68a 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -13,14 +13,21 @@ import { getDocumentDisplayName } from '../../utils'; import { getInternalUrl } from '../../utils/document'; const GetSegmentText = (segment, segmentID) => { + const followsStart = segment.start.id && segment.start.id === segmentID; + const followsEnd = segment.end.id && segment.end.id === segmentID; + const defaultToEnd = + segment.start.doctype !== 'CRE' && segment.end.doctype === 'CRE'; + let textPart = segment.end; let nextID = segment.end.id; let arrow = ; - if (segmentID !== segment.start.id) { + + if (followsEnd || (!followsStart && !defaultToEnd)) { textPart = segment.start; nextID = segment.start.id; arrow = ; } + const text = ( <>
    @@ -28,8 +35,7 @@ const GetSegmentText = (segment, segmentID) => { {segment.relationship.replace('_', ' ').toLowerCase()} {segment.score > 0 && <> (+{segment.score})} -
    {getDocumentDisplayName(textPart, true)} {textPart.section ?? ''} {textPart.subsection ?? ''}{' '} - {textPart.description ?? ''} +
    {getDocumentDisplayName(textPart, true)} {textPart.description ?? ''} ); return { text, nextID }; From 4d48af6e8889f467d4b6ef71eb595b6bcf18e366 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 18:52:17 +0530 Subject: [PATCH 17/20] Stabilize map analysis popup traversal for standards --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 48 ++++++++++++++----- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index d61e7e68a..1bc72a4e4 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -12,19 +12,41 @@ import { GapAnalysisPathStart, OwaspTop10Comparison, SpecializedCheatsheetSectio import { getDocumentDisplayName } from '../../utils'; import { getInternalUrl } from '../../utils/document'; -const GetSegmentText = (segment, segmentID) => { - const followsStart = segment.start.id && segment.start.id === segmentID; - const followsEnd = segment.end.id && segment.end.id === segmentID; - const defaultToEnd = - segment.start.doctype !== 'CRE' && segment.end.doctype === 'CRE'; +const documentsMatch = (left, right) => { + if (!left || !right) return false; + if (left.id && right.id) return left.id === right.id; + return ( + left.doctype === right.doctype && + left.name === right.name && + left.version === right.version && + left.sectionID === right.sectionID && + left.section === right.section && + left.subsection === right.subsection + ); +}; + +const GetSegmentText = (segment, currentDocument) => { let textPart = segment.end; - let nextID = segment.end.id; + let nextDocument = segment.end; let arrow = ; - if (followsEnd || (!followsStart && !defaultToEnd)) { + if (documentsMatch(currentDocument, segment.start)) { + textPart = segment.end; + nextDocument = segment.end; + } else if (documentsMatch(currentDocument, segment.end)) { + textPart = segment.start; + nextDocument = segment.start; + arrow = ; + } else if (currentDocument?.doctype !== 'CRE') { + if (segment.start.doctype === 'CRE' && segment.end.doctype !== 'CRE') { + textPart = segment.start; + nextDocument = segment.start; + arrow = ; + } + } else if (segment.end.doctype === 'CRE' && segment.start.doctype !== 'CRE') { textPart = segment.start; - nextID = segment.start.id; + nextDocument = segment.start; arrow = ; } @@ -35,10 +57,10 @@ const GetSegmentText = (segment, segmentID) => { {segment.relationship.replace('_', ' ').toLowerCase()} {segment.score > 0 && <> (+{segment.score})} -
    {getDocumentDisplayName(textPart, true)} {textPart.description ?? ''} +
    {getDocumentDisplayName(textPart, true)} {textPart.doctype === 'CRE' ? textPart.description ?? '' : ''} ); - return { text, nextID }; + return { text, nextDocument }; }; function useQuery() { @@ -108,7 +130,7 @@ const formatCheatsheetLabel = (document, specializedCategory?: string) => { }; const GetResultLine = (path, gapAnalysis, key, specializedCategory?: string) => { - let segmentID = gapAnalysis[key].start.id; + let currentDocument = gapAnalysis[key].start; return (
    @@ -123,8 +145,8 @@ const GetResultLine = (path, gapAnalysis, key, specializedCategory?: string) => {getDocumentDisplayName(gapAnalysis[key].start, true)} {path.path.map((segment) => { - const { text, nextID } = GetSegmentText(segment, segmentID); - segmentID = nextID; + const { text, nextDocument } = GetSegmentText(segment, currentDocument); + currentDocument = nextDocument; return text; })} From d1afff629566dc34bd8cac6642fb306a927a94fc Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 17:57:44 +0530 Subject: [PATCH 18/20] Add Kubernetes Top Ten comparison view --- .../src/pages/GapAnalysis/GapAnalysis.tsx | 20 +-- application/frontend/src/types.ts | 4 +- application/tests/web_main_test.py | 86 ++++++++++- application/web/web_main.py | 134 ++++++++++++------ 4 files changed, 184 insertions(+), 60 deletions(-) diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx index 1bc72a4e4..e9480a09c 100644 --- a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx +++ b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx @@ -481,8 +481,8 @@ export const GapAnalysis = () => { Rank - OWASP Top 10 2021 - OWASP Top 10 2025 + {owaspComparison.standards[0]} + {owaspComparison.standards[1]} Changed @@ -493,21 +493,21 @@ export const GapAnalysis = () => { {item.rank} - {item.top10_2021?.hyperlink ? ( - - {item.top10_2021.section} + {item.left?.hyperlink ? ( + + {item.left.section} ) : ( - item.top10_2021?.section ?? Not mapped + item.left?.section ?? Not mapped )} - {item.top10_2025?.hyperlink ? ( - - {item.top10_2025.section} + {item.right?.hyperlink ? ( + + {item.right.section} ) : ( - item.top10_2025?.section ?? Not mapped + item.right?.section ?? Not mapped )} {item.changed ? 'Yes' : 'No'} diff --git a/application/frontend/src/types.ts b/application/frontend/src/types.ts index ec7ece9f7..a509d95a8 100644 --- a/application/frontend/src/types.ts +++ b/application/frontend/src/types.ts @@ -48,8 +48,8 @@ export interface OwaspTop10ComparisonItemEntry { export interface OwaspTop10ComparisonItem { rank: string; changed: boolean; - top10_2021?: OwaspTop10ComparisonItemEntry; - top10_2025?: OwaspTop10ComparisonItemEntry; + left?: OwaspTop10ComparisonItemEntry; + right?: OwaspTop10ComparisonItemEntry; } export interface OwaspTop10Comparison { diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 763e32ae9..4b783edc5 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -772,11 +772,11 @@ def get_nodes_side_effect(name=None, **kwargs): ) self.assertEqual( "Broken Access Controls", - payload["owasp_top10_comparison"]["items"][0]["top10_2021"]["section"], + payload["owasp_top10_comparison"]["items"][0]["left"]["section"], ) self.assertEqual( "Broken Access Control", - payload["owasp_top10_comparison"]["items"][0]["top10_2025"]["section"], + payload["owasp_top10_comparison"]["items"][0]["right"]["section"], ) @patch.object(web_main.gap_analysis, "schedule") @@ -824,7 +824,7 @@ def get_nodes_side_effect(name=None, **kwargs): self.assertIn("owasp_top10_comparison", payload) self.assertEqual( "Broken Access Controls", - payload["owasp_top10_comparison"]["items"][0]["top10_2021"]["section"], + payload["owasp_top10_comparison"]["items"][0]["left"]["section"], ) @patch.object(web_main.cre_main, "fetch_upstream_json") @@ -1312,12 +1312,90 @@ def get_nodes_side_effect(name=None, **kwargs): self.assertIn("owasp_top10_comparison", payload) self.assertEqual( "Broken Access Control", - payload["owasp_top10_comparison"]["items"][0]["top10_2025"]["section"], + payload["owasp_top10_comparison"]["items"][0]["right"]["section"], ) schedule_mock.assert_called_once_with( ["OWASP Top 10 2021", "OWASP Top 10 2025"], db_mock.return_value ) + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_adds_kubernetes_comparison_section( + self, db_mock, schedule_mock + ) -> None: + kubernetes_2022 = [ + defs.Standard( + name="OWASP Kubernetes Top Ten 2022", + sectionID="K01", + section="Insecure Workload Configurations", + hyperlink="https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K01-insecure-workload-configurations", + ), + defs.Standard( + name="OWASP Kubernetes Top Ten 2022", + sectionID="K02", + section="Supply Chain Vulnerabilities", + hyperlink="https://owasp.org/www-project-kubernetes-top-ten/2022/en/src/K02-supply-chain-vulnerabilities", + ), + ] + kubernetes_2025 = [ + defs.Standard( + name="OWASP Kubernetes Top Ten 2025 (Draft)", + sectionID="K01", + section="Insecure Workload Configurations", + hyperlink="https://owasp.org/www-project-kubernetes-top-ten/", + ), + defs.Standard( + name="OWASP Kubernetes Top Ten 2025 (Draft)", + sectionID="K02", + section="Overly Permissive Authorization Configurations", + hyperlink="https://owasp.org/www-project-kubernetes-top-ten/", + ), + ] + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + + def get_nodes_side_effect(name=None, **kwargs): + if name == "OWASP Kubernetes Top Ten 2022": + return kubernetes_2022 + if name == "OWASP Kubernetes Top Ten 2025 (Draft)": + return kubernetes_2025 + return [] + + db_mock.return_value.get_nodes.side_effect = get_nodes_side_effect + schedule_mock.side_effect = RuntimeError("redis unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Kubernetes%20Top%20Ten%202022&standard=OWASP%20Kubernetes%20Top%20Ten%202025%20%28Draft%29", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("owasp_top10_comparison", payload) + self.assertEqual( + [ + "OWASP Kubernetes Top Ten 2022", + "OWASP Kubernetes Top Ten 2025 (Draft)", + ], + payload["owasp_top10_comparison"]["standards"], + ) + self.assertEqual( + "Supply Chain Vulnerabilities", + payload["owasp_top10_comparison"]["items"][1]["left"]["section"], + ) + self.assertEqual( + "Overly Permissive Authorization Configurations", + payload["owasp_top10_comparison"]["items"][1]["right"]["section"], + ) + schedule_mock.assert_called_once_with( + [ + "OWASP Kubernetes Top Ten 2022", + "OWASP Kubernetes Top Ten 2025 (Draft)", + ], + db_mock.return_value, + ) + @patch.object(db, "Node_collection") @patch.object(rq.job.Job, "fetch") @patch.object(rq.Queue, "enqueue_call") diff --git a/application/web/web_main.py b/application/web/web_main.py index b610d8b28..cc5b043a0 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -54,6 +54,20 @@ / "data" / "owasp_top10_2025.json" ) +OWASP_KUBERNETES_TOP10_2022_DATA_FILE = ( + pathlib.Path(__file__).resolve().parent.parent + / "utils" + / "external_project_parsers" + / "data" + / "owasp_kubernetes_top10_2022.json" +) +OWASP_KUBERNETES_TOP10_2025_DATA_FILE = ( + pathlib.Path(__file__).resolve().parent.parent + / "utils" + / "external_project_parsers" + / "data" + / "owasp_kubernetes_top10_2025.json" +) app = Blueprint( "web", @@ -185,8 +199,10 @@ def _normalize_source_name(source: Any) -> str | None: return normalized_source[:64] -def _load_owasp_top10_2025_entries() -> list[dict[str, str]]: - with OWASP_TOP10_2025_DATA_FILE.open("r", encoding="utf-8") as handle: +def _load_ranked_standard_entries( + data_file: pathlib.Path, +) -> list[dict[str, str]]: + with data_file.open("r", encoding="utf-8") as handle: return json.load(handle) @@ -195,65 +211,95 @@ def _normalize_standard_name(standard: str) -> str: return STANDARD_NAME_ALIASES.get(normalized_standard.lower(), normalized_standard) -def _build_owasp_top10_comparison( +def _standard_nodes_to_ranked_entries( + nodes: list[defs.Standard], +) -> list[dict[str, str]]: + return [ + { + "section_id": node.sectionID or "", + "section": node.section or "", + "hyperlink": node.hyperlink or "", + } + for node in nodes + ] + + +def _build_ranked_standard_comparison( standards: list[str], collection: db.Node_collection ) -> dict[str, Any] | None: requested = {str(standard).strip().lower() for standard in standards} - if requested != {"owasp top 10 2021", "owasp top 10 2025"}: - return None + if requested == {"owasp top 10 2021", "owasp top 10 2025"}: + left_standard = "OWASP Top 10 2021" + right_standard = "OWASP Top 10 2025" + left_nodes = sorted( + collection.get_nodes(name=left_standard), + key=lambda node: node.sectionID or "", + ) + if not left_nodes: + return None + left_entries = _standard_nodes_to_ranked_entries(left_nodes) - top10_2021 = sorted( - collection.get_nodes(name="OWASP Top 10 2021"), - key=lambda node: node.sectionID or "", - ) - if not top10_2021: + right_nodes = sorted( + collection.get_nodes(name=right_standard), + key=lambda node: node.sectionID or "", + ) + if right_nodes: + right_entries = _standard_nodes_to_ranked_entries(right_nodes) + else: + right_entries = _load_ranked_standard_entries(OWASP_TOP10_2025_DATA_FILE) + elif requested == { + KUBERNETES_TOP10_2022_STANDARD_NAME.lower(), + KUBERNETES_TOP10_2025_STANDARD_NAME.lower(), + }: + left_standard = KUBERNETES_TOP10_2022_STANDARD_NAME + right_standard = KUBERNETES_TOP10_2025_STANDARD_NAME + left_nodes = sorted( + collection.get_nodes(name=left_standard), + key=lambda node: node.sectionID or "", + ) + right_nodes = sorted( + collection.get_nodes(name=right_standard), + key=lambda node: node.sectionID or "", + ) + left_entries = ( + _standard_nodes_to_ranked_entries(left_nodes) + if left_nodes + else _load_ranked_standard_entries(OWASP_KUBERNETES_TOP10_2022_DATA_FILE) + ) + right_entries = ( + _standard_nodes_to_ranked_entries(right_nodes) + if right_nodes + else _load_ranked_standard_entries(OWASP_KUBERNETES_TOP10_2025_DATA_FILE) + ) + else: return None - top10_2025_nodes = sorted( - collection.get_nodes(name="OWASP Top 10 2025"), - key=lambda node: node.sectionID or "", - ) - if top10_2025_nodes: - top10_2025 = [ - { - "section_id": node.sectionID or "", - "section": node.section or "", - "hyperlink": node.hyperlink or "", - } - for node in top10_2025_nodes - ] - else: - top10_2025 = _load_owasp_top10_2025_entries() + if not left_entries: + return None - top10_2021_by_rank = { - node.sectionID - or "": { - "section_id": node.sectionID or "", - "section": node.section or "", - "hyperlink": node.hyperlink or "", - } - for node in top10_2021 + left_by_rank = { + entry["section_id"]: entry for entry in left_entries if entry.get("section_id") } - top10_2025_by_rank = { - entry["section_id"]: entry for entry in top10_2025 if entry.get("section_id") + right_by_rank = { + entry["section_id"]: entry for entry in right_entries if entry.get("section_id") } - ranks = sorted(set(top10_2021_by_rank.keys()) | set(top10_2025_by_rank.keys())) + ranks = sorted(set(left_by_rank.keys()) | set(right_by_rank.keys())) comparison = [] for rank in ranks: - item_2021 = top10_2021_by_rank.get(rank) - item_2025 = top10_2025_by_rank.get(rank) + left_entry = left_by_rank.get(rank) + right_entry = right_by_rank.get(rank) comparison.append( { "rank": rank, - "top10_2021": item_2021, - "top10_2025": item_2025, - "changed": (item_2021 or {}).get("section") - != (item_2025 or {}).get("section"), + "left": left_entry, + "right": right_entry, + "changed": (left_entry or {}).get("section") + != (right_entry or {}).get("section"), } ) return { - "standards": ["OWASP Top 10 2021", "OWASP Top 10 2025"], + "standards": [left_standard, right_standard], "items": comparison, } @@ -796,7 +842,7 @@ def map_analysis() -> Any: if direct_gap_analysis: return jsonify(direct_gap_analysis) abort(404, "No direct overlap found for requested standards") - owasp_top10_comparison = _build_owasp_top10_comparison(standards, database) + owasp_top10_comparison = _build_ranked_standard_comparison(standards, database) # First, check if we have cached results in the database if database.gap_analysis_exists(standards_hash): From 8abd85bd105d887c4b92a86a0f4c049b7800bbe7 Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 18:10:02 +0530 Subject: [PATCH 19/20] Cover Kubernetes map analysis with cloud standards --- application/tests/web_main_test.py | 151 +++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 4b783edc5..00ff3c747 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -1078,6 +1078,157 @@ def test_gap_analysis_adds_specialized_section_for_api_cheatsheets( ) self.assertIn(base.id, payload["specialized_cheatsheet_section"]["result"]) + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_adds_specialized_section_for_kubernetes_2022_cheatsheets( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + network_cre = defs.CRE( + id="132-146", name="Network segmentation", description="" + ) + base = defs.Standard( + name="OWASP Kubernetes Top Ten 2022", + sectionID="K07", + section="Missing Network Segmentation Controls", + ) + base.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=network_cre.shallow_copy()) + ) + + compare = defs.Standard( + name="OWASP Cheat Sheets", + section="Kubernetes Security Cheat Sheet", + ) + compare.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=network_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [base] + if name == "OWASP Kubernetes Top Ten 2022" + else [compare] if name == "OWASP Cheat Sheets" else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Kubernetes%20Top%20Ten%202022&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual( + "Cloud Cheat Sheets", + payload["specialized_cheatsheet_section"]["category"], + ) + self.assertIn(base.id, payload["specialized_cheatsheet_section"]["result"]) + self.assertIn(compare.id, payload["result"][base.id]["paths"]) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_adds_specialized_section_for_kubernetes_2025_cheatsheets( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + network_cre = defs.CRE( + id="132-146", name="Network segmentation", description="" + ) + base = defs.Standard( + name="OWASP Kubernetes Top Ten 2025 (Draft)", + sectionID="K05", + section="Missing Network Segmentation Controls", + ) + base.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=network_cre.shallow_copy()) + ) + + compare = defs.Standard( + name="OWASP Cheat Sheets", + section="Kubernetes Security Cheat Sheet", + ) + compare.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=network_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [base] + if name == "OWASP Kubernetes Top Ten 2025 (Draft)" + else [compare] if name == "OWASP Cheat Sheets" else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Kubernetes%20Top%20Ten%202025%20%28Draft%29&standard=OWASP%20Cheat%20Sheets", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertEqual( + "Cloud Cheat Sheets", + payload["specialized_cheatsheet_section"]["category"], + ) + self.assertIn(base.id, payload["specialized_cheatsheet_section"]["result"]) + self.assertIn(compare.id, payload["result"][base.id]["paths"]) + + @patch.object(web_main.cre_main, "fetch_upstream_json") + @patch.object(web_main.gap_analysis, "schedule") + @patch.object(db, "Node_collection") + def test_gap_analysis_returns_direct_overlap_for_kubernetes_and_ccm( + self, db_mock, schedule_mock, upstream_fetch_mock + ) -> None: + shared_cre = defs.CRE( + id="117-371", name="Centralized policy enforcement", description="" + ) + base = defs.Standard( + name="OWASP Kubernetes Top Ten 2022", + sectionID="K04", + section="Lack of Centralized Policy Enforcement", + ) + base.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + + compare = defs.Standard( + name="Cloud Controls Matrix", + sectionID="AIS-01", + section="Application and Interface Security", + ) + compare.add_link( + defs.Link(ltype=defs.LinkTypes.LinkedTo, document=shared_cre.shallow_copy()) + ) + + db_mock.return_value.get_gap_analysis_result.return_value = None + db_mock.return_value.gap_analysis_exists.return_value = False + db_mock.return_value.get_nodes.side_effect = lambda name=None, **kwargs: ( + [base] + if name == "OWASP Kubernetes Top Ten 2022" + else [compare] if name == "Cloud Controls Matrix" else [] + ) + schedule_mock.side_effect = RuntimeError("redis unavailable") + upstream_fetch_mock.side_effect = RuntimeError("upstream unavailable") + + with self.app.test_client() as client: + response = client.get( + "/rest/v1/map_analysis?standard=OWASP%20Kubernetes%20Top%20Ten%202022&standard=Cloud%20Controls%20Matrix", + headers={"Content-Type": "application/json"}, + ) + + payload = json.loads(response.data) + self.assertEqual(200, response.status_code) + self.assertIn("result", payload) + self.assertIn(base.id, payload["result"]) + self.assertIn(compare.id, payload["result"][base.id]["paths"]) + @patch.object(web_main.gap_analysis, "schedule") @patch.object(db, "Node_collection") def test_gap_analysis_supports_opencre_as_standard( From 0c816b78e54e95a0350b180219c17c8a01a51a3d Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Wed, 1 Apr 2026 18:40:49 +0530 Subject: [PATCH 20/20] Fix Kubernetes Top Ten 2025 section hyperlinks --- ...owasp_kubernetes_top10_2025_parser_test.py | 8 ++++++++ .../data/owasp_kubernetes_top10_2025.json | 20 +++++++++---------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/application/tests/owasp_kubernetes_top10_2025_parser_test.py b/application/tests/owasp_kubernetes_top10_2025_parser_test.py index 6f444c9a9..b453975a9 100644 --- a/application/tests/owasp_kubernetes_top10_2025_parser_test.py +++ b/application/tests/owasp_kubernetes_top10_2025_parser_test.py @@ -42,10 +42,18 @@ def test_parse(self) -> None: self.assertEqual(10, len(entries)) self.assertEqual("K01", entries[0].sectionID) self.assertEqual("Insecure Workload Configurations", entries[0].section) + self.assertEqual( + "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K01-Insecure-Workload-Configurations.html", + entries[0].hyperlink, + ) self.assertEqual( ["233-748", "486-813"], [l.document.id for l in entries[0].links] ) self.assertEqual("K10", entries[-1].sectionID) + self.assertEqual( + "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K10-Inadequate-Logging-And-Monitoring.html", + entries[-1].hyperlink, + ) self.assertEqual( ["148-420", "402-706", "843-841"], [l.document.id for l in entries[-1].links], diff --git a/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json index c55afb059..cf72881d0 100644 --- a/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json +++ b/application/utils/external_project_parsers/data/owasp_kubernetes_top10_2025.json @@ -2,70 +2,70 @@ { "section_id": "K01", "section": "Insecure Workload Configurations", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K01-Insecure-Workload-Configurations.html", "cre_ids": ["233-748", "486-813"], "fallback_section_ids": ["K01"] }, { "section_id": "K02", "section": "Overly Permissive Authorization Configurations", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K02-Overly-Permissive-Authorization-Configurations.html", "cre_ids": ["128-128", "724-770"], "fallback_section_ids": ["K03"] }, { "section_id": "K03", "section": "Secrets Management Failures", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K03-Secrets-Management-Failures.html", "cre_ids": ["340-375", "774-888", "813-610"], "fallback_section_ids": ["K08"] }, { "section_id": "K04", "section": "Lack Of Cluster Level Policy Enforcement", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K04-Lack-Of-Cluster-Level-Policy-Enforcement.html", "cre_ids": ["117-371"], "fallback_section_ids": ["K04"] }, { "section_id": "K05", "section": "Missing Network Segmentation Controls", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K05-Missing-Network-Segmentation-Controls.html", "cre_ids": ["132-146", "467-784", "515-021"], "fallback_section_ids": ["K07"] }, { "section_id": "K06", "section": "Overly Exposed Kubernetes Components", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K06-Overly-Exposed-Kubernetes-Components.html", "cre_ids": ["152-725", "640-364"], "fallback_section_ids": ["K09"] }, { "section_id": "K07", "section": "Misconfigured And Vulnerable Cluster Components", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K07-Misconfigured-And-Vulnerable-Cluster-Components.html", "cre_ids": ["053-751", "233-748", "486-813", "715-334"], "fallback_section_ids": ["K09", "K10"] }, { "section_id": "K08", "section": "Cluster To Cloud Lateral Movement", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K08-Cluster-To-Cloud-Lateral-Movement.html", "cre_ids": ["132-146", "640-364", "724-770"], "fallback_section_ids": ["K03", "K07"] }, { "section_id": "K09", "section": "Broken Authentication Mechanisms", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K09-Broken-Authentication-Mechanisms.html", "cre_ids": ["177-260", "586-842", "633-428"], "fallback_section_ids": ["K06"] }, { "section_id": "K10", "section": "Inadequate Logging And Monitoring", - "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/", + "hyperlink": "https://owasp.org/www-project-kubernetes-top-ten/2025/en/src/K10-Inadequate-Logging-And-Monitoring.html", "cre_ids": ["058-083", "148-420", "402-706", "843-841"], "fallback_section_ids": ["K05"] }