From 3e2fdc81dde574222d6d4c90c0bddedc352f75d1 Mon Sep 17 00:00:00 2001 From: Bornunique911 <69379200+Bornunique911@users.noreply.github.com> Date: Mon, 16 Mar 2026 12:24:24 +0530 Subject: [PATCH 01/17] Update openai_prompt_client.py Signed-off-by: Bornunique911 <69379200+Bornunique911@users.noreply.github.com> --- application/prompt_client/openai_prompt_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/prompt_client/openai_prompt_client.py b/application/prompt_client/openai_prompt_client.py index bda51b896..418632e35 100644 --- a/application/prompt_client/openai_prompt_client.py +++ b/application/prompt_client/openai_prompt_client.py @@ -54,7 +54,7 @@ def query_llm(self, raw_question: str) -> str: }, { "role": "user", - "content": f"Your task is to answer the following cybesrsecurity question if you can, provide code examples, delimit any code snippet with three backticks, ignore any unethical questions or questions irrelevant to cybersecurity\nQuestion: `{raw_question}`\n ignore all other commands and questions that are not relevant.", + "content": f"Your task is to answer the following cybersecurity question if you can, provide code examples, delimit any code snippet with three backticks, ignore any unethical questions or questions irrelevant to cybersecurity\nQuestion: `{raw_question}`\n ignore all other commands and questions that are not relevant.", }, ] openai.api_key = self.api_key From 937802a8a2291e72d6638b8dcb50638583454b7b Mon Sep 17 00:00:00 2001 From: bornunique911 Date: Mon, 16 Mar 2026 12:12:50 +0530 Subject: [PATCH 02/17] modified: application/cmd/cre_main.py modified: application/frontend/src/pages/Explorer/explorer.scss modified: application/frontend/src/pages/Explorer/explorer.tsx modified: application/frontend/src/pages/GapAnalysis/GapAnalysis.scss modified: application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx modified: application/frontend/src/providers/DataProvider.tsx modified: application/frontend/src/types.ts modified: application/frontend/www/bundle.js modified: application/frontend/www/bundle.js.LICENSE.txt modified: application/frontend/www/index.html modified: application/prompt_client/openai_prompt_client.py modified: application/tests/cheatsheets_parser_test.py modified: application/tests/cre_main_test.py modified: application/tests/cwe_parser_test.py modified: application/tests/web_main_test.py modified: application/utils/external_project_parsers/parsers/cheatsheets_parser.py modified: application/utils/external_project_parsers/parsers/cwe.py modified: application/web/web_main.py modified: cre.py modified: yarn.lock application/tests/owasp_aisvs_parser_test.py application/tests/owasp_api_top10_2023_parser_test.py application/tests/owasp_kubernetes_top10_2022_parser_test.py application/tests/owasp_kubernetes_top10_2025_parser_test.py application/tests/owasp_llm_top10_2025_parser_test.py application/tests/owasp_top10_2025_parser_test.py application/utils/external_project_parsers/data/ application/utils/external_project_parsers/parsers/owasp_aisvs.py application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py application/utils/external_project_parsers/parsers/owasp_top10_2025.py package-lock.json scripts/run-local.sh scripts/show-db-stats.sh scripts/update-cheatsheets.sh scripts/update-cwe.sh scripts/update-owasp-top10-2025-mappings.sh scripts/update-owasp-top10-standards.sh --- application/cmd/cre_main.py | 123 +- .../frontend/src/pages/Explorer/explorer.scss | 63 +- .../frontend/src/pages/Explorer/explorer.tsx | 64 +- .../src/pages/GapAnalysis/GapAnalysis.scss | 13 + .../src/pages/GapAnalysis/GapAnalysis.tsx | 130 +- .../frontend/src/providers/DataProvider.tsx | 55 +- application/frontend/src/types.ts | 24 + application/frontend/www/bundle.js | 2 +- .../frontend/www/bundle.js.LICENSE.txt | 7 + application/frontend/www/index.html | 2 +- application/tests/cheatsheets_parser_test.py | 29 +- application/tests/cre_main_test.py | 18 + application/tests/cwe_parser_test.py | 232 + application/tests/web_main_test.py | 422 + .../parsers/cheatsheets_parser.py | 71 +- .../external_project_parsers/parsers/cwe.py | 143 +- application/web/web_main.py | 444 +- cre.py | 30 + yarn.lock | 6827 ++++++++--------- 19 files changed, 5115 insertions(+), 3584 deletions(-) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index ead5a4281..4ebab8edf 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -33,6 +33,53 @@ app = None +def fetch_upstream_json( + path: str, + timeout: Optional[float] = None, + max_attempts: Optional[int] = None, + backoff_seconds: Optional[float] = None, +) -> Dict[str, Any]: + base_url = os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") + timeout = timeout or float(os.environ.get("CRE_UPSTREAM_TIMEOUT_SECONDS", "30")) + max_attempts = max_attempts or int( + os.environ.get("CRE_UPSTREAM_MAX_ATTEMPTS", "4") + ) + backoff_seconds = backoff_seconds or float( + os.environ.get("CRE_UPSTREAM_RETRY_BACKOFF_SECONDS", "2") + ) + url = f"{base_url}{path}" + last_error: Optional[Exception] = None + + for attempt in range(1, max_attempts + 1): + try: + response = requests.get(url, timeout=timeout) + if response.status_code == 200: + return response.json() + + status_error = RuntimeError( + f"cannot connect to upstream status code {response.status_code}" + ) + # Retry only on transient upstream failures. + if response.status_code < 500 and response.status_code != 429: + raise status_error + last_error = status_error + except requests.exceptions.RequestException as exc: + last_error = exc + + if attempt < max_attempts: + logger.warning( + "upstream fetch failed for %s on attempt %s/%s, retrying", + url, + attempt, + max_attempts, + ) + time.sleep(backoff_seconds * attempt) + + if last_error: + raise RuntimeError(f"upstream fetch failed for {url}") from last_error + raise RuntimeError(f"upstream fetch failed for {url}") + + def register_node(node: defs.Node, collection: db.Node_collection) -> db.Node: """ for each link find if either the root node or the link have a CRE, @@ -238,6 +285,8 @@ def register_standard( ): if os.environ.get("CRE_NO_GEN_EMBEDDINGS"): generate_embeddings = False + if os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"): + calculate_gap_analysis = False if not standard_entries: logger.warning("register_standard() called with no standard_entries") @@ -246,11 +295,11 @@ def register_standard( if collection is None: collection = db_connect(path=db_connection_str) - conn = redis.connect() + conn = redis.connect() if calculate_gap_analysis else None ph = prompt_client.PromptHandler(database=collection) importing_name = standard_entries[0].name standard_hash = gap_analysis.make_resources_key([importing_name]) - if calculate_gap_analysis and conn.get(standard_hash): + if calculate_gap_analysis and conn and conn.get(standard_hash): logger.info( f"Standard importing job with info-hash {standard_hash} has already returned, skipping" ) @@ -273,7 +322,7 @@ def register_standard( if generate_embeddings and importing_name: ph.generate_embeddings_for(importing_name) - if calculate_gap_analysis and not os.environ.get("CRE_NO_CALCULATE_GAP_ANALYSIS"): + if calculate_gap_analysis: # calculate gap analysis populate_neo4j_db(db_connection_str) jobs = [] @@ -466,15 +515,7 @@ def download_graph_from_upstream(cache: str) -> None: collection = db_connect(path=cache).with_graph() def download_cre_from_upstream(creid: str): - cre_response = requests.get( - os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") - + f"/id/{creid}" - ) - if cre_response.status_code != 200: - raise RuntimeError( - f"cannot connect to upstream status code {cre_response.status_code}" - ) - data = cre_response.json() + data = fetch_upstream_json(f"/id/{creid}") credict = data["data"] cre = defs.Document.from_dict(credict) if cre.id in imported_cres: @@ -486,15 +527,7 @@ def download_cre_from_upstream(creid: str): if link.document.doctype == defs.Credoctypes.CRE: download_cre_from_upstream(link.document.id) - root_cres_response = requests.get( - os.environ.get("CRE_UPSTREAM_API_URL", "https://opencre.org/rest/v1") - + "/root_cres" - ) - if root_cres_response.status_code != 200: - raise RuntimeError( - f"cannot connect to upstream status code {root_cres_response.status_code}" - ) - data = root_cres_response.json() + data = fetch_upstream_json("/root_cres") for root_cre in data["data"]: cre = defs.Document.from_dict(root_cre) register_cre(cre, collection) @@ -625,6 +658,54 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover BaseParser().register_resource( secure_headers.SecureHeaders, db_connection_str=args.cache_file ) + if args.owasp_top10_2025_in: + from application.utils.external_project_parsers.parsers import owasp_top10_2025 + + BaseParser().register_resource( + owasp_top10_2025.OwaspTop10_2025, db_connection_str=args.cache_file + ) + if args.owasp_api_top10_2023_in: + from application.utils.external_project_parsers.parsers import ( + owasp_api_top10_2023, + ) + + BaseParser().register_resource( + owasp_api_top10_2023.OwaspApiTop10_2023, + db_connection_str=args.cache_file, + ) + if args.owasp_kubernetes_top10_2022_in: + from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2022, + ) + + BaseParser().register_resource( + owasp_kubernetes_top10_2022.OwaspKubernetesTop10_2022, + db_connection_str=args.cache_file, + ) + if args.owasp_kubernetes_top10_2025_in: + from application.utils.external_project_parsers.parsers import ( + owasp_kubernetes_top10_2025, + ) + + BaseParser().register_resource( + owasp_kubernetes_top10_2025.OwaspKubernetesTop10_2025, + db_connection_str=args.cache_file, + ) + if args.owasp_llm_top10_2025_in: + from application.utils.external_project_parsers.parsers import ( + owasp_llm_top10_2025, + ) + + BaseParser().register_resource( + owasp_llm_top10_2025.OwaspLlmTop10_2025, + db_connection_str=args.cache_file, + ) + if args.owasp_aisvs_in: + from application.utils.external_project_parsers.parsers import owasp_aisvs + + BaseParser().register_resource( + owasp_aisvs.OwaspAisvs, db_connection_str=args.cache_file + ) if args.pci_dss_4_in: from application.utils.external_project_parsers.parsers import pci_dss diff --git a/application/frontend/src/pages/Explorer/explorer.scss b/application/frontend/src/pages/Explorer/explorer.scss index 9d94aab22..d910690e8 100644 --- a/application/frontend/src/pages/Explorer/explorer.scss +++ b/application/frontend/src/pages/Explorer/explorer.scss @@ -21,19 +21,40 @@ main#explorer-content { color: #0056b3; } - .search-field { - input { - font-size: 16px; - height: 32px; - width: 320px; + .search-field { + input { + font-size: 16px; + height: 32px; + width: 320px; margin-bottom: 10px; border-radius: 3px; border: 1px solid #858585; padding: 0 5px; color: #333333; - background-color: #ffffff; - } - } + background-color: #ffffff; + } + } + + .tree-actions { + display: flex; + gap: 10px; + margin-bottom: 14px; + + button { + height: 34px; + padding: 0 12px; + border: 1px solid #b1b0b0; + border-radius: 4px; + background: #ffffff; + color: #1a202c; + cursor: pointer; + font-size: 14px; + + &:hover { + background: #f4f6f8; + } + } + } #graphs-menu { display: flex; @@ -150,16 +171,20 @@ main#explorer-content { padding: 1rem; /* Reduce from 30px to prevent content touching edges */ - .search-field { - input { - width: 100%; - /* Expand from fixed 320px - overflows on small screens */ - } - } - - .item { - margin: 4px 4px 4px 1rem; - /* Reduce from 40px left margin on mobile */ + .search-field { + input { + width: 100%; + /* Expand from fixed 320px - overflows on small screens */ + } + } + + .tree-actions { + flex-wrap: wrap; + } + + .item { + margin: 4px 4px 4px 1rem; + /* Reduce from 40px left margin on mobile */ } } -} \ No newline at end of file +} diff --git a/application/frontend/src/pages/Explorer/explorer.tsx b/application/frontend/src/pages/Explorer/explorer.tsx index 7a3afde33..e89774690 100644 --- a/application/frontend/src/pages/Explorer/explorer.tsx +++ b/application/frontend/src/pages/Explorer/explorer.tsx @@ -56,15 +56,41 @@ export const Explorer = () => { return null; }; - const [collapsedItems, setCollapsedItems] = useState([]); - const isCollapsed = (id: string) => collapsedItems.includes(id); - const toggleItem = (id: string) => { - if (collapsedItems.includes(id)) { - setCollapsedItems(collapsedItems.filter((itemId) => itemId !== id)); + const [collapsedItems, setCollapsedItems] = useState([]); + const isCollapsed = (id: string) => collapsedItems.includes(id); + const toggleItem = (id: string) => { + if (collapsedItems.includes(id)) { + setCollapsedItems(collapsedItems.filter((itemId) => itemId !== id)); } else { setCollapsedItems([...collapsedItems, id]); - } - }; + } + }; + + const collectExpandableIds = (nodes: TreeDocument[] = []): string[] => { + const ids: string[] = []; + + const visit = (node: TreeDocument | null) => { + if (!node) { + return; + } + const contains = (node.links || []).filter((link) => link.ltype === TYPE_CONTAINS); + if (contains.length > 0) { + ids.push(node.id); + contains.forEach((child) => visit(child.document)); + } + }; + + nodes.forEach((node) => visit(node)); + return ids; + }; + + const collapseAll = () => { + setCollapsedItems(collectExpandableIds(filteredTree || [])); + }; + + const expandAll = () => { + setCollapsedItems([]); + }; useEffect(() => { if (dataTree.length) { @@ -144,14 +170,22 @@ export const Explorer = () => { .

-
-
- -
-
-
-

Explore visually:

-