From 9557506b020a04ffb4e4b412e02157ceb4c93ca1 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 07:57:41 +0900 Subject: [PATCH 1/7] feat: mark Android.bp as manifest without manifest license extraction. ScanCode already extracts licenses from Soong files, so only set is_manifest_file=True and skip get_manifest_licenses. --- src/fosslight_source/_scan_item.py | 22 ++++++++++-- src/fosslight_source/cli.py | 26 +++++++++----- tests/test_manifest_android_bp.py | 54 ++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 tests/test_manifest_android_bp.py diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index 0e96a8c..a93b53b 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -15,7 +15,7 @@ _notice_filename = ['licen[cs]e[s]?', 'notice[s]?', 'legal', 'copyright[s]?', 'copying*', 'patent[s]?', 'unlicen[cs]e', 'eula', '[a,l]?gpl[-]?[1-3]?[.,-,_]?[0-1]?', 'mit', 'bsd[-]?[0-4]?', 'bsd[-]?[0-4][-]?clause[s]?', 'apache[-,_]?[1-2]?[.,-,_]?[0-2]?'] -_manifest_filename = [ +_manifest_license_filename = [ r'.*\.pom$', r'package\.json$', r'composer\.json$', @@ -26,6 +26,10 @@ r'Cargo\.toml$', r'huggingface_hub_metadata\.json$', ] +# Manifest marker only: is_manifest_file=True, but license comes from ScanCode. +_manifest_marker_filename = [ + r'Android\.bp$', +] MAX_LICENSE_LENGTH = 200 MAX_LICENSE_TOTAL_LENGTH = 600 SUBSTRING_LICENSE_COMMENT = "Maximum character limit (License)" @@ -222,7 +226,19 @@ def is_notice_file(file_path: str) -> bool: return bool(re.match(pattern, filename, re.IGNORECASE)) -def is_manifest_file(file_path: str) -> bool: - pattern = r"({})$".format("|".join(_manifest_filename)) +def _matches_manifest_pattern(file_path: str, patterns: list) -> bool: + pattern = r"({})$".format("|".join(patterns)) filename = os.path.basename(file_path) return bool(re.match(pattern, filename, re.IGNORECASE)) + + +def extracts_manifest_license(file_path: str) -> bool: + return _matches_manifest_pattern(file_path, _manifest_license_filename) + + +def is_manifest_marker_file(file_path: str) -> bool: + return _matches_manifest_pattern(file_path, _manifest_marker_filename) + + +def is_manifest_file(file_path: str) -> bool: + return extracts_manifest_license(file_path) or is_manifest_marker_file(file_path) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 459ade3..3e6b2e0 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -29,7 +29,7 @@ import argparse from .run_spdx_extractor import get_spdx_downloads from .run_manifest_extractor import get_manifest_licenses -from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, is_manifest_file +from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, extracts_manifest_license, is_manifest_marker_file from ._kb_client import fetch_origin_urls_via_scan_job from fosslight_util.cover import dump_result_log from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename @@ -395,7 +395,7 @@ def _collect_kb_file_hashes( def merge_results( scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}, path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {}, - excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = "", + manifest_markers: set = None, excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = "", ui_mode: bool = False ) -> tuple[list, Optional[str], int, int]: @@ -422,6 +422,8 @@ def merge_results( """ if excluded_files is None: excluded_files = set() + if manifest_markers is None: + manifest_markers = set() # Merge ScanOSS into ScanCode results. # When ScanCode already detected a license for the same file, keep that license @@ -464,6 +466,10 @@ def merge_results( item.licenses = [] # clear existing licenses (setter clears when value falsy) item.licenses = valid_licenses + for file_name in manifest_markers: + item = _get_or_append_source_item(scancode_result, file_name) + item.is_manifest_file = True + kb_origin_urls: dict[str, str] = {} kb_status_message: Optional[str] = None kb_requested_count = 0 @@ -654,10 +660,10 @@ def run_scanners( run_kb = False run_kb_msg = f"KB({kb_url}) Unreachable" - spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files) + spdx_downloads, manifest_licenses, manifest_markers = metadata_collector(path_to_scan, excluded_files) merged_result, kb_status_message, kb_requested_count, _ = merge_results( scancode_result, scanoss_result, spdx_downloads, - path_to_scan, run_kb, manifest_licenses, excluded_files, + path_to_scan, run_kb, manifest_licenses, manifest_markers, excluded_files, hide_progress, kb_url, kb_token, ui_mode=ui_mode, ) @@ -705,19 +711,21 @@ def run_scanners( return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result -def metadata_collector(path_to_scan: str, excluded_files: set) -> dict: +def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, dict, set]: """ Collect metadata for merging. - Traverse files with exclusions applied - spdx_downloads: {rel_path: [download_urls]} - manifest_licenses: {rel_path: [license_names]} (empty list if extraction failed) + - manifest_markers: {rel_path} flagged as is_manifest_file without manifest license extraction - :return: (spdx_downloads, manifest_licenses) + :return: (spdx_downloads, manifest_licenses, manifest_markers) """ abs_path_to_scan = os.path.abspath(path_to_scan) spdx_downloads = {} manifest_licenses = {} + manifest_markers = set() for root, dirs, files in os.walk(path_to_scan): for file in files: @@ -730,10 +738,12 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> dict: if downloads: spdx_downloads[rel_path_file] = downloads - if is_manifest_file(file_path): + if extracts_manifest_license(file_path): manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] + elif is_manifest_marker_file(file_path): + manifest_markers.add(rel_path_file) - return spdx_downloads, manifest_licenses + return spdx_downloads, manifest_licenses, manifest_markers if __name__ == '__main__': diff --git a/tests/test_manifest_android_bp.py b/tests/test_manifest_android_bp.py new file mode 100644 index 0000000..7debb9a --- /dev/null +++ b/tests/test_manifest_android_bp.py @@ -0,0 +1,54 @@ +# Copyright (c) 2026 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for Android.bp manifest marker handling.""" + +from unittest.mock import patch + +from fosslight_source._scan_item import ( + SourceItem, + extracts_manifest_license, + is_manifest_file, + is_manifest_marker_file, +) +from fosslight_source.cli import merge_results, metadata_collector + + +def test_is_manifest_marker_file_recognizes_android_bp(): + assert is_manifest_marker_file("/tmp/module/Android.bp") is True + assert is_manifest_marker_file("/tmp/module/android.bp") is True + assert is_manifest_marker_file("/tmp/module/package.json") is False + + +def test_is_manifest_file_includes_android_bp_marker(): + assert is_manifest_file("/tmp/module/Android.bp") is True + assert extracts_manifest_license("/tmp/module/Android.bp") is False + assert extracts_manifest_license("/tmp/module/package.json") is True + + +def test_metadata_collector_marks_android_bp_without_license_extraction(tmp_path): + android_bp = tmp_path / "Android.bp" + android_bp.write_text('license { name: "test_license" }', encoding="utf-8") + package_json = tmp_path / "package.json" + package_json.write_text('{"license": "MIT"}', encoding="utf-8") + + with patch("fosslight_source.cli.get_manifest_licenses", return_value=["MIT"]) as mock_get: + spdx_downloads, manifest_licenses, manifest_markers = metadata_collector(str(tmp_path), set()) + + assert spdx_downloads == {} + assert manifest_licenses == {"package.json": ["MIT"]} + assert manifest_markers == {"Android.bp"} + mock_get.assert_called_once_with(str(package_json)) + + +def test_merge_results_sets_manifest_flag_without_overwriting_scancode_licenses(): + scancode_item = SourceItem("carrois-gothic-sc/Android.bp") + scancode_item.licenses = ["Apache-2.0", "MIT", "BSD"] + + merged, _, _, _ = merge_results( + scancode_result=[scancode_item], + manifest_markers={"carrois-gothic-sc/Android.bp"}, + ) + + assert len(merged) == 1 + assert merged[0].is_manifest_file is True + assert merged[0].licenses == ["Apache-2.0", "MIT", "BSD"] From 1db03424a463906bfa74c01464d999fdb6dfc78f Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 08:03:25 +0900 Subject: [PATCH 2/7] refactor: set Android.bp manifest flag in SourceItem init. Avoid extra merge/metadata loops by marking manifest marker files at creation time while keeping ScanCode licenses unchanged. --- src/fosslight_source/_scan_item.py | 37 +++++++++++++++--------------- src/fosslight_source/cli.py | 24 ++++++------------- tests/test_manifest_android_bp.py | 15 ++++++------ 3 files changed, 34 insertions(+), 42 deletions(-) diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index a93b53b..c2cc180 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -46,13 +46,31 @@ def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> tuple[str, str]: return f"{url.rstrip('/')}/", token +def _matches_manifest_pattern(file_path: str, patterns: list) -> bool: + pattern = r"({})$".format("|".join(patterns)) + filename = os.path.basename(file_path) + return bool(re.match(pattern, filename, re.IGNORECASE)) + + +def extracts_manifest_license(file_path: str) -> bool: + return _matches_manifest_pattern(file_path, _manifest_license_filename) + + +def is_manifest_marker_file(file_path: str) -> bool: + return _matches_manifest_pattern(file_path, _manifest_marker_filename) + + +def is_manifest_file(file_path: str) -> bool: + return extracts_manifest_license(file_path) or is_manifest_marker_file(file_path) + + class SourceItem(FileItem): def __init__(self, value: str) -> None: super().__init__("") self.source_name_or_path = value self.is_license_text = False - self.is_manifest_file = False + self.is_manifest_file = is_manifest_marker_file(value) self.scanoss_reference = {} self.matched_lines = "" # Only for SCANOSS results self.fileURL = "" # Only for SCANOSS results @@ -225,20 +243,3 @@ def is_notice_file(file_path: str) -> bool: filename = os.path.basename(file_path) return bool(re.match(pattern, filename, re.IGNORECASE)) - -def _matches_manifest_pattern(file_path: str, patterns: list) -> bool: - pattern = r"({})$".format("|".join(patterns)) - filename = os.path.basename(file_path) - return bool(re.match(pattern, filename, re.IGNORECASE)) - - -def extracts_manifest_license(file_path: str) -> bool: - return _matches_manifest_pattern(file_path, _manifest_license_filename) - - -def is_manifest_marker_file(file_path: str) -> bool: - return _matches_manifest_pattern(file_path, _manifest_marker_filename) - - -def is_manifest_file(file_path: str) -> bool: - return extracts_manifest_license(file_path) or is_manifest_marker_file(file_path) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 3e6b2e0..35a8a42 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -29,7 +29,7 @@ import argparse from .run_spdx_extractor import get_spdx_downloads from .run_manifest_extractor import get_manifest_licenses -from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, extracts_manifest_license, is_manifest_marker_file +from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, extracts_manifest_license from ._kb_client import fetch_origin_urls_via_scan_job from fosslight_util.cover import dump_result_log from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename @@ -395,7 +395,7 @@ def _collect_kb_file_hashes( def merge_results( scancode_result: list = [], scanoss_result: list = [], spdx_downloads: dict = {}, path_to_scan: str = "", run_kb: bool = False, manifest_licenses: dict = {}, - manifest_markers: set = None, excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = "", + excluded_files: set = None, hide_progress: bool = False, kb_url: str = "", kb_token: str = "", ui_mode: bool = False ) -> tuple[list, Optional[str], int, int]: @@ -422,8 +422,6 @@ def merge_results( """ if excluded_files is None: excluded_files = set() - if manifest_markers is None: - manifest_markers = set() # Merge ScanOSS into ScanCode results. # When ScanCode already detected a license for the same file, keep that license @@ -466,10 +464,6 @@ def merge_results( item.licenses = [] # clear existing licenses (setter clears when value falsy) item.licenses = valid_licenses - for file_name in manifest_markers: - item = _get_or_append_source_item(scancode_result, file_name) - item.is_manifest_file = True - kb_origin_urls: dict[str, str] = {} kb_status_message: Optional[str] = None kb_requested_count = 0 @@ -660,10 +654,10 @@ def run_scanners( run_kb = False run_kb_msg = f"KB({kb_url}) Unreachable" - spdx_downloads, manifest_licenses, manifest_markers = metadata_collector(path_to_scan, excluded_files) + spdx_downloads, manifest_licenses = metadata_collector(path_to_scan, excluded_files) merged_result, kb_status_message, kb_requested_count, _ = merge_results( scancode_result, scanoss_result, spdx_downloads, - path_to_scan, run_kb, manifest_licenses, manifest_markers, excluded_files, + path_to_scan, run_kb, manifest_licenses, excluded_files, hide_progress, kb_url, kb_token, ui_mode=ui_mode, ) @@ -711,21 +705,19 @@ def run_scanners( return success, result_log.get(RESULT_KEY, ""), scan_item, license_list, scanoss_result -def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, dict, set]: +def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, dict]: """ Collect metadata for merging. - Traverse files with exclusions applied - spdx_downloads: {rel_path: [download_urls]} - manifest_licenses: {rel_path: [license_names]} (empty list if extraction failed) - - manifest_markers: {rel_path} flagged as is_manifest_file without manifest license extraction - :return: (spdx_downloads, manifest_licenses, manifest_markers) + :return: (spdx_downloads, manifest_licenses) """ abs_path_to_scan = os.path.abspath(path_to_scan) spdx_downloads = {} manifest_licenses = {} - manifest_markers = set() for root, dirs, files in os.walk(path_to_scan): for file in files: @@ -740,10 +732,8 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di if extracts_manifest_license(file_path): manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] - elif is_manifest_marker_file(file_path): - manifest_markers.add(rel_path_file) - return spdx_downloads, manifest_licenses, manifest_markers + return spdx_downloads, manifest_licenses if __name__ == '__main__': diff --git a/tests/test_manifest_android_bp.py b/tests/test_manifest_android_bp.py index 7debb9a..93b76e4 100644 --- a/tests/test_manifest_android_bp.py +++ b/tests/test_manifest_android_bp.py @@ -25,29 +25,30 @@ def test_is_manifest_file_includes_android_bp_marker(): assert extracts_manifest_license("/tmp/module/package.json") is True -def test_metadata_collector_marks_android_bp_without_license_extraction(tmp_path): +def test_metadata_collector_skips_android_bp_license_extraction(tmp_path): android_bp = tmp_path / "Android.bp" android_bp.write_text('license { name: "test_license" }', encoding="utf-8") package_json = tmp_path / "package.json" package_json.write_text('{"license": "MIT"}', encoding="utf-8") with patch("fosslight_source.cli.get_manifest_licenses", return_value=["MIT"]) as mock_get: - spdx_downloads, manifest_licenses, manifest_markers = metadata_collector(str(tmp_path), set()) + spdx_downloads, manifest_licenses = metadata_collector(str(tmp_path), set()) assert spdx_downloads == {} assert manifest_licenses == {"package.json": ["MIT"]} - assert manifest_markers == {"Android.bp"} mock_get.assert_called_once_with(str(package_json)) +def test_source_item_marks_android_bp_at_creation(): + item = SourceItem("carrois-gothic-sc/Android.bp") + assert item.is_manifest_file is True + + def test_merge_results_sets_manifest_flag_without_overwriting_scancode_licenses(): scancode_item = SourceItem("carrois-gothic-sc/Android.bp") scancode_item.licenses = ["Apache-2.0", "MIT", "BSD"] - merged, _, _, _ = merge_results( - scancode_result=[scancode_item], - manifest_markers={"carrois-gothic-sc/Android.bp"}, - ) + merged, _, _, _ = merge_results(scancode_result=[scancode_item]) assert len(merged) == 1 assert merged[0].is_manifest_file is True From 1b0b06a35c3f8f6b08f8e42cb663d4705edf765f Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 08:09:31 +0900 Subject: [PATCH 3/7] refactor: handle Android.bp via manifest_licenses with empty list. Include Android.bp in manifest_licenses without calling get_manifest_licenses, set is_manifest_file only for existing ScanCode rows, and skip license overwrite when extraction is empty. --- src/fosslight_source/_scan_item.py | 17 ++++++------ src/fosslight_source/cli.py | 16 ++++++----- tests/test_manifest_android_bp.py | 43 +++++++++++++++++------------- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index c2cc180..7d395f1 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -15,7 +15,7 @@ _notice_filename = ['licen[cs]e[s]?', 'notice[s]?', 'legal', 'copyright[s]?', 'copying*', 'patent[s]?', 'unlicen[cs]e', 'eula', '[a,l]?gpl[-]?[1-3]?[.,-,_]?[0-1]?', 'mit', 'bsd[-]?[0-4]?', 'bsd[-]?[0-4][-]?clause[s]?', 'apache[-,_]?[1-2]?[.,-,_]?[0-2]?'] -_manifest_license_filename = [ +_manifest_filename = [ r'.*\.pom$', r'package\.json$', r'composer\.json$', @@ -25,9 +25,10 @@ r'.*\.podspec$', r'Cargo\.toml$', r'huggingface_hub_metadata\.json$', + r'Android\.bp$', ] -# Manifest marker only: is_manifest_file=True, but license comes from ScanCode. -_manifest_marker_filename = [ +# License extraction via get_manifest_licenses is skipped; ScanCode licenses are kept. +_manifest_skip_license_extraction = [ r'Android\.bp$', ] MAX_LICENSE_LENGTH = 200 @@ -53,15 +54,15 @@ def _matches_manifest_pattern(file_path: str, patterns: list) -> bool: def extracts_manifest_license(file_path: str) -> bool: - return _matches_manifest_pattern(file_path, _manifest_license_filename) + return _matches_manifest_pattern(file_path, _manifest_filename) -def is_manifest_marker_file(file_path: str) -> bool: - return _matches_manifest_pattern(file_path, _manifest_marker_filename) +def skips_manifest_license_extraction(file_path: str) -> bool: + return _matches_manifest_pattern(file_path, _manifest_skip_license_extraction) def is_manifest_file(file_path: str) -> bool: - return extracts_manifest_license(file_path) or is_manifest_marker_file(file_path) + return extracts_manifest_license(file_path) class SourceItem(FileItem): @@ -70,7 +71,7 @@ def __init__(self, value: str) -> None: super().__init__("") self.source_name_or_path = value self.is_license_text = False - self.is_manifest_file = is_manifest_marker_file(value) + self.is_manifest_file = False self.scanoss_reference = {} self.matched_lines = "" # Only for SCANOSS results self.fileURL = "" # Only for SCANOSS results diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 35a8a42..28e7916 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -29,7 +29,7 @@ import argparse from .run_spdx_extractor import get_spdx_downloads from .run_manifest_extractor import get_manifest_licenses -from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, extracts_manifest_license +from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, is_manifest_file, skips_manifest_license_extraction from ._kb_client import fetch_origin_urls_via_scan_job from fosslight_util.cover import dump_result_log from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename @@ -453,10 +453,9 @@ def merge_results( if manifest_licenses: for file_name, licenses in manifest_licenses.items(): valid_licenses = [lic.strip() for lic in licenses if isinstance(lic, str) and lic.strip()] - # Non-UI: skip manifests with no extracted licenses. - # UI: keep/create the row and mark is_manifest_file even without licenses. - if not valid_licenses and not ui_mode: - continue + if not valid_licenses and file_name not in scancode_result: + if not ui_mode: + continue item = _get_or_append_source_item(scancode_result, file_name) item.is_manifest_file = True if valid_licenses: @@ -730,8 +729,11 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di if downloads: spdx_downloads[rel_path_file] = downloads - if extracts_manifest_license(file_path): - manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] + if is_manifest_file(file_path): + if skips_manifest_license_extraction(file_path): + manifest_licenses[rel_path_file] = [] + else: + manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] return spdx_downloads, manifest_licenses diff --git a/tests/test_manifest_android_bp.py b/tests/test_manifest_android_bp.py index 93b76e4..d440a17 100644 --- a/tests/test_manifest_android_bp.py +++ b/tests/test_manifest_android_bp.py @@ -1,31 +1,29 @@ # Copyright (c) 2026 LG Electronics Inc. # SPDX-License-Identifier: Apache-2.0 -"""Tests for Android.bp manifest marker handling.""" +"""Tests for Android.bp manifest handling.""" from unittest.mock import patch from fosslight_source._scan_item import ( SourceItem, - extracts_manifest_license, is_manifest_file, - is_manifest_marker_file, + skips_manifest_license_extraction, ) from fosslight_source.cli import merge_results, metadata_collector -def test_is_manifest_marker_file_recognizes_android_bp(): - assert is_manifest_marker_file("/tmp/module/Android.bp") is True - assert is_manifest_marker_file("/tmp/module/android.bp") is True - assert is_manifest_marker_file("/tmp/module/package.json") is False +def test_is_manifest_file_recognizes_android_bp(): + assert is_manifest_file("/tmp/module/Android.bp") is True + assert is_manifest_file("/tmp/module/android.bp") is True + assert is_manifest_file("/tmp/module/package.json") is True -def test_is_manifest_file_includes_android_bp_marker(): - assert is_manifest_file("/tmp/module/Android.bp") is True - assert extracts_manifest_license("/tmp/module/Android.bp") is False - assert extracts_manifest_license("/tmp/module/package.json") is True +def test_skips_manifest_license_extraction_for_android_bp(): + assert skips_manifest_license_extraction("/tmp/module/Android.bp") is True + assert skips_manifest_license_extraction("/tmp/module/package.json") is False -def test_metadata_collector_skips_android_bp_license_extraction(tmp_path): +def test_metadata_collector_adds_android_bp_without_license_extraction(tmp_path): android_bp = tmp_path / "Android.bp" android_bp.write_text('license { name: "test_license" }', encoding="utf-8") package_json = tmp_path / "package.json" @@ -35,21 +33,28 @@ def test_metadata_collector_skips_android_bp_license_extraction(tmp_path): spdx_downloads, manifest_licenses = metadata_collector(str(tmp_path), set()) assert spdx_downloads == {} - assert manifest_licenses == {"package.json": ["MIT"]} + assert manifest_licenses == {"Android.bp": [], "package.json": ["MIT"]} mock_get.assert_called_once_with(str(package_json)) -def test_source_item_marks_android_bp_at_creation(): - item = SourceItem("carrois-gothic-sc/Android.bp") - assert item.is_manifest_file is True - - def test_merge_results_sets_manifest_flag_without_overwriting_scancode_licenses(): scancode_item = SourceItem("carrois-gothic-sc/Android.bp") scancode_item.licenses = ["Apache-2.0", "MIT", "BSD"] - merged, _, _, _ = merge_results(scancode_result=[scancode_item]) + merged, _, _, _ = merge_results( + scancode_result=[scancode_item], + manifest_licenses={"carrois-gothic-sc/Android.bp": []}, + ) assert len(merged) == 1 assert merged[0].is_manifest_file is True assert merged[0].licenses == ["Apache-2.0", "MIT", "BSD"] + + +def test_merge_results_skips_android_bp_not_in_scancode_result(): + merged, _, _, _ = merge_results( + scancode_result=[], + manifest_licenses={"module/Android.bp": []}, + ) + + assert merged == [] From 7be40441099185c7c64fd2c32ce3c8d47ab89bab Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 08:15:14 +0900 Subject: [PATCH 4/7] refactor: restore simple is_manifest_file and add scenario tests. Drop manifest pattern wrapper helpers and keep Android.bp skip inline in metadata_collector. --- src/fosslight_source/_scan_item.py | 28 ++------ src/fosslight_source/cli.py | 5 +- tests/test_manifest_android_bp.py | 6 -- tests/test_manifest_recommended_scenarios.py | 76 ++++++++++++++++++++ 4 files changed, 85 insertions(+), 30 deletions(-) create mode 100644 tests/test_manifest_recommended_scenarios.py diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index 7d395f1..955c136 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -27,10 +27,6 @@ r'huggingface_hub_metadata\.json$', r'Android\.bp$', ] -# License extraction via get_manifest_licenses is skipped; ScanCode licenses are kept. -_manifest_skip_license_extraction = [ - r'Android\.bp$', -] MAX_LICENSE_LENGTH = 200 MAX_LICENSE_TOTAL_LENGTH = 600 SUBSTRING_LICENSE_COMMENT = "Maximum character limit (License)" @@ -47,24 +43,6 @@ def resolve_kb_config(kb_url: str = "", kb_token: str = "") -> tuple[str, str]: return f"{url.rstrip('/')}/", token -def _matches_manifest_pattern(file_path: str, patterns: list) -> bool: - pattern = r"({})$".format("|".join(patterns)) - filename = os.path.basename(file_path) - return bool(re.match(pattern, filename, re.IGNORECASE)) - - -def extracts_manifest_license(file_path: str) -> bool: - return _matches_manifest_pattern(file_path, _manifest_filename) - - -def skips_manifest_license_extraction(file_path: str) -> bool: - return _matches_manifest_pattern(file_path, _manifest_skip_license_extraction) - - -def is_manifest_file(file_path: str) -> bool: - return extracts_manifest_license(file_path) - - class SourceItem(FileItem): def __init__(self, value: str) -> None: @@ -244,3 +222,9 @@ def is_notice_file(file_path: str) -> bool: filename = os.path.basename(file_path) return bool(re.match(pattern, filename, re.IGNORECASE)) + +def is_manifest_file(file_path: str) -> bool: + pattern = r"({})$".format("|".join(_manifest_filename)) + filename = os.path.basename(file_path) + return bool(re.match(pattern, filename, re.IGNORECASE)) + diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 28e7916..3fc4fe1 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -29,7 +29,7 @@ import argparse from .run_spdx_extractor import get_spdx_downloads from .run_manifest_extractor import get_manifest_licenses -from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, is_manifest_file, skips_manifest_license_extraction +from ._scan_item import SourceItem, resolve_kb_config, is_notice_file, is_manifest_file from ._kb_client import fetch_origin_urls_via_scan_job from fosslight_util.cover import dump_result_log from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename @@ -730,7 +730,8 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di spdx_downloads[rel_path_file] = downloads if is_manifest_file(file_path): - if skips_manifest_license_extraction(file_path): + # Android.bp: ScanCode licenses are kept; skip get_manifest_licenses. + if os.path.basename(file_path).lower() == 'android.bp': manifest_licenses[rel_path_file] = [] else: manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] diff --git a/tests/test_manifest_android_bp.py b/tests/test_manifest_android_bp.py index d440a17..b8ee8b4 100644 --- a/tests/test_manifest_android_bp.py +++ b/tests/test_manifest_android_bp.py @@ -7,7 +7,6 @@ from fosslight_source._scan_item import ( SourceItem, is_manifest_file, - skips_manifest_license_extraction, ) from fosslight_source.cli import merge_results, metadata_collector @@ -18,11 +17,6 @@ def test_is_manifest_file_recognizes_android_bp(): assert is_manifest_file("/tmp/module/package.json") is True -def test_skips_manifest_license_extraction_for_android_bp(): - assert skips_manifest_license_extraction("/tmp/module/Android.bp") is True - assert skips_manifest_license_extraction("/tmp/module/package.json") is False - - def test_metadata_collector_adds_android_bp_without_license_extraction(tmp_path): android_bp = tmp_path / "Android.bp" android_bp.write_text('license { name: "test_license" }', encoding="utf-8") diff --git a/tests/test_manifest_recommended_scenarios.py b/tests/test_manifest_recommended_scenarios.py new file mode 100644 index 0000000..f27f660 --- /dev/null +++ b/tests/test_manifest_recommended_scenarios.py @@ -0,0 +1,76 @@ +# Copyright (c) 2026 LG Electronics Inc. +# SPDX-License-Identifier: Apache-2.0 +"""Recommended verification scenarios for Android.bp manifest handling.""" + +import json +from pathlib import Path + +from fosslight_source._parsing_scancode_file_item import parsing_scancode +from fosslight_source._scan_item import SourceItem +from fosslight_source.cli import merge_results, metadata_collector + +REPO_ROOT = Path(__file__).resolve().parents[1] +CARROIS_SCAN_ROOT = REPO_ROOT / "temp" / "carrois-gothic-sc" +CARROIS_SCANCODE_JSON = REPO_ROOT / "temp" / "scancode_raw_result.json" + + +def test_scenario1_carrois_android_bp_keeps_scancode_licenses(): + """carrois-gothic-sc/Android.bp: is_manifest_file=True, ScanCode licenses preserved.""" + assert CARROIS_SCAN_ROOT.is_dir(), f"missing fixture dir: {CARROIS_SCAN_ROOT}" + assert CARROIS_SCANCODE_JSON.is_file(), f"missing fixture json: {CARROIS_SCANCODE_JSON}" + + scancode_files = json.loads(CARROIS_SCANCODE_JSON.read_text(encoding="utf-8"))["files"] + success, scancode_result, _messages, _license_list = parsing_scancode(scancode_files) + assert success is True + + android_bp = next(item for item in scancode_result if item.source_name_or_path.endswith("Android.bp")) + scancode_licenses = list(android_bp.licenses) + assert scancode_licenses, "expected ScanCode licenses on Android.bp" + + _, manifest_licenses = metadata_collector(str(CARROIS_SCAN_ROOT), set()) + assert "Android.bp" in manifest_licenses + assert manifest_licenses["Android.bp"] == [] + + merged, _, _, _ = merge_results( + scancode_result=list(scancode_result), + manifest_licenses=manifest_licenses, + ) + merged_android_bp = next(item for item in merged if item.source_name_or_path.endswith("Android.bp")) + + assert merged_android_bp.is_manifest_file is True + assert merged_android_bp.licenses == scancode_licenses + + +def test_scenario2_package_json_manifest_fail_keeps_scancode_licenses(): + """package.json manifest extraction fails but ScanCode row exists: manifest flag only.""" + scancode_item = SourceItem("app/package.json") + scancode_item.licenses = ["Apache-2.0"] + + merged, _, _, _ = merge_results( + scancode_result=[scancode_item], + manifest_licenses={"app/package.json": []}, + ) + + assert len(merged) == 1 + assert merged[0].is_manifest_file is True + assert merged[0].licenses == ["Apache-2.0"] + + +def test_scenario3_android_bp_not_in_scancode_no_row_non_ui(): + """Android.bp absent from ScanCode: no row in non-UI mode.""" + manifest = {"module/Android.bp": []} + + merged_non_ui, _, _, _ = merge_results(scancode_result=[], manifest_licenses=manifest, ui_mode=False) + assert merged_non_ui == [] + + +def test_scenario3_android_bp_not_in_scancode_ui_keeps_empty_row(): + """Android.bp absent from ScanCode: UI mode still creates manifest row.""" + manifest = {"module/Android.bp": []} + + merged_ui, _, _, _ = merge_results(scancode_result=[], manifest_licenses=manifest, ui_mode=True) + + assert len(merged_ui) == 1 + assert merged_ui[0].source_name_or_path == "module/Android.bp" + assert merged_ui[0].is_manifest_file is True + assert merged_ui[0].licenses == [] From 15bdf83fb6f53e0afc6545930b0b471ab096bc01 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 08:18:01 +0900 Subject: [PATCH 5/7] refactor: handle Android.bp skip in get_manifest_licenses and tighten manifest merge. Return empty licenses for Android.bp in run_manifest_extractor and only append manifest rows when licenses exist or UI mode is enabled. --- src/fosslight_source/cli.py | 21 ++++++++++--------- .../run_manifest_extractor.py | 3 +++ tests/test_manifest_android_bp.py | 11 ++++++++-- tests/test_manifest_recommended_scenarios.py | 16 ++++++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/src/fosslight_source/cli.py b/src/fosslight_source/cli.py index 3fc4fe1..e3d148b 100755 --- a/src/fosslight_source/cli.py +++ b/src/fosslight_source/cli.py @@ -453,10 +453,11 @@ def merge_results( if manifest_licenses: for file_name, licenses in manifest_licenses.items(): valid_licenses = [lic.strip() for lic in licenses if isinstance(lic, str) and lic.strip()] - if not valid_licenses and file_name not in scancode_result: - if not ui_mode: - continue - item = _get_or_append_source_item(scancode_result, file_name) + item = _get_or_append_source_item( + scancode_result, file_name, append=bool(valid_licenses) or ui_mode + ) + if item is None: + continue item.is_manifest_file = True if valid_licenses: # overwrite existing detected licenses with manifest-provided licenses @@ -499,9 +500,13 @@ def merge_results( return scancode_result, kb_status_message, kb_requested_count, kb_returned_count -def _get_or_append_source_item(scancode_result: list, file_name: str) -> SourceItem: +def _get_or_append_source_item( + scancode_result: list, file_name: str, append: bool = True +) -> Optional[SourceItem]: if file_name in scancode_result: return scancode_result[scancode_result.index(file_name)] + if not append: + return None item = SourceItem(file_name) scancode_result.append(item) return item @@ -730,11 +735,7 @@ def metadata_collector(path_to_scan: str, excluded_files: set) -> tuple[dict, di spdx_downloads[rel_path_file] = downloads if is_manifest_file(file_path): - # Android.bp: ScanCode licenses are kept; skip get_manifest_licenses. - if os.path.basename(file_path).lower() == 'android.bp': - manifest_licenses[rel_path_file] = [] - else: - manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] + manifest_licenses[rel_path_file] = get_manifest_licenses(file_path) or [] return spdx_downloads, manifest_licenses diff --git a/src/fosslight_source/run_manifest_extractor.py b/src/fosslight_source/run_manifest_extractor.py index 27d70f9..360c5d8 100644 --- a/src/fosslight_source/run_manifest_extractor.py +++ b/src/fosslight_source/run_manifest_extractor.py @@ -345,6 +345,9 @@ def append_license(value): def get_manifest_licenses(file_path: str) -> list[str]: + # Android.bp licenses come from ScanCode; manifest merge only sets is_manifest_file. + if os.path.basename(file_path).lower() == 'android.bp': + return [] if file_path.endswith('.pom'): try: pom_licenses = get_license_from_pom(group_id='', artifact_id='', version='', pom_path=file_path, check_parent=True) diff --git a/tests/test_manifest_android_bp.py b/tests/test_manifest_android_bp.py index b8ee8b4..278aa57 100644 --- a/tests/test_manifest_android_bp.py +++ b/tests/test_manifest_android_bp.py @@ -9,6 +9,7 @@ is_manifest_file, ) from fosslight_source.cli import merge_results, metadata_collector +from fosslight_source.run_manifest_extractor import get_manifest_licenses def test_is_manifest_file_recognizes_android_bp(): @@ -17,18 +18,24 @@ def test_is_manifest_file_recognizes_android_bp(): assert is_manifest_file("/tmp/module/package.json") is True +def test_get_manifest_licenses_returns_empty_for_android_bp(tmp_path): + android_bp = tmp_path / "Android.bp" + android_bp.write_text('license { name: "test_license" }', encoding="utf-8") + assert get_manifest_licenses(str(android_bp)) == [] + + def test_metadata_collector_adds_android_bp_without_license_extraction(tmp_path): android_bp = tmp_path / "Android.bp" android_bp.write_text('license { name: "test_license" }', encoding="utf-8") package_json = tmp_path / "package.json" package_json.write_text('{"license": "MIT"}', encoding="utf-8") - with patch("fosslight_source.cli.get_manifest_licenses", return_value=["MIT"]) as mock_get: + with patch("fosslight_source.cli.get_manifest_licenses", wraps=get_manifest_licenses) as mock_get: spdx_downloads, manifest_licenses = metadata_collector(str(tmp_path), set()) assert spdx_downloads == {} assert manifest_licenses == {"Android.bp": [], "package.json": ["MIT"]} - mock_get.assert_called_once_with(str(package_json)) + assert mock_get.call_count == 2 def test_merge_results_sets_manifest_flag_without_overwriting_scancode_licenses(): diff --git a/tests/test_manifest_recommended_scenarios.py b/tests/test_manifest_recommended_scenarios.py index f27f660..45d76fc 100644 --- a/tests/test_manifest_recommended_scenarios.py +++ b/tests/test_manifest_recommended_scenarios.py @@ -56,6 +56,22 @@ def test_scenario2_package_json_manifest_fail_keeps_scancode_licenses(): assert merged[0].licenses == ["Apache-2.0"] +def test_scenario3_android_bp_from_spdx_marks_manifest_without_new_row(): + """Android.bp added by spdx merge: manifest flag on existing item, no duplicate row.""" + spdx_item = SourceItem("module/Android.bp") + spdx_item.download_location = ["https://example.com/repo"] + + merged, _, _, _ = merge_results( + scancode_result=[spdx_item], + manifest_licenses={"module/Android.bp": []}, + ) + + assert len(merged) == 1 + assert merged[0].is_manifest_file is True + assert merged[0].download_location == ["https://example.com/repo"] + assert merged[0].licenses == [] + + def test_scenario3_android_bp_not_in_scancode_no_row_non_ui(): """Android.bp absent from ScanCode: no row in non-UI mode.""" manifest = {"module/Android.bp": []} From 97daf31cccd4038caa844b009e69fb25aa3effe0 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 09:44:02 +0900 Subject: [PATCH 6/7] style: remove trailing blank line in _scan_item.py for flake8 W391. --- src/fosslight_source/_scan_item.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fosslight_source/_scan_item.py b/src/fosslight_source/_scan_item.py index 955c136..fd71c04 100644 --- a/src/fosslight_source/_scan_item.py +++ b/src/fosslight_source/_scan_item.py @@ -227,4 +227,3 @@ def is_manifest_file(file_path: str) -> bool: pattern = r"({})$".format("|".join(_manifest_filename)) filename = os.path.basename(file_path) return bool(re.match(pattern, filename, re.IGNORECASE)) - From 2e9f9beca9235efbf5c3b551a06b5e0de628deb2 Mon Sep 17 00:00:00 2001 From: Soim Kim Date: Wed, 26 Aug 2026 09:51:15 +0900 Subject: [PATCH 7/7] test: drop temp/ fixture dependency from Android.bp scenario tests. --- tests/test_manifest_recommended_scenarios.py | 64 ++++++++------------ 1 file changed, 24 insertions(+), 40 deletions(-) diff --git a/tests/test_manifest_recommended_scenarios.py b/tests/test_manifest_recommended_scenarios.py index 45d76fc..e115cb1 100644 --- a/tests/test_manifest_recommended_scenarios.py +++ b/tests/test_manifest_recommended_scenarios.py @@ -2,43 +2,23 @@ # SPDX-License-Identifier: Apache-2.0 """Recommended verification scenarios for Android.bp manifest handling.""" -import json -from pathlib import Path - -from fosslight_source._parsing_scancode_file_item import parsing_scancode from fosslight_source._scan_item import SourceItem -from fosslight_source.cli import merge_results, metadata_collector - -REPO_ROOT = Path(__file__).resolve().parents[1] -CARROIS_SCAN_ROOT = REPO_ROOT / "temp" / "carrois-gothic-sc" -CARROIS_SCANCODE_JSON = REPO_ROOT / "temp" / "scancode_raw_result.json" - - -def test_scenario1_carrois_android_bp_keeps_scancode_licenses(): - """carrois-gothic-sc/Android.bp: is_manifest_file=True, ScanCode licenses preserved.""" - assert CARROIS_SCAN_ROOT.is_dir(), f"missing fixture dir: {CARROIS_SCAN_ROOT}" - assert CARROIS_SCANCODE_JSON.is_file(), f"missing fixture json: {CARROIS_SCANCODE_JSON}" +from fosslight_source.cli import merge_results - scancode_files = json.loads(CARROIS_SCANCODE_JSON.read_text(encoding="utf-8"))["files"] - success, scancode_result, _messages, _license_list = parsing_scancode(scancode_files) - assert success is True - android_bp = next(item for item in scancode_result if item.source_name_or_path.endswith("Android.bp")) - scancode_licenses = list(android_bp.licenses) - assert scancode_licenses, "expected ScanCode licenses on Android.bp" - - _, manifest_licenses = metadata_collector(str(CARROIS_SCAN_ROOT), set()) - assert "Android.bp" in manifest_licenses - assert manifest_licenses["Android.bp"] == [] +def test_scenario1_android_bp_keeps_scancode_licenses(): + """Android.bp with ScanCode licenses: is_manifest_file=True, licenses preserved.""" + scancode_item = SourceItem("Android.bp") + scancode_item.licenses = ["Apache-2.0", "unknown-license-reference", "BSD", "MIT", "OFL"] merged, _, _, _ = merge_results( - scancode_result=list(scancode_result), - manifest_licenses=manifest_licenses, + scancode_result=[scancode_item], + manifest_licenses={"Android.bp": []}, ) - merged_android_bp = next(item for item in merged if item.source_name_or_path.endswith("Android.bp")) - assert merged_android_bp.is_manifest_file is True - assert merged_android_bp.licenses == scancode_licenses + assert len(merged) == 1 + assert merged[0].is_manifest_file is True + assert merged[0].licenses == ["Apache-2.0", "unknown-license-reference", "BSD", "MIT", "OFL"] def test_scenario2_package_json_manifest_fail_keeps_scancode_licenses(): @@ -72,19 +52,23 @@ def test_scenario3_android_bp_from_spdx_marks_manifest_without_new_row(): assert merged[0].licenses == [] -def test_scenario3_android_bp_not_in_scancode_no_row_non_ui(): - """Android.bp absent from ScanCode: no row in non-UI mode.""" - manifest = {"module/Android.bp": []} - - merged_non_ui, _, _, _ = merge_results(scancode_result=[], manifest_licenses=manifest, ui_mode=False) +def test_scenario3_android_bp_not_in_result_no_row_non_ui(): + """Android.bp absent from merge result: no row in non-UI mode.""" + merged_non_ui, _, _, _ = merge_results( + scancode_result=[], + manifest_licenses={"module/Android.bp": []}, + ui_mode=False, + ) assert merged_non_ui == [] -def test_scenario3_android_bp_not_in_scancode_ui_keeps_empty_row(): - """Android.bp absent from ScanCode: UI mode still creates manifest row.""" - manifest = {"module/Android.bp": []} - - merged_ui, _, _, _ = merge_results(scancode_result=[], manifest_licenses=manifest, ui_mode=True) +def test_scenario3_android_bp_not_in_result_ui_keeps_empty_row(): + """Android.bp absent from merge result: UI mode still creates manifest row.""" + merged_ui, _, _, _ = merge_results( + scancode_result=[], + manifest_licenses={"module/Android.bp": []}, + ui_mode=True, + ) assert len(merged_ui) == 1 assert merged_ui[0].source_name_or_path == "module/Android.bp"