From bdbf16f720f394f1917c81eaa34ee069cb0152ca Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 14 Aug 2026 18:22:50 -0700 Subject: [PATCH 01/11] Create plugin for resource cache indices Signed-off-by: Jono Yang --- src/commoncode/plugin_resource_cache.py | 89 +++++++++++++++++++++++++ src/scancode_config.py | 5 ++ 2 files changed, 94 insertions(+) create mode 100644 src/commoncode/plugin_resource_cache.py diff --git a/src/commoncode/plugin_resource_cache.py b/src/commoncode/plugin_resource_cache.py new file mode 100644 index 0000000000..730ab90d88 --- /dev/null +++ b/src/commoncode/plugin_resource_cache.py @@ -0,0 +1,89 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import attr +import hashlib +import json +import os + +from plugincode.scan import ScanPlugin +from plugincode.scan import scan_impl +from commoncode.cliutils import PluggableCommandLineOption +from commoncode.cliutils import OTHER_SCAN_GROUP +from commoncode.hash import multi_checksums +from scancode_config import resource_cache_dir + + +RESOURCE_INDEX_DIR = "resource_cache_indices" + + +def compute_resource_cache_index(location, **kwargs): + """ + Compute resource cache index value for Resource at `location` + """ + result = {} + + md5 = multi_checksums( + location=location, + checksum_names=('md5') + ).values() + md5_bytes = bytes.fromhex(md5) + + # TODO: figure out if we can get the resource path through kwargs + resource_path = kwargs['path'] + + digest = hashlib.md5() + digest.update(md5_bytes) + digest.update( + resource_path.encode('utf-8', 'surrogateescape') + ) + result['resource_cache_index'] = digest.hexdigest() + + return result + + +@scan_impl +class ResourceCacheIndexScanner(ScanPlugin): + """ + Compute resource cache index value for Resources in Codebase + """ + resource_attributes = dict([ + ('resource_cache_index', attr.ib(default=None, repr=False)), + ]) + + run_order = 0 + sort_order = 0 + + options = [ + PluggableCommandLineOption(('-rc', '--resource_cache_index'), + is_flag=True, default=False, + help='Scan to compute resource cache index values for Resources in Codebase.', + help_group=OTHER_SCAN_GROUP, sort_order=10 + ) + ] + + def is_enabled(self, resource_cache_idx, **kwargs): + return resource_cache_idx + + def get_scanner(self, **kwargs): + return compute_resource_cache_index + + def process_codebase(self, codebase, **kwargs): + """ + Update resource cache + """ + + idx_cache_dir = os.path.join(resource_cache_dir, RESOURCE_INDEX_DIR) + + for resource in codebase: + # dump all resources to cache + cache_file = os.path.join(idx_cache_dir, resource.resource_cache_index) + with open(cache_file, 'wb') as f: + f.write(json.dumps(resource.serialize(), check_circular=False)) diff --git a/src/scancode_config.py b/src/scancode_config.py index 6e9f634b08..247f9c5206 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -190,6 +190,11 @@ def _create_dir(location): __env_package_cache_dir = os.getenv('SCANCODE_PACKAGE_INDEX_CACHE') packagedcode_cache_dir = (__env_package_cache_dir or std_package_cache_dir) + +std_resource_cache_dir = join(scancode_src_dir, 'commoncode', 'data', 'cache') +__env_resource_cache_dir = os.getenv('SCANCODE_RESOURCE_INDEX_CACHE') +resource_cache_dir = (__env_resource_cache_dir or std_resource_cache_dir) + _create_dir(licensedcode_cache_dir) _create_dir(packagedcode_cache_dir) _create_dir(scancode_cache_dir) From 5b3901e5eecc3e94c6657d1aff95ba5f947c5507 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 18 Aug 2026 18:35:44 -0700 Subject: [PATCH 02/11] Use pypi hash strategy for organizing resource cache * Use sha256 instead of md5 Signed-off-by: Jono Yang --- pyproject.toml | 2 +- .../plugin_resource_cache.py | 69 +++++++++++-------- src/scancode_config.py | 3 +- 3 files changed, 42 insertions(+), 32 deletions(-) rename src/{commoncode => scancode}/plugin_resource_cache.py (51%) diff --git a/pyproject.toml b/pyproject.toml index f2371c6bb2..acfc083728 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,7 +164,6 @@ files = [ { filename = "pyproject.toml" }, { filename = "pyproject-scancode-toolkit-mini.toml" }, { filename = "pyproject-packagedcode.toml" }, - ] @@ -276,6 +275,7 @@ packages = "packagedcode.plugin_package:PackageScanner" emails = "cluecode.plugin_email:EmailScanner" urls = "cluecode.plugin_url:UrlScanner" generated = "summarycode.generated:GeneratedCodeDetector" +resource_cache_index = "scancode.plugin_resource_cache:ResourceCacheIndexScanner" # scancode_post_scan is the entry point for post_scan plugins executed after the diff --git a/src/commoncode/plugin_resource_cache.py b/src/scancode/plugin_resource_cache.py similarity index 51% rename from src/commoncode/plugin_resource_cache.py rename to src/scancode/plugin_resource_cache.py index 730ab90d88..8e6581d796 100644 --- a/src/commoncode/plugin_resource_cache.py +++ b/src/scancode/plugin_resource_cache.py @@ -7,46 +7,56 @@ # See https://aboutcode.org for more information about nexB OSS projects. # - import attr import hashlib import json import os -from plugincode.scan import ScanPlugin -from plugincode.scan import scan_impl from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import OTHER_SCAN_GROUP -from commoncode.hash import multi_checksums +from commoncode.hash import binary_chunks +from plugincode.scan import ScanPlugin +from plugincode.scan import scan_impl from scancode_config import resource_cache_dir -RESOURCE_INDEX_DIR = "resource_cache_indices" +def hasher_from_chunks(chunks): + """ + Return a sha256 hasher loaded with `chunks` + """ + hasher = hashlib.sha256() + for chunk in chunks: + hasher.update(chunk) + return hasher -def compute_resource_cache_index(location, **kwargs): +def compute_resource_cache_index(location, path, **kwargs): """ Compute resource cache index value for Resource at `location` """ - result = {} + chunks = binary_chunks(location=location) + sha256_hasher = hasher_from_chunks(chunks=chunks) + # TODO: consider using filename instead of path + sha256_hasher.update(path.encode('utf-8', 'surrogateescape')) - md5 = multi_checksums( - location=location, - checksum_names=('md5') - ).values() - md5_bytes = bytes.fromhex(md5) + result = { + 'resource_cache_index': sha256_hasher.hexdigest() + } - # TODO: figure out if we can get the resource path through kwargs - resource_path = kwargs['path'] + return result - digest = hashlib.md5() - digest.update(md5_bytes) - digest.update( - resource_path.encode('utf-8', 'surrogateescape') - ) - result['resource_cache_index'] = digest.hexdigest() - return result +def get_resource_cache_file_location(resource_cache_index): + """ + Return the location of the cache file for a given `resource_cache_index` + hexstring. + """ + # Split the hash into two subdirectories using the first two prefix pairs + prefix1 = resource_cache_index[:2] + prefix2 = resource_cache_index[2:4] + filename = resource_cache_index[4:] + resource_cache_file_path = os.path.join(resource_cache_dir, prefix1, prefix2, filename) + return resource_cache_file_path @scan_impl @@ -65,25 +75,24 @@ class ResourceCacheIndexScanner(ScanPlugin): PluggableCommandLineOption(('-rc', '--resource_cache_index'), is_flag=True, default=False, help='Scan to compute resource cache index values for Resources in Codebase.', - help_group=OTHER_SCAN_GROUP, sort_order=10 + help_group=OTHER_SCAN_GROUP, sort_order=0 ) ] - def is_enabled(self, resource_cache_idx, **kwargs): - return resource_cache_idx + def is_enabled(self, resource_cache_index, **kwargs): + return resource_cache_index def get_scanner(self, **kwargs): return compute_resource_cache_index def process_codebase(self, codebase, **kwargs): """ - Update resource cache + Update Resource cache """ - - idx_cache_dir = os.path.join(resource_cache_dir, RESOURCE_INDEX_DIR) - for resource in codebase: # dump all resources to cache - cache_file = os.path.join(idx_cache_dir, resource.resource_cache_index) - with open(cache_file, 'wb') as f: + resource_cache_file_location = get_resource_cache_file_location( + resource_cache_index=resource.resource_cache_index + ) + with open(resource_cache_file_location, 'w') as f: f.write(json.dumps(resource.serialize(), check_circular=False)) diff --git a/src/scancode_config.py b/src/scancode_config.py index 247f9c5206..198e6b6d8e 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -191,13 +191,14 @@ def _create_dir(location): packagedcode_cache_dir = (__env_package_cache_dir or std_package_cache_dir) -std_resource_cache_dir = join(scancode_src_dir, 'commoncode', 'data', 'cache') +std_resource_cache_dir = join(scancode_src_dir, 'commoncode', 'data', 'cache', 'resource_cache_index') __env_resource_cache_dir = os.getenv('SCANCODE_RESOURCE_INDEX_CACHE') resource_cache_dir = (__env_resource_cache_dir or std_resource_cache_dir) _create_dir(licensedcode_cache_dir) _create_dir(packagedcode_cache_dir) _create_dir(scancode_cache_dir) +_create_dir(resource_cache_dir) # - scancode_temp_dir: for short-lived temporary files which are import- or run- # specific that may live for the duration of a function call or for the duration From 390510f7c32e0d74c2590d16e33d270f429db950 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 21 Aug 2026 15:41:32 -0700 Subject: [PATCH 03/11] Place resource cache into scancode cache dir Signed-off-by: Jono Yang --- src/scancode/plugin_resource_cache.py | 7 +++++-- src/scancode_config.py | 5 ----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/scancode/plugin_resource_cache.py b/src/scancode/plugin_resource_cache.py index 8e6581d796..2ebd598316 100644 --- a/src/scancode/plugin_resource_cache.py +++ b/src/scancode/plugin_resource_cache.py @@ -17,7 +17,10 @@ from commoncode.hash import binary_chunks from plugincode.scan import ScanPlugin from plugincode.scan import scan_impl -from scancode_config import resource_cache_dir +from scancode_config import scancode_cache_dir + + +RESOURCE_CACHE_DIR = os.path.join(scancode_cache_dir, "resource_cache_index") def hasher_from_chunks(chunks): @@ -55,7 +58,7 @@ def get_resource_cache_file_location(resource_cache_index): prefix1 = resource_cache_index[:2] prefix2 = resource_cache_index[2:4] filename = resource_cache_index[4:] - resource_cache_file_path = os.path.join(resource_cache_dir, prefix1, prefix2, filename) + resource_cache_file_path = os.path.join(RESOURCE_CACHE_DIR, prefix1, prefix2, filename) return resource_cache_file_path diff --git a/src/scancode_config.py b/src/scancode_config.py index 198e6b6d8e..6be7ea909c 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -191,14 +191,9 @@ def _create_dir(location): packagedcode_cache_dir = (__env_package_cache_dir or std_package_cache_dir) -std_resource_cache_dir = join(scancode_src_dir, 'commoncode', 'data', 'cache', 'resource_cache_index') -__env_resource_cache_dir = os.getenv('SCANCODE_RESOURCE_INDEX_CACHE') -resource_cache_dir = (__env_resource_cache_dir or std_resource_cache_dir) - _create_dir(licensedcode_cache_dir) _create_dir(packagedcode_cache_dir) _create_dir(scancode_cache_dir) -_create_dir(resource_cache_dir) # - scancode_temp_dir: for short-lived temporary files which are import- or run- # specific that may live for the duration of a function call or for the duration From 04bc04896d8ed5aa88e6c3e95233f23e7edbc35f Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 21 Aug 2026 18:31:46 -0700 Subject: [PATCH 04/11] Update scan_resource to get and save data to cache Signed-off-by: Jono Yang --- src/scancode/cli.py | 24 +++++- src/scancode/plugin_resource_cache.py | 101 -------------------------- src/scancode/resource_cache.py | 90 +++++++++++++++++++++++ 3 files changed, 113 insertions(+), 102 deletions(-) delete mode 100644 src/scancode/plugin_resource_cache.py create mode 100644 src/scancode/resource_cache.py diff --git a/src/scancode/cli.py b/src/scancode/cli.py index f7fe221c21..412e0fb793 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -67,6 +67,7 @@ class WindowsError(Exception): from scancode import notice from scancode import print_about from scancode import Scanner +from scancode import resource_cache from scancode.help import epilog_text from scancode.help import examples_text from scancode.interrupt import DEFAULT_TIMEOUT @@ -1493,6 +1494,7 @@ def scan_resource( results = {} scan_errors = [] timings = {} if with_timing else None + scanners_to_run = [] if not with_threading: interruptor = fake_interruptible @@ -1503,8 +1505,24 @@ def scan_resource( # and start returning values. The kill timeout is otherwise there # as a gatekeeper for runaway processes. - # run each scanner in sequence in its own interruptible + # compute resource_cache_index + resource_cache_index = resource_cache.compute_resource_cache_index(location=location, path=path) + + # update `results` with cached data or add scanner to scanners_to_run if no + # cache data is available for scanner in scanners: + # get resource_cache_data + resource_cache_data = resource_cache.get_resource_cache_data( + resource_cache_index=resource_cache_index, + plugin_name=scanner.name + ) + if resource_cache_data: + results.update(resource_cache_data) + else: + scanners_to_run.append(scanner) + + # run each scanner in sequence in its own interruptible + for scanner in scanners_to_run: if with_timing: start = time() @@ -1523,6 +1541,10 @@ def scan_resource( # the return value of a scanner fun MUST be a mapping if values_mapping: results.update(values_mapping) + resource_cache.update_resource_cache_data( + resource_cache_index=resource_cache_index, + plugin_name=scanner.name + ) except Exception: msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc() diff --git a/src/scancode/plugin_resource_cache.py b/src/scancode/plugin_resource_cache.py deleted file mode 100644 index 2ebd598316..0000000000 --- a/src/scancode/plugin_resource_cache.py +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import attr -import hashlib -import json -import os - -from commoncode.cliutils import PluggableCommandLineOption -from commoncode.cliutils import OTHER_SCAN_GROUP -from commoncode.hash import binary_chunks -from plugincode.scan import ScanPlugin -from plugincode.scan import scan_impl -from scancode_config import scancode_cache_dir - - -RESOURCE_CACHE_DIR = os.path.join(scancode_cache_dir, "resource_cache_index") - - -def hasher_from_chunks(chunks): - """ - Return a sha256 hasher loaded with `chunks` - """ - hasher = hashlib.sha256() - for chunk in chunks: - hasher.update(chunk) - return hasher - - -def compute_resource_cache_index(location, path, **kwargs): - """ - Compute resource cache index value for Resource at `location` - """ - chunks = binary_chunks(location=location) - sha256_hasher = hasher_from_chunks(chunks=chunks) - # TODO: consider using filename instead of path - sha256_hasher.update(path.encode('utf-8', 'surrogateescape')) - - result = { - 'resource_cache_index': sha256_hasher.hexdigest() - } - - return result - - -def get_resource_cache_file_location(resource_cache_index): - """ - Return the location of the cache file for a given `resource_cache_index` - hexstring. - """ - # Split the hash into two subdirectories using the first two prefix pairs - prefix1 = resource_cache_index[:2] - prefix2 = resource_cache_index[2:4] - filename = resource_cache_index[4:] - resource_cache_file_path = os.path.join(RESOURCE_CACHE_DIR, prefix1, prefix2, filename) - return resource_cache_file_path - - -@scan_impl -class ResourceCacheIndexScanner(ScanPlugin): - """ - Compute resource cache index value for Resources in Codebase - """ - resource_attributes = dict([ - ('resource_cache_index', attr.ib(default=None, repr=False)), - ]) - - run_order = 0 - sort_order = 0 - - options = [ - PluggableCommandLineOption(('-rc', '--resource_cache_index'), - is_flag=True, default=False, - help='Scan to compute resource cache index values for Resources in Codebase.', - help_group=OTHER_SCAN_GROUP, sort_order=0 - ) - ] - - def is_enabled(self, resource_cache_index, **kwargs): - return resource_cache_index - - def get_scanner(self, **kwargs): - return compute_resource_cache_index - - def process_codebase(self, codebase, **kwargs): - """ - Update Resource cache - """ - for resource in codebase: - # dump all resources to cache - resource_cache_file_location = get_resource_cache_file_location( - resource_cache_index=resource.resource_cache_index - ) - with open(resource_cache_file_location, 'w') as f: - f.write(json.dumps(resource.serialize(), check_circular=False)) diff --git a/src/scancode/resource_cache.py b/src/scancode/resource_cache.py new file mode 100644 index 0000000000..5ad957095f --- /dev/null +++ b/src/scancode/resource_cache.py @@ -0,0 +1,90 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import hashlib +import json +import os + +from commoncode.hash import binary_chunks +from scancode_config import scancode_cache_dir + + +RESOURCE_CACHE_DIR = os.path.join(scancode_cache_dir, "resource_cache_index") + + +def hasher_from_chunks(chunks): + """ + Return a sha256 hasher loaded with `chunks`. + """ + hasher = hashlib.sha256() + for chunk in chunks: + hasher.update(chunk) + return hasher + + +def compute_resource_cache_index(location, path): + """ + Compute resource_cache_index value for Resource at `location`. + """ + chunks = binary_chunks(location=location) + sha256_hasher = hasher_from_chunks(chunks=chunks) + # TODO: consider using filename instead of path + sha256_hasher.update(path.encode('utf-8', 'surrogateescape')) + return sha256_hasher.hexdigest() + + +def get_resource_cache_directory_location(resource_cache_index): + """ + Return the location of the directory containing the cache files for a given + `resource_cache_index` hexstring. + """ + # Split the hash into two subdirectories using the first two prefix pairs + prefix1 = resource_cache_index[:2] + prefix2 = resource_cache_index[2:4] + directory_name = resource_cache_index[4:] + return os.path.join(RESOURCE_CACHE_DIR, prefix1, prefix2, directory_name) + + +def get_resource_cache_file_location(resource_cache_index, plugin_name): + """ + Return the location of the file containing the cached results of the scanner + `plugin_name` for a resource keyed by `resource_cache_index` hexstring. + """ + resource_cache_directory_location = get_resource_cache_directory_location(resource_cache_index=resource_cache_index) + return os.path.join(resource_cache_directory_location, plugin_name) + + +def get_resource_cache_data(resource_cache_index, plugin_name): + """ + Return a mapping containing the results of scan plugin, `plugin_name`, for a + resource keyed by `resource_cache_index` hexstring. If the cache file does + not exist, an empty mapping is returned. + """ + resource_cache_file_location = get_resource_cache_file_location( + resource_cache_index=resource_cache_index, + plugin_name=plugin_name + ) + if os.path.exists(resource_cache_file_location): + with open(resource_cache_file_location) as f: + return json.load(f) + else: + return {} + + +def update_resource_cache_data(resource_cache_index, plugin_name, results): + """ + Update the resource cache with the `results` of the scanner `plugin_name` + for the resource keyed by `resource_cache_index`. + """ + resource_cache_file_location = get_resource_cache_file_location( + resource_cache_index=resource_cache_index, + plugin_name=plugin_name + ) + with open(resource_cache_file_location, 'w') as f: + json.dump(results, f) From 4149d826106ca1b798cb5dee0a041d7e9d9bb227 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 21 Aug 2026 19:36:10 -0700 Subject: [PATCH 05/11] Create directories before opening or creating cache files * Create envvar for controlling whether or not the scan results cache is used Signed-off-by: Jono Yang --- pyproject.toml | 1 - src/scancode/cli.py | 38 +++++++++++++++++++--------------- src/scancode/resource_cache.py | 9 +++++--- src/scancode_config.py | 3 +++ 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index acfc083728..9893c3c585 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -275,7 +275,6 @@ packages = "packagedcode.plugin_package:PackageScanner" emails = "cluecode.plugin_email:EmailScanner" urls = "cluecode.plugin_url:UrlScanner" generated = "summarycode.generated:GeneratedCodeDetector" -resource_cache_index = "scancode.plugin_resource_cache:ResourceCacheIndexScanner" # scancode_post_scan is the entry point for post_scan plugins executed after the diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 412e0fb793..ca8b133c82 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -74,6 +74,7 @@ class WindowsError(Exception): from scancode.interrupt import fake_interruptible from scancode.interrupt import interruptible from scancode.pool import ScanCodeTimeoutError +from scancode_config import USE_CACHED_RESULTS # Tracing flags TRACE = False @@ -1508,21 +1509,22 @@ def scan_resource( # compute resource_cache_index resource_cache_index = resource_cache.compute_resource_cache_index(location=location, path=path) - # update `results` with cached data or add scanner to scanners_to_run if no - # cache data is available - for scanner in scanners: - # get resource_cache_data - resource_cache_data = resource_cache.get_resource_cache_data( - resource_cache_index=resource_cache_index, - plugin_name=scanner.name - ) - if resource_cache_data: - results.update(resource_cache_data) - else: - scanners_to_run.append(scanner) + if USE_CACHED_RESULTS: + # update `results` with cached data or add scanner to scanners_to_run if no + # cache data is available + for scanner in scanners: + # get resource_cache_data + resource_cache_data = resource_cache.get_resource_cache_data( + resource_cache_index=resource_cache_index, + plugin_name=scanner.name + ) + if resource_cache_data: + results.update(resource_cache_data) + else: + scanners_to_run.append(scanner) # run each scanner in sequence in its own interruptible - for scanner in scanners_to_run: + for scanner in scanners_to_run or scanners: if with_timing: start = time() @@ -1541,10 +1543,12 @@ def scan_resource( # the return value of a scanner fun MUST be a mapping if values_mapping: results.update(values_mapping) - resource_cache.update_resource_cache_data( - resource_cache_index=resource_cache_index, - plugin_name=scanner.name - ) + if USE_CACHED_RESULTS: + resource_cache.update_resource_cache_data( + resource_cache_index=resource_cache_index, + plugin_name=scanner.name, + results=values_mapping, + ) except Exception: msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc() diff --git a/src/scancode/resource_cache.py b/src/scancode/resource_cache.py index 5ad957095f..4b3e796d2a 100644 --- a/src/scancode/resource_cache.py +++ b/src/scancode/resource_cache.py @@ -11,6 +11,7 @@ import json import os +from commoncode.fileutils import create_dir from commoncode.hash import binary_chunks from scancode_config import scancode_cache_dir @@ -82,9 +83,11 @@ def update_resource_cache_data(resource_cache_index, plugin_name, results): Update the resource cache with the `results` of the scanner `plugin_name` for the resource keyed by `resource_cache_index`. """ - resource_cache_file_location = get_resource_cache_file_location( - resource_cache_index=resource_cache_index, - plugin_name=plugin_name + resource_cache_directory_location = get_resource_cache_directory_location( + resource_cache_index=resource_cache_index ) + if not os.path.exists(resource_cache_directory_location): + create_dir(resource_cache_directory_location) + resource_cache_file_location = os.path.join(resource_cache_directory_location, plugin_name) with open(resource_cache_file_location, 'w') as f: json.dump(results, f) diff --git a/src/scancode_config.py b/src/scancode_config.py index 6be7ea909c..7119f028d0 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -221,3 +221,6 @@ def _create_dir(location): # Used for tests to regenerate fixtures with regen=True REGEN_TEST_FIXTURES = SCANCODE_REGEN_TEST_FIXTURES = os.getenv('SCANCODE_REGEN_TEST_FIXTURES', False) + +# Used to control whether or not we use cached results during scan time +USE_CACHED_RESULTS = SCANCODE_USE_CACHED_RESULTS = os.getenv('SCANCODE_USE_CACHED_RESULTS', True) From cc74305962196285da25db8d308d5d538c3c7aee Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 24 Aug 2026 17:44:23 -0700 Subject: [PATCH 06/11] Rename resource_cache_index to results_cache_index Signed-off-by: Jono Yang --- src/scancode/cli.py | 40 ++++++++++----- src/scancode/resource_cache.py | 93 ---------------------------------- src/scancode/results_cache.py | 90 ++++++++++++++++++++++++++++++++ src/scancode_config.py | 7 +-- 4 files changed, 122 insertions(+), 108 deletions(-) delete mode 100644 src/scancode/resource_cache.py create mode 100644 src/scancode/results_cache.py diff --git a/src/scancode/cli.py b/src/scancode/cli.py index ca8b133c82..877dcce897 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -67,14 +67,13 @@ class WindowsError(Exception): from scancode import notice from scancode import print_about from scancode import Scanner -from scancode import resource_cache +from scancode import results_cache from scancode.help import epilog_text from scancode.help import examples_text from scancode.interrupt import DEFAULT_TIMEOUT from scancode.interrupt import fake_interruptible from scancode.interrupt import interruptible from scancode.pool import ScanCodeTimeoutError -from scancode_config import USE_CACHED_RESULTS # Tracing flags TRACE = False @@ -415,6 +414,13 @@ def default_processes(): # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) + +@click.option('--no-cached-results', + is_flag=True, + default=False, + hidden=True, + help='ScanCode will not use cached results during scan time.', + help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption) def scancode( ctx, input, # NOQA @@ -435,6 +441,7 @@ def scancode( test_error_mode, keep_temp_files, check_version, + no_cached_results, echo_func=echo_stderr, *args, **kwargs, @@ -551,6 +558,7 @@ def scancode( return_results=False, echo_func=echo_func, outdated=outdated, + use_cached_results=not no_cached_results, *args, **kwargs ) @@ -596,6 +604,7 @@ def run_scan( pretty_params=None, plugin_options=plugin_options, outdated=None, + use_cached_results=False, *args, **kwargs ): @@ -1008,6 +1017,7 @@ def echo_func(*_args, **_kwargs): verbose=verbose, kwargs=requested_options, echo_func=echo_func, + use_cached_results=use_cached_results, ) success = success and scan_success @@ -1228,6 +1238,7 @@ def run_scanners( verbose=False, kwargs=None, echo_func=echo_stderr, + use_cached_results=False, ): """ Run the list of `stage` ScanPlugin `plugins` on `codebase`. @@ -1269,7 +1280,8 @@ def run_scanners( # TODO: add CLI option to bypass cache entirely? scan_success = scan_codebase( codebase, scanners, processes, timeout, - with_timing=timing, progress_manager=progress_manager) + with_timing=timing, progress_manager=progress_manager, + use_cached_results=use_cached_results) # TODO: add progress indicator # run the process codebase of each scan plugin (most often a no-op) @@ -1307,6 +1319,7 @@ def scan_codebase( with_timing=False, progress_manager=None, echo_func=echo_stderr, + use_cached_results=False, ): """ Run the `scanners` Scanner objects on the `codebase` Codebase. Return True @@ -1336,7 +1349,8 @@ def scan_codebase( scanners=scanners, timeout=timeout, with_timing=with_timing, - with_threading=use_threading + with_threading=use_threading, + use_cached_results=use_cached_results, ) if TRACE: @@ -1469,6 +1483,7 @@ def scan_resource( timeout=DEFAULT_TIMEOUT, with_timing=False, with_threading=True, + use_cached_results=False, ): """ Given a ``location_path`` tuple pf (location, path), return a tuple of: @@ -1506,16 +1521,17 @@ def scan_resource( # and start returning values. The kill timeout is otherwise there # as a gatekeeper for runaway processes. - # compute resource_cache_index - resource_cache_index = resource_cache.compute_resource_cache_index(location=location, path=path) + results_cache_index = '' + if use_cached_results: + # compute results_cache_index + results_cache_index = results_cache.compute_results_cache_index(location=location, path=path) - if USE_CACHED_RESULTS: # update `results` with cached data or add scanner to scanners_to_run if no # cache data is available for scanner in scanners: # get resource_cache_data - resource_cache_data = resource_cache.get_resource_cache_data( - resource_cache_index=resource_cache_index, + resource_cache_data = results_cache.get_results_cache_data( + results_cache_index=results_cache_index, plugin_name=scanner.name ) if resource_cache_data: @@ -1543,9 +1559,9 @@ def scan_resource( # the return value of a scanner fun MUST be a mapping if values_mapping: results.update(values_mapping) - if USE_CACHED_RESULTS: - resource_cache.update_resource_cache_data( - resource_cache_index=resource_cache_index, + if use_cached_results: + results_cache.update_results_cache_data( + results_cache_index=results_cache_index, plugin_name=scanner.name, results=values_mapping, ) diff --git a/src/scancode/resource_cache.py b/src/scancode/resource_cache.py deleted file mode 100644 index 4b3e796d2a..0000000000 --- a/src/scancode/resource_cache.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import hashlib -import json -import os - -from commoncode.fileutils import create_dir -from commoncode.hash import binary_chunks -from scancode_config import scancode_cache_dir - - -RESOURCE_CACHE_DIR = os.path.join(scancode_cache_dir, "resource_cache_index") - - -def hasher_from_chunks(chunks): - """ - Return a sha256 hasher loaded with `chunks`. - """ - hasher = hashlib.sha256() - for chunk in chunks: - hasher.update(chunk) - return hasher - - -def compute_resource_cache_index(location, path): - """ - Compute resource_cache_index value for Resource at `location`. - """ - chunks = binary_chunks(location=location) - sha256_hasher = hasher_from_chunks(chunks=chunks) - # TODO: consider using filename instead of path - sha256_hasher.update(path.encode('utf-8', 'surrogateescape')) - return sha256_hasher.hexdigest() - - -def get_resource_cache_directory_location(resource_cache_index): - """ - Return the location of the directory containing the cache files for a given - `resource_cache_index` hexstring. - """ - # Split the hash into two subdirectories using the first two prefix pairs - prefix1 = resource_cache_index[:2] - prefix2 = resource_cache_index[2:4] - directory_name = resource_cache_index[4:] - return os.path.join(RESOURCE_CACHE_DIR, prefix1, prefix2, directory_name) - - -def get_resource_cache_file_location(resource_cache_index, plugin_name): - """ - Return the location of the file containing the cached results of the scanner - `plugin_name` for a resource keyed by `resource_cache_index` hexstring. - """ - resource_cache_directory_location = get_resource_cache_directory_location(resource_cache_index=resource_cache_index) - return os.path.join(resource_cache_directory_location, plugin_name) - - -def get_resource_cache_data(resource_cache_index, plugin_name): - """ - Return a mapping containing the results of scan plugin, `plugin_name`, for a - resource keyed by `resource_cache_index` hexstring. If the cache file does - not exist, an empty mapping is returned. - """ - resource_cache_file_location = get_resource_cache_file_location( - resource_cache_index=resource_cache_index, - plugin_name=plugin_name - ) - if os.path.exists(resource_cache_file_location): - with open(resource_cache_file_location) as f: - return json.load(f) - else: - return {} - - -def update_resource_cache_data(resource_cache_index, plugin_name, results): - """ - Update the resource cache with the `results` of the scanner `plugin_name` - for the resource keyed by `resource_cache_index`. - """ - resource_cache_directory_location = get_resource_cache_directory_location( - resource_cache_index=resource_cache_index - ) - if not os.path.exists(resource_cache_directory_location): - create_dir(resource_cache_directory_location) - resource_cache_file_location = os.path.join(resource_cache_directory_location, plugin_name) - with open(resource_cache_file_location, 'w') as f: - json.dump(results, f) diff --git a/src/scancode/results_cache.py b/src/scancode/results_cache.py new file mode 100644 index 0000000000..fcb6db87ef --- /dev/null +++ b/src/scancode/results_cache.py @@ -0,0 +1,90 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import hashlib +import json +import os + +from commoncode.fileutils import create_dir +from commoncode.hash import binary_chunks +from scancode_config import results_cache_dir + + +def hasher_from_chunks(chunks): + """ + Return a sha256 hasher loaded with `chunks`. + """ + hasher = hashlib.sha256() + for chunk in chunks: + hasher.update(chunk) + return hasher + + +def compute_results_cache_index(location, path): + """ + Compute results_cache_index value for a Resource at `location`. + """ + chunks = binary_chunks(location=location) + sha256_hasher = hasher_from_chunks(chunks=chunks) + # TODO: consider using filename instead of path + sha256_hasher.update(path.encode('utf-8', 'surrogateescape')) + return sha256_hasher.hexdigest() + + +def get_results_cache_directory_location(results_cache_index): + """ + Return the location of the directory containing the cache files for a given + `results_cache_index` hexstring. + """ + # Split the hash into two subdirectories using the first two prefix pairs + prefix1 = results_cache_index[:2] + prefix2 = results_cache_index[2:4] + directory_name = results_cache_index[4:] + return os.path.join(results_cache_dir, prefix1, prefix2, directory_name) + + +def get_results_cache_file_location(results_cache_index, plugin_name): + """ + Return the location of the file containing the cached results of the scanner + `plugin_name` for a resource keyed by `results_cache_index` hexstring. + """ + results_cache_directory_location = get_results_cache_directory_location(results_cache_index=results_cache_index) + return os.path.join(results_cache_directory_location, plugin_name) + + +def get_results_cache_data(results_cache_index, plugin_name): + """ + Return a mapping containing the results of scan plugin, `plugin_name`, for a + resource keyed by `resource_cache_index` hexstring. If the cache file does + not exist, an empty mapping is returned. + """ + results_cache_file_location = get_results_cache_file_location( + results_cache_index=results_cache_index, + plugin_name=plugin_name + ) + if os.path.exists(results_cache_file_location): + with open(results_cache_file_location) as f: + return json.load(f) + else: + return {} + + +def update_results_cache_data(results_cache_index, plugin_name, results): + """ + Update the results cache with the `results` of the scanner `plugin_name` + for the resource keyed by `results_cache_index`. + """ + results_cache_directory_location = get_results_cache_directory_location( + results_cache_index=results_cache_index + ) + if not os.path.exists(results_cache_directory_location): + create_dir(results_cache_directory_location) + results_cache_file_location = os.path.join(results_cache_directory_location, plugin_name) + with open(results_cache_file_location, 'w') as f: + json.dump(results, f) diff --git a/src/scancode_config.py b/src/scancode_config.py index 7119f028d0..c020494f6f 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -190,10 +190,14 @@ def _create_dir(location): __env_package_cache_dir = os.getenv('SCANCODE_PACKAGE_INDEX_CACHE') packagedcode_cache_dir = (__env_package_cache_dir or std_package_cache_dir) +std_results_cache_dir = join(scancode_cache_dir, 'results') +__env_results_cache_dir = os.getenv('SCANCODE_RESULTS_CACHE') +results_cache_dir = (__env_results_cache_dir or std_results_cache_dir) _create_dir(licensedcode_cache_dir) _create_dir(packagedcode_cache_dir) _create_dir(scancode_cache_dir) +_create_dir(results_cache_dir) # - scancode_temp_dir: for short-lived temporary files which are import- or run- # specific that may live for the duration of a function call or for the duration @@ -221,6 +225,3 @@ def _create_dir(location): # Used for tests to regenerate fixtures with regen=True REGEN_TEST_FIXTURES = SCANCODE_REGEN_TEST_FIXTURES = os.getenv('SCANCODE_REGEN_TEST_FIXTURES', False) - -# Used to control whether or not we use cached results during scan time -USE_CACHED_RESULTS = SCANCODE_USE_CACHED_RESULTS = os.getenv('SCANCODE_USE_CACHED_RESULTS', True) From d032e0d6337b1d86c760f0de3644a402a50c0ab9 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 24 Aug 2026 18:37:25 -0700 Subject: [PATCH 07/11] Change option no-cached-results to use-cached-results Signed-off-by: Jono Yang --- src/scancode/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 877dcce897..00695097c7 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -415,11 +415,11 @@ def default_processes(): hidden=True, help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) -@click.option('--no-cached-results', +@click.option('--use-cached-results', is_flag=True, default=False, hidden=True, - help='ScanCode will not use cached results during scan time.', + help='ScanCode will use cached results during scan time.', help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption) def scancode( ctx, @@ -441,7 +441,7 @@ def scancode( test_error_mode, keep_temp_files, check_version, - no_cached_results, + use_cached_results, echo_func=echo_stderr, *args, **kwargs, @@ -558,7 +558,7 @@ def scancode( return_results=False, echo_func=echo_func, outdated=outdated, - use_cached_results=not no_cached_results, + use_cached_results=use_cached_results, *args, **kwargs ) From 8ef121c0dec425a52ddd17c91bb1f9d7d4516d3a Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 25 Aug 2026 01:48:13 -0700 Subject: [PATCH 08/11] Fix logic where we run scanners after getting cached results Signed-off-by: Jono Yang --- src/scancode/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 00695097c7..5d186177ef 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -1538,9 +1538,11 @@ def scan_resource( results.update(resource_cache_data) else: scanners_to_run.append(scanner) + else: + scanners_to_run = scanners # run each scanner in sequence in its own interruptible - for scanner in scanners_to_run or scanners: + for scanner in scanners_to_run: if with_timing: start = time() From 51cf6d744bd393910608f334851926c1fe055f8d Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 25 Aug 2026 13:15:36 -0700 Subject: [PATCH 09/11] Create test for use-cached-results cli option Signed-off-by: Jono Yang --- .../scancode/data/results_cache/expected.json | 605 ++++++++++++++++++ .../scancode/data/results_cache/package.json | 96 +++ .../copyrights | 1 + .../emails | 1 + .../info | 1 + .../licenses | 1 + .../packages | 1 + .../urls | 1 + tests/scancode/test_cli.py | 25 + 9 files changed, 732 insertions(+) create mode 100755 tests/scancode/data/results_cache/expected.json create mode 100644 tests/scancode/data/results_cache/package.json create mode 100644 tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights create mode 100644 tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails create mode 100644 tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info create mode 100644 tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses create mode 100644 tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages create mode 100644 tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls diff --git a/tests/scancode/data/results_cache/expected.json b/tests/scancode/data/results_cache/expected.json new file mode 100755 index 0000000000..3385fe4869 --- /dev/null +++ b/tests/scancode/data/results_cache/expected.json @@ -0,0 +1,605 @@ +{ + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "async", + "version": "1.2.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": "Higher-order functions and common patterns for asynchronous code", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Caolan McMahon", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "caolan", + "email": "caolan.mcmahon@gmail.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "beaugunderson", + "email": "beau@beaugunderson.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "aearly", + "email": "alexander.early@gmail.com", + "url": null + } + ], + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "homepage_url": "https://github.com/caolan/async#readme", + "download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "size": null, + "sha1": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/caolan/async/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/caolan/async.git@b66e85d1cca8c8056313253f22d18f571e7001d2", + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- MIT\n", + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "repository_homepage_url": "https://www.npmjs.com/package/async", + "repository_download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/async/1.2.1", + "package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "package.json" + ], + "datasource_ids": [ + "npm_package_json" + ], + "purl": "pkg:npm/async@1.2.1" + } + ], + "dependencies": [ + { + "purl": "pkg:npm/benchmark", + "extracted_requirement": "github:bestiejs/benchmark.js", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/benchmark?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/coveralls", + "extracted_requirement": "^2.11.2", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/coveralls?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/jshint", + "extracted_requirement": "~2.7.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/jshint?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": ">=2.4.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/lodash?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/mkdirp", + "extracted_requirement": "~0.5.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/mkdirp?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/nodeunit", + "extracted_requirement": ">0.0.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/nodeunit?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/nyc", + "extracted_requirement": "^2.1.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/nyc?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/uglify-js", + "extracted_requirement": "1.2.x", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/uglify-js?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/yargs", + "extracted_requirement": "~3.9.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/yargs?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + } + ], + "license_detections": [ + { + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", + "license_expression": "mit", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 22, + "end_line": 22, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "mit_30.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE" + } + ] + }, + { + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf", + "license_expression": "mit", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null + } + ] + } + ], + "files": [ + { + "path": "package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 2361, + "sha1": "29c3cacb4c3abe6f69dc87a77d9c9105546f6978", + "md5": "b8be873c4fb54a835bfac7eb47a6531e", + "sha256": "cb6f5a82e473620da4d1aecf82dd4d4fa9ada393a7679b28a42cac86f0a83c92", + "sha1_git": "247e1784187cc3929165ec2d534f2134b9147ecd", + "mime_type": "application/json", + "file_type": "JSON data", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "package_data": [ + { + "type": "npm", + "namespace": null, + "name": "async", + "version": "1.2.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": "Higher-order functions and common patterns for asynchronous code", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Caolan McMahon", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "caolan", + "email": "caolan.mcmahon@gmail.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "beaugunderson", + "email": "beau@beaugunderson.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "aearly", + "email": "alexander.early@gmail.com", + "url": null + } + ], + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "homepage_url": "https://github.com/caolan/async#readme", + "download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "size": null, + "sha1": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/caolan/async/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/caolan/async.git@b66e85d1cca8c8056313253f22d18f571e7001d2", + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- MIT\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/benchmark", + "extracted_requirement": "github:bestiejs/benchmark.js", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/coveralls", + "extracted_requirement": "^2.11.2", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/jshint", + "extracted_requirement": "~2.7.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": ">=2.4.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/mkdirp", + "extracted_requirement": "~0.5.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/nodeunit", + "extracted_requirement": ">0.0.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/nyc", + "extracted_requirement": "^2.1.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/uglify-js", + "extracted_requirement": "1.2.x", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/yargs", + "extracted_requirement": "~3.9.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/async", + "repository_download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/async/1.2.1", + "datasource_id": "npm_package_json", + "purl": "pkg:npm/async@1.2.1" + } + ], + "for_packages": [ + "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 22, + "end_line": 22, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "mit_30.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE" + } + ], + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee" + } + ], + "license_clues": [], + "percentage_of_license_text": 0.83, + "copyrights": [], + "holders": [], + "authors": [ + { + "author": "Caolan McMahon", + "start_line": 5, + "end_line": 6 + } + ], + "emails": [ + { + "email": "alexander.early@gmail.com", + "start_line": 73, + "end_line": 73 + }, + { + "email": "caolan.mcmahon@gmail.com", + "start_line": 78, + "end_line": 78 + }, + { + "email": "beau@beaugunderson.com", + "start_line": 82, + "end_line": 82 + } + ], + "urls": [ + { + "url": "https://github.com/caolan/async.git", + "start_line": 17, + "end_line": 17 + }, + { + "url": "https://github.com/caolan/async/issues", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/caolan/async#readme", + "start_line": 65, + "end_line": 65 + }, + { + "url": "http://registry.npmjs.org/async/-/async-1.2.1.tgz", + "start_line": 91, + "end_line": 91 + }, + { + "url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "start_line": 94, + "end_line": 94 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/package.json b/tests/scancode/data/results_cache/package.json new file mode 100644 index 0000000000..247e178418 --- /dev/null +++ b/tests/scancode/data/results_cache/package.json @@ -0,0 +1,96 @@ +{ + "name": "async", + "description": "Higher-order functions and common patterns for asynchronous code", + "main": "lib/async.js", + "author": { + "name": "Caolan McMahon" + }, + "version": "1.2.1", + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/caolan/async.git" + }, + "bugs": { + "url": "https://github.com/caolan/async/issues" + }, + "license": "MIT", + "devDependencies": { + "benchmark": "github:bestiejs/benchmark.js", + "coveralls": "^2.11.2", + "jshint": "~2.7.0", + "lodash": ">=2.4.1", + "mkdirp": "~0.5.1", + "nodeunit": ">0.0.0", + "nyc": "^2.1.0", + "uglify-js": "1.2.x", + "yargs": "~3.9.1" + }, + "jam": { + "main": "lib/async.js", + "include": [ + "lib/async.js", + "README.md", + "LICENSE" + ], + "categories": [ + "Utilities" + ] + }, + "scripts": { + "test": "npm run-script lint && nodeunit test/test-async.js", + "lint": "jshint lib/*.js test/*.js perf/*.js", + "coverage": "nyc npm test && nyc report", + "coveralls": "nyc npm test && nyc report --reporter=text-lcov | coveralls" + }, + "spm": { + "main": "lib/async.js" + }, + "volo": { + "main": "lib/async.js", + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ] + }, + "gitHead": "b66e85d1cca8c8056313253f22d18f571e7001d2", + "homepage": "https://github.com/caolan/async#readme", + "_id": "async@1.2.1", + "_shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "_from": "async@*", + "_npmVersion": "2.9.0", + "_nodeVersion": "2.0.2", + "_npmUser": { + "name": "aearly", + "email": "alexander.early@gmail.com" + }, + "maintainers": [ + { + "name": "caolan", + "email": "caolan.mcmahon@gmail.com" + }, + { + "name": "beaugunderson", + "email": "beau@beaugunderson.com" + }, + { + "name": "aearly", + "email": "alexander.early@gmail.com" + } + ], + "dist": { + "shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "tarball": "http://registry.npmjs.org/async/-/async-1.2.1.tgz" + }, + "directories": {}, + "_resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "readme": "ERROR: No README data found!" +} diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights new file mode 100644 index 0000000000..08d741e004 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights @@ -0,0 +1 @@ +{"copyrights": [], "holders": [], "authors": [{"author": "Caolan McMahon", "start_line": 5, "end_line": 6}]} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails new file mode 100644 index 0000000000..affed944a5 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails @@ -0,0 +1 @@ +{"emails": [{"email": "alexander.early@gmail.com", "start_line": 73, "end_line": 73}, {"email": "caolan.mcmahon@gmail.com", "start_line": 78, "end_line": 78}, {"email": "beau@beaugunderson.com", "start_line": 82, "end_line": 82}]} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info new file mode 100644 index 0000000000..d17acb8452 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info @@ -0,0 +1 @@ +{"date": "2024-06-23", "size": 2361, "sha1": "29c3cacb4c3abe6f69dc87a77d9c9105546f6978", "md5": "b8be873c4fb54a835bfac7eb47a6531e", "sha256": "cb6f5a82e473620da4d1aecf82dd4d4fa9ada393a7679b28a42cac86f0a83c92", "sha1_git": "247e1784187cc3929165ec2d534f2134b9147ecd", "mime_type": "application/json", "file_type": "JSON data", "programming_language": null, "is_binary": false, "is_text": true, "is_archive": false, "is_media": false, "is_source": false, "is_script": false} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses new file mode 100644 index 0000000000..dbe7ffb67b --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses @@ -0,0 +1 @@ +{"detected_license_expression": "mit", "detected_license_expression_spdx": "MIT", "license_detections": [{"license_expression": "mit", "license_expression_spdx": "MIT", "matches": [{"license_expression": "mit", "license_expression_spdx": "MIT", "from_file": null, "start_line": 22, "end_line": 22, "matcher": "2-aho", "score": 100.0, "matched_length": 2, "match_coverage": 100.0, "rule_relevance": 100, "rule_identifier": "mit_30.RULE", "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE"}], "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee"}], "license_clues": [], "percentage_of_license_text": 0.83} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages new file mode 100644 index 0000000000..f1833bd934 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages @@ -0,0 +1 @@ +{"package_data": [{"type": "npm", "namespace": null, "name": "async", "version": "1.2.1", "qualifiers": {}, "subpath": null, "primary_language": "JavaScript", "description": "Higher-order functions and common patterns for asynchronous code", "release_date": null, "parties": [{"type": "person", "role": "author", "name": "Caolan McMahon", "email": null, "url": null}, {"type": "person", "role": "maintainer", "name": "caolan", "email": "caolan.mcmahon@gmail.com", "url": null}, {"type": "person", "role": "maintainer", "name": "beaugunderson", "email": "beau@beaugunderson.com", "url": null}, {"type": "person", "role": "maintainer", "name": "aearly", "email": "alexander.early@gmail.com", "url": null}], "keywords": ["async", "callback", "utility", "module"], "homepage_url": "https://github.com/caolan/async#readme", "download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", "size": null, "sha1": "a4816a17cd5ff516dfa2c7698a453369b9790de0", "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": "https://github.com/caolan/async/issues", "code_view_url": null, "vcs_url": "git+https://github.com/caolan/async.git@b66e85d1cca8c8056313253f22d18f571e7001d2", "copyright": null, "holder": null, "declared_license_expression": "mit", "declared_license_expression_spdx": "MIT", "license_detections": [{"license_expression": "mit", "license_expression_spdx": "MIT", "matches": [{"license_expression": "mit", "license_expression_spdx": "MIT", "from_file": null, "start_line": 1, "end_line": 1, "matcher": "1-spdx-id", "score": 100.0, "matched_length": 1, "match_coverage": 100.0, "rule_relevance": 100, "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", "rule_url": null, "matched_text": "MIT"}], "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf"}], "other_license_expression": null, "other_license_expression_spdx": null, "other_license_detections": [], "extracted_license_statement": "- MIT\n", "notice_text": null, "source_packages": [], "file_references": [], "is_private": false, "is_virtual": false, "extra_data": {}, "dependencies": [{"purl": "pkg:npm/benchmark", "extracted_requirement": "github:bestiejs/benchmark.js", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/coveralls", "extracted_requirement": "^2.11.2", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/jshint", "extracted_requirement": "~2.7.0", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/lodash", "extracted_requirement": ">=2.4.1", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/mkdirp", "extracted_requirement": "~0.5.1", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/nodeunit", "extracted_requirement": ">0.0.0", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/nyc", "extracted_requirement": "^2.1.0", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/uglify-js", "extracted_requirement": "1.2.x", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/yargs", "extracted_requirement": "~3.9.1", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}], "repository_homepage_url": "https://www.npmjs.com/package/async", "repository_download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", "api_data_url": "https://registry.npmjs.org/async/1.2.1", "datasource_id": "npm_package_json", "purl": "pkg:npm/async@1.2.1"}]} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls new file mode 100644 index 0000000000..8df21ee182 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls @@ -0,0 +1 @@ +{"urls": [{"url": "https://github.com/caolan/async.git", "start_line": 17, "end_line": 17}, {"url": "https://github.com/caolan/async/issues", "start_line": 20, "end_line": 20}, {"url": "https://github.com/caolan/async#readme", "start_line": 65, "end_line": 65}, {"url": "http://registry.npmjs.org/async/-/async-1.2.1.tgz", "start_line": 91, "end_line": 91}, {"url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", "start_line": 94, "end_line": 94}]} \ No newline at end of file diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index b5512dee31..9f97f8fb45 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -1052,3 +1052,28 @@ def test_scan_does_validate_input_and_fails_on_faulty_json_input(test_file, expe def test_scan_does_validate_input_and_does_not_fail_on_valid_json_input(): test_file = test_env.get_test_loc('various-inputs/true-scan-json.json') run_scan_click(['--from-json', test_file, '--json-pp', '-'], retry=False) + + +def test_use_cached_results(): + results_cache_location = test_env.get_test_loc(u'results_cache/results/') + test_file = test_env.get_test_loc(u'results_cache/package.json') + result_file = test_env.get_temp_file('json') + env = { + 'SCANCODE_RESULTS_CACHE': results_cache_location + } + args = [ + '--info', + '--license', + '--copyright', + '--package', + '--email', + '--url', + '--strip-root', + '--use-cached-results', + test_file , + '--json', + result_file + ] + results = run_scan_plain(options=args, env=env) + expected = test_env.get_test_loc('results_cache/expected.json') + check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES) From 68ec4be2671efbca2839fbdd6531a545c7eb309a Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 25 Aug 2026 13:37:11 -0700 Subject: [PATCH 10/11] Create results_cache_index from filename instead of path Signed-off-by: Jono Yang --- src/scancode/cli.py | 8 ++++---- src/scancode/results_cache.py | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/scancode/cli.py b/src/scancode/cli.py index 5d186177ef..fe94d7da4b 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -1341,7 +1341,7 @@ def scan_codebase( """ # NOTE: we never scan directories - resources = ((r.location, r.path) for r in codebase.walk() if r.is_file) + resources = ((r.location, r.path, r.name) for r in codebase.walk() if r.is_file) use_threading = processes >= 0 runner = partial( @@ -1478,7 +1478,7 @@ def terminate_pool_with_backoff(pool, number_of_trials=3): def scan_resource( - location_path, + location_path_name, scanners, timeout=DEFAULT_TIMEOUT, with_timing=False, @@ -1506,7 +1506,7 @@ def scan_resource( processing and threading works. """ scan_time = time() - location, path = location_path + location, path, name = location_path_name results = {} scan_errors = [] timings = {} if with_timing else None @@ -1524,7 +1524,7 @@ def scan_resource( results_cache_index = '' if use_cached_results: # compute results_cache_index - results_cache_index = results_cache.compute_results_cache_index(location=location, path=path) + results_cache_index = results_cache.compute_results_cache_index(location=location, filename=name) # update `results` with cached data or add scanner to scanners_to_run if no # cache data is available diff --git a/src/scancode/results_cache.py b/src/scancode/results_cache.py index fcb6db87ef..15549e259e 100644 --- a/src/scancode/results_cache.py +++ b/src/scancode/results_cache.py @@ -26,14 +26,13 @@ def hasher_from_chunks(chunks): return hasher -def compute_results_cache_index(location, path): +def compute_results_cache_index(location, filename): """ Compute results_cache_index value for a Resource at `location`. """ chunks = binary_chunks(location=location) sha256_hasher = hasher_from_chunks(chunks=chunks) - # TODO: consider using filename instead of path - sha256_hasher.update(path.encode('utf-8', 'surrogateescape')) + sha256_hasher.update(filename.encode('utf-8', 'surrogateescape')) return sha256_hasher.hexdigest() From 7336b824b20f4ca0f0d25c9db254af66bd57cd56 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 25 Aug 2026 14:54:53 -0700 Subject: [PATCH 11/11] Update CHANGELOG.rst Signed-off-by: Jono Yang --- CHANGELOG.rst | 2 ++ src/scancode/cli.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9a5a6b402..dbb8d073fd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Add experimental option for using cached results during scan time. + v33.0.0rc1 - 2026-05-14 ------------------------ diff --git a/src/scancode/cli.py b/src/scancode/cli.py index fe94d7da4b..2aca3d81c9 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -408,8 +408,8 @@ def default_processes(): help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) @click.option( - "--check-version/--no-check-version", - help="Whether to check for new versions. Defaults to true.", + '--check-version/--no-check-version', + help='Whether to check for new versions. Defaults to true.', default=True, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, @@ -419,7 +419,7 @@ def default_processes(): is_flag=True, default=False, hidden=True, - help='ScanCode will use cached results during scan time.', + help='(EXPERIMENTAL) ScanCode will use cached results during scan time.', help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption) def scancode( ctx, @@ -580,7 +580,7 @@ def scancode( def run_scan( - input, # + input, # config_file=None, ignore=[], from_json=False, @@ -1153,7 +1153,7 @@ def load_configuration_file(path): click.echo(f"Loading env from {path}") try: - + config_values = saneyaml.load(path.read()) ignores = config_values.get("ignored_patterns", []) except (saneyaml.YAMLError, Exception):