diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9a5a6b402..dbb8d073fd 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,8 @@ Next release ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 +- Add experimental option for using cached results during scan time. + v33.0.0rc1 - 2026-05-14 ------------------------ diff --git a/pyproject.toml b/pyproject.toml index f2371c6bb2..9893c3c585 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -164,7 +164,6 @@ files = [ { filename = "pyproject.toml" }, { filename = "pyproject-scancode-toolkit-mini.toml" }, { filename = "pyproject-packagedcode.toml" }, - ] diff --git a/src/scancode/cli.py b/src/scancode/cli.py index f7fe221c21..2aca3d81c9 100644 --- a/src/scancode/cli.py +++ b/src/scancode/cli.py @@ -67,6 +67,7 @@ class WindowsError(Exception): from scancode import notice from scancode import print_about from scancode import Scanner +from scancode import results_cache from scancode.help import epilog_text from scancode.help import examples_text from scancode.interrupt import DEFAULT_TIMEOUT @@ -407,12 +408,19 @@ def default_processes(): help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) @click.option( - "--check-version/--no-check-version", - help="Whether to check for new versions. Defaults to true.", + '--check-version/--no-check-version', + help='Whether to check for new versions. Defaults to true.', default=True, # not yet supported in Click 6.7 but added in PluggableCommandLineOption hidden=True, help_group=cliutils.MISC_GROUP, sort_order=1000, cls=PluggableCommandLineOption) + +@click.option('--use-cached-results', + is_flag=True, + default=False, + hidden=True, + help='(EXPERIMENTAL) ScanCode will use cached results during scan time.', + help_group=cliutils.CORE_GROUP, sort_order=250, cls=PluggableCommandLineOption) def scancode( ctx, input, # NOQA @@ -433,6 +441,7 @@ def scancode( test_error_mode, keep_temp_files, check_version, + use_cached_results, echo_func=echo_stderr, *args, **kwargs, @@ -549,6 +558,7 @@ def scancode( return_results=False, echo_func=echo_func, outdated=outdated, + use_cached_results=use_cached_results, *args, **kwargs ) @@ -570,7 +580,7 @@ def scancode( def run_scan( - input, # + input, # config_file=None, ignore=[], from_json=False, @@ -594,6 +604,7 @@ def run_scan( pretty_params=None, plugin_options=plugin_options, outdated=None, + use_cached_results=False, *args, **kwargs ): @@ -1006,6 +1017,7 @@ def echo_func(*_args, **_kwargs): verbose=verbose, kwargs=requested_options, echo_func=echo_func, + use_cached_results=use_cached_results, ) success = success and scan_success @@ -1141,7 +1153,7 @@ def load_configuration_file(path): click.echo(f"Loading env from {path}") try: - + config_values = saneyaml.load(path.read()) ignores = config_values.get("ignored_patterns", []) except (saneyaml.YAMLError, Exception): @@ -1226,6 +1238,7 @@ def run_scanners( verbose=False, kwargs=None, echo_func=echo_stderr, + use_cached_results=False, ): """ Run the list of `stage` ScanPlugin `plugins` on `codebase`. @@ -1267,7 +1280,8 @@ def run_scanners( # TODO: add CLI option to bypass cache entirely? scan_success = scan_codebase( codebase, scanners, processes, timeout, - with_timing=timing, progress_manager=progress_manager) + with_timing=timing, progress_manager=progress_manager, + use_cached_results=use_cached_results) # TODO: add progress indicator # run the process codebase of each scan plugin (most often a no-op) @@ -1305,6 +1319,7 @@ def scan_codebase( with_timing=False, progress_manager=None, echo_func=echo_stderr, + use_cached_results=False, ): """ Run the `scanners` Scanner objects on the `codebase` Codebase. Return True @@ -1326,7 +1341,7 @@ def scan_codebase( """ # NOTE: we never scan directories - resources = ((r.location, r.path) for r in codebase.walk() if r.is_file) + resources = ((r.location, r.path, r.name) for r in codebase.walk() if r.is_file) use_threading = processes >= 0 runner = partial( @@ -1334,7 +1349,8 @@ def scan_codebase( scanners=scanners, timeout=timeout, with_timing=with_timing, - with_threading=use_threading + with_threading=use_threading, + use_cached_results=use_cached_results, ) if TRACE: @@ -1462,11 +1478,12 @@ def terminate_pool_with_backoff(pool, number_of_trials=3): def scan_resource( - location_path, + location_path_name, scanners, timeout=DEFAULT_TIMEOUT, with_timing=False, with_threading=True, + use_cached_results=False, ): """ Given a ``location_path`` tuple pf (location, path), return a tuple of: @@ -1489,10 +1506,11 @@ def scan_resource( processing and threading works. """ scan_time = time() - location, path = location_path + location, path, name = location_path_name results = {} scan_errors = [] timings = {} if with_timing else None + scanners_to_run = [] if not with_threading: interruptor = fake_interruptible @@ -1503,8 +1521,28 @@ def scan_resource( # and start returning values. The kill timeout is otherwise there # as a gatekeeper for runaway processes. + results_cache_index = '' + if use_cached_results: + # compute results_cache_index + results_cache_index = results_cache.compute_results_cache_index(location=location, filename=name) + + # update `results` with cached data or add scanner to scanners_to_run if no + # cache data is available + for scanner in scanners: + # get resource_cache_data + resource_cache_data = results_cache.get_results_cache_data( + results_cache_index=results_cache_index, + plugin_name=scanner.name + ) + if resource_cache_data: + results.update(resource_cache_data) + else: + scanners_to_run.append(scanner) + else: + scanners_to_run = scanners + # run each scanner in sequence in its own interruptible - for scanner in scanners: + for scanner in scanners_to_run: if with_timing: start = time() @@ -1523,6 +1561,12 @@ def scan_resource( # the return value of a scanner fun MUST be a mapping if values_mapping: results.update(values_mapping) + if use_cached_results: + results_cache.update_results_cache_data( + results_cache_index=results_cache_index, + plugin_name=scanner.name, + results=values_mapping, + ) except Exception: msg = 'ERROR: for scanner: ' + scanner.name + ':\n' + traceback.format_exc() diff --git a/src/scancode/results_cache.py b/src/scancode/results_cache.py new file mode 100644 index 0000000000..15549e259e --- /dev/null +++ b/src/scancode/results_cache.py @@ -0,0 +1,89 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import hashlib +import json +import os + +from commoncode.fileutils import create_dir +from commoncode.hash import binary_chunks +from scancode_config import results_cache_dir + + +def hasher_from_chunks(chunks): + """ + Return a sha256 hasher loaded with `chunks`. + """ + hasher = hashlib.sha256() + for chunk in chunks: + hasher.update(chunk) + return hasher + + +def compute_results_cache_index(location, filename): + """ + Compute results_cache_index value for a Resource at `location`. + """ + chunks = binary_chunks(location=location) + sha256_hasher = hasher_from_chunks(chunks=chunks) + sha256_hasher.update(filename.encode('utf-8', 'surrogateescape')) + return sha256_hasher.hexdigest() + + +def get_results_cache_directory_location(results_cache_index): + """ + Return the location of the directory containing the cache files for a given + `results_cache_index` hexstring. + """ + # Split the hash into two subdirectories using the first two prefix pairs + prefix1 = results_cache_index[:2] + prefix2 = results_cache_index[2:4] + directory_name = results_cache_index[4:] + return os.path.join(results_cache_dir, prefix1, prefix2, directory_name) + + +def get_results_cache_file_location(results_cache_index, plugin_name): + """ + Return the location of the file containing the cached results of the scanner + `plugin_name` for a resource keyed by `results_cache_index` hexstring. + """ + results_cache_directory_location = get_results_cache_directory_location(results_cache_index=results_cache_index) + return os.path.join(results_cache_directory_location, plugin_name) + + +def get_results_cache_data(results_cache_index, plugin_name): + """ + Return a mapping containing the results of scan plugin, `plugin_name`, for a + resource keyed by `resource_cache_index` hexstring. If the cache file does + not exist, an empty mapping is returned. + """ + results_cache_file_location = get_results_cache_file_location( + results_cache_index=results_cache_index, + plugin_name=plugin_name + ) + if os.path.exists(results_cache_file_location): + with open(results_cache_file_location) as f: + return json.load(f) + else: + return {} + + +def update_results_cache_data(results_cache_index, plugin_name, results): + """ + Update the results cache with the `results` of the scanner `plugin_name` + for the resource keyed by `results_cache_index`. + """ + results_cache_directory_location = get_results_cache_directory_location( + results_cache_index=results_cache_index + ) + if not os.path.exists(results_cache_directory_location): + create_dir(results_cache_directory_location) + results_cache_file_location = os.path.join(results_cache_directory_location, plugin_name) + with open(results_cache_file_location, 'w') as f: + json.dump(results, f) diff --git a/src/scancode_config.py b/src/scancode_config.py index 6e9f634b08..c020494f6f 100644 --- a/src/scancode_config.py +++ b/src/scancode_config.py @@ -190,9 +190,14 @@ def _create_dir(location): __env_package_cache_dir = os.getenv('SCANCODE_PACKAGE_INDEX_CACHE') packagedcode_cache_dir = (__env_package_cache_dir or std_package_cache_dir) +std_results_cache_dir = join(scancode_cache_dir, 'results') +__env_results_cache_dir = os.getenv('SCANCODE_RESULTS_CACHE') +results_cache_dir = (__env_results_cache_dir or std_results_cache_dir) + _create_dir(licensedcode_cache_dir) _create_dir(packagedcode_cache_dir) _create_dir(scancode_cache_dir) +_create_dir(results_cache_dir) # - scancode_temp_dir: for short-lived temporary files which are import- or run- # specific that may live for the duration of a function call or for the duration diff --git a/tests/scancode/data/results_cache/expected.json b/tests/scancode/data/results_cache/expected.json new file mode 100755 index 0000000000..3385fe4869 --- /dev/null +++ b/tests/scancode/data/results_cache/expected.json @@ -0,0 +1,605 @@ +{ + "packages": [ + { + "type": "npm", + "namespace": null, + "name": "async", + "version": "1.2.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": "Higher-order functions and common patterns for asynchronous code", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Caolan McMahon", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "caolan", + "email": "caolan.mcmahon@gmail.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "beaugunderson", + "email": "beau@beaugunderson.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "aearly", + "email": "alexander.early@gmail.com", + "url": null + } + ], + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "homepage_url": "https://github.com/caolan/async#readme", + "download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "size": null, + "sha1": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/caolan/async/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/caolan/async.git@b66e85d1cca8c8056313253f22d18f571e7001d2", + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- MIT\n", + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "repository_homepage_url": "https://www.npmjs.com/package/async", + "repository_download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/async/1.2.1", + "package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "package.json" + ], + "datasource_ids": [ + "npm_package_json" + ], + "purl": "pkg:npm/async@1.2.1" + } + ], + "dependencies": [ + { + "purl": "pkg:npm/benchmark", + "extracted_requirement": "github:bestiejs/benchmark.js", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/benchmark?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/coveralls", + "extracted_requirement": "^2.11.2", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/coveralls?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/jshint", + "extracted_requirement": "~2.7.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/jshint?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": ">=2.4.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/lodash?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/mkdirp", + "extracted_requirement": "~0.5.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/mkdirp?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/nodeunit", + "extracted_requirement": ">0.0.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/nodeunit?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/nyc", + "extracted_requirement": "^2.1.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/nyc?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/uglify-js", + "extracted_requirement": "1.2.x", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/uglify-js?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + }, + { + "purl": "pkg:npm/yargs", + "extracted_requirement": "~3.9.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:npm/yargs?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "package.json", + "datasource_id": "npm_package_json" + } + ], + "license_detections": [ + { + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", + "license_expression": "mit", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 22, + "end_line": 22, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "mit_30.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE" + } + ] + }, + { + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf", + "license_expression": "mit", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null + } + ] + } + ], + "files": [ + { + "path": "package.json", + "type": "file", + "name": "package.json", + "base_name": "package", + "extension": ".json", + "size": 2361, + "sha1": "29c3cacb4c3abe6f69dc87a77d9c9105546f6978", + "md5": "b8be873c4fb54a835bfac7eb47a6531e", + "sha256": "cb6f5a82e473620da4d1aecf82dd4d4fa9ada393a7679b28a42cac86f0a83c92", + "sha1_git": "247e1784187cc3929165ec2d534f2134b9147ecd", + "mime_type": "application/json", + "file_type": "JSON data", + "programming_language": null, + "is_binary": false, + "is_text": true, + "is_archive": false, + "is_media": false, + "is_source": false, + "is_script": false, + "package_data": [ + { + "type": "npm", + "namespace": null, + "name": "async", + "version": "1.2.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": "Higher-order functions and common patterns for asynchronous code", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Caolan McMahon", + "email": null, + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "caolan", + "email": "caolan.mcmahon@gmail.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "beaugunderson", + "email": "beau@beaugunderson.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "aearly", + "email": "alexander.early@gmail.com", + "url": null + } + ], + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "homepage_url": "https://github.com/caolan/async#readme", + "download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "size": null, + "sha1": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/caolan/async/issues", + "code_view_url": null, + "vcs_url": "git+https://github.com/caolan/async.git@b66e85d1cca8c8056313253f22d18f571e7001d2", + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- MIT\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/benchmark", + "extracted_requirement": "github:bestiejs/benchmark.js", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/coveralls", + "extracted_requirement": "^2.11.2", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/jshint", + "extracted_requirement": "~2.7.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/lodash", + "extracted_requirement": ">=2.4.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/mkdirp", + "extracted_requirement": "~0.5.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/nodeunit", + "extracted_requirement": ">0.0.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/nyc", + "extracted_requirement": "^2.1.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/uglify-js", + "extracted_requirement": "1.2.x", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/yargs", + "extracted_requirement": "~3.9.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/async", + "repository_download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "api_data_url": "https://registry.npmjs.org/async/1.2.1", + "datasource_id": "npm_package_json", + "purl": "pkg:npm/async@1.2.1" + } + ], + "for_packages": [ + "pkg:npm/async@1.2.1?uuid=fixed-uid-done-for-testing-5642512d1758" + ], + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "package.json", + "start_line": 22, + "end_line": 22, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 2, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "mit_30.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE" + } + ], + "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee" + } + ], + "license_clues": [], + "percentage_of_license_text": 0.83, + "copyrights": [], + "holders": [], + "authors": [ + { + "author": "Caolan McMahon", + "start_line": 5, + "end_line": 6 + } + ], + "emails": [ + { + "email": "alexander.early@gmail.com", + "start_line": 73, + "end_line": 73 + }, + { + "email": "caolan.mcmahon@gmail.com", + "start_line": 78, + "end_line": 78 + }, + { + "email": "beau@beaugunderson.com", + "start_line": 82, + "end_line": 82 + } + ], + "urls": [ + { + "url": "https://github.com/caolan/async.git", + "start_line": 17, + "end_line": 17 + }, + { + "url": "https://github.com/caolan/async/issues", + "start_line": 20, + "end_line": 20 + }, + { + "url": "https://github.com/caolan/async#readme", + "start_line": 65, + "end_line": 65 + }, + { + "url": "http://registry.npmjs.org/async/-/async-1.2.1.tgz", + "start_line": 91, + "end_line": 91 + }, + { + "url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "start_line": 94, + "end_line": 94 + } + ], + "files_count": 0, + "dirs_count": 0, + "size_count": 0, + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/package.json b/tests/scancode/data/results_cache/package.json new file mode 100644 index 0000000000..247e178418 --- /dev/null +++ b/tests/scancode/data/results_cache/package.json @@ -0,0 +1,96 @@ +{ + "name": "async", + "description": "Higher-order functions and common patterns for asynchronous code", + "main": "lib/async.js", + "author": { + "name": "Caolan McMahon" + }, + "version": "1.2.1", + "keywords": [ + "async", + "callback", + "utility", + "module" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/caolan/async.git" + }, + "bugs": { + "url": "https://github.com/caolan/async/issues" + }, + "license": "MIT", + "devDependencies": { + "benchmark": "github:bestiejs/benchmark.js", + "coveralls": "^2.11.2", + "jshint": "~2.7.0", + "lodash": ">=2.4.1", + "mkdirp": "~0.5.1", + "nodeunit": ">0.0.0", + "nyc": "^2.1.0", + "uglify-js": "1.2.x", + "yargs": "~3.9.1" + }, + "jam": { + "main": "lib/async.js", + "include": [ + "lib/async.js", + "README.md", + "LICENSE" + ], + "categories": [ + "Utilities" + ] + }, + "scripts": { + "test": "npm run-script lint && nodeunit test/test-async.js", + "lint": "jshint lib/*.js test/*.js perf/*.js", + "coverage": "nyc npm test && nyc report", + "coveralls": "nyc npm test && nyc report --reporter=text-lcov | coveralls" + }, + "spm": { + "main": "lib/async.js" + }, + "volo": { + "main": "lib/async.js", + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ] + }, + "gitHead": "b66e85d1cca8c8056313253f22d18f571e7001d2", + "homepage": "https://github.com/caolan/async#readme", + "_id": "async@1.2.1", + "_shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "_from": "async@*", + "_npmVersion": "2.9.0", + "_nodeVersion": "2.0.2", + "_npmUser": { + "name": "aearly", + "email": "alexander.early@gmail.com" + }, + "maintainers": [ + { + "name": "caolan", + "email": "caolan.mcmahon@gmail.com" + }, + { + "name": "beaugunderson", + "email": "beau@beaugunderson.com" + }, + { + "name": "aearly", + "email": "alexander.early@gmail.com" + } + ], + "dist": { + "shasum": "a4816a17cd5ff516dfa2c7698a453369b9790de0", + "tarball": "http://registry.npmjs.org/async/-/async-1.2.1.tgz" + }, + "directories": {}, + "_resolved": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", + "readme": "ERROR: No README data found!" +} diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights new file mode 100644 index 0000000000..08d741e004 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/copyrights @@ -0,0 +1 @@ +{"copyrights": [], "holders": [], "authors": [{"author": "Caolan McMahon", "start_line": 5, "end_line": 6}]} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails new file mode 100644 index 0000000000..affed944a5 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/emails @@ -0,0 +1 @@ +{"emails": [{"email": "alexander.early@gmail.com", "start_line": 73, "end_line": 73}, {"email": "caolan.mcmahon@gmail.com", "start_line": 78, "end_line": 78}, {"email": "beau@beaugunderson.com", "start_line": 82, "end_line": 82}]} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info new file mode 100644 index 0000000000..d17acb8452 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/info @@ -0,0 +1 @@ +{"date": "2024-06-23", "size": 2361, "sha1": "29c3cacb4c3abe6f69dc87a77d9c9105546f6978", "md5": "b8be873c4fb54a835bfac7eb47a6531e", "sha256": "cb6f5a82e473620da4d1aecf82dd4d4fa9ada393a7679b28a42cac86f0a83c92", "sha1_git": "247e1784187cc3929165ec2d534f2134b9147ecd", "mime_type": "application/json", "file_type": "JSON data", "programming_language": null, "is_binary": false, "is_text": true, "is_archive": false, "is_media": false, "is_source": false, "is_script": false} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses new file mode 100644 index 0000000000..dbe7ffb67b --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/licenses @@ -0,0 +1 @@ +{"detected_license_expression": "mit", "detected_license_expression_spdx": "MIT", "license_detections": [{"license_expression": "mit", "license_expression_spdx": "MIT", "matches": [{"license_expression": "mit", "license_expression_spdx": "MIT", "from_file": null, "start_line": 22, "end_line": 22, "matcher": "2-aho", "score": 100.0, "matched_length": 2, "match_coverage": 100.0, "rule_relevance": 100, "rule_identifier": "mit_30.RULE", "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/mit_30.RULE"}], "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee"}], "license_clues": [], "percentage_of_license_text": 0.83} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages new file mode 100644 index 0000000000..f1833bd934 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/packages @@ -0,0 +1 @@ +{"package_data": [{"type": "npm", "namespace": null, "name": "async", "version": "1.2.1", "qualifiers": {}, "subpath": null, "primary_language": "JavaScript", "description": "Higher-order functions and common patterns for asynchronous code", "release_date": null, "parties": [{"type": "person", "role": "author", "name": "Caolan McMahon", "email": null, "url": null}, {"type": "person", "role": "maintainer", "name": "caolan", "email": "caolan.mcmahon@gmail.com", "url": null}, {"type": "person", "role": "maintainer", "name": "beaugunderson", "email": "beau@beaugunderson.com", "url": null}, {"type": "person", "role": "maintainer", "name": "aearly", "email": "alexander.early@gmail.com", "url": null}], "keywords": ["async", "callback", "utility", "module"], "homepage_url": "https://github.com/caolan/async#readme", "download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", "size": null, "sha1": "a4816a17cd5ff516dfa2c7698a453369b9790de0", "md5": null, "sha256": null, "sha512": null, "bug_tracking_url": "https://github.com/caolan/async/issues", "code_view_url": null, "vcs_url": "git+https://github.com/caolan/async.git@b66e85d1cca8c8056313253f22d18f571e7001d2", "copyright": null, "holder": null, "declared_license_expression": "mit", "declared_license_expression_spdx": "MIT", "license_detections": [{"license_expression": "mit", "license_expression_spdx": "MIT", "matches": [{"license_expression": "mit", "license_expression_spdx": "MIT", "from_file": null, "start_line": 1, "end_line": 1, "matcher": "1-spdx-id", "score": 100.0, "matched_length": 1, "match_coverage": 100.0, "rule_relevance": 100, "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", "rule_url": null, "matched_text": "MIT"}], "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf"}], "other_license_expression": null, "other_license_expression_spdx": null, "other_license_detections": [], "extracted_license_statement": "- MIT\n", "notice_text": null, "source_packages": [], "file_references": [], "is_private": false, "is_virtual": false, "extra_data": {}, "dependencies": [{"purl": "pkg:npm/benchmark", "extracted_requirement": "github:bestiejs/benchmark.js", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/coveralls", "extracted_requirement": "^2.11.2", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/jshint", "extracted_requirement": "~2.7.0", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/lodash", "extracted_requirement": ">=2.4.1", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/mkdirp", "extracted_requirement": "~0.5.1", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/nodeunit", "extracted_requirement": ">0.0.0", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/nyc", "extracted_requirement": "^2.1.0", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/uglify-js", "extracted_requirement": "1.2.x", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}, {"purl": "pkg:npm/yargs", "extracted_requirement": "~3.9.1", "scope": "devDependencies", "is_runtime": false, "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, "extra_data": {}}], "repository_homepage_url": "https://www.npmjs.com/package/async", "repository_download_url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", "api_data_url": "https://registry.npmjs.org/async/1.2.1", "datasource_id": "npm_package_json", "purl": "pkg:npm/async@1.2.1"}]} \ No newline at end of file diff --git a/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls new file mode 100644 index 0000000000..8df21ee182 --- /dev/null +++ b/tests/scancode/data/results_cache/results/b0/e3/dd13b9b5980bb1ca7aab89e5490cb136952374489a755947ac004309b035/urls @@ -0,0 +1 @@ +{"urls": [{"url": "https://github.com/caolan/async.git", "start_line": 17, "end_line": 17}, {"url": "https://github.com/caolan/async/issues", "start_line": 20, "end_line": 20}, {"url": "https://github.com/caolan/async#readme", "start_line": 65, "end_line": 65}, {"url": "http://registry.npmjs.org/async/-/async-1.2.1.tgz", "start_line": 91, "end_line": 91}, {"url": "https://registry.npmjs.org/async/-/async-1.2.1.tgz", "start_line": 94, "end_line": 94}]} \ No newline at end of file diff --git a/tests/scancode/test_cli.py b/tests/scancode/test_cli.py index b5512dee31..9f97f8fb45 100644 --- a/tests/scancode/test_cli.py +++ b/tests/scancode/test_cli.py @@ -1052,3 +1052,28 @@ def test_scan_does_validate_input_and_fails_on_faulty_json_input(test_file, expe def test_scan_does_validate_input_and_does_not_fail_on_valid_json_input(): test_file = test_env.get_test_loc('various-inputs/true-scan-json.json') run_scan_click(['--from-json', test_file, '--json-pp', '-'], retry=False) + + +def test_use_cached_results(): + results_cache_location = test_env.get_test_loc(u'results_cache/results/') + test_file = test_env.get_test_loc(u'results_cache/package.json') + result_file = test_env.get_temp_file('json') + env = { + 'SCANCODE_RESULTS_CACHE': results_cache_location + } + args = [ + '--info', + '--license', + '--copyright', + '--package', + '--email', + '--url', + '--strip-root', + '--use-cached-results', + test_file , + '--json', + result_file + ] + results = run_scan_plain(options=args, env=env) + expected = test_env.get_test_loc('results_cache/expected.json') + check_json_scan(expected, result_file, remove_file_date=True, regen=REGEN_TEST_FIXTURES)