Enable inclusion of "All rights reserved" into extracted copyrights

pyproject.toml

@@ -272,6 +272,7 @@ facet = "summarycode.facet:AddFacet"
 info = "scancode.plugin_info:InfoScanner"
 licenses = "licensedcode.plugin_license:LicenseScanner"
 copyrights = "cluecode.plugin_copyright:CopyrightScanner"
+allrights = "cluecode.plugin_allrights:AllrightsCopyrightScanner"
 packages = "packagedcode.plugin_package:PackageScanner"
 emails = "cluecode.plugin_email:EmailScanner"
 urls = "cluecode.plugin_url:UrlScanner"

src/cluecode/copyrights.py

@@ -336,20 +336,25 @@ def detect(self,
             tree_node_label = tree_node.label
 
             if (include_copyrights or include_holders) and 'COPYRIGHT' in tree_node_label:
-                copyrght = build_detection_from_node(
+                if include_copyright_allrights:
+                    refiner = refine_allrights
+                else:
+                    refiner = refine_copyright
+
+                copyright = build_detection_from_node(
                     node=tree_node,
                     cls=CopyrightDetection,
                     ignored_labels=non_copyright_labels,
                     include_copyright_allrights=include_copyright_allrights,
-                    refiner=refine_copyright,
+                    refiner=refiner,
                 )
 
                 if TRACE or TRACE_DEEP:
-                    logger_debug(f'CopyrightDetector: final copyright: {copyrght}')
+                    logger_debug(f'CopyrightDetector: final copyright: {copyright}')
 
-                if copyrght:
+                if copyright:
                     if include_copyrights:
-                        yield copyrght
+                        yield copyright
 
                     if include_holders:
                         # By default we strip email and urls from holders ....
@@ -3562,6 +3567,31 @@ def refine_copyright(c):
     return c.strip()
 
 
+def refine_allrights(c):
+    """
+    Refine a detected copyright string.
+    FIXME: the grammar should not allow this to happen.
+    """
+    if not c:
+        return
+    c = ' '.join(c.split())
+    c = strip_some_punct(c)
+    c = strip_solo_quotes(c)
+    # this catches trailing slashes in URL for consistency
+    c = c.strip('/ ~')
+    c = strip_all_unbalanced_parens(c)
+    c = remove_some_extra_words_and_punct(c)
+    c = ' '.join(c.split())
+    c = remove_dupe_copyright_words(c)
+    c = strip_prefixes(c, prefixes=set(['by', 'c']))
+    c = c.strip()
+    c = c.strip('+')
+    c = strip_balanced_edge_parens(c)
+    c = strip_suffixes(c, suffixes=COPYRIGHTS_SUFFIXES)
+    c = c.strip("'")
+    return c.strip()
+
+
 def remove_dupe_holder(h):
     """
     Remove duplicated holders
@@ -4452,7 +4482,7 @@ def collect_candidate_lines(numbered_lines):
 remove_weird_comment_markers = re.compile(r'^(rem|\@rem|dnl)\s+').sub
 
 # common comment line prefix in man pages
-remove_man_comment_markers = re.compile(r'\."').sub
+remove_man_comment_markers = re.compile(r'^\.\\"').sub
 
 
 def remove_code_comment_markers(s):

src/cluecode/plugin_allrights.py

@@ -0,0 +1,61 @@
+#
+# This code is heavily inspired by original code from ScanCode Toolkit.
+# The copyright header is therefore propagated. The content is primarily
+# taken from following packages in the ScanCode Toolkit:
+# - cluecode.plugin_copyright
+# - scancode.api
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import attr
+from plugincode.scan import ScanPlugin
+from plugincode.scan import scan_impl
+
+from commoncode.cliutils import PluggableCommandLineOption
+from commoncode.cliutils import SCAN_GROUP
+
+
+@scan_impl
+class AllrightsCopyrightScanner(ScanPlugin):
+    """Scan a Resource for copyrights with all rights reserved. This class is an adapted version of
+    scancodes cluecode.plugin_copyright.
+    """
+
+    resource_attributes = dict(
+        [
+            ("copyrights", attr.ib(default=attr.Factory(list))),
+            ("holders", attr.ib(default=attr.Factory(list))),
+            ("authors", attr.ib(default=attr.Factory(list))),
+        ]
+    )
+
+    run_order = 6
+    sort_order = 6
+
+    options = [
+        PluggableCommandLineOption(
+            (
+                "-a",
+                "--allrights",
+            ),
+            is_flag=True,
+            default=False,
+            help="Scan <input> for copyrights and all rights reserved.",
+            help_group=SCAN_GROUP,
+            sort_order=50,
+        ),
+    ]
+
+    def is_enabled(self, allrights, **kwargs):  # NOQA
+        return allrights
+
+    def get_scanner(self, **kwargs):
+        from scancode.api import allrights_scanner
+        return allrights_scanner
+

src/scancode/api.py

@@ -6,16 +6,17 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-from itertools import islice
-from os.path import getsize
 import logging
 import os
 import sys
+from itertools import islice
+from os.path import getsize
+
+from typecode.contenttype import get_type
 
 from commoncode.filetype import get_last_modified_date
 from commoncode.hash import multi_checksums
 from scancode import ScancodeError
-from typecode.contenttype import get_type
 
 TRACE = os.environ.get('SCANCODE_DEBUG_API', False)
 
@@ -30,6 +31,7 @@ def logger_debug(*args):
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
+
     def logger_debug(*args):
         return logger.debug(
             ' '.join(isinstance(a, str) and a or repr(a) for a in args)
@@ -46,9 +48,9 @@ def logger_debug(*args):
 
 
 def get_copyrights(
-    location,
-    deadline=sys.maxsize,
-    **kwargs,
+        location,
+        deadline=sys.maxsize,
+        **kwargs,
 ):
     """
     Return a mapping with a single 'copyrights' key with a value that is a list
@@ -79,12 +81,45 @@ def get_copyrights(
     return results
 
 
+def allrights_scanner(
+        location,
+        deadline=sys.maxsize,
+        **kwargs,
+):
+    """Return a mapping with a single 'copyrights' key with a value that is a list
+    of mappings for copyright detected in the file at `location`.
+    Adapted copy of scancodes scancode.api.get_copyrights.
+    """
+    from cluecode.copyrights import detect_copyrights
+    from cluecode.copyrights import Detection
+
+    detections = detect_copyrights(
+        location,
+        include_copyrights=True,
+        include_holders=True,
+        include_authors=True,
+        include_copyright_years=True,
+        include_copyright_allrights=True,
+        deadline=deadline,
+    )
+
+    copyrights, holders, authors = Detection.split(detections, to_dict=True)
+
+    results = dict([
+        ('copyrights', copyrights),
+        ('holders', holders),
+        ('authors', authors),
+    ])
+
+    return results
+
+
 def get_emails(
-    location,
-    threshold=50,
-    test_slow_mode=False,
-    test_error_mode=False,
-    **kwargs,
+        location,
+        threshold=50,
+        test_slow_mode=False,
+        test_error_mode=False,
+        **kwargs,
 ):
     """
     Return a mapping with a single 'emails' key with a value that is a list of
@@ -148,14 +183,14 @@ def get_urls(location, threshold=50, **kwargs):
 
 
 def get_licenses(
-    location,
-    min_score=0,
-    include_text=False,
-    license_text_diagnostics=False,
-    license_diagnostics=False,
-    deadline=sys.maxsize,
-    unknown_licenses=False,
-    **kwargs,
+        location,
+        min_score=0,
+        include_text=False,
+        license_text_diagnostics=False,
+        license_diagnostics=False,
+        deadline=sys.maxsize,
+        unknown_licenses=False,
+        **kwargs,
 ):
     """
     Return a mapping or license_detections for licenses detected in the file at
@@ -257,12 +292,12 @@ def get_licenses(
 
 
 def _get_package_data(
-    location,
-    application=True,
-    system=False,
-    compiled=False,
-    package_only=False, 
-    **kwargs
+        location,
+        application=True,
+        system=False,
+        compiled=False,
+        package_only=False,
+        **kwargs
 ):
     """
     Return a mapping of package manifest information detected in the file at ``location``.
@@ -309,12 +344,12 @@ def get_package_info(location, **kwargs):
 
 
 def get_package_data(
-    location,
-    application=True,
-    system=False,
-    compiled=False,
-    package_only=False,
-    **kwargs
+        location,
+        application=True,
+        system=False,
+        compiled=False,
+        package_only=False,
+        **kwargs
 ):
     """
     Return a mapping of package manifest information detected in the file at

tests/cluecode/cluecode_test_utils.py

@@ -55,6 +55,7 @@ class CopyrightTest(object):
     # one of holders, copyrights, authors
     what = attr.ib(default=attr.Factory(list))
     copyrights = attr.ib(default=attr.Factory(list))
+    allrights = attr.ib(default=attr.Factory(list))
     holders = attr.ib(default=attr.Factory(list))
     authors = attr.ib(default=attr.Factory(list))
 
@@ -175,6 +176,7 @@ def make_copyright_test_functions(
     index,
     test_data_dir=test_env.test_data_dir,
     regen=REGEN_TEST_FIXTURES,
+    with_allrights=False,
 ):
     """
     Build and return a test function closing on tests arguments and the function
@@ -188,7 +190,7 @@ def make_copyright_test_functions(
     from summarycode.copyright_tallies import tally_persons
 
     def closure_test_function(*args, **kwargs):
-        detections = detect_copyrights(test_file)
+        detections = detect_copyrights(test_file, include_copyright_allrights=with_allrights)
         copyrights, holders, authors = Detection.split_values(detections)
 
         holders_summary = []
@@ -215,7 +217,12 @@ def closure_test_function(*args, **kwargs):
         expected_yaml = test.dumps()
 
         for wht in test.what:
-            setattr(test, wht, results.get(wht))
+            # use the fact we are overriding attributes, leave copyrights as is
+            # and add allrights section. This shortcut needs a better solution.
+            if test.allrights and with_allrights and wht == 'copyrights':
+                setattr(test, 'allrights', results.get(wht))
+            else:
+                setattr(test, wht, results.get(wht))
         results_yaml = test.dumps()
 
         if regen:
@@ -246,6 +253,7 @@ def build_tests(
     clazz,
     test_data_dir=test_env.test_data_dir,
     regen=REGEN_TEST_FIXTURES,
+    with_allrights=False,
 ):
     """
     Dynamically build test methods from a sequence of CopyrightTest and attach
@@ -263,6 +271,7 @@ def build_tests(
             index=i,
             test_data_dir=test_data_dir,
             regen=actual_regen,
+            with_allrights=with_allrights,
         )
 
         # attach that method to our test class

tests/cluecode/data/authors/author-config.rpath.yml

@@ -6,3 +6,5 @@ authors:
 authors_summary:
   - value: Gordon Matzigkeit <gord@gnu.ai.mit.edu>
     count: 1
+allrights:
+  - Copyright 1996-2006 Free Software Foundation, Inc.

tests/cluecode/data/authors/author_addr_c-addr_c.c.yml

@@ -6,3 +6,6 @@ authors:
 authors_summary:
   - value: John Doe
     count: 1
+allrights:
+  - Copyright 1999 Cornell University. All rights reserved.
+  - Copyright 2000 Jon Doe. All rights reserved.

tests/cluecode/data/authors/author_avinash-BitVector_py.py.yml

@@ -7,3 +7,5 @@ authors:
 authors_summary:
   - value: Avinash Kak (kak@purdue.edu)
     count: 2
+allrights:
+  - (c) 2008 Avinash Kak. Python Software Foundation.

tests/cluecode/data/authors/author_avinash_kak-BitVector_py.py.yml

@@ -7,3 +7,5 @@ authors:
 authors_summary:
   - value: Avinash Kak (kak@purdue.edu)
     count: 2
+allrights:
+  - (c) 2008 Avinash Kak. Python Software Foundation.

tests/cluecode/data/authors/author_complex_author-strtol_c.c.yml

@@ -6,3 +6,5 @@ authors:
 authors_summary:
   - value: the University of California, Berkeley and its contributors
     count: 1
+allrights:
+  - Copyright (c) 1990 The Regents of the University of California. All rights reserved.

tests/cluecode/data/authors/author_correct-detail_9_html.html.yml

@@ -1,3 +1,5 @@
 what:
   - authors
   - authors_summary
+allrights:
+  - (c) Oss http://s.pudn.com/upload_log.asp?e