Merge pull request #13847 from harmut01/pbl_scancode_fix

Add workaround for files with permissive binary licenses

tools/test/travis-ci/scancode-evaluate.py

@@ -30,6 +30,7 @@ MISSING_SPDX_TEXT = "Missing SPDX license identifier"
 
 userlog = logging.getLogger("scancode-evaluate")
 
+
 class ReturnCode(Enum):
     """Return codes."""
 
@@ -55,7 +56,7 @@ def path_leaf(path):
 
 
 def has_permissive_text_in_scancode_output(scancode_output_data_file_licenses):
-    """Returns true if at list one license in the scancode output is permissive."""
+    """Returns true if at least one license in the scancode output is permissive"""
     return any(
         scancode_output_data_file_license['category'] == 'Permissive'
         for scancode_output_data_file_license in scancode_output_data_file_licenses
@@ -75,6 +76,22 @@ def has_spdx_text_in_analysed_file(scanned_file_content):
     return bool(re.findall("SPDX-License-Identifier:?", scanned_file_content))
 
 
+def has_binary_license(scanned_file_content):
+    """Returns true if the file analysed by ScanCode contains a Permissive Binary License."""
+    return bool(re.findall("Permissive Binary License", scanned_file_content))
+
+
+def get_file_text(scancode_output_data_file):
+    """Returns file text for scancode output file"""
+    file_path = os.path.abspath(scancode_output_data_file['path'])
+    try:
+        with open(file_path, 'r') as read_file:
+            return read_file.read()
+    except UnicodeDecodeError:
+        userlog.warning("Unable to decode file text in: %s" % file_path)
+        # Ignore files that cannot be decoded
+
+
 def license_check(scancode_output_path):
     """Check licenses in the scancode json file for specified directory.
 
@@ -85,7 +102,7 @@ def license_check(scancode_output_path):
 
     Returns:
     0 if nothing found
-    >0 - count how many license isses found
+    >0 - count how many license issues found
     ReturnCode.ERROR.value if any error in file licenses found
     """
 
@@ -113,6 +130,8 @@ def license_check(scancode_output_path):
             continue
 
         if not has_permissive_text_in_scancode_output(scancode_output_data_file['licenses']):
+            scanned_file_content = get_file_text(scancode_output_data_file)
+            if not (scanned_file_content and has_binary_license(scanned_file_content)):
                 scancode_output_data_file['fail_reason'] = MISSING_PERMISSIVE_LICENSE_TEXT
                 license_offenders.append(scancode_output_data_file)
 
@@ -120,17 +139,11 @@ def license_check(scancode_output_path):
             # Scancode does not recognize license notice in Python file headers.
             # Issue: https://github.com/nexB/scancode-toolkit/issues/1913
             # Therefore check if the file tested by ScanCode actually has a licence notice.
-            file_path = os.path.abspath(scancode_output_data_file['path'])
+            scanned_file_content = get_file_text(scancode_output_data_file)
-            try:
-                with open(file_path, 'r') as read_file:
-                    scanned_file_content = read_file.read()
-            except UnicodeDecodeError:
-                userlog.warning("Unable to look for SPDX text in `{}`:".format(file_path))
-                # Ignore files that cannot be decoded
-                # check the next file in the scancode output
-                continue
 
-            if not has_spdx_text_in_analysed_file(scanned_file_content):
+            if not scanned_file_content:
+                continue
+            elif not has_spdx_text_in_analysed_file(scanned_file_content):
                 scancode_output_data_file['fail_reason'] = MISSING_SPDX_TEXT
                 spdx_offenders.append(scancode_output_data_file)
 

tools/test/travis-ci/scancode_evaluate_test.py

@@ -34,22 +34,36 @@ HEADER_WITH_SPDX = "/* mbed Microcontroller Library\
  * limitations under the License.\
  */"
 
+HEADER_WITH_BINARY_LICENSE = "/*\
+ * Copyright (c) 2019, Arm Limited, All Rights Reserved\
+ * SPDX-License-Identifier: LicenseRef-PBL\
+ *\
+ * This file and the related binary are licensed under the\
+ * Permissive Binary License, Version 1.0 (the \"License\");\
+ * you may not use these files except in compliance with the License.\
+ *\
+ */"
+
 @pytest.fixture()
 def create_scanned_files():
     """Create stub files.
     test3.h missing license notice
     test4.h with license notice
     test5.h with license notice
+    test6.h with permissive binary license
     """
     file_paths = [
         os.path.join(STUBS_PATH, "test3.h"),
         os.path.join(STUBS_PATH, "test4.h"),
-        os.path.join(STUBS_PATH, "test5.h")
+        os.path.join(STUBS_PATH, "test5.h"),
+        os.path.join(STUBS_PATH, "test6.h")
     ]
     for file_path in file_paths:
         with open(file_path, "w") as new_file:
             if file_path in [os.path.join(STUBS_PATH, "test3.h")]:
                 new_file.write(HEADER_WITHOUT_SPDX)
+            elif file_path in [os.path.join(STUBS_PATH, "test6.h")]:
+                new_file.write(HEADER_WITH_BINARY_LICENSE)
             else:
                 new_file.write(HEADER_WITH_SPDX)
     yield
@@ -81,6 +95,7 @@ class TestScancodeEvaluate:
             test3.h: Missing `Permissive` license text and `spdx` in match.identifier and not in file tested by ScanCode (error count += 1)
             test4.h: Missing `Permissive` license text and `spdx` in match.identifier but found in file tested by ScanCode (error count += 1)
             test5.h: Missing `spdx` in match.identifier but found in file tested by ScanCode. (error count += 0)
+            test6.h: Matching `spdx` in match.identifier but Permissive Binary License header (error count += 0)
             @inputs scancode_test/scancode_test_2.json
             @output 3
         """

tools/test/travis-ci/scancode_test/scancode_test_3.json

@@ -170,6 +170,50 @@
       ],
       "scan_errors":[
          
+      ]
+   },
+     {
+      "path":"tools/test/travis-ci/scancode_test/test6.h",
+      "type":"file",
+      "licenses":[
+         {
+            "key": "unknown-spdx",
+            "score": 100.0,
+             "name": "Unknown SPDX license detected but not recognized",
+             "short_name": "unknown SPDX",
+             "category": "Unstated License",
+             "is_exception": false,
+             "owner": "Unspecified",
+             "homepage_url": null,
+             "text_url": "",
+             "reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown-spdx",
+             "spdx_license_key": null,
+             "spdx_url": "",
+             "start_line": 3,
+             "end_line": 3,
+             "matched_rule": {
+               "identifier": "spdx-license-identifier: unknown-spdx",
+               "license_expression": "unknown-spdx",
+               "licenses": [
+                 "unknown-spdx"
+               ],
+               "is_license_text": false,
+               "is_license_notice": false,
+               "is_license_reference": false,
+               "is_license_tag": true,
+               "matcher": "1-spdx-id",
+               "rule_length": 1,
+               "matched_length": 1,
+               "match_coverage": 100.0,
+               "rule_relevance": 100
+          }
+         }
+      ],
+      "license_expressions":[
+         "unknown-spdx"
+      ],
+      "scan_errors":[
+
       ]
    }
   ]