ARMmbed
/
mbed-os
mirror of https://github.com/ARMmbed/mbed-os.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor workaround for scancode evaluation of PBL 

Add function "has_binary_license" to check if a file has a
non-permissive license contains one. PBL is not recognized by scancode,
causing it to be flagged as a non-permissive license. CI doesn't allow
any non-permissive licenses, although, files flageed as SPDX are
allowed. Workaround causes all files with a valid PBL to be flagged as
missing an SPDX.

Add condition in "has_spdx_text_in_scancode_output" to ignore any spdx
identifier with "unknown" in the name. Scancode erroneously matches PBL
to matched_rule.identifer "spdx-license-identifier: unknown-spdx". This
prevents the workaround from working.
pull/13847/head
Harrison Mutai 2020-11-06 09:44:43 +00:00
parent d085e9f1ed
commit e93a3e2beb
2 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
tools/test/travis-ci/scancode-evaluate.py
View File

@ -30,6 +30,7 @@ MISSING_SPDX_TEXT = "Missing SPDX license identifier"
userlog = logging.getLogger("scancode-evaluate")
class ReturnCode(Enum):
"""Return codes."""
@ -54,13 +55,12 @@ def path_leaf(path):
return tail or os.path.basename(head)
def has_permissive_text_in_scancode_output(scancode_output_data_file):
"""Returns true if at least one license in the scancode output is permissive or is a Permissive Binary License"""
# temporary workaround for files with Permissive Binary Licenses
def has_permissive_text_in_scancode_output(scancode_output_data_file_licenses):
"""Returns true if at least one license in the scancode output is permissive"""
return any(
scancode_output_data_file_license['category'] == 'Permissive'
for scancode_output_data_file_license in scancode_output_data_file['licenses']
) or has_binary_license(scancode_output_data_file)
for scancode_output_data_file_license in scancode_output_data_file_licenses
)
def has_spdx_text_in_scancode_output(scancode_output_data_file_licenses):
@ -76,16 +76,20 @@ def has_spdx_text_in_analysed_file(scanned_file_content):
return bool(re.findall("SPDX-License-Identifier:?", scanned_file_content))
def has_binary_license(scancode_output_data_file):
def has_binary_license(scanned_file_content):
"""Returns true if the file analysed by ScanCode contains a Permissive Binary License."""
return bool(re.findall("Permissive Binary License", scanned_file_content))
def get_file_text(scancode_output_data_file):
"""Returns file text for scancode output file"""
file_path = os.path.abspath(scancode_output_data_file['path'])
try:
with open(file_path, 'r') as read_file:
scanned_file_content = read_file.read()
return bool(re.findall("Permissive Binary License", scanned_file_content))
return read_file.read()
except UnicodeDecodeError:
userlog.warning("Unable to look for PBL text in `{}`:".format(file_path))
return False
userlog.warning("Unable to decode file text in: %s" % file_path)
# Ignore files that cannot be decoded
def license_check(scancode_output_path):
@ -98,7 +102,7 @@ def license_check(scancode_output_path):
Returns:
0 if nothing found
>0 - count how many license isses found
>0 - count how many license issues found
ReturnCode.ERROR.value if any error in file licenses found
"""
@ -125,25 +129,21 @@ def license_check(scancode_output_path):
# check the next file in the scancode output
continue
if not has_permissive_text_in_scancode_output(scancode_output_data_file):
scancode_output_data_file['fail_reason'] = MISSING_PERMISSIVE_LICENSE_TEXT
license_offenders.append(scancode_output_data_file)
if not has_permissive_text_in_scancode_output(scancode_output_data_file['licenses']):
scanned_file_content = get_file_text(scancode_output_data_file)
if not (scanned_file_content and has_binary_license(scanned_file_content)):
scancode_output_data_file['fail_reason'] = MISSING_PERMISSIVE_LICENSE_TEXT
license_offenders.append(scancode_output_data_file)
if not has_spdx_text_in_scancode_output(scancode_output_data_file['licenses']):
# Scancode does not recognize license notice in Python file headers.
# Issue: https://github.com/nexB/scancode-toolkit/issues/1913
# Therefore check if the file tested by ScanCode actually has a licence notice.
file_path = os.path.abspath(scancode_output_data_file['path'])
try:
with open(file_path, 'r') as read_file:
scanned_file_content = read_file.read()
except UnicodeDecodeError:
userlog.warning("Unable to look for SPDX text in `{}`:".format(file_path))
# Ignore files that cannot be decoded
# check the next file in the scancode output
continue
scanned_file_content = get_file_text(scancode_output_data_file)
if not has_spdx_text_in_analysed_file(scanned_file_content):
if not scanned_file_content:
continue
elif not has_spdx_text_in_analysed_file(scanned_file_content):
scancode_output_data_file['fail_reason'] = MISSING_SPDX_TEXT
spdx_offenders.append(scancode_output_data_file)

8
tools/test/travis-ci/scancode_evaluate_test.py
View File

@ -34,7 +34,7 @@ HEADER_WITH_SPDX = "/* mbed Microcontroller Library\
* limitations under the License.\
*/"
BINARY_HEADER = "/*\
HEADER_WITH_BINARY_LICENSE = "/*\
* Copyright (c) 2019, Arm Limited, All Rights Reserved\
* SPDX-License-Identifier: LicenseRef-PBL\
*\
@ -63,7 +63,7 @@ def create_scanned_files():
if file_path in [os.path.join(STUBS_PATH, "test3.h")]:
new_file.write(HEADER_WITHOUT_SPDX)
elif file_path in [os.path.join(STUBS_PATH, "test6.h")]:
new_file.write(BINARY_HEADER)
new_file.write(HEADER_WITH_BINARY_LICENSE)
else:
new_file.write(HEADER_WITH_SPDX)
yield
@ -95,7 +95,7 @@ class TestScancodeEvaluate:
test3.h: Missing `Permissive` license text and `spdx` in match.identifier and not in file tested by ScanCode (error count += 1)
test4.h: Missing `Permissive` license text and `spdx` in match.identifier but found in file tested by ScanCode (error count += 1)
test5.h: Missing `spdx` in match.identifier but found in file tested by ScanCode. (error count += 0)
test6.h: Matching 'unknown-spdx' in match.identifier and Permissive Binary License in header (error count += 1)
test6.h: Matching `spdx` in match.identifier but Permissive Binary License header (error count += 0)
@inputs scancode_test/scancode_test_2.json
@output 3
"""
@ -107,4 +107,4 @@ class TestScancodeEvaluate:
@inputs scancode_test/scancode_test_2.json
@outputs 0
"""
assert license_check(os.path.join(STUBS_PATH, "scancode_test_4.json")) == 0
assert license_check(os.path.join(STUBS_PATH, "scancode_test_4.json")) == 0