Migrating TravisCI scripts to GitHub Actions

pull/15011/head
Saheer 2021-08-17 18:08:50 +01:00
parent faac0ed034
commit 6c918b2468
15 changed files with 1672 additions and 500 deletions

196
.github/workflows/basic_checks.yml vendored Normal file
View File

@ -0,0 +1,196 @@
# This workflow performs the checks like license check,
# doxygen, unit tests etc.
name: Basic Checks
on:
pull_request:
push:
branches:
- mbed-os-5.15
jobs:
license-check:
runs-on: ubuntu-latest
container:
image: ghcr.io/armmbed/mbed-os-env:mbed-os-5.15-latest
steps:
- name: Checkout repo
uses: actions/checkout@v2
with:
fetch-depth: 0
-
name: license check
run: |
set -x
mkdir -p SCANCODE
git diff --name-only --diff-filter=d origin/${GITHUB_BASE_REF} \
| ( grep '.\(c\|cpp\|h\|hpp\|py\)$' || true )
echo $?
git diff --name-only --diff-filter=d origin/${GITHUB_BASE_REF} \
| ( grep '.\(c\|cpp\|h\|hpp\|py\)$' || true ) \
| ( grep -v '^tools/test/toolchains/api_test.py' || true ) \
| while read file; do cp --parents "${file}" SCANCODE; done
ls SCANCODE
scancode -l --json-pp scancode.json SCANCODE
python ./tools/test/ci/scancode-evaluate.py scancode.json || true
cat scancode-evaluate.log
COUNT=$(cat scancode-evaluate.log | grep 'File:' | grep -v 'SPDX' | wc -l) || true
if [ $COUNT = 0 ]; then
echo "License check OK";
true;
else
echo "License check failed, please review license issues found in files";
false;
fi
include-check:
runs-on: ubuntu-latest
container:
image: ghcr.io/armmbed/mbed-os-env:mbed-os-5.15-latest
steps:
- name: Checkout repo
uses: actions/checkout@v2
-
name: "include check"
run: |
! git grep '^#include\s["'"']mbed.h['"'"]$' -- '*.c' '*.h' '*.cpp' '*.hpp' \
':!*platform_mbed.h' ':!*TESTS/*' ':!TEST_APPS/' ':!UNITTESTS/' \
':!*tests/*' ':!*targets/*' ':!*TARGET_*' ':!*unsupported/*'
docs-check:
runs-on: ubuntu-latest
container:
image: ghcr.io/armmbed/mbed-os-env:mbed-os-5.15-latest
steps:
- name: Checkout repo
uses: actions/checkout@v2
-
name: spell checks
run: |
./tools/test/ci/doxy-spellchecker/spell.sh drivers
./tools/test/ci/doxy-spellchecker/spell.sh platform
./tools/test/ci/doxy-spellchecker/spell.sh events
./tools/test/ci/doxy-spellchecker/spell.sh rtos
./tools/test/ci/doxy-spellchecker/spell.sh connectivity/netsocket
-
name: doxygen
run: |
mkdir BUILD
# Assert that the Doxygen build produced no warnings.
# The strange command below asserts that the Doxygen command had an
# output of zero length
doxygen doxyfile_options 2>&1
# Once Mbed OS has been fixed, enable the full test by replacing the top line with this:
# - ( ! doxygen doxyfile_options 2>&1 | grep . )
# Assert that all binary libraries are named correctly
# The strange command below asserts that there are exactly 0 libraries
# that do not start with lib
find "(" -name "*.a" -or -name "*.ar" ")" -and -not -name "lib*" |
tee BUILD/badlibs |
sed -e "s/^/Bad library name found: /" && [ ! -s BUILD/badlibs ]
# Assert that all assembler files are named correctly
# The strange command below asserts that there are exactly 0 libraries
# that do end with .s
find -name "*.s" | tee BUILD/badasm |
sed -e "s/^/Bad Assembler file name found: /" && [ ! -s BUILD/badasm ]
style-check:
runs-on: ubuntu-latest
container:
image: ghcr.io/armmbed/mbed-os-env:mbed-os-5.15-latest
steps:
- name: Checkout repo
uses: actions/checkout@v2
with:
fetch-depth: 0
-
name: astyle checks
run: |
git diff --name-only --diff-filter=d origin/${GITHUB_BASE_REF} \
| ( grep '.*\.\(c\|cpp\|h\|hpp\)$' || true ) \
| ( grep -v -f .astyleignore || true ) \
| while read file; do astyle -n --options=.astylerc "${file}"; done
git diff --exit-code --diff-filter=d --color
-
name: "UTF-8 Check"
run: |
# Make sure we're not introducing any text which is not UTF-8 encoded
git diff origin/${GITHUB_BASE_REF} -U0 | ( grep -a '^+' || true ) | ( ! grep -axv '.*' )
python-tests:
# these tests run in 3.7, hence running in vm not in pre-built docker
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ '3.5', '3.6', '3.7' ]
steps:
-
name: Checkout repo
uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
-
name: install dependencies
run: |
pip install -r requirements.txt
pip install mock==2.0.0 attrs==19.1.0 pytest==3.3.0 'pylint>=1.9,<2' 'hypothesis>=3,<4' 'coverage>=4.5,<5'
-
name: pytest
run: |
# PYTHONPATH=.
coverage run -a -m pytest tools/test
python tools/test/pylint.py
coverage run -a tools/project.py -S | sed -n '/^Total/p'
coverage html
events-library:
runs-on: ubuntu-latest
container:
image: ghcr.io/armmbed/mbed-os-env:mbed-os-5.15-latest
steps:
-
name: Checkout repo
uses: actions/checkout@v2
with:
fetch-depth: 0
-
name: run test
shell: bash
run: |
# Check that example compiles
sed -n '/``` cpp/,/```/{/```$/Q;/```/d;p;}' events/README.md > main.cpp
python tools/make.py -t GCC_ARM -m K64F --source=. --build=BUILD/K64F/GCC_ARM -j0
# Check that example compiles without rtos
sed -n '/``` cpp/,/```/{/```$/Q;/```/d;p;}' events/README.md > main.cpp
rm -r rtos drivers/source/usb features/cellular features/netsocket features/nanostack \
features/lwipstack features/frameworks/greentea-client \
features/frameworks/utest features/frameworks/unity components BUILD
python tools/make.py -t GCC_ARM -m DISCO_F401VC --source=. --build=BUILD/DISCO_F401VC/GCC_ARM -j0
# Run local equeue tests
make -C events/source test
# Run profiling tests
make -C events/source prof | tee prof

View File

@ -1,297 +0,0 @@
# Copyright (c) 2013-2019 Arm Limited. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the License); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
language: sh
os: linux
dist: xenial
env:
global:
- deps_url="https://mbed-os-ci-public.s3-eu-west-1.amazonaws.com/jenkins-ci/deps"
- deps_dir="${HOME}/.cache/deps"
cache:
pip: true
directories:
- ${HOME}/.cache/deps
before_install:
- source tools/test/travis-ci/functions.sh
- set_status "pending" "Test started."
after_success:
- set_status "success" "Success!"
after_failure:
- set_status "failure" "Test failed."
matrix:
include:
### Basic Tests ###
- &basic-vm
stage: "Basic"
name: "file attributes"
env: NAME=gitattributestest
script:
- git diff --exit-code
- <<: *basic-vm
name: "license check"
env: NAME=licence_check
script:
- |
! grep --recursive --max-count=100 --ignore-case --exclude .travis.yml \
"gnu general\|gnu lesser\|lesser general\|public license"
- <<: *basic-vm
name: "include check"
env: NAME=include_check
script:
- |
! git grep '^#include\s["'"']mbed.h['"'"]$' -- '*.c' '*.h' '*.cpp' '*.hpp' \
':!*platform_mbed.h' ':!*TESTS/*' ':!TEST_APPS/' ':!UNITTESTS/' \
':!*tests/*' ':!*targets/*' ':!*TARGET_*' ':!*unsupported/*'
### Docs Tests ###
- &docs-vm
stage: "Docs"
name: "astyle"
env: NAME=astyle
install:
- >-
curl -L0 https://mbed-os-ci-public.s3-eu-west-1.amazonaws.com/jenkins-ci/deps/astyle_3.1_linux.tar.gz --output astyle.tar.gz;
mkdir -p BUILD && tar xf astyle.tar.gz -C BUILD;
cd BUILD/astyle/build/gcc;
make;
export PATH="${PWD}/bin:${PATH}";
cd -
- astyle --version
# Fetch remaining information needed for branch comparison
- git fetch --all --unshallow --tags
- git fetch origin "${TRAVIS_BRANCH}"
script:
- >-
git diff --name-only --diff-filter=d FETCH_HEAD..HEAD \
| ( grep '.\(c\|cpp\|h\|hpp\)$' || true ) \
| ( grep -v -f .astyleignore || true ) \
| while read file; do astyle -n --options=.astylerc "${file}"; done
- git diff --exit-code --diff-filter=d --color
- <<: *docs-vm
name: "spellcheck"
env: NAME=doxy-spellcheck
install:
- source_pkg aspell
script:
- ./tools/test/travis-ci/doxy-spellchecker/spell.sh drivers
- ./tools/test/travis-ci/doxy-spellchecker/spell.sh platform
- ./tools/test/travis-ci/doxy-spellchecker/spell.sh events
- ./tools/test/travis-ci/doxy-spellchecker/spell.sh rtos
- ./tools/test/travis-ci/doxy-spellchecker/spell.sh features/netsocket
- <<: *docs-vm
name: "doxygen"
env: NAME=docs
install:
# Build doxygen
- >
(git clone --depth=1 --single-branch --branch Release_1_8_14 https://github.com/doxygen/doxygen;
cd doxygen;
mkdir build;
cd build;
cmake -G "Unix Makefiles" ..;
make;
sudo make install)
# Create BUILD directory for tests
- mkdir BUILD
script:
# Assert that the Doxygen build produced no warnings.
# The strange command below asserts that the Doxygen command had an
# output of zero length
- doxygen doxyfile_options 2>&1
# Once Mbed OS has been fixed, enable the full test by replacing the top line with this:
# - ( ! doxygen doxyfile_options 2>&1 | grep . )
# Assert that all binary libraries are named correctly
# The strange command below asserts that there are exactly 0 libraries
# that do not start with lib
- >
find "(" -name "*.a" -or -name "*.ar" ")" -and -not -name "lib*" |
tee BUILD/badlibs |
sed -e "s/^/Bad library name found: /" && [ ! -s BUILD/badlibs ]
# Assert that all assebler files are named correctly
# The strange command below asserts that there are exactly 0 libraries
# that do end with .s
- >
find -name "*.s" | tee BUILD/badasm |
sed -e "s/^/Bad Assembler file name found: /" && [ ! -s BUILD/badasm ]
### Python Tests ###
- &pytools-vm
stage: "Pytest"
name: "tools-py35"
env: NAME=tools-py3.5
language: python
python: 3.5
install:
# Install gcc
- source_pkg gcc
- arm-none-eabi-gcc --version
# Install additional python modules
- python --version
- |-
tr -d ' ' >> requirements.txt <<< "
mock==2.0.0
attrs==19.1.0
pytest==3.3.0
pylint>=1.9,<2
hypothesis>=3,<4
coverage>=4.5,<5
"
- python -m pip install --upgrade pip==18.1
- python -m pip install --upgrade setuptools==40.4.3
- pip install -r requirements.txt
- pip list --verbose
script:
# Run local testing on tools
- PYTHONPATH=. coverage run -a -m pytest tools/test
- python tools/test/pylint.py
- coverage run -a tools/project.py -S | sed -n '/^Total/p'
- coverage html
- <<: *pytools-vm
name: "tools-py36"
env: NAME=tools-py3.6
python: 3.6
- <<: *pytools-vm
name: "tools-py37"
env: NAME=tools-py3.7
python: 3.7
### Extended Tests ###
- <<: *extended-vm
name: "events"
env: NAME=events EVENTS=events
script:
# Check that example compiles
- sed -n '/``` cpp/,/```/{/```$/Q;/```/d;p;}' ${EVENTS}/README.md > main.cpp
- python tools/make.py -t GCC_ARM -m K64F --source=. --build=BUILD/K64F/GCC_ARM -j0
# Check that example compiles without rtos
- sed -n '/``` cpp/,/```/{/```$/Q;/```/d;p;}' ${EVENTS}/README.md > main.cpp
- |
rm -r rtos drivers/source/usb features/cellular features/netsocket features/nanostack \
features/lwipstack features/frameworks/greentea-client \
features/frameworks/utest features/frameworks/unity components BUILD
- python tools/make.py -t GCC_ARM -m DISCO_F401VC --source=. --build=BUILD/DISCO_F401VC/GCC_ARM -j0
# Run local equeue tests
- make -C ${EVENTS}/source test
# Run profiling tests
- make -C ${EVENTS}/source prof | tee prof
after_success:
# Update status, comparing with master if possible.
- |
CURR=$(grep -o '[0-9]\+ cycles' prof | awk '{sum += $1} END {print sum}')
PREV=$(curl -u "${MBED_BOT}" https://api.github.com/repos/${TRAVIS_REPO_SLUG}/status/master \
| jq -re "select(.sha != \"${TRAVIS_COMMIT}\")
| .statuses[] | select(.context == \"travis-ci/${NAME}\").description
| capture(\"runtime is (?<runtime>[0-9]+)\").runtime" \
|| echo 0)
delta=""
[ "${PREV}" -ne 0 ] && delta="($(printf "%+d" "$(( ${CURR} - ${PREV} ))" cycles)"
set_status "success" "Success! Runtime is ${CURR} cycles. ${delta}"
- <<: *extended-vm
name: "littlefs"
env: NAME=littlefs LITTLEFS=features/storage/filesystem/littlefs
install:
# Install gcc
- source_pkg gcc
- arm-none-eabi-gcc --version
# Install python modules
- python -m pip install --upgrade pip==18.1
- python -m pip install --upgrade setuptools==40.4.3
- pip install -r requirements.txt
- pip list --verbose
# Install test-specific packages
- source_pkg fuse
- source_pkg libfuse-dev
- fusermount --version
before_script:
# Setup and patch littlefs-fuse
- git clone https://github.com/armmbed/littlefs-fuse littlefs_fuse
- git -C littlefs_fuse checkout 3f1ed6e37799e49e3710830dc6abb926d5503cf2
- echo '*' > littlefs_fuse/.mbedignore
- rm -rf littlefs_fuse/littlefs/*
- cp -r $(git ls-tree --name-only HEAD ${LITTLEFS}/littlefs/) littlefs_fuse/littlefs
# Create file-backed disk
- mkdir MOUNT
- sudo chmod a+rw /dev/loop0
- dd if=/dev/zero bs=512 count=2048 of=DISK
- losetup /dev/loop0 DISK
- CFLAGS="-Werror -Wno-format"
script:
# Check that example compiles
- export CFLAGS="-Werror -Wno-format"
- sed -n '/``` c++/,/```/{/```/d;p;}' ${LITTLEFS}/README.md > main.cpp
- python tools/make.py -t GCC_ARM -m K82F --source=. --build=BUILD/K82F/GCC_ARM -j0
# Run local littlefs tests
- make -C${LITTLEFS}/littlefs test QUIET=1
# Run local littlefs tests with set of variations
- make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=64 -DLFS_PROG_SIZE=64"
- make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_PROG_SIZE=1"
- make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512"
- make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048"
- make -C${LITTLEFS}/littlefs clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS"
# Self-hosting littlefs fuzz test with littlefs-fuse
- make -Clittlefs_fuse
- littlefs_fuse/lfs --format /dev/loop0
- littlefs_fuse/lfs /dev/loop0 MOUNT
- ls MOUNT
- mkdir MOUNT/littlefs
- cp -r $(git ls-tree --name-only HEAD ${LITTLEFS}/littlefs/) MOUNT/littlefs
- ls MOUNT/littlefs
- CFLAGS="-Wno-format" make -CMOUNT/littlefs -B test_dirs test_files QUIET=1
# Compile and find the code size with smallest configuration
- cd ${TRAVIS_BUILD_DIR}/${LITTLEFS}/littlefs
- make clean size
CC='arm-none-eabi-gcc -mthumb'
OBJ="$(ls lfs*.o | tr '\n' ' ')"
CFLAGS+="-DLFS_NO_ASSERT -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR"
| tee sizes
after_success:
# Update status, comparing with master if possible.
- |
CURR=$(tail -n1 sizes | awk '{print $1}')
PREV=$(curl -u "${MBED_BOT}" https://api.github.com/repos/${TRAVIS_REPO_SLUG}/status/master \
| jq -re "select(.sha != \"${TRAVIS_COMMIT}\")
| .statuses[] | select(.context == \"travis-ci/${NAME}\").description
| capture(\"code size is (?<size>[0-9]+)\").size" \
|| echo 0)
delta=""
[ "${PREV}" -ne 0 ] && delta="($(printf "%+0.2f%%" "$(<<< "100 * ((${CURR} - ${PREV})/${PREV})" bc -l)"))"
set_status "success" "Success! Code size is ${CURR}B. ${delta}"

View File

@ -1,15 +1,13 @@
[![Mbed OS][mbed-os-logo]][mbed-os-link]
[![Build status release][mbed-travis-release-svg]][mbed-travis-release]
[![Build status master][mbed-travis-master-svg]][mbed-travis-master]
[![Build status][mbed-os-5.15-svg]][mbed-os-5.15]
[![Tools coverage status][mbed-coveralls-tools-svg]][mbed-coveralls-tools]
[mbed-os-logo]: logo.png
[mbed-os-link]: https://www.mbed.com/en/platform/mbed-os/
[mbed-travis-master]: https://travis-ci.org/ARMmbed/mbed-os
[mbed-travis-master-svg]: https://travis-ci.org/ARMmbed/mbed-os.svg?branch=master
[mbed-travis-release]: https://travis-ci.org/ARMmbed/mbed-os/branches
[mbed-travis-release-svg]: https://travis-ci.org/ARMmbed/mbed-os.svg?branch=latest
[mbed-os-5.15]: https://github.com/armmbed/mbed-os/actions/workflows/basic_checks.yml
[mbed-os-5.15-svg]: https://github.com/armmbed/mbed-os/actions/workflows/basic_checks.yml/badge.svg?branch=mbed-os-5.15
[mbed-coveralls-tools]: https://coveralls.io/github/ARMmbed/mbed-os?branch=master
[mbed-coveralls-tools-svg]: https://coveralls.io/repos/github/ARMmbed/mbed-os/badge.svg?branch=master

View File

@ -121,6 +121,6 @@ echo "Total Errors Found: ${ERRORS}"
if [ ${ERRORS} -ne 0 ]; then
echo "If any of the failed words should be considered valid please add them to the ignore.en.pws file"\
"found in tools/test/travis-ci/doxy-spellchecker between the _code_ and _doxy_ tags."
"found in tools/test/ci/doxy-spellchecker between the _code_ and _doxy_ tags."
exit 1
fi

View File

@ -0,0 +1,182 @@
"""
SPDX-License-Identifier: Apache-2.0
Copyright (c) 2020 Arm Limited. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations
"""
import argparse
import json
import logging
import os.path
import re
import sys
from enum import Enum
MISSING_LICENSE_TEXT = "Missing license header"
MISSING_PERMISSIVE_LICENSE_TEXT = "Non-permissive license"
MISSING_SPDX_TEXT = "Missing SPDX license identifier"
userlog = logging.getLogger("scancode-evaluate")
class ReturnCode(Enum):
"""Return codes."""
SUCCESS = 0
ERROR = -1
def init_logger():
"""Initialise the logger."""
userlog.setLevel(logging.INFO)
userlog.addHandler(
logging.FileHandler(
os.path.join(os.getcwd(), 'scancode-evaluate.log'), mode='w'
)
)
def path_leaf(path):
"""Return the leaf of a path."""
head, tail = os.path.split(path)
# Ensure the correct file name is returned if the file ends with a slash
return tail or os.path.basename(head)
def has_permissive_text_in_scancode_output(scancode_output_data_file_licenses):
"""Returns true if at least one license in the scancode output is permissive"""
return any(
scancode_output_data_file_license['category'] == 'Permissive'
for scancode_output_data_file_license in scancode_output_data_file_licenses
)
def has_spdx_text_in_scancode_output(scancode_output_data_file_licenses):
"""Returns true if at least one license in the scancode output has the spdx identifier."""
return any(
'spdx' in scancode_output_data_file_license['matched_rule']['identifier']
for scancode_output_data_file_license in scancode_output_data_file_licenses
)
def has_spdx_text_in_analysed_file(scanned_file_content):
"""Returns true if the file analysed by ScanCode contains SPDX identifier."""
return bool(re.findall("SPDX-License-Identifier:?", scanned_file_content))
def has_binary_license(scanned_file_content):
"""Returns true if the file analysed by ScanCode contains a Permissive Binary License."""
return bool(re.findall("Permissive Binary License", scanned_file_content))
def get_file_text(scancode_output_data_file):
"""Returns file text for scancode output file"""
file_path = os.path.abspath(scancode_output_data_file['path'])
try:
with open(file_path, 'r') as read_file:
return read_file.read()
except UnicodeDecodeError:
userlog.warning("Unable to decode file text in: %s" % file_path)
# Ignore files that cannot be decoded
def license_check(scancode_output_path):
"""Check licenses in the scancode json file for specified directory.
This function does not verify if file exists, should be done prior the call.
Args:
scancode_output_path: path to the scancode json output file (output from scancode --license --json-pp)
Returns:
0 if nothing found
>0 - count how many license issues found
ReturnCode.ERROR.value if any error in file licenses found
"""
license_offenders = []
spdx_offenders = []
try:
with open(scancode_output_path, 'r') as read_file:
scancode_output_data = json.load(read_file)
except json.JSONDecodeError as jex:
userlog.warning("JSON could not be decoded, Invalid JSON in body: %s", jex)
return ReturnCode.ERROR.value
if 'files' not in scancode_output_data:
userlog.warning("Missing `files` attribute in %s" % (scancode_output_path))
return ReturnCode.ERROR.value
for scancode_output_data_file in scancode_output_data['files']:
if scancode_output_data_file['type'] != 'file':
continue
if not scancode_output_data_file['licenses']:
scancode_output_data_file['fail_reason'] = MISSING_LICENSE_TEXT
license_offenders.append(scancode_output_data_file)
# check the next file in the scancode output
continue
if not has_permissive_text_in_scancode_output(scancode_output_data_file['licenses']):
scanned_file_content = get_file_text(scancode_output_data_file)
if not (scanned_file_content and has_binary_license(scanned_file_content)):
scancode_output_data_file['fail_reason'] = MISSING_PERMISSIVE_LICENSE_TEXT
license_offenders.append(scancode_output_data_file)
if not has_spdx_text_in_scancode_output(scancode_output_data_file['licenses']):
# Scancode does not recognize license notice in Python file headers.
# Issue: https://github.com/nexB/scancode-toolkit/issues/1913
# Therefore check if the file tested by ScanCode actually has a licence notice.
scanned_file_content = get_file_text(scancode_output_data_file)
if not scanned_file_content:
continue
elif not has_spdx_text_in_analysed_file(scanned_file_content):
scancode_output_data_file['fail_reason'] = MISSING_SPDX_TEXT
spdx_offenders.append(scancode_output_data_file)
if license_offenders:
userlog.warning("Found files with missing license details, please review and fix")
for offender in license_offenders:
userlog.warning("File: %s reason: %s" % (path_leaf(offender['path']), offender['fail_reason']))
if spdx_offenders:
userlog.warning("Found files with missing SPDX identifier, please review and fix")
for offender in spdx_offenders:
userlog.warning("File: %s reason: %s" % (path_leaf(offender['path']), offender['fail_reason']))
return len(license_offenders)
def parse_args():
"""Parse command line arguments."""
parser = argparse.ArgumentParser(description="License check.")
parser.add_argument(
'scancode_output_path',
help="scancode-toolkit output json file"
)
return parser.parse_args()
if __name__ == "__main__":
init_logger()
args = parse_args()
if os.path.isfile(args.scancode_output_path):
sys.exit(
ReturnCode.SUCCESS.value
if license_check(args.scancode_output_path) == 0
else ReturnCode.ERROR.value
)
else:
userlog.warning("Could not find the scancode json file")
sys.exit(ReturnCode.ERROR.value)

View File

@ -0,0 +1,110 @@
#!/usr/bin/env python
# Copyright (c) 2020 Arm Limited and Contributors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
import importlib
import os
import pytest
license_check = importlib.import_module("scancode-evaluate").license_check
STUBS_PATH = os.path.join(
os.path.abspath(os.path.join(os.path.dirname(__file__))), "scancode_test"
)
HEADER_WITHOUT_SPDX = "/* Copyright (C) Arm Limited, Inc - All Rights Reserved\
* Unauthorized copying of this. file, via any medium is strictly prohibited\
* Proprietary and confidential\
*/"
HEADER_WITH_SPDX = "/* mbed Microcontroller Library\
* Copyright (c) 2006-2013 ARM Limited\
*\
* SPDX-License-Identifier: Apache-2.0\
* Licensed under the Apache License, Version 2.0 (the \"License\");\
* you may not use this file except in compliance with the License.\
* You may obtain a copy of the License at\
*\
* http://www.apache.org/licenses/LICENSE-2.0\
*\
* Unless required by applicable law or agreed to in writing, software\
* distributed under the License is distributed on an \"AS IS\" BASIS,\
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\
* See the License for the specific language governing permissions and\
* limitations under the License.\
*/"
HEADER_WITH_BINARY_LICENSE = "/*\
* Copyright (c) 2019, Arm Limited, All Rights Reserved\
* SPDX-License-Identifier: LicenseRef-PBL\
*\
* This file and the related binary are licensed under the\
* Permissive Binary License, Version 1.0 (the \"License\");\
* you may not use these files except in compliance with the License.\
*\
*/"
@pytest.fixture()
def create_scanned_files():
"""Create stub files.
test3.h missing license notice
test4.h with license notice
test5.h with license notice
test6.h with permissive binary license
"""
file_paths = [
os.path.join(STUBS_PATH, "test3.h"),
os.path.join(STUBS_PATH, "test4.h"),
os.path.join(STUBS_PATH, "test5.h"),
os.path.join(STUBS_PATH, "test6.h")
]
for file_path in file_paths:
with open(file_path, "w") as new_file:
if file_path in [os.path.join(STUBS_PATH, "test3.h")]:
new_file.write(HEADER_WITHOUT_SPDX)
elif file_path in [os.path.join(STUBS_PATH, "test6.h")]:
new_file.write(HEADER_WITH_BINARY_LICENSE)
else:
new_file.write(HEADER_WITH_SPDX)
yield
for file_path in file_paths:
os.remove(file_path)
class TestScancodeEvaluate:
def test_missing_files_attribute(self):
""" Missing `files` attribute in JSON.
@inputs scancode_test/scancode_test_1.json
@outputs -1
"""
assert license_check(os.path.join(STUBS_PATH, "scancode_test_1.json")) == -1
def test_various_combinations_permissive_license_with_spdx(self):
""" Various combinations where at least one license in
a file is permissive and has spdx in the match.identifier
attribute.
@inputs scancode_test/scancode_test_2.json
@outputs 0
"""
assert license_check(os.path.join(STUBS_PATH, "scancode_test_2.json")) == 0
def test_missing_license_permissive_license_and_spdx(self, create_scanned_files):
""" Test four files scanned with various issues.
test.h: Missing license text (error count += 1)
test3.h: Missing `Permissive` license text and `spdx` in match.identifier and not in file tested by ScanCode (error count += 1)
test4.h: Missing `Permissive` license text and `spdx` in match.identifier but found in file tested by ScanCode (error count += 1)
test5.h: Missing `spdx` in match.identifier but found in file tested by ScanCode. (error count += 0)
test6.h: Matching `spdx` in match.identifier but Permissive Binary License header (error count += 0)
@inputs scancode_test/scancode_test_2.json
@output 3
"""
assert license_check(os.path.join(STUBS_PATH, "scancode_test_3.json")) == 3
def test_permissive_license_no_spdx(self, create_scanned_files):
""" Multiple `Permissive` licenses in one file but none with `spdx` in
match.identifier and not in file tested by ScanCode (error count += 0)
@inputs scancode_test/scancode_test_2.json
@outputs 0
"""
assert license_check(os.path.join(STUBS_PATH, "scancode_test_4.json")) == 0

View File

@ -0,0 +1,7 @@
{
"headers": [
{
"tool_name": "scancode test fail"
}
]
}

View File

@ -0,0 +1,782 @@
{
"headers":[
{
"tool_name":"scancode-toolkit",
"tool_version":"3.1.1",
"options":{
"input":[
"test.h",
"test2.h",
"test3.h",
"test4.h",
"test5.h",
"test6.h",
"test7.h",
"test8.h"
],
"--json-pp":"scancode.json",
"--license":true
},
"notice":"Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
"start_timestamp":"2020-10-01T135040.106260",
"end_timestamp":"2020-10-01T135047.793497",
"message":null,
"errors":[
],
"extra_data":{
"files_count":8
}
}
],
"files":[
{
"path":"tools/test/ci/scancode_test/test.h",
"type":"file",
"licenses":[
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":2,
"end_line":2,
"matched_rule":{
"identifier":"spdx-license-identifier: bsd-new",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":8,
"end_line":8,
"matched_rule":{
"identifier":"bsd-new_360.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":4,
"matched_length":4,
"match_coverage":100.0,
"rule_relevance":100.0
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":11,
"end_line":11,
"matched_rule":{
"identifier":"bsd-new_10.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100.0
}
}
],
"license_expressions":[
"bsd-new",
"bsd-new",
"bsd-new"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test2.h",
"type":"file",
"licenses":[
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":2,
"end_line":2,
"matched_rule":{
"identifier":"spdx-license-identifier: bsd-new",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":8,
"end_line":8,
"matched_rule":{
"identifier":"bsd-new_360.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":4,
"matched_length":4,
"match_coverage":100.0,
"rule_relevance":100.0
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":11,
"end_line":11,
"matched_rule":{
"identifier":"bsd-new_10.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100.0
}
}
],
"license_expressions":[
"bsd-new",
"bsd-new",
"bsd-new"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test3.h",
"type":"file",
"licenses":[
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":2,
"end_line":2,
"matched_rule":{
"identifier":"spdx-license-identifier: bsd-new",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":8,
"end_line":8,
"matched_rule":{
"identifier":"bsd-new_360.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":4,
"matched_length":4,
"match_coverage":100.0,
"rule_relevance":100.0
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":11,
"end_line":11,
"matched_rule":{
"identifier":"bsd-new_10.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100.0
}
}
],
"license_expressions":[
"bsd-new",
"bsd-new",
"bsd-new"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test4.h",
"type":"file",
"licenses":[
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":2,
"end_line":2,
"matched_rule":{
"identifier":"spdx-license-identifier: bsd-new",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":8,
"end_line":8,
"matched_rule":{
"identifier":"bsd-new_360.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":4,
"matched_length":4,
"match_coverage":100.0,
"rule_relevance":100.0
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":11,
"end_line":11,
"matched_rule":{
"identifier":"bsd-new_10.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100.0
}
}
],
"license_expressions":[
"bsd-new",
"bsd-new",
"bsd-new"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test5.c",
"type":"file",
"licenses":[
{
"key":"apache-2.0",
"score":89.58,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":17,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"3-seq",
"rule_length":96,
"matched_length":86,
"match_coverage":89.58,
"rule_relevance":100.0
}
},
{
"key":"apache-2.0",
"score":100.0,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":5,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
}
],
"license_expressions":[
"apache-2.0",
"apache-2.0"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test6.c",
"type":"file",
"licenses":[
{
"key":"apache-2.0",
"score":89.58,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":17,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"3-seq",
"rule_length":96,
"matched_length":86,
"match_coverage":89.58,
"rule_relevance":100.0
}
},
{
"key":"apache-2.0",
"score":100.0,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":5,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
}
],
"license_expressions":[
"apache-2.0",
"apache-2.0"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test7.c",
"type":"file",
"licenses":[
{
"key":"apache-2.0",
"score":89.58,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":17,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"3-seq",
"rule_length":96,
"matched_length":86,
"match_coverage":89.58,
"rule_relevance":100.0
}
},
{
"key":"apache-2.0",
"score":100.0,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":5,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
}
],
"license_expressions":[
"apache-2.0",
"apache-2.0"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test8.c",
"type":"file",
"licenses":[
{
"key":"apache-2.0",
"score":89.58,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":17,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"3-seq",
"rule_length":96,
"matched_length":86,
"match_coverage":89.58,
"rule_relevance":100.0
}
},
{
"key":"apache-2.0",
"score":100.0,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":5,
"end_line":5,
"matched_rule":{
"identifier":"spdx-license-identifier: apache-2.0",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
}
],
"license_expressions":[
"apache-2.0",
"apache-2.0"
],
"scan_errors":[
]
}
]
}

View File

@ -0,0 +1,220 @@
{
"headers":[
{
"tool_name":"scancode-toolkit",
"tool_version":"3.1.1",
"options":{
"input":[
"tools/test/ci/scancode_test/test.h",
"tools/test/ci/scancode_test/test2.h",
"tools/test/ci/scancode_test/test3.h",
"tools/test/ci/scancode_test/test4.h",
"tools/test/ci/scancode_test/test5.h"
],
"--json-pp":"scancode.json",
"--license":true
},
"notice":"Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
"start_timestamp":"2020-10-01T135040.106260",
"end_timestamp":"2020-10-01T135047.793497",
"message":null,
"errors":[
],
"extra_data":{
"files_count":5
}
}
],
"files":[
{
"path":"tools/test/ci/scancode_test/test.h",
"type":"file",
"licenses":[
],
"license_expressions":[
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test3.h",
"type":"file",
"licenses":[
{
"key":"proprietary-license",
"score":77.0,
"name":"Proprietary License",
"short_name":"Proprietary License",
"category":"Proprietary Free",
"is_exception":false,
"owner":"Unspecified",
"homepage_url":null,
"text_url":"",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:proprietary-license",
"spdx_license_key":null,
"spdx_url":"",
"start_line":2,
"end_line":3,
"matched_rule":{
"identifier":"proprietary_12.RULE",
"license_expression":"proprietary-license",
"licenses":[
"proprietary-license"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":14,
"matched_length":14,
"match_coverage":100.0,
"rule_relevance":77
}
}
],
"license_expressions":[
"proprietary-license"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test4.h",
"type":"file",
"licenses":[
{
"key":"proprietary-license",
"score":77.0,
"name":"Proprietary License",
"short_name":"Proprietary License",
"category":"Proprietary Free",
"is_exception":false,
"owner":"Unspecified",
"homepage_url":null,
"text_url":"",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:proprietary-license",
"spdx_license_key":null,
"spdx_url":"",
"start_line":2,
"end_line":3,
"matched_rule":{
"identifier":"proprietary_12.RULE",
"license_expression":"proprietary-license",
"licenses":[
"proprietary-license"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":14,
"matched_length":14,
"match_coverage":100.0,
"rule_relevance":77
}
}
],
"license_expressions":[
"proprietary-license"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test5.h",
"type":"file",
"licenses":[
{
"key":"apache-2.0",
"score":100.0,
"name":"Apache License 2.0",
"short_name":"Apache 2.0",
"category":"Permissive",
"is_exception":false,
"owner":"Apache Software Foundation",
"homepage_url":"http://www.apache.org/licenses/",
"text_url":"http://www.apache.org/licenses/LICENSE-2.0",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:apache-2.0",
"spdx_license_key":"Apache-2.0",
"spdx_url":"https://spdx.org/licenses/Apache-2.0",
"start_line":4,
"end_line":14,
"matched_rule":{
"identifier":"apache-2.0_7.RULE",
"license_expression":"apache-2.0",
"licenses":[
"apache-2.0"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":85,
"matched_length":85,
"match_coverage":100.0,
"rule_relevance":100
}
}
],
"license_expressions":[
"apache-2.0"
],
"scan_errors":[
]
},
{
"path":"tools/test/ci/scancode_test/test6.h",
"type":"file",
"licenses":[
{
"key": "unknown-spdx",
"score": 100.0,
"name": "Unknown SPDX license detected but not recognized",
"short_name": "unknown SPDX",
"category": "Unstated License",
"is_exception": false,
"owner": "Unspecified",
"homepage_url": null,
"text_url": "",
"reference_url": "https://enterprise.dejacode.com/urn/urn:dje:license:unknown-spdx",
"spdx_license_key": null,
"spdx_url": "",
"start_line": 3,
"end_line": 3,
"matched_rule": {
"identifier": "spdx-license-identifier: unknown-spdx",
"license_expression": "unknown-spdx",
"licenses": [
"unknown-spdx"
],
"is_license_text": false,
"is_license_notice": false,
"is_license_reference": false,
"is_license_tag": true,
"matcher": "1-spdx-id",
"rule_length": 1,
"matched_length": 1,
"match_coverage": 100.0,
"rule_relevance": 100
}
}
],
"license_expressions":[
"unknown-spdx"
],
"scan_errors":[
]
}
]
}

View File

@ -0,0 +1,170 @@
{
"headers":[
{
"tool_name":"scancode-toolkit",
"tool_version":"3.1.1",
"options":{
"input":[
"test4.h"
],
"--json-pp":"scancode.json",
"--license":true
},
"notice":"Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
"start_timestamp":"2020-10-01T135040.106260",
"end_timestamp":"2020-10-01T135047.793497",
"message":null,
"errors":[
],
"extra_data":{
"files_count":1
}
}
],
"files":[
{
"path":"tools/test/ci/scancode_test/test3.h",
"type":"file",
"licenses":[
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":2,
"end_line":2,
"matched_rule":{
"identifier":"sprdx-license-identifier: bsd-new",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":false,
"is_license_tag":true,
"matcher":"1-spdx-id",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":8,
"end_line":8,
"matched_rule":{
"identifier":"bsd-new_360.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":4,
"matched_length":4,
"match_coverage":100.0,
"rule_relevance":100.0
}
},
{
"key":"elastic-license-2018",
"score":72.22,
"name":"Elastic License Agreement 2018",
"short_name":"Elastic License 2018",
"category":"Source-available",
"is_exception":false,
"owner":"Elastic",
"homepage_url":"https://github.com/elastic/elasticsearch/blob/0d8aa7527e242fbda9d84867ab8bc955758eebce/licenses/ELASTIC-LICENSE.txt",
"text_url":"https://github.com/elastic/elasticsearch/blob/0d8aa7527e242fbda9d84867ab8bc955758eebce/licenses/ELASTIC-LICENSE.txt",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:elastic-license-2018",
"spdx_license_key":null,
"spdx_url":"",
"start_line":9,
"end_line":10,
"matched_rule":{
"identifier":"elastic_1.RULE",
"license_expression":"elastic-license-2018",
"licenses":[
"elastic-license-2018"
],
"is_license_text":false,
"is_license_notice":true,
"is_license_reference":false,
"is_license_tag":false,
"matcher":"3-seq",
"rule_length":18,
"matched_length":13,
"match_coverage":72.22,
"rule_relevance":100.0
}
},
{
"key":"bsd-new",
"score":100.0,
"name":"BSD-3-Clause",
"short_name":"BSD-3-Clause",
"category":"Permissive",
"is_exception":false,
"owner":"Regents of the University of California",
"homepage_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"text_url":"http://www.opensource.org/licenses/BSD-3-Clause",
"reference_url":"https://enterprise.dejacode.com/urn/urn:dje:license:bsd-new",
"spdx_license_key":"BSD-3-Clause",
"spdx_url":"https://spdx.org/licenses/BSD-3-Clause",
"start_line":11,
"end_line":11,
"matched_rule":{
"identifier":"bsd-new_10.RULE",
"license_expression":"bsd-new",
"licenses":[
"bsd-new"
],
"is_license_text":false,
"is_license_notice":false,
"is_license_reference":true,
"is_license_tag":false,
"matcher":"2-aho",
"rule_length":3,
"matched_length":3,
"match_coverage":100.0,
"rule_relevance":100.0
}
}
],
"license_expressions":[
"bsd-new",
"bsd-new",
"elastic-license-2018",
"bsd-new"
],
"scan_errors":[
]
}
]
}

View File

@ -1,196 +0,0 @@
#!/bin/bash -euf
#
# Copyright (c) 2013-2018 Arm Limited. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the License); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o pipefail
#
# Helper functions for printing status information.
# Uses 'echo' instead of 'printf' due to Travis CI stdout sync issues.
#
info() { echo -e "I: ${1}"; }
die() { echo -e "E: ${1}" 1>&2; exit "${2:-1}"; }
#
# Sets the GitHub job status for a given commit.
#
set_status()
{
local job_name="${NAME}"
local payload=""
payload=$(<<< "
{
'state': '${1}',
'description': '${2}',
'context': 'travis-ci/${job_name}',
'target_url': 'https://travis-ci.org/${TRAVIS_REPO_SLUG}/jobs/${TRAVIS_JOB_ID}'
}" tr "'" '"')
curl --silent --output /dev/null --user "${MBED_BOT}" --request POST \
"https://api.github.com/repos/${TRAVIS_REPO_SLUG}/statuses/${TRAVIS_PULL_REQUEST_SHA:-${TRAVIS_COMMIT}}" \
--data @- <<< "${payload}"
}
#
# Sources a pre-compiled GCC installation from AWS, installing the archive by
# extracting and prepending the executable directory to PATH.
#
# Note: Expects 'deps_url' and 'deps_dir' to already be defined.
#
_install_gcc()
{
# Ignore shellcheck warnings: Variables defined in .travis.yml
# shellcheck disable=SC2154
local url="${deps_url}/gcc9-linux.tar.bz2"
# shellcheck disable=SC2154
local gcc_path="${deps_dir}/gcc/gcc-arm-none-eabi-9-2019-q4-major/"
local archive="gcc.tar.bz2"
info "URL: ${url}"
if [ ! -d "${deps_dir}/gcc" ]; then
info "Downloading archive"
curl --location "${url}" --output "${deps_dir}/${archive}"
ls -al "${deps_dir}"
info "Extracting 'gcc'"
mkdir -p "${deps_dir}/gcc"
tar -xf "${deps_dir}/${archive}" -C "${deps_dir}/gcc"
rm "${deps_dir}/${archive}"
fi
info "Installing 'gcc'"
export "PATH=${gcc_path}/bin:${PATH}"
}
#
# Downloads a list of packages from AWS, really fast.
#
_fetch_deps()
{
local pkg="${1}"
local dep_list="${2}"
local pid_list=""
local PID;
info "Fetching '${pkg}' archives"
while read -r dep; do
curl --location "${deps_url}/${dep}.deb" \
--output "${deps_dir}/${dep}.deb" \
|| die "Download failed ('${dep}')" \
&& info "Fetched ${deps_url}/${dep}.deb" &
PID=$!
pid_list="${pid_list} ${PID}"
done <<< "${dep_list}"
# Ignoring shellcheck warning, since we need to allow parameter expansion to
# turn the list string into parametesr.
# shellcheck disable=SC2086
wait ${pid_list}
info "Fetch completed."
}
#
# Installs a list of Debian packages, fetching them if not locally found.
#
_install_deps()
{
local pkg="${1}"
local dep_list="${2}"
local first_dep=""
# Assume that if the first package isn't cached, none are.
first_dep=$(<<< "${dep_list}" head -n1)
[ ! -f "${deps_dir}/${first_dep}.deb" ] && _fetch_deps "${pkg}" "${dep_list}"
# Install dependencies
info "Installing '${pkg}' packages"
# Ignore shellcheck warnings: Word splitting is specifically used to build
# command in one go, and expression non-expansion
# is intentional.
# shellcheck disable=SC2046 disable=SC2016
sudo dpkg -i $(<<< "${dep_list}" sed -e 's_^ *__' -e 's_^\(.*\)$_'"${deps_dir}"'/\1.deb_' | tr $'\n' ' ')
}
#
# Wrapper for installing a given package.
#
source_pkg()
{
# Debian dependencies needed for a single package.
local aspell_deps="aspell
aspell-en
dictionaries-common
libaspell15"
local libfuse_deps="libfuse-dev
libpcre3-dev
libpcre32-3
libpcrecpp0v5
libselinux1-dev
libsepol1-dev
libc-bin"
local pkg="${1}"
case "${pkg}" in
"fuse" )
# 'fuse' does not require an 'apt-get update' to install in Travis CI, so
# there's no reason to upload it or its dependencies into AWS.
sudo apt-get -o=dir::cache="${deps_dir}/apt-get" install fuse \
|| die "Installation failed"
;;
"aspell" )
_install_deps aspell "${aspell_deps}" \
|| die "Installation failed"
;;
"libfuse-dev" )
_install_deps libfuse-dev "${libfuse_deps}" \
|| die "Installation failed"
;;
"gcc" )
_install_gcc \
|| die "Installation failed"
;;
* )
die "Package not supported: '${pkg}'"
;;
esac
}