Issue #3386076 by fjgarlin, catch, longwave, larowlan, mstrelan, el7cosmos, RoSk0, xurizaemon, poker10, alexpott, bbrala, nick_schuch: GitLab CI integration for core

merge-requests/4724/merge
catch 2023-09-13 10:32:38 +01:00
parent 8e42cef117
commit dfa16b8aca
5 changed files with 490 additions and 1 deletions

114
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,114 @@
################
# Drupal GitLabCI template.
#
# Based off GitlabCI templates project: https://git.drupalcode.org/project/gitlab_templates
# Guide: https://www.drupal.org/docs/develop/git/using-gitlab-to-contribute-to-drupal/gitlab-ci
#
# With thanks to:
# - The GitLab Acceleration Initiative participants
# - DrupalSpoons
################
################
# Includes
#
# Additional configuration can be provided through includes.
# One advantage of include files is that if they are updated upstream, the
# changes affect all pipelines using that include.
#
# Includes can be overridden by re-declaring anything provided in an include,
# here in gitlab-ci.yml.
# https://docs.gitlab.com/ee/ci/yaml/includes.html#override-included-configuration-values
################
include:
- project: $_GITLAB_TEMPLATES_REPO
ref: $_GITLAB_TEMPLATES_REF
file:
- '/includes/include.drupalci.variables.yml'
- '/includes/include.drupalci.workflows.yml'
################
# Variables
#
# Overriding variables
# - To override one or more of these variables, simply declare your own variables keyword.
# - Keywords declared directly in .gitlab-ci.yml take precedence over include files.
# - Documentation: https://docs.gitlab.com/ee/ci/variables/
# - Predefined variables: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
#
################
variables:
_TARGET_PHP: "8.2"
COMPOSER: composer.json
# Let composer know what self.version means.
COMPOSER_ROOT_VERSION: "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}-dev"
CONCURRENCY: 32
################
# Stages
#
# Each job is assigned to a stage, defining the order in which the jobs are executed.
# Jobs in the same stage run in parallel.
#
# If all jobs in a stage succeed, the pipeline will proceed to the next stage.
# If any job in the stage fails, the pipeline will exit early.
################
.default-stage: &default-stage
stage: test
trigger:
# Rely on the status of the child pipeline.
strategy: depend
include:
- local: .gitlab-ci/pipeline.yml
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
.run-manually: &run-manually
when: manual
'PHP 8.2 MySQL 5.7':
<<: [ *default-stage, *run-manually ]
variables:
_TARGET_DB_TYPE: "mysql"
_TARGET_DB_VERSION: "5.7"
_DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
'PHP 8.2 MySQL 8':
<<: *default-stage
variables:
_TARGET_DB_TYPE: "mysql"
_TARGET_DB_VERSION: "8"
_DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
'PHP 8.2 PostgreSQL 13.5':
<<: [ *default-stage, *run-manually ]
variables:
_TARGET_DB_TYPE: "pgsql"
_TARGET_DB_VERSION: "13.5"
_DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
'PHP 8.2 PostgreSQL 14.1':
<<: [ *default-stage, *run-manually ]
variables:
_TARGET_DB_TYPE: "pgsql"
_TARGET_DB_VERSION: "14.1"
_DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production
# 'PHP 8.2 SQLite 3.26.0':
# <<: [ *default-stage, *run-manually ]
# variables:
# _TARGET_DB_TYPE: "sqlite"
# _TARGET_DB_VERSION: "3.26.0"
# _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production
# 'PHP 8.2 MariaDB 10.3.22':
# <<: [ *default-stage, *run-manually ]
# variables:
# _TARGET_DB_TYPE: "mariadb"
# _TARGET_DB_VERSION: "10.3.22"
# _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production

364
.gitlab-ci/pipeline.yml Normal file
View File

@ -0,0 +1,364 @@
# cspell:ignore drupaltestbot drupaltestbotpw
stages:
################
# Build
#
# Assemble the test environment.
################
- 🏗️ Build
################
# Code quality checks
#
# This stage includes any codebase validation before running tests.
################
- 🪄 Lint
################
# Test
#
# The test phase actually executes the tests, as well as gathering results
# and artifacts.
################
- 🗜️ Test
#############
# Templates #
#############
.default-job-settings: &default-job-settings
interruptible: true
allow_failure: false
image:
name: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production
rules:
- if: $CI_PIPELINE_SOURCE == "parent_pipeline"
.composer-cache: &composer-cache
key:
files:
- ./composer.json
- ./composer.lock
paths:
- ./vendor
.yarn-cache: &yarn-cache
key:
files:
- ./core/package.json
- ./core/yarn.lock
paths:
- ./core/node_modules
.pull-composer-cache: &pull-composer-cache
cache:
policy: pull
<<: *composer-cache
dependencies:
- '📦️ Composer'
.with-composer-cache: &with-composer-cache
needs:
- '📦️ Composer'
<<: *pull-composer-cache
.with-yarn-cache: &with-yarn-cache
dependencies:
- '📦️ Yarn'
needs:
- '📦️ Yarn'
cache:
policy: pull
<<: *yarn-cache
.junit-artifacts: &junit-artifacts
artifacts:
expose_as: junit
expire_in: 6 mos
paths:
- junit.xml
reports:
junit: junit.xml
.with-linting: &with-linting
needs:
- '📦️ Composer'
- '🧹 PHP Static Analysis (phpstan)'
- '🧹 PHP Coding standards (PHPCS)'
- '🧹 Compilation check'
- '📦️ Yarn'
- '📔 Spell-checking'
- '🧹 JavaScript linting (eslint)'
- '🧹 CSS linting (stylelint)'
<<: *pull-composer-cache
.with-unit-tests: &with-unit-tests
needs:
- '⚡️ PHPUnit Unit'
- '📦️ Composer'
<<: *pull-composer-cache
.with-composer-yarn-and-unit-tests: &with-composer-yarn-and-unit-tests
needs:
- '📦️ Composer'
- '⚡️ PHPUnit Unit'
- '📦️ Yarn'
dependencies:
- '📦️ Yarn'
- '📦️ Composer'
.test-variables: &test-variables
FF_NETWORK_PER_BUILD: 1
SIMPLETEST_BASE_URL: http://localhost/subdirectory
DB_DRIVER: mysql
MYSQL_ROOT_PASSWORD: root
MYSQL_DATABASE: mysql
MYSQL_USER: drupaltestbot
MYSQL_PASSWORD: drupaltestbotpw
MARIADB_TAG: $_TARGET_DB_VERSION
POSTGRES_TAG: $_TARGET_DB_VERSION
POSTGRES_DB: drupaltestbot
POSTGRES_USER: drupaltestbot
POSTGRES_PASSWORD: drupaltestbotpw
MINK_DRIVER_ARGS_WEBDRIVER: '["chrome", {"browserName":"chrome","chromeOptions":{"args":["--disable-dev-shm-usage","--disable-gpu","--headless"]}}, "http://localhost:9515"]'
CI_PARALLEL_NODE_INDEX: $CI_NODE_INDEX
CI_PARALLEL_NODE_TOTAL: $CI_NODE_TOTAL
.with-database: &with-database
name: $_DB_IMAGE
alias: database
.with-chrome: &with-chrome
name: $_CONFIG_DOCKERHUB_ROOT/chromedriver:production
alias: chrome
entrypoint:
- chromedriver
- "--no-sandbox"
- "--log-path=/tmp/chromedriver.log"
- "--verbose"
- "--whitelisted-ips="
.phpunit-artifacts: &phpunit-artifacts
artifacts:
when: always
expire_in: 6 mos
reports:
junit: ./sites/default/files/simpletest/phpunit-*.xml
paths:
- ./sites/default/files/simpletest/phpunit-*.xml
- ./sites/simpletest/browser_output
.setup-webroot: &setup-webserver
before_script:
- ln -s $CI_PROJECT_DIR /var/www/html/subdirectory
- sudo service apache2 start
.run-tests: &run-tests
script:
# Determine DB driver.
- |
[[ $_TARGET_DB_TYPE == "sqlite" ]] && export SIMPLETEST_DB=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite
[[ $_TARGET_DB_TYPE == "mysql" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
[[ $_TARGET_DB_TYPE == "mariadb" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
[[ $_TARGET_DB_TYPE == "pgsql" ]] && export SIMPLETEST_DB=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql
- export
- mkdir -p ./sites/simpletest ./sites/default/files ./build/logs/junit /var/www/.composer
- chown -R www-data:www-data ./sites ./build/logs/junit ./vendor /var/www/
- sudo -u www-data git config --global --add safe.directory $CI_PROJECT_DIR
# Need to pass this along directly.
- sudo MINK_DRIVER_ARGS_WEBDRIVER="$MINK_DRIVER_ARGS_WEBDRIVER" -u www-data php ./core/scripts/run-tests.sh --color --keep-results --types "$TESTSUITE" --concurrency "$CONCURRENCY" --repeat "1" --sqlite "./sites/default/files/.sqlite" --dburl $SIMPLETEST_DB --url $SIMPLETEST_BASE_URL --verbose --non-html --all --ci-parallel-node-index $CI_PARALLEL_NODE_INDEX --ci-parallel-node-total $CI_PARALLEL_NODE_TOTAL
################
# Jobs
#
# Jobs define what scripts are actually executed in each stage.
#
# The 'rules' keyword can also be used to define conditions for each job.
#
# Documentation: https://docs.gitlab.com/ee/ci/jobs/
################
################
# Build Jobs
################
' Output build parameters':
<<: *default-job-settings
stage: 🏗️ Build
script:
- echo "Checking variables"
- echo $_TARGET_PHP
- echo $_TARGET_DB_TYPE
- echo $_TARGET_DB_VERSION
'📦️ Composer':
<<: *default-job-settings
stage: 🏗️ Build
cache:
<<: *composer-cache
artifacts:
expire_in: 1 week
expose_as: 'web-vendor'
paths:
- vendor/
script:
- export
- composer validate
- composer install
'📦️ Yarn':
<<: *default-job-settings
stage: 🏗️ Build
cache:
<<: *yarn-cache
artifacts:
expire_in: 1 week
expose_as: 'yarn-vendor'
paths:
- core/node_modules/
script:
# Installs all core javascript dependencies and adds junit formatter.
- yarn --cwd ./core add stylelint-junit-formatter
################
# Lint Jobs
################
'🧹 PHP Coding standards (PHPCS)':
<<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ]
stage: 🪄 Lint
script:
- composer phpcs -- --report-junit=junit.xml --report-full --report-summary
'🧹 PHP Static Analysis (phpstan)':
<<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ]
stage: 🪄 Lint
script:
# Turn off apc to avoid corrupt composer cache.
- php -d apc.enabled=0 -d apc.enable_cli=0 vendor/bin/phpstan analyze --configuration=./core/phpstan.neon.dist --error-format=junit > junit.xml
'🧹 CSS linting (stylelint)':
<<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ]
stage: 🪄 Lint
script:
- yarn run --cwd=./core lint:css --color --custom-formatter node_modules/stylelint-junit-formatter > junit.xml
'🧹 Compilation check':
<<: [ *with-yarn-cache, *default-job-settings ]
stage: 🪄 Lint
script:
- yarn run --cwd=./core build:css --check
- cd core && yarn run -s check:ckeditor5
'🧹 JavaScript linting (eslint)':
<<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ]
stage: 🪄 Lint
script:
- yarn --cwd=./core run -s lint:core-js-passing --format junit > junit.xml
'📔 Spell-checking':
<<: [ *with-yarn-cache, *default-job-settings ]
stage: 🪄 Lint
script:
- git fetch origin
- export MODIFIED=`git diff --name-only origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}|while read r;do echo "$CI_PROJECT_DIR/$r";done|tr "\n" " "`
- echo $MODIFIED | tr ' ' '\n' | yarn --cwd=./core run -s spellcheck:core --no-must-find-files --file-list stdin
################
# Test Jobs
################
'⚡️ PHPUnit Unit':
<<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
stage: 🗜️ Test
services:
# There are some unit tests that need a database.
# @todo Remove after https://www.drupal.org/project/drupal/issues/3386217
- <<: *with-database
variables:
<<: *test-variables
TESTSUITE: PHPUnit-Unit
CONCURRENCY: "$CONCURRENCY"
'⚙️️ PHPUnit Kernel':
<<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
stage: 🗜️ Test
parallel: 3
variables:
<<: *test-variables
TESTSUITE: PHPUnit-Kernel
CONCURRENCY: "$CONCURRENCY"
services:
- <<: *with-database
'🖱️️️ PHPUnit Functional Javascript':
<<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
stage: 🗜️ Test
variables:
<<: *test-variables
TESTSUITE: PHPUnit-FunctionalJavascript
CONCURRENCY: 15
services:
- <<: *with-database
- <<: *with-chrome
'👷️️️ PHPUnit Build':
<<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
stage: 🗜️ Test
variables:
<<: *test-variables
TESTSUITE: PHPUnit-Build
CONCURRENCY: "$CONCURRENCY"
services:
- <<: *with-database
'🌐️️ PHPUnit Functional':
<<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ]
stage: 🗜️ Test
parallel: 8
variables:
<<: *test-variables
TESTSUITE: PHPUnit-Functional
CONCURRENCY: "$CONCURRENCY"
services:
- <<: *with-database
'🦉️️️ Nightwatch':
<<: [ *with-composer-yarn-and-unit-tests, *setup-webserver, *default-job-settings ]
stage: 🗜️ Test
variables:
<<: *test-variables
services:
- <<: *with-database
- <<: *with-chrome
script:
# Determine DB driver.
- |
[[ $_TARGET_DB_TYPE == "sqlite" ]] && export DRUPAL_TEST_DB_URL=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite
[[ $_TARGET_DB_TYPE == "mysql" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
[[ $_TARGET_DB_TYPE == "mariadb" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql
[[ $_TARGET_DB_TYPE == "pgsql" ]] && export DRUPAL_TEST_DB_URL=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql
- export
- cp ./core/.env.example ./core/.env
# dotenv-safe/config does not support environment variables
# @see https://github.com/rolodato/dotenv-safe/issues/126
# @todo move this to `variables` when the above is resolved
- echo "DRUPAL_TEST_BASE_URL='http://localhost/subdirectory'" >> ./core/.env
- echo "DRUPAL_TEST_CHROMEDRIVER_AUTOSTART=false" >> ./core/.env
- echo "DRUPAL_TEST_DB_URL='${DRUPAL_TEST_DB_URL}'" >> ./core/.env
- echo "DRUPAL_TEST_WEBDRIVER_HOSTNAME='localhost'" >> ./core/.env
- echo "DRUPAL_TEST_WEBDRIVER_CHROME_ARGS='--disable-dev-shm-usage --disable-gpu --headless'" >> ./core/.env
- echo "DRUPAL_TEST_WEBDRIVER_PORT='9515'" >> ./core/.env
- echo "DRUPAL_NIGHTWATCH_OUTPUT='"nightwatch_output"'" >> ./core/.env
- cat ./core/.env
- mkdir -p ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output
- chown -R www-data:www-data ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output ./core/.env
- sudo BABEL_DISABLE_CACHE=1 -u www-data yarn --cwd ./core test:nightwatch
artifacts:
when: always
expire_in: 6 mos
reports:
junit: ./nightwatch_output/*.xml
paths:
- ./nightwatch_output

View File

@ -322,9 +322,11 @@ distro
ditka
divs
dnumber
dockerhub
docroot
docroots
dolore
dotenv
downcasting
doxygen
dragtable
@ -540,6 +542,7 @@ insidekeyword
instaclick
instantiator
interactable
interruptible
introspectable
invalidators
invalididentifier

View File

@ -22,7 +22,7 @@
"prettier": "prettier --write \"./**/*.js\"",
"spellcheck": "cspell -c .cspell.json",
"spellcheck:make-drupal-dict": "rm -f misc/cspell/dictionary.txt && touch misc/cspell/dictionary.txt && yarn -s spellcheck:core --unique --words-only | perl -Mopen=locale -pe '$_=lc$_' | LC_ALL=en_US.UTF-8 tr -d \\\\\\\\ | LC_ALL=C sort -u -o misc/cspell/dictionary.txt",
"spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\"",
"spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\" \".gitlab-ci/*\" \".gitlab-ci.yml\"",
"vendor-update": "node ./scripts/js/vendor-update.js",
"watch:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js --watch",
"build:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js",

View File

@ -404,6 +404,8 @@ function simpletest_script_parse_args() {
'execute-test' => '',
'xml' => '',
'non-html' => FALSE,
'ci-parallel-node-index' => 1,
'ci-parallel-node-total' => 1,
];
// Override with set values.
@ -1024,6 +1026,12 @@ function simpletest_script_get_test_list() {
simpletest_script_print_error('No valid tests were specified.');
exit(SIMPLETEST_SCRIPT_EXIT_FAILURE);
}
if ((int) $args['ci-parallel-node-total'] > 1) {
$tests_per_job = ceil(count($test_list) / $args['ci-parallel-node-total']);
$test_list = array_slice($test_list, ($args['ci-parallel-node-index'] - 1) * $tests_per_job, $tests_per_job);
}
return $test_list;
}