From dfa16b8aca8e92be1f00a021c520a645f11f0227 Mon Sep 17 00:00:00 2001 From: catch Date: Wed, 13 Sep 2023 10:32:38 +0100 Subject: [PATCH] Issue #3386076 by fjgarlin, catch, longwave, larowlan, mstrelan, el7cosmos, RoSk0, xurizaemon, poker10, alexpott, bbrala, nick_schuch: GitLab CI integration for core --- .gitlab-ci.yml | 114 ++++++++++ .gitlab-ci/pipeline.yml | 364 ++++++++++++++++++++++++++++++++ core/misc/cspell/dictionary.txt | 3 + core/package.json | 2 +- core/scripts/run-tests.sh | 8 + 5 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 .gitlab-ci.yml create mode 100644 .gitlab-ci/pipeline.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..e5f5b946746 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,114 @@ +################ +# Drupal GitLabCI template. +# +# Based off GitlabCI templates project: https://git.drupalcode.org/project/gitlab_templates +# Guide: https://www.drupal.org/docs/develop/git/using-gitlab-to-contribute-to-drupal/gitlab-ci +# +# With thanks to: +# - The GitLab Acceleration Initiative participants +# - DrupalSpoons +################ + +################ +# Includes +# +# Additional configuration can be provided through includes. +# One advantage of include files is that if they are updated upstream, the +# changes affect all pipelines using that include. +# +# Includes can be overridden by re-declaring anything provided in an include, +# here in gitlab-ci.yml. +# https://docs.gitlab.com/ee/ci/yaml/includes.html#override-included-configuration-values +################ + +include: + - project: $_GITLAB_TEMPLATES_REPO + ref: $_GITLAB_TEMPLATES_REF + file: + - '/includes/include.drupalci.variables.yml' + - '/includes/include.drupalci.workflows.yml' + +################ +# Variables +# +# Overriding variables +# - To override one or more of these variables, simply declare your own variables keyword. +# - Keywords declared directly in .gitlab-ci.yml take precedence over include files. +# - Documentation: https://docs.gitlab.com/ee/ci/variables/ +# - Predefined variables: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html +# +################ + +variables: + _TARGET_PHP: "8.2" + COMPOSER: composer.json + # Let composer know what self.version means. + COMPOSER_ROOT_VERSION: "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}-dev" + CONCURRENCY: 32 + +################ +# Stages +# +# Each job is assigned to a stage, defining the order in which the jobs are executed. +# Jobs in the same stage run in parallel. +# +# If all jobs in a stage succeed, the pipeline will proceed to the next stage. +# If any job in the stage fails, the pipeline will exit early. +################ + +.default-stage: &default-stage + stage: test + trigger: + # Rely on the status of the child pipeline. + strategy: depend + include: + - local: .gitlab-ci/pipeline.yml + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + +.run-manually: &run-manually + when: manual + +'PHP 8.2 MySQL 5.7': + <<: [ *default-stage, *run-manually ] + variables: + _TARGET_DB_TYPE: "mysql" + _TARGET_DB_VERSION: "5.7" + _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production + +'PHP 8.2 MySQL 8': + <<: *default-stage + variables: + _TARGET_DB_TYPE: "mysql" + _TARGET_DB_VERSION: "8" + _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + +'PHP 8.2 PostgreSQL 13.5': + <<: [ *default-stage, *run-manually ] + variables: + _TARGET_DB_TYPE: "pgsql" + _TARGET_DB_VERSION: "13.5" + _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production + +'PHP 8.2 PostgreSQL 14.1': + <<: [ *default-stage, *run-manually ] + variables: + _TARGET_DB_TYPE: "pgsql" + _TARGET_DB_VERSION: "14.1" + _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production + +# 'PHP 8.2 SQLite 3.26.0': +# <<: [ *default-stage, *run-manually ] +# variables: +# _TARGET_DB_TYPE: "sqlite" +# _TARGET_DB_VERSION: "3.26.0" +# _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production + +# 'PHP 8.2 MariaDB 10.3.22': +# <<: [ *default-stage, *run-manually ] +# variables: +# _TARGET_DB_TYPE: "mariadb" +# _TARGET_DB_VERSION: "10.3.22" +# _DB_IMAGE: $_CONFIG_DOCKERHUB_ROOT/$_TARGET_DB_TYPE-$_TARGET_DB_VERSION:production diff --git a/.gitlab-ci/pipeline.yml b/.gitlab-ci/pipeline.yml new file mode 100644 index 00000000000..77009bf197b --- /dev/null +++ b/.gitlab-ci/pipeline.yml @@ -0,0 +1,364 @@ +# cspell:ignore drupaltestbot drupaltestbotpw + +stages: + ################ + # Build + # + # Assemble the test environment. + ################ + - ๐Ÿ—๏ธ Build + + ################ + # Code quality checks + # + # This stage includes any codebase validation before running tests. + ################ + - ๐Ÿช„ Lint + + ################ + # Test + # + # The test phase actually executes the tests, as well as gathering results + # and artifacts. + ################ + - ๐Ÿ—œ๏ธ Test + +############# +# Templates # +############# + +.default-job-settings: &default-job-settings + interruptible: true + allow_failure: false + image: + name: $_CONFIG_DOCKERHUB_ROOT/php-$_TARGET_PHP-apache:production + rules: + - if: $CI_PIPELINE_SOURCE == "parent_pipeline" + +.composer-cache: &composer-cache + key: + files: + - ./composer.json + - ./composer.lock + paths: + - ./vendor + +.yarn-cache: &yarn-cache + key: + files: + - ./core/package.json + - ./core/yarn.lock + paths: + - ./core/node_modules + +.pull-composer-cache: &pull-composer-cache + cache: + policy: pull + <<: *composer-cache + dependencies: + - '๐Ÿ“ฆ๏ธ Composer' + +.with-composer-cache: &with-composer-cache + needs: + - '๐Ÿ“ฆ๏ธ Composer' + <<: *pull-composer-cache + +.with-yarn-cache: &with-yarn-cache + dependencies: + - '๐Ÿ“ฆ๏ธ Yarn' + needs: + - '๐Ÿ“ฆ๏ธ Yarn' + cache: + policy: pull + <<: *yarn-cache + +.junit-artifacts: &junit-artifacts + artifacts: + expose_as: junit + expire_in: 6 mos + paths: + - junit.xml + reports: + junit: junit.xml + +.with-linting: &with-linting + needs: + - '๐Ÿ“ฆ๏ธ Composer' + - '๐Ÿงน PHP Static Analysis (phpstan)' + - '๐Ÿงน PHP Coding standards (PHPCS)' + - '๐Ÿงน Compilation check' + - '๐Ÿ“ฆ๏ธ Yarn' + - '๐Ÿ“” Spell-checking' + - '๐Ÿงน JavaScript linting (eslint)' + - '๐Ÿงน CSS linting (stylelint)' + <<: *pull-composer-cache + +.with-unit-tests: &with-unit-tests + needs: + - 'โšก๏ธ PHPUnit Unit' + - '๐Ÿ“ฆ๏ธ Composer' + <<: *pull-composer-cache + +.with-composer-yarn-and-unit-tests: &with-composer-yarn-and-unit-tests + needs: + - '๐Ÿ“ฆ๏ธ Composer' + - 'โšก๏ธ PHPUnit Unit' + - '๐Ÿ“ฆ๏ธ Yarn' + dependencies: + - '๐Ÿ“ฆ๏ธ Yarn' + - '๐Ÿ“ฆ๏ธ Composer' + +.test-variables: &test-variables + FF_NETWORK_PER_BUILD: 1 + SIMPLETEST_BASE_URL: http://localhost/subdirectory + DB_DRIVER: mysql + MYSQL_ROOT_PASSWORD: root + MYSQL_DATABASE: mysql + MYSQL_USER: drupaltestbot + MYSQL_PASSWORD: drupaltestbotpw + MARIADB_TAG: $_TARGET_DB_VERSION + POSTGRES_TAG: $_TARGET_DB_VERSION + POSTGRES_DB: drupaltestbot + POSTGRES_USER: drupaltestbot + POSTGRES_PASSWORD: drupaltestbotpw + MINK_DRIVER_ARGS_WEBDRIVER: '["chrome", {"browserName":"chrome","chromeOptions":{"args":["--disable-dev-shm-usage","--disable-gpu","--headless"]}}, "http://localhost:9515"]' + CI_PARALLEL_NODE_INDEX: $CI_NODE_INDEX + CI_PARALLEL_NODE_TOTAL: $CI_NODE_TOTAL + +.with-database: &with-database + name: $_DB_IMAGE + alias: database + +.with-chrome: &with-chrome + name: $_CONFIG_DOCKERHUB_ROOT/chromedriver:production + alias: chrome + entrypoint: + - chromedriver + - "--no-sandbox" + - "--log-path=/tmp/chromedriver.log" + - "--verbose" + - "--whitelisted-ips=" + +.phpunit-artifacts: &phpunit-artifacts + artifacts: + when: always + expire_in: 6 mos + reports: + junit: ./sites/default/files/simpletest/phpunit-*.xml + paths: + - ./sites/default/files/simpletest/phpunit-*.xml + - ./sites/simpletest/browser_output + +.setup-webroot: &setup-webserver + before_script: + - ln -s $CI_PROJECT_DIR /var/www/html/subdirectory + - sudo service apache2 start + +.run-tests: &run-tests + script: + # Determine DB driver. + - | + [[ $_TARGET_DB_TYPE == "sqlite" ]] && export SIMPLETEST_DB=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite + [[ $_TARGET_DB_TYPE == "mysql" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql + [[ $_TARGET_DB_TYPE == "mariadb" ]] && export SIMPLETEST_DB=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql + [[ $_TARGET_DB_TYPE == "pgsql" ]] && export SIMPLETEST_DB=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql + - export + - mkdir -p ./sites/simpletest ./sites/default/files ./build/logs/junit /var/www/.composer + - chown -R www-data:www-data ./sites ./build/logs/junit ./vendor /var/www/ + - sudo -u www-data git config --global --add safe.directory $CI_PROJECT_DIR + # Need to pass this along directly. + - sudo MINK_DRIVER_ARGS_WEBDRIVER="$MINK_DRIVER_ARGS_WEBDRIVER" -u www-data php ./core/scripts/run-tests.sh --color --keep-results --types "$TESTSUITE" --concurrency "$CONCURRENCY" --repeat "1" --sqlite "./sites/default/files/.sqlite" --dburl $SIMPLETEST_DB --url $SIMPLETEST_BASE_URL --verbose --non-html --all --ci-parallel-node-index $CI_PARALLEL_NODE_INDEX --ci-parallel-node-total $CI_PARALLEL_NODE_TOTAL + +################ +# Jobs +# +# Jobs define what scripts are actually executed in each stage. +# +# The 'rules' keyword can also be used to define conditions for each job. +# +# Documentation: https://docs.gitlab.com/ee/ci/jobs/ +################ + +################ +# Build Jobs +################ + +'โ„น๏ธ Output build parameters': + <<: *default-job-settings + stage: ๐Ÿ—๏ธ Build + script: + - echo "Checking variables" + - echo $_TARGET_PHP + - echo $_TARGET_DB_TYPE + - echo $_TARGET_DB_VERSION + +'๐Ÿ“ฆ๏ธ Composer': + <<: *default-job-settings + stage: ๐Ÿ—๏ธ Build + cache: + <<: *composer-cache + artifacts: + expire_in: 1 week + expose_as: 'web-vendor' + paths: + - vendor/ + script: + - export + - composer validate + - composer install + +'๐Ÿ“ฆ๏ธ Yarn': + <<: *default-job-settings + stage: ๐Ÿ—๏ธ Build + cache: + <<: *yarn-cache + artifacts: + expire_in: 1 week + expose_as: 'yarn-vendor' + paths: + - core/node_modules/ + script: + # Installs all core javascript dependencies and adds junit formatter. + - yarn --cwd ./core add stylelint-junit-formatter + +################ +# Lint Jobs +################ + +'๐Ÿงน PHP Coding standards (PHPCS)': + <<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ] + stage: ๐Ÿช„ Lint + script: + - composer phpcs -- --report-junit=junit.xml --report-full --report-summary + +'๐Ÿงน PHP Static Analysis (phpstan)': + <<: [ *with-composer-cache, *junit-artifacts, *default-job-settings ] + stage: ๐Ÿช„ Lint + script: + # Turn off apc to avoid corrupt composer cache. + - php -d apc.enabled=0 -d apc.enable_cli=0 vendor/bin/phpstan analyze --configuration=./core/phpstan.neon.dist --error-format=junit > junit.xml + +'๐Ÿงน CSS linting (stylelint)': + <<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ] + stage: ๐Ÿช„ Lint + script: + - yarn run --cwd=./core lint:css --color --custom-formatter node_modules/stylelint-junit-formatter > junit.xml + +'๐Ÿงน Compilation check': + <<: [ *with-yarn-cache, *default-job-settings ] + stage: ๐Ÿช„ Lint + script: + - yarn run --cwd=./core build:css --check + - cd core && yarn run -s check:ckeditor5 + +'๐Ÿงน JavaScript linting (eslint)': + <<: [ *with-yarn-cache, *junit-artifacts, *default-job-settings ] + stage: ๐Ÿช„ Lint + script: + - yarn --cwd=./core run -s lint:core-js-passing --format junit > junit.xml + +'๐Ÿ“” Spell-checking': + <<: [ *with-yarn-cache, *default-job-settings ] + stage: ๐Ÿช„ Lint + script: + - git fetch origin + - export MODIFIED=`git diff --name-only origin/${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}|while read r;do echo "$CI_PROJECT_DIR/$r";done|tr "\n" " "` + - echo $MODIFIED | tr ' ' '\n' | yarn --cwd=./core run -s spellcheck:core --no-must-find-files --file-list stdin + +################ +# Test Jobs +################ + +'โšก๏ธ PHPUnit Unit': + <<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ] + stage: ๐Ÿ—œ๏ธ Test + services: + # There are some unit tests that need a database. + # @todo Remove after https://www.drupal.org/project/drupal/issues/3386217 + - <<: *with-database + variables: + <<: *test-variables + TESTSUITE: PHPUnit-Unit + CONCURRENCY: "$CONCURRENCY" + +'โš™๏ธ๏ธ PHPUnit Kernel': + <<: [ *with-composer-cache, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ] + stage: ๐Ÿ—œ๏ธ Test + parallel: 3 + variables: + <<: *test-variables + TESTSUITE: PHPUnit-Kernel + CONCURRENCY: "$CONCURRENCY" + services: + - <<: *with-database + +'๐Ÿ–ฑ๏ธ๏ธ๏ธ PHPUnit Functional Javascript': + <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ] + stage: ๐Ÿ—œ๏ธ Test + variables: + <<: *test-variables + TESTSUITE: PHPUnit-FunctionalJavascript + CONCURRENCY: 15 + services: + - <<: *with-database + - <<: *with-chrome + +'๐Ÿ‘ท๏ธ๏ธ๏ธ PHPUnit Build': + <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ] + stage: ๐Ÿ—œ๏ธ Test + variables: + <<: *test-variables + TESTSUITE: PHPUnit-Build + CONCURRENCY: "$CONCURRENCY" + services: + - <<: *with-database + +'๐ŸŒ๏ธ๏ธ PHPUnit Functional': + <<: [ *with-linting, *with-unit-tests, *phpunit-artifacts, *setup-webserver, *run-tests, *default-job-settings ] + stage: ๐Ÿ—œ๏ธ Test + parallel: 8 + variables: + <<: *test-variables + TESTSUITE: PHPUnit-Functional + CONCURRENCY: "$CONCURRENCY" + services: + - <<: *with-database + +'๐Ÿฆ‰๏ธ๏ธ๏ธ Nightwatch': + <<: [ *with-composer-yarn-and-unit-tests, *setup-webserver, *default-job-settings ] + stage: ๐Ÿ—œ๏ธ Test + variables: + <<: *test-variables + services: + - <<: *with-database + - <<: *with-chrome + script: + # Determine DB driver. + - | + [[ $_TARGET_DB_TYPE == "sqlite" ]] && export DRUPAL_TEST_DB_URL=sqlite://localhost/subdirectory/sites/default/files/db.sqlite?module=sqlite + [[ $_TARGET_DB_TYPE == "mysql" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql + [[ $_TARGET_DB_TYPE == "mariadb" ]] && export DRUPAL_TEST_DB_URL=mysql://$MYSQL_USER:$MYSQL_PASSWORD@database/$MYSQL_DATABASE?module=mysql + [[ $_TARGET_DB_TYPE == "pgsql" ]] && export DRUPAL_TEST_DB_URL=pgsql://$POSTGRES_USER:$POSTGRES_PASSWORD@database/$POSTGRES_DB?module=pgsql + - export + - cp ./core/.env.example ./core/.env + # dotenv-safe/config does not support environment variables + # @see https://github.com/rolodato/dotenv-safe/issues/126 + # @todo move this to `variables` when the above is resolved + - echo "DRUPAL_TEST_BASE_URL='http://localhost/subdirectory'" >> ./core/.env + - echo "DRUPAL_TEST_CHROMEDRIVER_AUTOSTART=false" >> ./core/.env + - echo "DRUPAL_TEST_DB_URL='${DRUPAL_TEST_DB_URL}'" >> ./core/.env + - echo "DRUPAL_TEST_WEBDRIVER_HOSTNAME='localhost'" >> ./core/.env + - echo "DRUPAL_TEST_WEBDRIVER_CHROME_ARGS='--disable-dev-shm-usage --disable-gpu --headless'" >> ./core/.env + - echo "DRUPAL_TEST_WEBDRIVER_PORT='9515'" >> ./core/.env + - echo "DRUPAL_NIGHTWATCH_OUTPUT='"nightwatch_output"'" >> ./core/.env + - cat ./core/.env + - mkdir -p ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output + - chown -R www-data:www-data ./sites/simpletest ./sites/default/files /var/www/.cache/yarn /var/www/.yarn ./nightwatch_output ./core/.env + - sudo BABEL_DISABLE_CACHE=1 -u www-data yarn --cwd ./core test:nightwatch + artifacts: + when: always + expire_in: 6 mos + reports: + junit: ./nightwatch_output/*.xml + paths: + - ./nightwatch_output diff --git a/core/misc/cspell/dictionary.txt b/core/misc/cspell/dictionary.txt index a4cd7d4c83f..0d890fb02be 100644 --- a/core/misc/cspell/dictionary.txt +++ b/core/misc/cspell/dictionary.txt @@ -322,9 +322,11 @@ distro ditka divs dnumber +dockerhub docroot docroots dolore +dotenv downcasting doxygen dragtable @@ -540,6 +542,7 @@ insidekeyword instaclick instantiator interactable +interruptible introspectable invalidators invalididentifier diff --git a/core/package.json b/core/package.json index 42580071317..148ab19a17a 100644 --- a/core/package.json +++ b/core/package.json @@ -22,7 +22,7 @@ "prettier": "prettier --write \"./**/*.js\"", "spellcheck": "cspell -c .cspell.json", "spellcheck:make-drupal-dict": "rm -f misc/cspell/dictionary.txt && touch misc/cspell/dictionary.txt && yarn -s spellcheck:core --unique --words-only | perl -Mopen=locale -pe '$_=lc$_' | LC_ALL=en_US.UTF-8 tr -d \\\\\\\\ | LC_ALL=C sort -u -o misc/cspell/dictionary.txt", - "spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\"", + "spellcheck:core": "cspell -c .cspell.json --root .. \"core/**/*\" \"composer/**/*\" \"composer.json\" \".gitlab-ci/*\" \".gitlab-ci.yml\"", "vendor-update": "node ./scripts/js/vendor-update.js", "watch:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js --watch", "build:ckeditor5": "webpack --config ./modules/ckeditor5/webpack.config.js", diff --git a/core/scripts/run-tests.sh b/core/scripts/run-tests.sh index 7cf7a856c6f..5317ba5a655 100755 --- a/core/scripts/run-tests.sh +++ b/core/scripts/run-tests.sh @@ -404,6 +404,8 @@ function simpletest_script_parse_args() { 'execute-test' => '', 'xml' => '', 'non-html' => FALSE, + 'ci-parallel-node-index' => 1, + 'ci-parallel-node-total' => 1, ]; // Override with set values. @@ -1024,6 +1026,12 @@ function simpletest_script_get_test_list() { simpletest_script_print_error('No valid tests were specified.'); exit(SIMPLETEST_SCRIPT_EXIT_FAILURE); } + + if ((int) $args['ci-parallel-node-total'] > 1) { + $tests_per_job = ceil(count($test_list) / $args['ci-parallel-node-total']); + $test_list = array_slice($test_list, ($args['ci-parallel-node-index'] - 1) * $tests_per_job, $tests_per_job); + } + return $test_list; }