chore: replace "package builder" shell implemention with python (#24306)
parent
a6bb6a85d6
commit
75a8bcfae2
|
@ -122,24 +122,19 @@ workflows:
|
||||||
exclude:
|
exclude:
|
||||||
- { os: darwin, arch: arm64 }
|
- { os: darwin, arch: arm64 }
|
||||||
- { os: windows, arch: arm64 }
|
- { os: windows, arch: arm64 }
|
||||||
- build-package:
|
- build-packages:
|
||||||
<<: *any_filter
|
<<: *any_filter
|
||||||
name: build-package-<< matrix.os >>-<< matrix.arch >>
|
|
||||||
requires:
|
requires:
|
||||||
- build-<< matrix.os >>-<< matrix.arch >>
|
- build-linux-amd64
|
||||||
matrix:
|
- build-linux-arm64
|
||||||
parameters:
|
- build-darwin-amd64
|
||||||
os: [ linux, darwin, windows ]
|
- build-windows-amd64
|
||||||
arch: [ amd64, arm64 ]
|
|
||||||
exclude:
|
|
||||||
- { os: darwin, arch: arm64 }
|
|
||||||
- { os: windows, arch: arm64 }
|
|
||||||
- check_package_deb_amd64:
|
- check_package_deb_amd64:
|
||||||
requires:
|
requires:
|
||||||
- build-package-linux-amd64
|
- build-packages
|
||||||
- check_package_deb_arm64:
|
- check_package_deb_arm64:
|
||||||
requires:
|
requires:
|
||||||
- build-package-linux-arm64
|
- build-packages
|
||||||
- check_package_rpm:
|
- check_package_rpm:
|
||||||
<<: *nofork_filter
|
<<: *nofork_filter
|
||||||
name:
|
name:
|
||||||
|
@ -148,8 +143,7 @@ workflows:
|
||||||
parameters:
|
parameters:
|
||||||
arch: [ x86_64, aarch64 ]
|
arch: [ x86_64, aarch64 ]
|
||||||
requires:
|
requires:
|
||||||
- build-package-linux-amd64
|
- build-packages
|
||||||
- build-package-linux-arm64
|
|
||||||
- test-downgrade:
|
- test-downgrade:
|
||||||
<<: *any_filter
|
<<: *any_filter
|
||||||
requires:
|
requires:
|
||||||
|
@ -161,22 +155,16 @@ workflows:
|
||||||
- test-linux-packages:
|
- test-linux-packages:
|
||||||
<<: *nofork_filter
|
<<: *nofork_filter
|
||||||
requires:
|
requires:
|
||||||
- build-package-linux-amd64
|
- build-packages
|
||||||
- sign-packages:
|
- sign-packages:
|
||||||
<<: *release_filter
|
<<: *release_filter
|
||||||
requires:
|
requires:
|
||||||
- build-package-linux-amd64
|
- build-packages
|
||||||
- build-package-linux-arm64
|
|
||||||
- build-package-darwin-amd64
|
|
||||||
- build-package-windows-amd64
|
|
||||||
- s3-publish-packages:
|
- s3-publish-packages:
|
||||||
<<: *release_filter
|
<<: *release_filter
|
||||||
requires:
|
requires:
|
||||||
- test-linux-packages
|
- test-linux-packages
|
||||||
- build-package-darwin-amd64
|
- build-packages
|
||||||
- build-package-linux-amd64
|
|
||||||
- build-package-linux-arm64
|
|
||||||
- build-package-windows-amd64
|
|
||||||
- sign-packages
|
- sign-packages
|
||||||
- changelog:
|
- changelog:
|
||||||
<<: *release_filter
|
<<: *release_filter
|
||||||
|
@ -189,10 +177,7 @@ workflows:
|
||||||
- perf-test:
|
- perf-test:
|
||||||
record_results: true
|
record_results: true
|
||||||
requires:
|
requires:
|
||||||
- build-package-darwin-amd64
|
- build-packages
|
||||||
- build-package-linux-amd64
|
|
||||||
- build-package-linux-arm64
|
|
||||||
- build-package-windows-amd64
|
|
||||||
filters:
|
filters:
|
||||||
branches:
|
branches:
|
||||||
only:
|
only:
|
||||||
|
@ -309,18 +294,12 @@ workflows:
|
||||||
requires:
|
requires:
|
||||||
- build-docker-nightly-amd64
|
- build-docker-nightly-amd64
|
||||||
- build-docker-nightly-arm64
|
- build-docker-nightly-arm64
|
||||||
- build-package:
|
- build-packages:
|
||||||
name: build-package-<< matrix.os >>-<< matrix.arch >>
|
|
||||||
requires:
|
requires:
|
||||||
- build-nightly-<< matrix.os >>-<< matrix.arch >>
|
- build-nightly-linux-amd64
|
||||||
- changelog
|
- build-nightly-linux-arm64
|
||||||
matrix:
|
- build-nightly-darwin-amd64
|
||||||
parameters:
|
- build-nightly-windows-amd64
|
||||||
os: [ linux, darwin, windows ]
|
|
||||||
arch: [ amd64, arm64 ]
|
|
||||||
exclude:
|
|
||||||
- { os: darwin, arch: arm64 }
|
|
||||||
- { os: windows, arch: arm64 }
|
|
||||||
- litmus-full-test:
|
- litmus-full-test:
|
||||||
requires:
|
requires:
|
||||||
- build-nightly-linux-amd64
|
- build-nightly-linux-amd64
|
||||||
|
@ -488,47 +467,38 @@ jobs:
|
||||||
paths:
|
paths:
|
||||||
- bin
|
- bin
|
||||||
|
|
||||||
build-package:
|
build-packages:
|
||||||
executor: linux-amd64
|
machine:
|
||||||
parameters:
|
image: ubuntu-2204:current
|
||||||
os:
|
|
||||||
type: string
|
|
||||||
arch:
|
|
||||||
type: string
|
|
||||||
steps:
|
steps:
|
||||||
- checkout
|
|
||||||
- attach_workspace:
|
- attach_workspace:
|
||||||
at: .
|
at: /tmp/workspace
|
||||||
- run:
|
- checkout
|
||||||
name: Install Package Dependencies
|
- run: |
|
||||||
command: |
|
|
||||||
export DEBIAN_FRONTEND=noninteractive
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
sudo -E apt-get update
|
sudo -E apt-get update
|
||||||
sudo -E apt-get install --yes \
|
sudo -E apt-get install --no-install-recommends --yes \
|
||||||
|
asciidoc \
|
||||||
build-essential \
|
build-essential \
|
||||||
git \
|
git \
|
||||||
|
python3 \
|
||||||
rpm \
|
rpm \
|
||||||
ruby-dev
|
ruby-dev \
|
||||||
|
xmlto
|
||||||
|
|
||||||
gem install fpm
|
sudo gem install fpm
|
||||||
- run:
|
|
||||||
name: Get InfluxDB Version
|
python3 -m pip install -r .circleci/scripts/package/requirements.txt
|
||||||
command: |
|
|
||||||
PREFIX=2.x .circleci/scripts/get-version
|
# Unfortunately, this must be executed as root. This is so permission
|
||||||
- run:
|
# modifying commands (chown, chmod, etc.) succeed.
|
||||||
name: Build Package
|
sudo --preserve-env=CIRCLE_TAG,CIRCLE_SHA1 .circleci/scripts/package/build.py
|
||||||
command: |
|
- store_artifacts:
|
||||||
export PLAT=<< parameters.os >>
|
path: artifacts/
|
||||||
export ARCH=<< parameters.arch >>
|
|
||||||
.circleci/scripts/build-package
|
|
||||||
- persist_to_workspace:
|
- persist_to_workspace:
|
||||||
root: /
|
root: .
|
||||||
paths:
|
paths:
|
||||||
- artifacts
|
- artifacts
|
||||||
- store_artifacts:
|
|
||||||
path: /artifacts
|
|
||||||
|
|
||||||
destination: artifacts
|
|
||||||
|
|
||||||
sign-packages:
|
sign-packages:
|
||||||
circleci_ip_ranges: true
|
circleci_ip_ranges: true
|
||||||
|
@ -671,7 +641,7 @@ jobs:
|
||||||
name: Terraform apply
|
name: Terraform apply
|
||||||
command: |
|
command: |
|
||||||
set -x
|
set -x
|
||||||
export DEBNAME="$(find /tmp/workspace/artifacts/influxdb2-*-amd64.deb)"
|
export DEBNAME="$(find /tmp/workspace/artifacts/influxdb2*amd64.deb)"
|
||||||
terraform -chdir=scripts/ci init -input=false
|
terraform -chdir=scripts/ci init -input=false
|
||||||
AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \
|
AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \
|
||||||
-chdir=scripts/ci \
|
-chdir=scripts/ci \
|
||||||
|
@ -969,7 +939,7 @@ jobs:
|
||||||
name: Validate Debian Package (AMD64)
|
name: Validate Debian Package (AMD64)
|
||||||
command: |
|
command: |
|
||||||
sudo .circleci/scripts/package-validation/debian \
|
sudo .circleci/scripts/package-validation/debian \
|
||||||
/tmp/workspace/artifacts/influxdb2*-amd64.deb
|
/tmp/workspace/artifacts/influxdb2*amd64.deb
|
||||||
|
|
||||||
check_package_deb_arm64:
|
check_package_deb_arm64:
|
||||||
machine:
|
machine:
|
||||||
|
@ -983,7 +953,7 @@ jobs:
|
||||||
name: Validate Debian Package (ARM64)
|
name: Validate Debian Package (ARM64)
|
||||||
command: |
|
command: |
|
||||||
sudo .circleci/scripts/package-validation/debian \
|
sudo .circleci/scripts/package-validation/debian \
|
||||||
/tmp/workspace/artifacts/influxdb2*-arm64.deb
|
/tmp/workspace/artifacts/influxdb2*arm64.deb
|
||||||
|
|
||||||
check_package_rpm:
|
check_package_rpm:
|
||||||
executor: linux-amd64
|
executor: linux-amd64
|
||||||
|
|
|
@ -1,139 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -o errexit \
|
|
||||||
-o nounset \
|
|
||||||
-o pipefail
|
|
||||||
|
|
||||||
REGEX_RELEASE_VERSION='[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+'
|
|
||||||
|
|
||||||
if [[ ${RELEASE:-} ]]
|
|
||||||
then
|
|
||||||
# This ensures that release packages are built with valid versions.
|
|
||||||
# Unfortunately, `fpm` is fairly permissive with what version tags
|
|
||||||
# it accepts. This becomes a problem when `apt` or `dpkg` is used
|
|
||||||
# to install the package (both have strict version requirements).
|
|
||||||
if ! [[ ${VERSION} =~ ^${REGEX_RELEASE_VERSION}$ ]]
|
|
||||||
then
|
|
||||||
printf 'Release version is invalid!\n' >&2 && exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
function run_fpm()
|
|
||||||
{
|
|
||||||
if [[ ${1} == rpm ]]
|
|
||||||
then
|
|
||||||
case ${ARCH} in
|
|
||||||
arm64)
|
|
||||||
ARCH=aarch64
|
|
||||||
;;
|
|
||||||
amd64)
|
|
||||||
ARCH=x86_64
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
fi
|
|
||||||
|
|
||||||
pushd "${workspace}"
|
|
||||||
|
|
||||||
fpm \
|
|
||||||
--log error \
|
|
||||||
`# package description` \
|
|
||||||
--name influxdb2 \
|
|
||||||
--vendor InfluxData \
|
|
||||||
--description 'Distributed time-series database.' \
|
|
||||||
--url https://influxdata.com \
|
|
||||||
--maintainer support@influxdb.com \
|
|
||||||
--license MIT \
|
|
||||||
`# package configuration` \
|
|
||||||
--input-type dir \
|
|
||||||
--output-type "${1}" \
|
|
||||||
--architecture "${ARCH}" \
|
|
||||||
--version "${VERSION}" \
|
|
||||||
--iteration 1 \
|
|
||||||
`# package relationships` \
|
|
||||||
--deb-recommends influxdb2-cli \
|
|
||||||
--conflicts influxdb \
|
|
||||||
--depends curl \
|
|
||||||
`# package scripts` \
|
|
||||||
--before-install control/preinst \
|
|
||||||
--after-install control/postinst \
|
|
||||||
--after-remove control/postrm \
|
|
||||||
`# package files` \
|
|
||||||
--chdir fs/ \
|
|
||||||
--package /artifacts
|
|
||||||
|
|
||||||
popd
|
|
||||||
|
|
||||||
# `goreleaser` stripped off the package revision and replaced '_' with
|
|
||||||
# '-'. Since the dockerfiles expect the previous naming convention,
|
|
||||||
# this rewrites the package names to match. Version information is
|
|
||||||
# also stored as metadata within the package.
|
|
||||||
case ${1} in
|
|
||||||
deb)
|
|
||||||
mv "/artifacts/influxdb2_${VERSION}-1_${ARCH}.deb" \
|
|
||||||
"/artifacts/influxdb2-${VERSION}-${ARCH}.deb"
|
|
||||||
;;
|
|
||||||
rpm)
|
|
||||||
mv "/artifacts/influxdb2-${VERSION//-/_}-1.${ARCH}.rpm" \
|
|
||||||
"/artifacts/influxdb2-${VERSION//-/_}.${ARCH}.rpm"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
sudo bash <<'EOF'
|
|
||||||
mkdir /artifacts && chown -R circleci: /artifacts
|
|
||||||
EOF
|
|
||||||
|
|
||||||
build_archive()
|
|
||||||
{
|
|
||||||
workspace="$(mktemp -d)"
|
|
||||||
|
|
||||||
mkdir "${workspace}/influxdb2_${PLAT}_${ARCH}"
|
|
||||||
|
|
||||||
# `failglob` is required because `bin/influxd_${PLAT}_${ARCH}/*` may
|
|
||||||
# not expand. This will prevent the package from being built without
|
|
||||||
# the included binary files. This will also display as an error
|
|
||||||
# from CircleCI interface.
|
|
||||||
shopt -s failglob
|
|
||||||
cp -p LICENSE README.md "bin/influxd_${PLAT}_${ARCH}/"* \
|
|
||||||
"${workspace}/influxdb2_${PLAT}_${ARCH}/"
|
|
||||||
|
|
||||||
pushd "${workspace}"
|
|
||||||
|
|
||||||
if [[ ${PLAT} != windows ]]
|
|
||||||
then
|
|
||||||
# Using `find .. -type f` to supply a list of files to `tar` serves two
|
|
||||||
# purposes. The first being that `tar` wont construct a '.' directory
|
|
||||||
# in the root of the tarfile. The second being that this excludes
|
|
||||||
# empty directories from the tarfile.
|
|
||||||
find "influxdb2_${PLAT}_${ARCH}/" -type f \
|
|
||||||
| tar -czf "/artifacts/influxdb2-${VERSION}-${PLAT}-${ARCH}.tar.gz" -T -
|
|
||||||
else
|
|
||||||
# windows uses zip
|
|
||||||
find "influxdb2_${PLAT}_${ARCH}/" -type f \
|
|
||||||
| zip -r "/artifacts/influxdb2-${VERSION}-${PLAT}-${ARCH}.zip" -@
|
|
||||||
fi
|
|
||||||
|
|
||||||
popd
|
|
||||||
}
|
|
||||||
|
|
||||||
build_package_linux()
|
|
||||||
{
|
|
||||||
if [[ ${PLAT} != linux ]]
|
|
||||||
then
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
workspace="$(mktemp -d)"
|
|
||||||
|
|
||||||
mkdir -p "${workspace}/fs/usr/bin"
|
|
||||||
|
|
||||||
# (see reasoning above)
|
|
||||||
shopt -s failglob
|
|
||||||
cp -rp .circleci/package/. "${workspace}/"
|
|
||||||
cp -p "bin/influxd_${PLAT}_${ARCH}/"* "${workspace}/fs/usr/bin"
|
|
||||||
|
|
||||||
run_fpm deb
|
|
||||||
run_fpm rpm
|
|
||||||
}
|
|
||||||
|
|
||||||
build_archive
|
|
||||||
build_package_linux
|
|
|
@ -0,0 +1,383 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
|
||||||
|
def build_linux_archive(source, package, version):
|
||||||
|
"""
|
||||||
|
Builds a Linux Archive.
|
||||||
|
|
||||||
|
This archive contains the binary artifacts, configuration, and scripts
|
||||||
|
installed by the DEB and RPM packages. This mimics the file-system. So,
|
||||||
|
binaries are installed into "/usr/bin", configuration into "/etc", and
|
||||||
|
scripts into their relevant directories. Permissions match those of
|
||||||
|
the DEB and RPM packages.
|
||||||
|
"""
|
||||||
|
with tempfile.TemporaryDirectory() as workspace:
|
||||||
|
# fmt: off
|
||||||
|
shutil.copytree(os.path.join(package["source"], "fs"),
|
||||||
|
workspace, dirs_exist_ok=True, ignore=shutil.ignore_patterns(".keepdir"))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for extra in package["extras"]:
|
||||||
|
# fmt: off
|
||||||
|
shutil.copy(extra["source"],
|
||||||
|
os.path.join(workspace, extra["target"]))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for binary in package["binaries"]:
|
||||||
|
target = os.path.join(source["binary"], binary)
|
||||||
|
|
||||||
|
if os.path.exists(target):
|
||||||
|
# fmt: off
|
||||||
|
shutil.copy(target,
|
||||||
|
os.path.join(workspace, "usr/bin", os.path.basename(target)))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
# After the package contents are copied into the working directory,
|
||||||
|
# the permissions must be updated. Since the CI executor may change
|
||||||
|
# occasionally (images/ORBs deprecated over time), the umask may
|
||||||
|
# not be what we expect. This allows this packaging script to be
|
||||||
|
# agnostic to umask/system configuration.
|
||||||
|
for root, dirs, files in os.walk(workspace):
|
||||||
|
for target in [os.path.join(root, f) for f in files]:
|
||||||
|
# files in "usr/bin" are executable
|
||||||
|
if os.path.relpath(root, workspace) == "usr/bin":
|
||||||
|
os.chmod(target, 0o0755)
|
||||||
|
else:
|
||||||
|
# standard file permissions
|
||||||
|
os.chmod(target, 0o0644)
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = "root",
|
||||||
|
group = "root")
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for target in [os.path.join(root, d) for d in dirs]:
|
||||||
|
# standard directory permissions
|
||||||
|
os.chmod(target, 0o0755)
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = "root",
|
||||||
|
group = "root")
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for override in package["perm_overrides"]:
|
||||||
|
target = os.path.join(workspace, override["target"])
|
||||||
|
os.chmod(target, override["perms"])
|
||||||
|
# "owner" and "group" should be a system account and group with
|
||||||
|
# a well-defined UID and GID. Otherwise, the UID/GID might vary
|
||||||
|
# between systems. When the archive is extracted/package is
|
||||||
|
# installed, things may not behave as we would expect.
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = override["owner"],
|
||||||
|
group = override["group"])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
os.makedirs(source["target"], exist_ok=True)
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
subprocess.check_call([
|
||||||
|
"tar", "-czf",
|
||||||
|
os.path.join(
|
||||||
|
source["target"],
|
||||||
|
"{:s}-{:s}_{:s}_{:s}.tar.gz".format(
|
||||||
|
package["name"],
|
||||||
|
version,
|
||||||
|
source["plat"],
|
||||||
|
source["arch"]
|
||||||
|
)
|
||||||
|
),
|
||||||
|
# ".keepdir" allows Git to track otherwise empty directories. The presence
|
||||||
|
# of the directories allows `package["extras"]` and `package["binaries"]`
|
||||||
|
# to be copied into the archive without requiring "mkdir". These should
|
||||||
|
# directories are excluded from the final archive.
|
||||||
|
"--exclude", ".keepdir",
|
||||||
|
# This re-parents the contents of the archive with `package["name"]-version`.
|
||||||
|
# It is undocumented, however, when matching, "--transform" always removes
|
||||||
|
# the trailing slash. This regex must handle "./" and "./<more components>".
|
||||||
|
"--transform",
|
||||||
|
"s#^.\(/\|$\)#{:s}-{:s}/#".format(
|
||||||
|
package["name"],
|
||||||
|
version
|
||||||
|
),
|
||||||
|
# compress everything within `workspace`
|
||||||
|
"-C", workspace, '.'
|
||||||
|
])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
|
||||||
|
def build_archive(source, package, version):
|
||||||
|
"""
|
||||||
|
Builds Archive for other (not-Linux) Platforms.
|
||||||
|
|
||||||
|
This archive contains binary artifacts and configuration. Unlike the
|
||||||
|
linux archive, which contains the configuration and matches the file-
|
||||||
|
system of the DEB and RPM packages, everything is located within the
|
||||||
|
root of the archive. However, permissions do match those of the DEB
|
||||||
|
and RPM packages.
|
||||||
|
"""
|
||||||
|
with tempfile.TemporaryDirectory() as workspace:
|
||||||
|
for extra in package["extras"]:
|
||||||
|
# fmt: off
|
||||||
|
target = os.path.join(workspace,
|
||||||
|
os.path.basename(extra["target"]))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
shutil.copy(extra["source"], target)
|
||||||
|
os.chmod(target, 0o0644)
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = "root",
|
||||||
|
group = "root")
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for binary in package["binaries"]:
|
||||||
|
target = os.path.join(source["binary"], binary)
|
||||||
|
|
||||||
|
if os.path.exists(target):
|
||||||
|
# fmt: off
|
||||||
|
shutil.copy(target,
|
||||||
|
os.path.join(workspace, os.path.basename(target)))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
os.chmod(target, 0o0755)
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = "root",
|
||||||
|
group = "root")
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
os.makedirs(source["target"], exist_ok=True)
|
||||||
|
|
||||||
|
if source["plat"] == "darwin":
|
||||||
|
# fmt: off
|
||||||
|
subprocess.check_call([
|
||||||
|
"tar", "-czf",
|
||||||
|
os.path.join(
|
||||||
|
source["target"],
|
||||||
|
"{:s}-{:s}_{:s}_{:s}.tar.gz".format(
|
||||||
|
package["name"],
|
||||||
|
version,
|
||||||
|
source["plat"],
|
||||||
|
source["arch"]
|
||||||
|
)
|
||||||
|
),
|
||||||
|
# This re-parents the contents of the archive with `package["name"]-version`.
|
||||||
|
# It is undocumented, however, when matching, "--transform" always removes
|
||||||
|
# the trailing slash. This regex must handle "./" and "./<more components>".
|
||||||
|
"--transform",
|
||||||
|
"s#^.\(/\|$\)#{:s}-{:s}/#".format(
|
||||||
|
package["name"],
|
||||||
|
version
|
||||||
|
),
|
||||||
|
# compress everything within `workspace`
|
||||||
|
"-C", workspace, '.'
|
||||||
|
])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
if source["plat"] == "windows":
|
||||||
|
# preserve current working directory
|
||||||
|
current = os.getcwd()
|
||||||
|
|
||||||
|
for root, dirs, files in os.walk(workspace):
|
||||||
|
for file in files:
|
||||||
|
# Unfortunately, it looks like "-r" cannot be combined with
|
||||||
|
# "-j" (which strips the path of input files). This changes
|
||||||
|
# directory to the current input file and *then* appends it
|
||||||
|
# to the archive.
|
||||||
|
os.chdir(os.path.join(workspace, root))
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
subprocess.check_call([
|
||||||
|
"zip", "-r",
|
||||||
|
os.path.join(
|
||||||
|
os.path.join(current, source["target"]),
|
||||||
|
"{:s}-{:s}-{:s}.zip".format(
|
||||||
|
package["name"],
|
||||||
|
version,
|
||||||
|
source["plat"],
|
||||||
|
source["arch"]
|
||||||
|
)
|
||||||
|
),
|
||||||
|
file
|
||||||
|
])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
# restore current working directory
|
||||||
|
os.chdir(current)
|
||||||
|
|
||||||
|
|
||||||
|
def build_linux_package(source, package, version):
|
||||||
|
"""
|
||||||
|
Constructs a DEB or RPM Package.
|
||||||
|
"""
|
||||||
|
with tempfile.TemporaryDirectory() as workspace:
|
||||||
|
# fmt: off
|
||||||
|
shutil.copytree(package["source"], workspace,
|
||||||
|
dirs_exist_ok=True, ignore=shutil.ignore_patterns(".keepdir"))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for extra in package["extras"]:
|
||||||
|
# fmt: off
|
||||||
|
shutil.copy(extra["source"],
|
||||||
|
os.path.join(workspace, "fs", extra["target"]))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for binary in package["binaries"]:
|
||||||
|
target = os.path.join(source["binary"], binary)
|
||||||
|
|
||||||
|
if os.path.exists(target):
|
||||||
|
# fmt: off
|
||||||
|
shutil.copy(target,
|
||||||
|
os.path.join(workspace, "fs/usr/bin", os.path.basename(target)))
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
# After the package contents are copied into the working directory,
|
||||||
|
# the permissions must be updated. Since the CI executor may change
|
||||||
|
# occasionally (images/ORBs deprecated over time), the umask may
|
||||||
|
# not be what we expect. This allows this packaging script to be
|
||||||
|
# agnostic to umask/system configuration.
|
||||||
|
for root, dirs, files in os.walk(workspace):
|
||||||
|
for target in [os.path.join(root, f) for f in files]:
|
||||||
|
# files in "fs/usr/bin" are executable
|
||||||
|
if os.path.relpath(root, workspace) == "fs/usr/bin":
|
||||||
|
os.chmod(target, 0o0755)
|
||||||
|
else:
|
||||||
|
# standard file permissions
|
||||||
|
os.chmod(target, 0o0644)
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = "root",
|
||||||
|
group = "root")
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for target in [os.path.join(root, d) for d in dirs]:
|
||||||
|
# standard directory permissions
|
||||||
|
os.chmod(target, 0o0755)
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = "root",
|
||||||
|
group = "root")
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
for override in package["perm_overrides"]:
|
||||||
|
target = os.path.join(workspace, "fs", override["target"])
|
||||||
|
os.chmod(target, override["perms"])
|
||||||
|
# "owner" and "group" should be a system account and group with
|
||||||
|
# a well-defined UID and GID. Otherwise, the UID/GID might vary
|
||||||
|
# between systems. When the archive is extracted/package is
|
||||||
|
# installed, things may not behave as we would expect.
|
||||||
|
# fmt: off
|
||||||
|
shutil.chown(
|
||||||
|
target,
|
||||||
|
user = override["owner"],
|
||||||
|
group = override["group"])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
os.makedirs(source["target"], exist_ok=True)
|
||||||
|
fpm_wrapper(source, package, version, workspace, "rpm")
|
||||||
|
fpm_wrapper(source, package, version, workspace, "deb")
|
||||||
|
|
||||||
|
|
||||||
|
def fpm_wrapper(source, package, version, workspace, package_type):
|
||||||
|
"""
|
||||||
|
Constructs either a DEB/RPM Package.
|
||||||
|
|
||||||
|
This wraps some configuration settings that are *only* relevant
|
||||||
|
to `fpm`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
conffiles = []
|
||||||
|
for root, dirs, files in os.walk(os.path.join(workspace, "fs/etc")):
|
||||||
|
for file in files:
|
||||||
|
# fmt: off
|
||||||
|
conffiles.extend([
|
||||||
|
"--config-files", os.path.join("/", os.path.relpath(root, os.path.join(workspace, "fs")), file)
|
||||||
|
])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
# `source["arch"]` matches DEB architecture names. When building RPMs, it must
|
||||||
|
# be converted into RPM architecture names.
|
||||||
|
architecture = source["arch"]
|
||||||
|
if package_type == "rpm":
|
||||||
|
if architecture == "amd64":
|
||||||
|
architecture = "x86_64"
|
||||||
|
if architecture == "arm64":
|
||||||
|
architecture = "aarch64"
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
p = subprocess.check_call([
|
||||||
|
"fpm",
|
||||||
|
"--log", "error",
|
||||||
|
# package description
|
||||||
|
"--name", package["name"],
|
||||||
|
"--vendor", "InfluxData",
|
||||||
|
"--description", "Distributed time-series database.",
|
||||||
|
"--url", "https://influxdata.com",
|
||||||
|
"--maintainer", "support@influxdb.com",
|
||||||
|
"--license", "MIT",
|
||||||
|
# package configuration
|
||||||
|
"--input-type", "dir",
|
||||||
|
"--output-type", package_type,
|
||||||
|
"--architecture", architecture,
|
||||||
|
"--version", version,
|
||||||
|
"--iteration", "1",
|
||||||
|
# maintainer scripts
|
||||||
|
"--after-install", os.path.join(workspace, "control/postinst"),
|
||||||
|
"--after-remove", os.path.join(workspace, "control/postrm"),
|
||||||
|
"--before-install", os.path.join(workspace, "control/preinst"),
|
||||||
|
# package relationships
|
||||||
|
"--deb-recommends", "influxdb2-cli",
|
||||||
|
"--conflicts", "influxdb",
|
||||||
|
"--depends", "curl",
|
||||||
|
# package conffiles
|
||||||
|
*conffiles,
|
||||||
|
# package options
|
||||||
|
"--chdir", os.path.join(workspace, "fs/"),
|
||||||
|
"--package", source["target"]
|
||||||
|
])
|
||||||
|
# fmt: on
|
||||||
|
|
||||||
|
|
||||||
|
circle_tag = os.getenv("CIRCLE_TAG", default="")
|
||||||
|
circle_sha = os.getenv("CIRCLE_SHA1", default="DEADBEEF")
|
||||||
|
# Determine if `circle_tag` matches the semantic version regex. Otherwise,
|
||||||
|
# assume that `circle_tag` is not intended to tag a release. The regex is
|
||||||
|
# permissive of what occurs after the semantic version. This allows for
|
||||||
|
# alphas, betas, and release candidates.
|
||||||
|
if re.match("^v[0-9]+.[0-9]+.[0-9]+", circle_tag):
|
||||||
|
version = circle_tag[1:]
|
||||||
|
else:
|
||||||
|
# When `circle_tag` cannot be used to construct the package version,
|
||||||
|
# use `circle_sha`. Since `circle_sha` can start with an alpha (non-
|
||||||
|
# -numeric) character, prefix it with "2.x-".
|
||||||
|
version = "2.x-" + circle_sha[:8]
|
||||||
|
|
||||||
|
with open(".circleci/scripts/package/config.yaml") as file:
|
||||||
|
document = yaml.load(file, Loader=yaml.SafeLoader)
|
||||||
|
|
||||||
|
# fmt: off
|
||||||
|
for s, p in [
|
||||||
|
(s, p)
|
||||||
|
for s in document["sources" ]
|
||||||
|
for p in document["packages"]
|
||||||
|
]:
|
||||||
|
# fmt: on
|
||||||
|
if s["plat"] == "linux":
|
||||||
|
build_linux_archive(s, p, version)
|
||||||
|
build_linux_package(s, p, version)
|
||||||
|
if s["plat"] == "darwin" or s["plat"] == "windows":
|
||||||
|
build_archive(s, p, version)
|
|
@ -0,0 +1,49 @@
|
||||||
|
---
|
||||||
|
sources:
|
||||||
|
- binary: /tmp/workspace/bin/influxd_linux_amd64/
|
||||||
|
target: artifacts/
|
||||||
|
arch: amd64
|
||||||
|
plat: linux
|
||||||
|
|
||||||
|
- binary: /tmp/workspace/bin/influxd_linux_arm64/
|
||||||
|
target: artifacts/
|
||||||
|
arch: arm64
|
||||||
|
plat: linux
|
||||||
|
|
||||||
|
- binary: /tmp/workspace/bin/influxd_darwin_amd64/
|
||||||
|
target: artifacts/
|
||||||
|
arch: amd64
|
||||||
|
plat: darwin
|
||||||
|
|
||||||
|
- binary: /tmp/workspace/bin/influxd_windows_amd64/
|
||||||
|
target: artifacts/
|
||||||
|
arch: amd64
|
||||||
|
plat: windows
|
||||||
|
|
||||||
|
packages:
|
||||||
|
- name: influxdb2
|
||||||
|
binaries:
|
||||||
|
- influxd
|
||||||
|
- influxd.exe
|
||||||
|
extras:
|
||||||
|
- source: LICENSE
|
||||||
|
target: usr/share/influxdb/LICENSE
|
||||||
|
|
||||||
|
- source: README.md
|
||||||
|
target: usr/share/influxdb/README.md
|
||||||
|
perm_overrides:
|
||||||
|
- owner: root
|
||||||
|
group: root
|
||||||
|
perms: 0755
|
||||||
|
target: usr/share/influxdb/influxdb2-upgrade.sh
|
||||||
|
|
||||||
|
- owner: root
|
||||||
|
group: root
|
||||||
|
perms: 0755
|
||||||
|
target: usr/lib/influxdb/scripts/init.sh
|
||||||
|
|
||||||
|
- owner: root
|
||||||
|
group: root
|
||||||
|
perms: 0755
|
||||||
|
target: usr/lib/influxdb/scripts/influxd-systemd-start.sh
|
||||||
|
source: .circleci/scripts/package/influxdb2
|
|
@ -0,0 +1 @@
|
||||||
|
This prevents Git from removing this directory.
|
|
@ -0,0 +1,2 @@
|
||||||
|
PyYAML==6.0
|
||||||
|
regex==2023.6.3
|
Loading…
Reference in New Issue