From da688cf51ce0f071e8b61661513d2432b3676d0d Mon Sep 17 00:00:00 2001 From: Kevin Daudt <kdaudt@alpinelinux.org> Date: Sun, 16 Jan 2022 16:14:44 +0100 Subject: [PATCH 1/6] tests: remove abuild-keygen interactive cp workound This has been fixed in abuild-keygen, so we no longer need to work around it. --- test/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/Dockerfile b/test/Dockerfile index fa6f67e..b69f10a 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -6,7 +6,4 @@ RUN doas apk upgrade -U RUN doas apk add --no-cache bats curl ncurses gettext \ && doas install-bats-helpers.sh -# Fixes occasional hangs during testing due to waiting for stdin -RUN doas sed -i 's/cp -i/cp/' /usr/bin/abuild-keygen - ENV PATH=/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -- GitLab From 01e7d1750efa0abb1dcc3682cad5616f986612b2 Mon Sep 17 00:00:00 2001 From: Kevin Daudt <kdaudt@alpinelinux.org> Date: Sun, 16 Jan 2022 16:34:59 +0100 Subject: [PATCH 2/6] build: extract common functions to a library This way the functionality can be reused --- overlay/usr/local/bin/build.sh | 22 ++------------------ overlay/usr/local/lib/functions.sh | 32 ++++++++++++++++++++++++++++++ test/docker-compose.yml | 3 +++ 3 files changed, 37 insertions(+), 20 deletions(-) create mode 100644 overlay/usr/local/lib/functions.sh diff --git a/overlay/usr/local/bin/build.sh b/overlay/usr/local/bin/build.sh index 6910cb6..efdd4f1 100755 --- a/overlay/usr/local/bin/build.sh +++ b/overlay/usr/local/bin/build.sh @@ -1,6 +1,8 @@ #!/bin/sh # shellcheck disable=SC3043 +. /usr/local/lib/functions.sh + # shellcheck disable=SC3040 set -eu -o pipefail @@ -92,14 +94,6 @@ check_aport() { fi } -changed_repos() { - cd "$APORTSDIR" - for repo in $REPOS; do - git diff --exit-code "$BASEBRANCH"...HEAD -- "$repo" >/dev/null \ - || echo "$repo" - done -} - set_repositories_for() { local target_repo="$1" repos='' repo='' local release @@ -115,18 +109,6 @@ set_repositories_for() { doas apk update } -changed_aports() { - cd "$APORTSDIR" - local repo="$1" - local aports - - aports=$(git diff --name-only --diff-filter=ACMR --relative="$repo" \ - "$BASEBRANCH"...HEAD -- "*/APKBUILD" | xargs -I% dirname %) - # $aports should remain unquoted - # shellcheck disable=2086 - ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -I% basename % | xargs -} - apply_offset_limit() { start=$1 limit=$2 diff --git a/overlay/usr/local/lib/functions.sh b/overlay/usr/local/lib/functions.sh new file mode 100644 index 0000000..188f7d5 --- /dev/null +++ b/overlay/usr/local/lib/functions.sh @@ -0,0 +1,32 @@ +# shellcheck disable=SC3043 + +: + +# shellcheck disable=SC3040 +set -eu -o pipefail + +changed_repos() { + : "${APORTSDIR?APORTSDIR missing}" + : "${BASEBRANCH?BASEBRANCH missing}" + + cd "$APORTSDIR" + for repo in $REPOS; do + git diff --exit-code "$BASEBRANCH"...HEAD -- "$repo" >/dev/null \ + || echo "$repo" + done +} + +changed_aports() { + : "${APORTSDIR?APORTSDIR missing}" + : "${BASEBRANCH?BASEBRANCH missing}" + + cd "$APORTSDIR" + local repo="$1" + local aports + + aports=$(git diff --name-only --diff-filter=ACMR --relative="$repo" \ + "$BASEBRANCH"...HEAD -- "*/APKBUILD" | xargs -I% dirname %) + + # shellcheck disable=2086 + ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -I% basename % | xargs +} diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 8b4c290..34c6ad2 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -10,9 +10,12 @@ services: build: . working_dir: /tests/ command: [/usr/bin/bats, build.bats] + depends_on: + - alpine-gitlab-ci test-local: <<: *test volumes: - ./overlay/tests:/tests - ../overlay/usr/local/bin/:/local/bin/ + - ../overlay/usr/local/lib/:/usr/local/lib/ - ./overlay/usr/local/bin/:/local/tests/bin/ -- GitLab From 159d70674f5300e7d973fcb39ec7b14d49aab2c4 Mon Sep 17 00:00:00 2001 From: Kevin Daudt <kdaudt@alpinelinux.org> Date: Sun, 16 Jan 2022 16:36:48 +0100 Subject: [PATCH 3/6] git: set default branch name This suppresses the hint that happens in every CI build. --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 0d720ce..7ec42ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,6 @@ FROM alpinelinux/build-base:latest$ALPINE_ARCH COPY overlay/ / +RUN git config --global init.defaultBranch master + ENV SUDO=doas -- GitLab From 1ac30a9a05f8fd3a851eafead46111ae656a8043 Mon Sep 17 00:00:00 2001 From: Kevin Daudt <kdaudt@alpinelinux.org> Date: Sun, 16 Jan 2022 16:46:04 +0100 Subject: [PATCH 4/6] build: simplify changed_aports changed_aports pipes the result to `xargs` to make it space delimited like the output of `ap builddirs`, for the consumer to pipe it to `xargs -n1 echo` to make it line delimited again for sed. Remove these redundant steps. --- overlay/usr/local/bin/build.sh | 2 +- overlay/usr/local/lib/functions.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/overlay/usr/local/bin/build.sh b/overlay/usr/local/bin/build.sh index efdd4f1..81c4164 100755 --- a/overlay/usr/local/bin/build.sh +++ b/overlay/usr/local/bin/build.sh @@ -188,7 +188,7 @@ build_limit=$CI_ALPINE_BUILD_LIMIT for repo in $(changed_repos); do set_repositories_for "$repo" built_aports=0 - changed_aports_in_repo=$(changed_aports "$repo" | xargs -rn1 echo) + changed_aports_in_repo=$(changed_aports "$repo") changed_aports_in_repo_count=$(echo "$changed_aports_in_repo" | wc -l) changed_aports_to_build=$(echo "$changed_aports_in_repo" | apply_offset_limit "$build_start" "$build_limit") diff --git a/overlay/usr/local/lib/functions.sh b/overlay/usr/local/lib/functions.sh index 188f7d5..e20ab17 100644 --- a/overlay/usr/local/lib/functions.sh +++ b/overlay/usr/local/lib/functions.sh @@ -25,8 +25,8 @@ changed_aports() { local aports aports=$(git diff --name-only --diff-filter=ACMR --relative="$repo" \ - "$BASEBRANCH"...HEAD -- "*/APKBUILD" | xargs -I% dirname %) + "$BASEBRANCH"...HEAD -- "*/APKBUILD" | xargs -rn1 dirname) # shellcheck disable=2086 - ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -I% basename % | xargs + ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -rn1 basename } -- GitLab From b2b884d518185d4afb91c4b09784165671286039 Mon Sep 17 00:00:00 2001 From: Kevin Daudt <kdaudt@alpinelinux.org> Date: Sun, 16 Jan 2022 17:17:16 +0100 Subject: [PATCH 5/6] build: fix warning about REPODEST being read-only `REPODEST` was declared read-only in build.sh. Later, 'abuild.conf', which also defines `REPODEST`, is being sourced in a subshell. This resulted in a warning about `REPODEST` being read-only. Fix this by defining REPODEST as a normal variable, not a read-only variable. --- overlay/usr/local/bin/build.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/overlay/usr/local/bin/build.sh b/overlay/usr/local/bin/build.sh index 81c4164..5b4a445 100755 --- a/overlay/usr/local/bin/build.sh +++ b/overlay/usr/local/bin/build.sh @@ -7,12 +7,11 @@ set -eu -o pipefail readonly APORTSDIR=$CI_PROJECT_DIR -readonly REPODEST=${REPODEST-$HOME/packages} readonly REPOS="main community testing non-free" -readonly ARCH=$(apk --print-arch) # gitlab variables readonly BASEBRANCH=$CI_MERGE_REQUEST_TARGET_BRANCH_NAME +: "${REPODEST:=$HOME/packages}" : "${MIRROR:=https://dl-cdn.alpinelinux.org/alpine}" : "${MAX_ARTIFACT_SIZE:=300000000}" #300M : "${CI_DEBUG_BUILD:=}" -- GitLab From c4235a8feeb9584b37ad7112b272129973637071 Mon Sep 17 00:00:00 2001 From: Kevin Daudt <kdaudt@alpinelinux.org> Date: Sun, 16 Jan 2022 18:48:10 +0100 Subject: [PATCH 6/6] build: add sections for log output gitlab supports section markers which can be used specify collapsible sections to make the build log better readable. --- overlay/usr/local/bin/build.sh | 11 +++++++++++ overlay/usr/local/lib/functions.sh | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/overlay/usr/local/bin/build.sh b/overlay/usr/local/bin/build.sh index 5b4a445..abf0515 100755 --- a/overlay/usr/local/bin/build.sh +++ b/overlay/usr/local/bin/build.sh @@ -154,6 +154,8 @@ copy_artifacts() { fi } +section_start setup "Setting up the system" collapse + if debugging; then set -x fi @@ -181,6 +183,8 @@ if debugging; then git --no-pager log -200 --oneline --graph --decorate --all fi +section_end setup + build_start=$CI_ALPINE_BUILD_OFFSET build_limit=$CI_ALPINE_BUILD_LIMIT @@ -195,10 +199,12 @@ for repo in $(changed_repos); do # shellcheck disable=SC2086 # Splitting is expected here printf " - %s\n" $changed_aports_to_build for pkgname in $changed_aports_to_build; do + section_start "build_$pkgname" "Building package $pkgname" built_aports=$((built_aports+1)) if check_aport "$repo" "$pkgname"; then build_aport "$repo" "$pkgname" fi + section_end "build_$pkgname" done build_start=$((build_start-(changed_aports_in_repo_count-built_aports))) @@ -210,7 +216,11 @@ for repo in $(changed_repos); do fi done +section_start artifacts "Handeling artifacts" collapse copy_artifacts || true +section_end artifacts + +section_start summary "Build summary" echo "### Build summary ###" @@ -226,6 +236,7 @@ for ng in $aport_ng; do msg "$ng: build failed" red failed=true done +section_end summary if [ "$failed" = true ]; then exit 1 diff --git a/overlay/usr/local/lib/functions.sh b/overlay/usr/local/lib/functions.sh index e20ab17..670bf04 100644 --- a/overlay/usr/local/lib/functions.sh +++ b/overlay/usr/local/lib/functions.sh @@ -30,3 +30,24 @@ changed_aports() { # shellcheck disable=2086 ap builddirs -d "$APORTSDIR/$repo" $aports 2>/dev/null | xargs -rn1 basename } + +section_start() { + name=${1?arg 1 name missing} + header=${2?arg 2 header missing} + collapsed=$2 + timestamp=$(date +%s) + + options="" + case $collapsed in + yes|on|collapsed|true) options="[collapsed=true]";; + esac + + printf "\e[0Ksection_start:%d:%s%s\r\e[0K%s\n" "$timestamp" "$name" "$options" "$header" +} + +section_end() { + name=$1 + timestamp=$(date +%s) + + printf "\e[0Ksection_end:%d:%s\r\e[0K" "$timestamp" "$name" +} -- GitLab