diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 26a42d4..773f12a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,9 +1,8 @@ # Keeps GitHub Actions pinned versions current. # -# Only repos with their own workflow steps need this. The ~95 image repos call -# daemonless/dbuild/.github/workflows/daemonless-build.yaml@main, which is a -# branch ref -- there is no version for dependabot to bump, and the actions they -# actually run live in dbuild's reusable workflow instead. +# Only repos with their own workflow steps need this. The image repos call +# daemonless/workflows/.github/workflows/daemonless-build.yaml@v1, and the +# actions they actually run are pinned there, not here. version: 2 updates: diff --git a/.github/workflows/daemonless-build.yaml b/.github/workflows/daemonless-build.yaml index facc4fa..486728b 100644 --- a/.github/workflows/daemonless-build.yaml +++ b/.github/workflows/daemonless-build.yaml @@ -1,5 +1,9 @@ name: Reusable App Build +# Moved to daemonless/workflows. Call +# daemonless/workflows/.github/workflows/daemonless-build.yaml@v1 +# This pass-through keeps old callers building until they move. + on: workflow_call: inputs: @@ -58,913 +62,19 @@ on: type: number default: 180 -env: - REGISTRY: ghcr.io - DBUILD_REF: v1.13.1 - jobs: - # Detect which variants exist and build dynamic matrix - detect: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.detect.outputs.matrix }} - compose_only: ${{ steps.detect.outputs.compose_only }} - architectures: ${{ steps.detect.outputs.architectures }} - declared_architectures: ${{ steps.detect.outputs.declared_architectures }} - manifest_tags: ${{ steps.detect.outputs.manifest_tags }} - build_tools_ref: ${{ steps.resolve-build-tools.outputs.ref }} - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Resolve build-tools ref - id: resolve-build-tools - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ -n "${{ inputs.build_tools_ref }}" ]; then - echo "ref=${{ inputs.build_tools_ref }}" >> $GITHUB_OUTPUT - else - REF=$(gh api repos/daemonless/build-tools/releases/latest --jq '.tag_name') - echo "ref=$REF" >> $GITHUB_OUTPUT - fi - - - name: Install dependencies - run: pip install PyYAML - - - name: Detect build matrix - id: detect - run: | - # Fetch dbuild - curl -sSL "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" | tar xz -C /tmp - DBUILD_DIR=$(ls -d /tmp/dbuild-* | head -1) - export PYTHONPATH="$DBUILD_DIR" - ARCH_FILTER="${{ inputs.arch }}" - # daemonless org defaults CI to amd64-only -- non-amd64 archs are - # QEMU-emulated on GHA runners and not something we want auto- - # building on every push. A caller can still opt back in by - # passing the `arch` input explicitly. Other orgs reusing this - # workflow get every architecture listed in .daemonless/config.yaml, - # unrestricted, by default. - if [ -z "$ARCH_FILTER" ] && [ "${{ github.repository_owner }}" = "daemonless" ]; then - ARCH_FILTER="amd64" - fi - python3 -m dbuild ${ARCH_FILTER:+--arch "$ARCH_FILTER"} detect --format github ${{ inputs.variant && format('--variant {0}', inputs.variant) || '' }} - - # Fails the check if dbuild lint reports errors. build doesn't need it: - # a metadata error shows red but never holds back a rebuild. - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Install dependencies - run: pip install PyYAML jinja2 - - - name: dbuild lint - run: | - set -o pipefail - curl -fsSL "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" | tar xz -C /tmp - DBUILD_DIR=$(ls -d /tmp/dbuild-* | head -1) - export PYTHONPATH="$DBUILD_DIR" - # rc captured on its own: piping the { ...; echo } block through tee - # reported the last echo's status, so lint errors never failed the job. - rc=0 - python3 -m dbuild lint > /tmp/lint.txt 2>&1 || rc=$? - cat /tmp/lint.txt - { - echo '## dbuild lint' - echo '' - echo '```' - cat /tmp/lint.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - exit "$rc" - build: - needs: detect - if: ${{ needs.detect.outputs.matrix != '' && needs.detect.outputs.compose_only != 'true' }} - runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.build_timeout }} - permissions: - contents: read - packages: write - - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.detect.outputs.matrix) }} - - steps: - - name: Free disk space - uses: endersonmenezes/free-disk-space@v4 - with: - remove_tool_cache: true - remove_android: true - remove_dotnet: true - remove_haskell: true - remove_swap: true - remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox mono-complete" - remove_packages_one_command: true - rm_cmd: rmz - - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Generate Containerfile/README from templates - if: inputs.generate - run: | - curl -sSL "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" | tar xz -C /tmp - DBUILD_DIR=$(ls -d /tmp/dbuild-* | head -1) - export PYTHONPATH="$DBUILD_DIR" - pip install -q PyYAML jinja2 - python3 -m dbuild generate - - - name: Download pre-build artifact - if: inputs.pre_artifact_name != '' - uses: actions/download-artifact@v8 - with: - name: ${{ inputs.pre_artifact_name }} - path: ${{ inputs.pre_artifact_path }} - - - name: Prepare VM data directory - run: sudo mkdir -p /mnt/freebsd-vm && sudo chmod 777 /mnt/freebsd-vm - - # Remote VNC only on a run with debug logging on (Re-run jobs -> Enable - # debug logging). The password is made for this run and printed only - # encrypted to the SSH keys of whoever started or re-ran it; no keys, no - # remote VNC. A run with remote VNC is never published. - - name: One-time VNC password - id: vnc - if: runner.debug == '1' - env: - ACTOR: ${{ github.triggering_actor }} - run: | - set -eu - if ! curl -fsSL "https://github.com/${ACTOR}.keys" | grep -E '^(ssh-ed25519|ssh-rsa) ' > /tmp/vnc-keys; then - echo "::warning::${ACTOR} has no ssh-ed25519 or ssh-rsa key on GitHub; remote VNC is off for this run." - exit 0 - fi - pw=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | cut -c1-8) - echo "::add-mask::$pw" - command -v age >/dev/null || sudo apt-get install -y -qq age >/dev/null - enc=$(printf '%s' "$pw" | age -a -R /tmp/vnc-keys) - { - echo "### One-time VNC password" - echo "Encrypted to the SSH keys of ${ACTOR}. Save the block below to a file and run" - echo "\`age -d -i ~/.ssh/id_ed25519 \`." - echo '```' - echo "$enc" - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - echo "$enc" - echo "pw=$pw" >> "$GITHUB_OUTPUT" - - - name: Build in FreeBSD VM (${{ matrix.tag }}${{ matrix.arch_suffix }}) - uses: vmactions/freebsd-vm@v1.5.9 - env: - # A debug run publishes nothing, so its VM, which a console can reach, - # gets no credentials; builds fall back to unauthenticated API calls. - GITHUB_TOKEN: ${{ steps.vnc.outputs.pw == '' && secrets.GITHUB_TOKEN || '' }} - GITHUB_ACTOR: ${{ github.actor }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ steps.vnc.outputs.pw == '' && secrets.DOCKERHUB_TOKEN || '' }} - DBUILD_REGISTRY: ${{ inputs.registry }} - with: - release: "15.1" - vnc-password: ${{ steps.vnc.outputs.pw }} - arch: ${{ matrix.vm_arch }} - sync: ${{ matrix.vm_sync }} - # The action's 6144 MB OOMs large Rust builds; the public runner has 16 GB. - mem: 12288 - usesh: true - copyback: true - data-dir: /mnt/freebsd-vm - debug-on-error: ${{ steps.vnc.outputs.pw != '' }} - envs: "GITHUB_TOKEN GITHUB_ACTOR DOCKERHUB_USERNAME DOCKERHUB_TOKEN DBUILD_REGISTRY" - prepare: | - set -e - # Create 4GB swap for heavy builds - dd if=/dev/zero of=/swapfile bs=1m count=4096 - chmod 0600 /swapfile - mdconfig -a -t vnode -f /swapfile -u 0 - swapon /dev/md0 - - # build-tools assets are amd64-bare/-aarch64/-riscv64 -- fixed - # naming, independent of dbuild's registry-tag arch_suffix. - fetch -qo /tmp/build-tools.tar.gz \ - "https://github.com/daemonless/build-tools/releases/download/${{ needs.detect.outputs.build_tools_ref }}/${{ needs.detect.outputs.build_tools_ref }}${{ matrix.arch != 'amd64' && format('-{0}', matrix.arch) || '' }}.tar.gz" - mkdir -p /tmp/build-tools - tar -xzf /tmp/build-tools.tar.gz -C /tmp/build-tools - mkdir -p /etc/pkg - printf 'local: { url: "file:///tmp/build-tools", enabled: yes }\nFreeBSD: { enabled: no }\n' > /etc/pkg/FreeBSD.conf - pkg repo /tmp/build-tools - pkg install -y podman sysutils/podman-compose jq skopeo buildah trivy python3 devel/py-pyyaml appjail ocijail - mkdir -p /var/log/appjail/jails /usr/local/appjail/jails - - rm -rf /var/db/containers /var/lib/containers 2>/dev/null || true - kldload pf - kldload linux64 # linuxulator images: linux_base pre-install and CIT boot need it - sysctl net.inet.ip.forwarding=1 - sysctl kern.ipc.shm_allow_removed=1 - run: | - set -e - - # Fetch dbuild - fetch -qo /tmp/dbuild.tar.gz \ - "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" - mkdir -p /tmp/dbuild - tar -xzf /tmp/dbuild.tar.gz -C /tmp/dbuild --strip-components=1 - export PYTHONPATH="/tmp/dbuild" - DBUILD="python3 -m dbuild" - - echo "=== dbuild CI (GitHub Actions) ===" - $DBUILD --version - - VARIANT="${{ matrix.tag }}" - ARCH="${{ matrix.arch }}" - IMAGE_NAME="${{ inputs.image_name }}" - - # Read old version from sbom.json before build - OLD_VERSION=$(jq -r --arg tag "$VARIANT" '.tags[$tag].app_version // ""' sbom.json 2>/dev/null || echo "") - echo "$OLD_VERSION" > .version-old - - # Build - $DBUILD -v build --variant "$VARIANT" --arch "$ARCH" - - # Extract new version from built image - NEW_VERSION=$(podman run --rm --entrypoint "" "ghcr.io/daemonless/${IMAGE_NAME}:build-${VARIANT}" cat /app/version 2>/dev/null || echo "") - echo "$NEW_VERSION" > .version-new - - # Screenshot deps: amd64 only (no aarch64 chromium); dbuild then - # auto-downgrades screenshot -> health on aarch64. - if [ "${{ matrix.arch }}" = "amd64" ] && { grep -q 'mode:.*screenshot' .daemonless/config.yaml 2>/dev/null || grep -q 'mode:.*screenshot' .dbuild.yaml 2>/dev/null; }; then - SS_REF=$(echo "${{ needs.detect.outputs.build_tools_ref }}" | sed 's/\(freebsd-[0-9.]*-\)/\1screenshot-/') - fetch -qo /tmp/build-tools-screenshot.tar.gz \ - "https://github.com/daemonless/build-tools/releases/download/${{ needs.detect.outputs.build_tools_ref }}/${SS_REF}.tar.gz" - tar -xzf /tmp/build-tools-screenshot.tar.gz -C /tmp/build-tools - pkg repo /tmp/build-tools - pkg install -y chromium www/py-selenium graphics/py-scikit-image - fi - - # Test (capture exit code for copyback) - mkdir -p cit-results - cit_exit=0 - $DBUILD -v test --variant "$VARIANT" \ - --json cit-results/${{ inputs.image_name }}-${{ matrix.tag }}${{ matrix.arch_suffix }}.json \ - || cit_exit=$? - - # Copy any temp screenshots to results - cp /tmp/cit-screenshot*.png cit-results/ 2>/dev/null || true - - # Post-mortem: dbuild removes the CIT container on cleanup and only - # prints its last 10 lines, which a crash loop fills with s6 restart - # notices -- so the failure itself never reaches the log. Re-run the - # image the way CIT does and copy the full output back. - if [ $cit_exit -ne 0 ]; then - PM="postmortem-${{ matrix.tag }}" - podman rm -f "$PM" >/dev/null 2>&1 || true - podman run -d --name "$PM" -e PUID=1000 -e PGID=1000 \ - "${DBUILD_REGISTRY}/${{ inputs.image_name }}:build-${VARIANT}" \ - >/dev/null 2>&1 || true - sleep 45 - { - echo "### podman logs (full)" - podman logs "$PM" 2>&1 - echo "### inspect" - podman inspect "$PM" --format 'exit={{.State.ExitCode}} oom={{.State.OOMKilled}} err={{.State.Error}}' 2>&1 - echo "### df"; df -h / - } > "cit-results/${PM}.log" 2>&1 - podman rm -f "$PM" >/dev/null 2>&1 || true - fi - - # Save exit code (don't exit here - let copyback happen first) - echo "$cit_exit" > cit-results/cit-exit-code - - # Only push + sbom if tests passed and not a PR - if [ $cit_exit -ne 0 ]; then - echo "Tests failed with exit code $cit_exit - skipping push" - elif [ "${{ github.event_name }}" = "pull_request" ]; then - echo "PR build - skipping push" - elif [ "${{ steps.vnc.outputs.pw != '' }}" = "true" ]; then - echo "Remote VNC was enabled for this run - skipping push" - else - # Free disk before push - swapoff /dev/md0 2>/dev/null; mdconfig -d -u 0 2>/dev/null; rm -f /swapfile - podman image prune -f - pkg clean -ay - rm -rf /tmp/cit-* - - # SBOM before push: --embed bakes the CycloneDX/SPDX docs into - # the image at /usr/share/sbom/, so the pushed image ships them. - $DBUILD -v sbom --variant "$VARIANT" --arch "$ARCH" --format all --embed - $DBUILD -v push --variant "$VARIANT" --arch "$ARCH" - fi - - # The VM step reads as passed once a debug session ends, and anyone at - # the console could have changed the build: fail the job so nothing - # downstream (manifests, signing, SBOM commit) publishes it. - - name: Refuse to publish a run with remote VNC - if: ${{ !cancelled() && steps.vnc.outputs.pw != '' }} - run: | - echo "::error::Remote VNC was enabled for this run, so nothing from it is published. Run it again without debug logging to publish." - exit 1 - - - name: Upload test artifacts - uses: actions/upload-artifact@v7 - if: always() - with: - name: cit-results-${{ matrix.tag }}${{ matrix.arch_suffix }} - path: cit-results/ - retention-days: 30 - - - name: Upload SBOM - uses: actions/upload-artifact@v7 - if: success() && github.event_name != 'pull_request' - with: - name: sbom-${{ matrix.tag }}${{ matrix.arch_suffix }} - path: sbom-results/ - retention-days: 90 - - - name: Check test result - if: always() - run: | - if [ ! -f cit-results/cit-exit-code ]; then - echo "ERROR: cit-exit-code file not found - copyback may have failed" - exit 1 - fi - exit_code=$(cat cit-results/cit-exit-code) - if [ "$exit_code" != "0" ]; then - echo "Tests failed with exit code $exit_code" - exit $exit_code - fi - - - name: Update Docker Hub description - if: success() && matrix.tag == 'latest' && matrix.arch == 'amd64' && github.event_name != 'pull_request' - continue-on-error: true - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - uses: daemonless/dbuild/.github/actions/dockerhub-desc@main - with: - image_name: ${{ inputs.image_name }} - dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Determine version label - id: ver - if: always() - run: | - OLD=$(cat .version-old 2>/dev/null | tr -d '[:space:]' || echo "") - NEW=$(cat .version-new 2>/dev/null | tr -d '[:space:]' || echo "") - if [ -n "$NEW" ] && [ -n "$OLD" ] && [ "$OLD" != "$NEW" ]; then - echo "label=${OLD} → ${NEW}" >> $GITHUB_OUTPUT - elif [ -n "$NEW" ]; then - echo "label=${NEW}" >> $GITHUB_OUTPUT - else - echo "label=" >> $GITHUB_OUTPUT - fi - - - name: Discord notification - if: always() - continue-on-error: true - env: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - uses: daemonless/dbuild/.github/actions/discord-notify@main - with: - webhook_url: ${{ secrets.DISCORD_WEBHOOK }} - image_name: ${{ inputs.image_name }} - tag: ${{ matrix.tag }}${{ matrix.arch_suffix }} - status: ${{ job.status }} - commit_sha: ${{ github.sha }} - commit_url: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - run_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - branch: ${{ github.ref_name }} - has_dockerhub: ${{ secrets.DOCKERHUB_USERNAME != '' }} - version: ${{ steps.ver.outputs.label }} - pushed: ${{ github.event_name != 'pull_request' }} - - # Linux half (inputs.linux): Containerfile.linux built natively on an amd64 - # and an arm64 runner, checked like cit says, pushed as -linux-. - # dbuild manifest (any builder's) adds it to when its version is the - # FreeBSD images'. Never through dbuild's command line: Linux images are - # built here only. - build-linux: - needs: detect - if: ${{ inputs.linux && needs.detect.outputs.compose_only != 'true' }} - runs-on: ${{ matrix.runner }} - timeout-minutes: ${{ inputs.build_timeout }} - permissions: - contents: read - packages: write - strategy: - fail-fast: false - matrix: - include: - - arch: amd64 - runner: ubuntu-24.04 - - arch: arm64 - runner: ubuntu-24.04-arm - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Fetch dbuild - run: | - curl -fsSL "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" | tar xz -C /tmp - mv /tmp/dbuild-* /tmp/dbuild - pip install -q PyYAML - - - name: Build and test (linux/${{ matrix.arch }}) - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - export PYTHONPATH=/tmp/dbuild - python3 /tmp/dbuild/.github/scripts/linux-image.py ${{ matrix.arch }} \ - ${{ github.event_name != 'pull_request' && '--push' || '' }} - - # Create multi-arch manifests - create-manifests: - needs: [detect, build, build-linux] - # build-linux skipped (no inputs.linux) must not hold this back; failed must. - if: >- - !cancelled() && needs.build.result == 'success' && needs.build-linux.result != 'failure' - && github.event_name != 'pull_request' - && (contains(needs.detect.outputs.declared_architectures, 'aarch64') || inputs.linux) - runs-on: ubuntu-latest - steps: - - name: Install skopeo - run: sudo apt-get update -qq && sudo apt-get install -y skopeo - - - name: Fetch dbuild - run: | - curl -sSL "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" | tar xz -C /tmp - mv /tmp/dbuild-* /tmp/dbuild - - - name: Checkout repository - uses: actions/checkout@v7 - - - name: dbuild manifest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - run: | - export PYTHONPATH="/tmp/dbuild" - python3 -m dbuild -v manifest - - # Compose-only test (no build, just test pre-built images) - test-compose: - needs: detect - if: ${{ needs.detect.outputs.compose_only == 'true' }} - runs-on: ubuntu-latest - - steps: - - name: Free disk space - uses: endersonmenezes/free-disk-space@v4 - with: - remove_tool_cache: true - remove_android: true - remove_dotnet: true - remove_haskell: true - remove_swap: true - remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox mono-complete" - remove_packages_one_command: true - rm_cmd: rmz - - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Prepare VM data directory - run: sudo mkdir -p /mnt/freebsd-vm && sudo chmod 777 /mnt/freebsd-vm - - # Same one-time VNC password as the build job. - - name: One-time VNC password - id: vnc - if: runner.debug == '1' - env: - ACTOR: ${{ github.triggering_actor }} - run: | - set -eu - if ! curl -fsSL "https://github.com/${ACTOR}.keys" | grep -E '^(ssh-ed25519|ssh-rsa) ' > /tmp/vnc-keys; then - echo "::warning::${ACTOR} has no ssh-ed25519 or ssh-rsa key on GitHub; remote VNC is off for this run." - exit 0 - fi - pw=$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | cut -c1-8) - echo "::add-mask::$pw" - command -v age >/dev/null || sudo apt-get install -y -qq age >/dev/null - enc=$(printf '%s' "$pw" | age -a -R /tmp/vnc-keys) - { - echo "### One-time VNC password" - echo "Encrypted to the SSH keys of ${ACTOR}. Save the block below to a file and run" - echo "\`age -d -i ~/.ssh/id_ed25519 \`." - echo '```' - echo "$enc" - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - echo "$enc" - echo "pw=$pw" >> "$GITHUB_OUTPUT" - - - name: Test in FreeBSD VM - uses: vmactions/freebsd-vm@v1.5.9 - with: - release: "15.1" - vnc-password: ${{ steps.vnc.outputs.pw }} - usesh: true - copyback: true - data-dir: /mnt/freebsd-vm - debug-on-error: ${{ steps.vnc.outputs.pw != '' }} - prepare: | - set -e - # Install build tools from pinned bundle - fetch -qo /tmp/build-tools.tar.gz \ - "https://github.com/daemonless/build-tools/releases/download/${{ needs.detect.outputs.build_tools_ref }}/${{ needs.detect.outputs.build_tools_ref }}.tar.gz" - mkdir -p /tmp/build-tools - tar -xzf /tmp/build-tools.tar.gz -C /tmp/build-tools - mkdir -p /etc/pkg - printf 'local: { url: "file:///tmp/build-tools", enabled: yes }\nFreeBSD: { enabled: no }\n' > /etc/pkg/FreeBSD.conf - pkg repo /tmp/build-tools - pkg install -y podman sysutils/podman-compose jq python3 devel/py-pyyaml appjail ocijail - - rm -rf /var/db/containers /var/lib/containers 2>/dev/null || true - kldload pf - kldload linux64 # linuxulator images: linux_base pre-install and CIT boot need it - sysctl net.inet.ip.forwarding=1 - sysctl kern.ipc.shm_allow_removed=1 - run: | - set -e - - # Fetch dbuild - fetch -qo /tmp/dbuild.tar.gz \ - "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" - mkdir -p /tmp/dbuild - tar -xzf /tmp/dbuild.tar.gz -C /tmp/dbuild --strip-components=1 - export PYTHONPATH="/tmp/dbuild" - DBUILD="python3 -m dbuild" - - mkdir -p cit-results - $DBUILD -v test \ - --json cit-results/${{ inputs.image_name }}.json - - - name: Upload test artifacts - uses: actions/upload-artifact@v7 - if: always() - with: - name: cit-results - path: cit-results/ - retention-days: 30 - - - name: Discord notification - if: always() - continue-on-error: true - env: - DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} - run: | - [ -z "$DISCORD_WEBHOOK" ] && exit 0 - if [ "${{ job.status }}" = "success" ]; then - COLOR=3066993 - STATUS="Success" - else - COLOR=15158332 - STATUS="Failed" - fi - - SHORT_SHA="${{ github.sha }}" - SHORT_SHA="${SHORT_SHA:0:7}" - - curl -sS -H "Content-Type: application/json" \ - -d "{\"embeds\":[{\"title\":\"${{ inputs.image_name }} compose test $STATUS\",\"url\":\"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\",\"color\":$COLOR,\"fields\":[{\"name\":\"Commit\",\"value\":\"[$SHORT_SHA](${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }})\",\"inline\":true},{\"name\":\"Branch\",\"value\":\"${{ github.ref_name }}\",\"inline\":true}]}]}" \ - "$DISCORD_WEBHOOK" - - # Merge SBOMs and commit to repo - commit-sbom: - needs: build - if: success() && github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - contents: write - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Download all SBOM artifacts - uses: actions/download-artifact@v8 - with: - pattern: sbom-* - path: sbom-artifacts/ - - - name: Merge SBOMs - run: | - jq -n \ - --arg image "${{ inputs.image_name }}" \ - --arg generated "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - '{image: $image, generated: $generated, tags: {}}' > sbom.json - - for f in $(find sbom-artifacts/ -name '*-sbom.json'); do - [ -f "$f" ] || continue - tag=$(jq -r '.tag' "$f") - [ -z "$tag" ] && continue - echo "Processing: $tag" - jq --arg tag "$tag" \ - --slurpfile data "$f" \ - '.tags[$tag] = ($data[0] | del(.image, .tag))' \ - sbom.json > tmp.json && mv tmp.json sbom.json - done - - # Sort all keys and package lists alphabetically for stable ordering - jq -S '.tags[].packages[] |= sort_by(.name)' sbom.json > tmp.json && mv tmp.json sbom.json - - echo "Tags: $(jq -r '.tags | keys | join(", ")' sbom.json)" - jq -r '.tags | to_entries[] | "\(.key): \(.value.summary.total) packages"' sbom.json - - - name: Generate Containerfile/README from templates - id: generate - if: inputs.generate - run: | - curl -sSL "https://github.com/daemonless/dbuild/archive/${{ inputs.dbuild_ref || env.DBUILD_REF }}.tar.gz" | tar xz -C /tmp - DBUILD_DIR=$(ls -d /tmp/dbuild-* | head -1) - export PYTHONPATH="$DBUILD_DIR" - pip install -q PyYAML jinja2 - # "=== Generated ===" is dbuild's own step log for each file - # it writes (Containerfile, Containerfile.pkg, Containerfile.krb, - # README.md, ...) -- parse it instead of guessing variant names. - FILES=$(python3 -m dbuild generate | tee /dev/stderr | sed -n 's/^=== Generated \(.*\) ===$/\1/p' | tr '\n' ' ') - echo "files=$FILES" >> "$GITHUB_OUTPUT" - - - name: Commit SBOM and generated files - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Keep this run's files aside: main moves while the build runs (the - # aarch64 leg and overlapping runs commit too), so each attempt - # starts from the current tip and lays our files on top. The newest - # build's SBOM is the truth; there is nothing to merge. - FILES="sbom.json" - for f in ${{ steps.generate.outputs.files }}; do - [ -f "$f" ] && FILES="$FILES $f" - done - mkdir -p /tmp/ours - for f in $FILES; do - mkdir -p "/tmp/ours/$(dirname "$f")" && cp "$f" "/tmp/ours/$f" - done - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - for attempt in 1 2 3 4 5; do - git fetch -q origin main - git reset -q --hard origin/main - for f in $FILES; do cp "/tmp/ours/$f" "$f"; done - - # The top-level and per-tag `generated` timestamps change every - # run; only commit when the versions or package lists actually - # changed, so no-op rebuilds stay silent. - git show HEAD:sbom.json 2>/dev/null | jq -S 'del(.generated, .tags[].generated)' > /tmp/sbom-old.json || echo '{}' > /tmp/sbom-old.json - jq -S 'del(.generated, .tags[].generated)' sbom.json > /tmp/sbom-new.json - if cmp -s /tmp/sbom-old.json /tmp/sbom-new.json; then - git checkout -q -- sbom.json - fi - git add $FILES - if git diff --cached --quiet; then - echo "No content change to sbom.json or generated files" - exit 0 - fi - - git commit -q -m "Update sbom.json and generated files [skip ci]" - if git push; then - exit 0 - fi - echo "Push rejected (attempt $attempt); main moved, retrying" - sleep $((attempt * 10)) - done - echo "Could not push after 5 attempts" - exit 1 - - # Sign the pushed images, and sign the SBOMs onto them as cosign - # attestations. Runs on Linux (keyless needs the GitHub OIDC token) against - # the already-pushed ghcr images -- keyless (GitHub OIDC), so no keys to - # manage. A signature is on the image's digest, so every tag of that - # digest (the versioned ones push.py adds too) verifies. Consumers: - # cosign verify ghcr.io/daemonless/: \ - # --certificate-identity-regexp 'github.com/daemonless/' \ - # --certificate-oidc-issuer https://token.actions.githubusercontent.com - # cosign verify-attestation --type cyclonedx ghcr.io/daemonless/: ... - attest-sbom: - needs: build - if: success() && github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - packages: write # push attestations to ghcr - id-token: write # keyless OIDC signing - contents: read - steps: - - name: Install cosign - uses: sigstore/cosign-installer@v4.1.3 - - - name: Login to GHCR - # cosign's own login writes credentials where cosign's keychain reads - # them (podman login, used by other jobs, writes a path cosign ignores). - run: cosign login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - - - name: Download SBOM artifacts - uses: actions/download-artifact@v8 - with: - pattern: sbom-* - path: sbom-artifacts/ - - - name: Sign pushed images and attest SBOMs onto them - env: - IMAGE_NAME: ${{ inputs.image_name }} - run: | - set -euo pipefail - shopt -s nullglob - signed="" - # actions/download-artifact only nests matched artifacts under a - # per-artifact subdirectory when 2+ artifacts match the pattern -- - # with exactly one match (single-variant images) it flattens the - # files straight into sbom-artifacts/, so a dir-name-based glob - # silently matches nothing there. Parse the tag out of each SBOM - # filename instead (dbuild/sbom.py writes - # --cyclonedx.json / -spdx.json), which - # works regardless of nesting. - for f in $(find sbom-artifacts -name "${IMAGE_NAME}-*-cyclonedx.json" -o -name "${IMAGE_NAME}-*-spdx.json"); do - base="$(basename "$f")" - rest="${base#"${IMAGE_NAME}"-}" - if [[ "$rest" == *-cyclonedx.json ]]; then - tag="${rest%-cyclonedx.json}"; predicate_type="cyclonedx" - else - tag="${rest%-spdx.json}"; predicate_type="spdxjson" - fi - ref="ghcr.io/daemonless/${IMAGE_NAME}:${tag}" - # Once per tag: both SBOM formats name the same image. - if [[ " $signed " != *" $tag "* ]]; then - echo "=== Signing ${ref} ===" - cosign sign --yes "$ref" - signed="$signed $tag" - fi - echo "=== Attesting ${predicate_type} SBOM onto ${ref} (${f}) ===" - cosign attest --yes --type "$predicate_type" --predicate "$f" "$ref" - done - - # A multi-arch image's bare tag is a manifest list assembled after the - # builds; it has a digest of its own, so it is signed once it exists. - sign-manifests: - needs: [attest-sbom, create-manifests] - # success() is false when any job upstream was skipped, and build-linux - # (behind create-manifests) is skipped for every FreeBSD-only build: - # the two jobs this needs are asked by name. - if: >- - !cancelled() && needs.attest-sbom.result == 'success' && needs.create-manifests.result == 'success' - && github.event_name != 'pull_request' - runs-on: ubuntu-latest - permissions: - packages: write - id-token: write - contents: read - steps: - - name: Install cosign - uses: sigstore/cosign-installer@v4.1.3 - - - name: Login to GHCR - run: cosign login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} - - - name: Download SBOM artifacts - uses: actions/download-artifact@v8 - with: - pattern: sbom-* - path: sbom-artifacts/ - - - name: Sign the manifest lists - env: - IMAGE_NAME: ${{ inputs.image_name }} - run: | - set -euo pipefail - # The bare tag is the arch-suffixed one the SBOM is named for, - # less its suffix (see dbuild/config.py arch_tag_suffix). - signed="" - for f in $(find sbom-artifacts -name "${IMAGE_NAME}-*-cyclonedx.json"); do - tag="$(basename "$f")"; tag="${tag#"${IMAGE_NAME}"-}"; tag="${tag%-cyclonedx.json}" - bare="${tag%-amd64}"; bare="${bare%-aarch64}" - [[ "$bare" != "$tag" && " $signed " != *" $bare "* ]] || continue - ref="ghcr.io/daemonless/${IMAGE_NAME}:${bare}" - echo "=== Signing manifest list ${ref} ===" - cosign sign --yes "$ref" - signed="$signed $bare" - done - - # Trigger status page update after successful builds - notify-status: - needs: [build, test-compose, commit-sbom, create-manifests] - if: always() && !failure() && !cancelled() && github.event_name != 'pull_request' - runs-on: ubuntu-latest - steps: - # Dispatch as the daemonless-dispatch App (installed on daemonless-io - # only) so the deploy run is attributed to a bot, not to a PAT owner. - - uses: actions/create-github-app-token@v3 - id: dispatch-token - with: - app-id: ${{ secrets.DISPATCH_APP_ID }} - private-key: ${{ secrets.DISPATCH_APP_PRIVATE_KEY }} - owner: daemonless - repositories: daemonless-io - - name: Trigger status update - env: - GH_TOKEN: ${{ steps.dispatch-token.outputs.token }} - IMAGE: ${{ inputs.image_name }} - run: | - gh api repos/daemonless/daemonless-io/dispatches \ - -f event_type=status-update \ - -f "client_payload[image]=$IMAGE" - - # Announce genuinely new images to a dedicated Discord channel. Runs once - # after the whole build matrix (not per variant/arch leg) so a new image - # with multiple variants doesn't post multiple announcements. - announce-new-container: - needs: build - if: success() && github.event_name != 'pull_request' && github.ref_name == 'main' - runs-on: ubuntu-latest - permissions: - contents: read - actions: read - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Announce new container - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ANNOUNCE_WEBHOOK: ${{ secrets.DISCORD_ANNOUNCE_WEBHOOK || secrets.DISCORD_WEBHOOK }} - run: | - set -e - [ -f .daemonless/config.yaml ] || exit 0 - [ -n "$ANNOUNCE_WEBHOOK" ] || exit 0 - - # Announce a new container fleet addition exactly once: on its very first - # successful build on main. Query GitHub Actions API for any prior successful - # run on main (ignoring the current run and any runs with run_number >= current). - PRIOR_SUCCESSES=$(curl -sS \ - -H "Authorization: Bearer $GH_TOKEN" \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs?branch=main&status=success&per_page=10" \ - | jq --arg current_id "${{ github.run_id }}" --arg current_run "${{ github.run_number }}" \ - '[.workflow_runs[] | select((.id | tostring) != $current_id and .run_number < ($current_run | tonumber))] | length') - - [ "$PRIOR_SUCCESSES" -eq 0 ] || exit 0 - - SHORT_SHA="${{ github.sha }}" - SHORT_SHA="${SHORT_SHA:0:7}" - COMMIT_URL="${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}" - - DESCRIPTION="" - if [ -f compose.yaml ]; then - DESCRIPTION=$(yq '.["x-daemonless"].description // ""' compose.yaml) || DESCRIPTION="" - fi - - # Discord can't render SVG thumbnails, so rasterize those to PNG. - LOGO="" - for ext in png jpg webp svg; do - [ -f ".daemonless/logo.$ext" ] || continue - if [ "$ext" = svg ]; then - { sudo apt-get install -y -qq librsvg2-bin >/dev/null \ - && rsvg-convert -w 256 -h 256 --keep-aspect-ratio \ - -o "$RUNNER_TEMP/logo.png" .daemonless/logo.svg \ - && LOGO="$RUNNER_TEMP/logo.png"; } || echo "::warning::logo.svg conversion failed; announcing without logo" - else - LOGO=".daemonless/logo.$ext" - fi - break - done - LOGO_NAME="logo.${LOGO##*.}" - - jq -n \ - --arg title "🚀 New Container Fleet Addition: ${{ inputs.image_name }}" \ - --arg desc "$DESCRIPTION" \ - --arg url "$COMMIT_URL" \ - --arg registry "[ghcr.io](https://ghcr.io/daemonless/${{ inputs.image_name }})" \ - --arg commit "[$SHORT_SHA]($COMMIT_URL)" \ - --arg branch "${{ github.ref_name }}" \ - --arg thumb "${LOGO:+attachment://$LOGO_NAME}" \ - '{username: "Daemonless New Containers", embeds: [{ - title: $title, url: $url, color: 15844367, - fields: [ - {name: "Registry", value: $registry, inline: true}, - {name: "Commit", value: $commit, inline: true}, - {name: "Branch", value: $branch, inline: true} - ], - footer: {text: "Daemonless FreeBSD Native OCI Containers"} - } - + (if $desc != "" then {description: $desc} else {} end) - + (if $thumb != "" then {thumbnail: {url: $thumb}} else {} end) - ]}' > "$RUNNER_TEMP/announce.json" - - if [ -n "$LOGO" ]; then - curl -sS \ - -F "payload_json=<$RUNNER_TEMP/announce.json;type=application/json" \ - -F "files[0]=@$LOGO;filename=$LOGO_NAME" \ - "$ANNOUNCE_WEBHOOK" - else - curl -sS -H "Content-Type: application/json" \ - --data-binary "@$RUNNER_TEMP/announce.json" \ - "$ANNOUNCE_WEBHOOK" - fi + uses: daemonless/workflows/.github/workflows/daemonless-build.yaml@v1 + with: + image_name: ${{ inputs.image_name }} + variant: ${{ inputs.variant }} + pre_artifact_name: ${{ inputs.pre_artifact_name }} + pre_artifact_path: ${{ inputs.pre_artifact_path }} + registry: ${{ inputs.registry }} + build_tools_ref: ${{ inputs.build_tools_ref }} + generate: ${{ inputs.generate }} + arch: ${{ inputs.arch }} + dbuild_ref: ${{ inputs.dbuild_ref }} + linux: ${{ inputs.linux }} + build_timeout: ${{ inputs.build_timeout }} + secrets: inherit diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 9da9a98..24d8e61 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,16 +26,14 @@ jobs: with: ref: ${{ github.event.inputs.tag || github.ref }} - # Image CI fetches DBUILD_REF by tag, so a release whose bump missed - # either place would ship one version and run another. - - name: Tag matches dbuild.VERSION and DBUILD_REF + # Image CI pins the dbuild release in daemonless/workflows (DBUILD_REF), + # bumped there by PR after this release. + - name: Tag matches dbuild.VERSION run: | VERSION="${TAG#v}" PKG=$(sed -n 's/^VERSION = "\(.*\)"$/\1/p' dbuild/__init__.py) - REF=$(sed -n 's/^ DBUILD_REF: \(.*\)$/\1/p' .github/workflows/daemonless-build.yaml) - echo "tag ${TAG} dbuild.VERSION ${PKG} DBUILD_REF ${REF}" + echo "tag ${TAG} dbuild.VERSION ${PKG}" [ "$PKG" = "$VERSION" ] || { echo "::error::dbuild/__init__.py says ${PKG}, tag is ${TAG}"; exit 1; } - [ "$REF" = "$TAG" ] || { echo "::error::DBUILD_REF is ${REF}, tag is ${TAG}"; exit 1; } - uses: actions/setup-python@v7 with: diff --git a/dbuild/templates/github-workflow.yaml b/dbuild/templates/github-workflow.yaml index 9807f5c..84b8f65 100644 --- a/dbuild/templates/github-workflow.yaml +++ b/dbuild/templates/github-workflow.yaml @@ -16,7 +16,7 @@ permissions: jobs: build: - uses: daemonless/dbuild/.github/workflows/daemonless-build.yaml@main + uses: daemonless/workflows/.github/workflows/daemonless-build.yaml@v1 with: image_name: {{ name }} secrets: inherit