From ec9e228247b8819ad8469caafd271aa8c2ae197a Mon Sep 17 00:00:00 2001 From: bybrooklyn Date: Sat, 28 Mar 2026 20:23:01 -0400 Subject: [PATCH] Plan Nightly artifact-based Docker packaging --- .dockerignore | 16 ++++ .github/workflows/build.yml | 148 ++++++++++++++++++++++++++++++---- .github/workflows/ci.yml | 58 +++++++++++-- .github/workflows/docker.yml | 20 ++++- .github/workflows/docs.yml | 2 + .github/workflows/nightly.yml | 9 ++- Dockerfile.runtime | 55 +++++++++++++ tests/integration_ffmpeg.rs | 24 +++++- 8 files changed, 301 insertions(+), 31 deletions(-) create mode 100644 Dockerfile.runtime diff --git a/.dockerignore b/.dockerignore index a2fb0bf..ca7ef80 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,7 +3,23 @@ alchemist.db .git .github *.log +*.md config.toml +docs +redoc +tests +web-e2e +static +src/css +docusaurus.config.ts +sidebars.ts +package.json +package-lock.json +bun.lock +tsconfig.json +.claude +.idea +.runtime web/node_modules web/dist web/.astro diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e931818..4835454 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,6 +46,7 @@ on: type: boolean required: false default: false + env: IMAGE_NAME: ghcr.io/bybrooklyn/alchemist @@ -63,6 +64,8 @@ jobs: uses: actions/setup-node@v4 with: node-version: 20 + cache: npm + cache-dependency-path: package-lock.json - name: Install docs dependencies run: npm ci @@ -112,12 +115,15 @@ jobs: with: bun-version: latest - - name: Restore Bun cache + - name: Restore Bun dependency cache uses: actions/cache@v4 with: - path: web/node_modules - key: bun-${{ github.workflow }}-${{ hashFiles('web/bun.lock') }} - restore-keys: bun-${{ github.workflow }}- + path: | + web/node_modules + ~/.bun/install/cache + key: bun-web-${{ runner.os }}-${{ hashFiles('web/bun.lock') }} + restore-keys: | + bun-web-${{ runner.os }}- - name: Install dependencies run: bun install --frozen-lockfile @@ -150,6 +156,10 @@ jobs: artifact_name: alchemist-linux-x86_64 - target: aarch64-unknown-linux-gnu artifact_name: alchemist-linux-aarch64 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G steps: - name: Checkout repository uses: actions/checkout@v4 @@ -174,6 +184,9 @@ jobs: with: targets: ${{ matrix.target }} + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: @@ -214,6 +227,10 @@ jobs: needs: [build-frontend] runs-on: windows-latest timeout-minutes: 45 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G steps: - name: Checkout repository uses: actions/checkout@v4 @@ -238,6 +255,9 @@ jobs: with: targets: x86_64-pc-windows-msvc + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: @@ -277,6 +297,10 @@ jobs: artifact_name: alchemist-macos-x86_64 - target: aarch64-apple-darwin artifact_name: alchemist-macos-arm64 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G steps: - name: Checkout repository uses: actions/checkout@v4 @@ -301,6 +325,9 @@ jobs: with: targets: ${{ matrix.target }} + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: @@ -328,17 +355,46 @@ jobs: ${{ matrix.artifact_name }}.tar.gz ${{ matrix.artifact_name }}.tar.gz.sha256 - build-docker: - needs: [build-frontend] + build-docker-image: + needs: [build-linux] runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 45 if: inputs.push_docker + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + artifact_name: alchemist-linux-x86_64 + cache_scope: docker-amd64 + cache_ref: ghcr.io/bybrooklyn/alchemist:buildcache-amd64 + - arch: arm64 + artifact_name: alchemist-linux-aarch64 + cache_scope: docker-arm64 + cache_ref: ghcr.io/bybrooklyn/alchemist:buildcache-arm64 steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 1 + - name: Download Linux artifact + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: artifacts/${{ matrix.arch }} + + - name: Prepare runtime context + shell: bash + run: | + set -euo pipefail + CONTEXT_DIR="runtime-context/${{ matrix.arch }}" + mkdir -p "${CONTEXT_DIR}" + tar -xzf "artifacts/${{ matrix.arch }}/${{ matrix.artifact_name }}.tar.gz" \ + -C "${CONTEXT_DIR}" + cp Dockerfile.runtime "${CONTEXT_DIR}/Dockerfile" + chmod +x "${CONTEXT_DIR}/alchemist" + - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -352,20 +408,82 @@ jobs: username: ${{ github.actor }} password: ${{ github.token }} - - name: Build and push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ inputs.docker_tags }} - cache-from: type=gha - cache-to: type=gha,mode=max + context: runtime-context/${{ matrix.arch }} + file: runtime-context/${{ matrix.arch }}/Dockerfile + platforms: linux/${{ matrix.arch }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: | + type=gha,scope=${{ matrix.cache_scope }} + type=registry,ref=${{ matrix.cache_ref }} + cache-to: | + type=gha,scope=${{ matrix.cache_scope }},mode=max + type=registry,ref=${{ matrix.cache_ref }},mode=max,image-manifest=true,oci-mediatypes=true provenance: false + sbom: false + + - name: Export digest + shell: bash + run: | + set -euo pipefail + mkdir -p digests + digest="${{ steps.build.outputs.digest }}" + touch "digests/${digest#sha256:}" + + - name: Upload digest artifact + uses: actions/upload-artifact@v4 + with: + name: docker-digests-${{ matrix.arch }} + path: digests/* + retention-days: 1 + + publish-docker-manifest: + needs: [build-docker-image] + runs-on: ubuntu-latest + timeout-minutes: 20 + if: inputs.push_docker + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: docker-digests-* + merge-multiple: true + path: digests + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Create and inspect manifests + env: + DOCKER_TAGS: ${{ inputs.docker_tags }} + shell: bash + run: | + set -euo pipefail + mapfile -t refs < <(find digests -type f -printf "${IMAGE_NAME}@sha256:%f\n" | sort) + if [ "${#refs[@]}" -eq 0 ]; then + echo "No image digests were downloaded" >&2 + exit 1 + fi + + while IFS= read -r tag; do + [ -n "${tag}" ] || continue + docker buildx imagetools create -t "${tag}" "${refs[@]}" + docker buildx imagetools inspect "${tag}" + done <<< "${DOCKER_TAGS}" publish-release: needs: - [build-docs, build-linux, build-windows, build-macos, build-docker] + [build-docs, build-linux, build-windows, build-macos, publish-docker-manifest] runs-on: ubuntu-latest timeout-minutes: 20 if: inputs.publish_release diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 831cd18..cbbb179 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,10 @@ jobs: rust-check: runs-on: ubuntu-latest timeout-minutes: 20 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G steps: - name: Checkout repository uses: actions/checkout@v4 @@ -37,10 +41,13 @@ jobs: with: components: rustfmt, clippy + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: - prefix-key: ci-check + prefix-key: rust-linux-shared - name: Check formatting run: cargo fmt --all -- --check @@ -54,6 +61,10 @@ jobs: rust-test: runs-on: ubuntu-latest timeout-minutes: 30 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G steps: - name: Checkout repository uses: actions/checkout@v4 @@ -63,6 +74,9 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install FFmpeg shell: bash run: | @@ -73,7 +87,7 @@ jobs: - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: - prefix-key: ci-test + prefix-key: rust-linux-shared - name: Run tests run: cargo test --locked --all-targets -- --test-threads=4 @@ -103,12 +117,15 @@ jobs: with: bun-version: latest - - name: Restore Bun cache + - name: Restore Bun dependency cache uses: actions/cache@v4 with: - path: web/node_modules - key: bun-ci-${{ hashFiles('web/bun.lock') }} - restore-keys: bun-ci- + path: | + web/node_modules + ~/.bun/install/cache + key: bun-web-${{ runner.os }}-${{ hashFiles('web/bun.lock') }} + restore-keys: | + bun-web-${{ runner.os }}- - name: Install dependencies run: bun install --frozen-lockfile @@ -126,6 +143,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 needs: [frontend-check] + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G defaults: run: working-directory: web-e2e @@ -138,19 +159,42 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Install FFmpeg run: sudo apt-get update && sudo apt-get install -y ffmpeg - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: - prefix-key: ci-e2e + prefix-key: rust-linux-shared - name: Install Bun uses: oven-sh/setup-bun@v2 with: bun-version: latest + - name: Restore Bun dependency cache + uses: actions/cache@v4 + with: + path: | + web/node_modules + web-e2e/node_modules + ~/.bun/install/cache + key: bun-e2e-${{ runner.os }}-${{ hashFiles('web/bun.lock', 'web-e2e/bun.lock') }} + restore-keys: | + bun-e2e-${{ runner.os }}- + bun-web-${{ runner.os }}- + + - name: Restore Playwright browser cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('web-e2e/package.json', 'web-e2e/bun.lock') }} + restore-keys: | + playwright-${{ runner.os }}- + - name: Build frontend run: | cd ../web diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6b4b58f..cd5df28 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -40,6 +40,7 @@ jobs: | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" - name: Set up QEMU + if: github.event_name == 'workflow_dispatch' uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -60,8 +61,11 @@ jobs: context: . platforms: linux/amd64 push: false - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: | + type=gha,scope=docker-amd64 + type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-amd64 + cache-to: | + type=gha,scope=docker-amd64,mode=max provenance: false - name: Build and push (manual dispatch) @@ -72,6 +76,14 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.image.outputs.name }}:dev - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: | + type=gha,scope=docker-amd64 + type=gha,scope=docker-arm64 + type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-amd64 + type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-arm64 + cache-to: | + type=gha,scope=docker-amd64,mode=max + type=gha,scope=docker-arm64,mode=max + type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-amd64,mode=max,image-manifest=true,oci-mediatypes=true + type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-arm64,mode=max,image-manifest=true,oci-mediatypes=true provenance: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6217670..d9d88db 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -52,6 +52,8 @@ jobs: uses: actions/setup-node@v4 with: node-version: 20 + cache: npm + cache-dependency-path: package-lock.json - name: Install dependencies run: npm ci diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cd5f9e0..537b1b1 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,6 +18,10 @@ jobs: ci-gate: runs-on: ubuntu-latest timeout-minutes: 20 + env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + SCCACHE_CACHE_SIZE: 2G steps: - name: Checkout repository uses: actions/checkout@v4 @@ -29,10 +33,13 @@ jobs: with: components: rustfmt, clippy + - name: Install sccache + uses: mozilla-actions/sccache-action@v0.0.9 + - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: - prefix-key: nightly-gate + prefix-key: rust-linux-shared - name: Check formatting run: cargo fmt --all -- --check diff --git a/Dockerfile.runtime b/Dockerfile.runtime new file mode 100644 index 0000000..0b12e56 --- /dev/null +++ b/Dockerfile.runtime @@ -0,0 +1,55 @@ +FROM debian:testing-slim AS runtime + +ARG TARGETARCH + +WORKDIR /app + +RUN mkdir -p /app/config /app/data + +RUN apt-get update && \ + sed -i 's/main/main contrib non-free non-free-firmware/g' /etc/apt/sources.list.d/debian.sources && \ + apt-get update && apt-get install -y --no-install-recommends \ + wget \ + xz-utils \ + libva-drm2 \ + libva2 \ + va-driver-all \ + libsqlite3-0 \ + ca-certificates \ + && if [ "${TARGETARCH}" = "amd64" ]; then \ + apt-get install -y --no-install-recommends \ + intel-media-va-driver-non-free \ + i965-va-driver || true; \ + fi \ + && rm -rf /var/lib/apt/lists/* + +RUN set -e; \ + if [ "${TARGETARCH}" = "amd64" ]; then \ + ARCHIVE="ffmpeg-release-amd64-static.tar.xz"; \ + URL="https://johnvansickle.com/ffmpeg/releases/${ARCHIVE}"; \ + SHA256="abda8d77ce8309141f83ab8edf0596834087c52467f6badf376a6a2a4c87cf67"; \ + elif [ "${TARGETARCH}" = "arm64" ]; then \ + ARCHIVE="ffmpeg-release-arm64-static.tar.xz"; \ + URL="https://johnvansickle.com/ffmpeg/releases/${ARCHIVE}"; \ + SHA256="f4149bb2b0784e30e99bdda85471c9b5930d3402014e934a5098b41d0f7201b1"; \ + else \ + echo "Unsupported architecture: ${TARGETARCH}" >&2; \ + exit 1; \ + fi; \ + wget -O "${ARCHIVE}" "${URL}"; \ + echo "${SHA256} ${ARCHIVE}" | sha256sum -c -; \ + tar xf "${ARCHIVE}"; \ + mv ffmpeg-*-static/ffmpeg /usr/local/bin/; \ + mv ffmpeg-*-static/ffprobe /usr/local/bin/; \ + rm -rf "${ARCHIVE}" ffmpeg-*-static + +COPY alchemist /usr/local/bin/alchemist + +ENV LIBVA_DRIVER_NAME=iHD +ENV RUST_LOG=info +ENV ALCHEMIST_CONFIG_PATH=/app/config/config.toml +ENV ALCHEMIST_DB_PATH=/app/data/alchemist.db + +EXPOSE 3000 + +ENTRYPOINT ["alchemist"] diff --git a/tests/integration_ffmpeg.rs b/tests/integration_ffmpeg.rs index d346b12..e1d9b49 100644 --- a/tests/integration_ffmpeg.rs +++ b/tests/integration_ffmpeg.rs @@ -345,7 +345,9 @@ async fn test_audio_stream_handling() -> Result<()> { let output = temp_dir.join("output_with_audio.mp4"); let (db, pipeline, db_path) = build_test_pipeline(|config| { - config.transcode.output_codec = OutputCodec::H264; + // Force a transcode so this test covers audio handling + // instead of the planner's same-codec skip path. + config.transcode.output_codec = OutputCodec::Hevc; }) .await?; @@ -391,7 +393,9 @@ async fn test_subtitle_extraction() -> Result<()> { let (db, pipeline, db_path) = build_test_pipeline(|config| { config.transcode.subtitle_mode = SubtitleMode::Extract; - config.transcode.output_codec = OutputCodec::H264; + // Force a transcode so subtitle extraction is exercised + // instead of skipping the already-H.264 fixture. + config.transcode.output_codec = OutputCodec::Hevc; }) .await?; @@ -459,9 +463,21 @@ async fn test_multiple_input_formats() -> Result<()> { let temp_dir = temp_output_dir(&format!("multi_format_{}", expected_input_codec))?; let output = temp_dir.join("output.mp4"); + let target_codec = match expected_input_codec { + "h264" => OutputCodec::Hevc, + "hevc" => OutputCodec::H264, + other => anyhow::bail!("Unexpected fixture codec: {}", other), + }; + let expected_output_codec = match target_codec { + OutputCodec::Hevc => "hevc", + OutputCodec::H264 => "h264", + OutputCodec::Av1 => "av1", + }; let (db, pipeline, db_path) = build_test_pipeline(|config| { - config.transcode.output_codec = OutputCodec::H264; + // Pick the opposite codec so both fixtures exercise a + // completed transcode rather than a planner skip. + config.transcode.output_codec = target_codec; }) .await?; @@ -473,7 +489,7 @@ async fn test_multiple_input_formats() -> Result<()> { "Job should complete successfully for {}", filename ); - verify_output_codec(&output, "h264")?; + verify_output_codec(&output, expected_output_codec)?; // Cleanup let _ = std::fs::remove_file(db_path);