Plan Nightly artifact-based Docker packaging

.dockerignore

@@ -3,7 +3,23 @@ alchemist.db
 .git
 .github
 *.log
+*.md
 config.toml
+docs
+redoc
+tests
+web-e2e
+static
+src/css
+docusaurus.config.ts
+sidebars.ts
+package.json
+package-lock.json
+bun.lock
+tsconfig.json
+.claude
+.idea
+.runtime
 web/node_modules
 web/dist
 web/.astro

.github/workflows/build.yml

@@ -46,6 +46,7 @@ on:
         type: boolean
         required: false
         default: false
+
 env:
   IMAGE_NAME: ghcr.io/bybrooklyn/alchemist
 
@@ -63,6 +64,8 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 20
+          cache: npm
+          cache-dependency-path: package-lock.json
 
       - name: Install docs dependencies
         run: npm ci
@@ -112,12 +115,15 @@ jobs:
         with:
           bun-version: latest
 
-      - name: Restore Bun cache
+      - name: Restore Bun dependency cache
         uses: actions/cache@v4
         with:
-          path: web/node_modules
+          path: |
-          key: bun-${{ github.workflow }}-${{ hashFiles('web/bun.lock') }}
+            web/node_modules
-          restore-keys: bun-${{ github.workflow }}-
+            ~/.bun/install/cache
+          key: bun-web-${{ runner.os }}-${{ hashFiles('web/bun.lock') }}
+          restore-keys: |
+            bun-web-${{ runner.os }}-
 
       - name: Install dependencies
         run: bun install --frozen-lockfile
@@ -150,6 +156,10 @@ jobs:
             artifact_name: alchemist-linux-x86_64
           - target: aarch64-unknown-linux-gnu
             artifact_name: alchemist-linux-aarch64
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -174,6 +184,9 @@ jobs:
         with:
           targets: ${{ matrix.target }}
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -214,6 +227,10 @@ jobs:
     needs: [build-frontend]
     runs-on: windows-latest
     timeout-minutes: 45
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -238,6 +255,9 @@ jobs:
         with:
           targets: x86_64-pc-windows-msvc
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -277,6 +297,10 @@ jobs:
             artifact_name: alchemist-macos-x86_64
           - target: aarch64-apple-darwin
             artifact_name: alchemist-macos-arm64
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -301,6 +325,9 @@ jobs:
         with:
           targets: ${{ matrix.target }}
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -328,17 +355,46 @@ jobs:
             ${{ matrix.artifact_name }}.tar.gz
             ${{ matrix.artifact_name }}.tar.gz.sha256
 
-  build-docker:
+  build-docker-image:
-    needs: [build-frontend]
+    needs: [build-linux]
     runs-on: ubuntu-latest
-    timeout-minutes: 90
+    timeout-minutes: 45
     if: inputs.push_docker
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            artifact_name: alchemist-linux-x86_64
+            cache_scope: docker-amd64
+            cache_ref: ghcr.io/bybrooklyn/alchemist:buildcache-amd64
+          - arch: arm64
+            artifact_name: alchemist-linux-aarch64
+            cache_scope: docker-arm64
+            cache_ref: ghcr.io/bybrooklyn/alchemist:buildcache-arm64
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
+      - name: Download Linux artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: artifacts/${{ matrix.arch }}
+
+      - name: Prepare runtime context
+        shell: bash
+        run: |
+          set -euo pipefail
+          CONTEXT_DIR="runtime-context/${{ matrix.arch }}"
+          mkdir -p "${CONTEXT_DIR}"
+          tar -xzf "artifacts/${{ matrix.arch }}/${{ matrix.artifact_name }}.tar.gz" \
+            -C "${CONTEXT_DIR}"
+          cp Dockerfile.runtime "${CONTEXT_DIR}/Dockerfile"
+          chmod +x "${CONTEXT_DIR}/alchemist"
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -352,20 +408,82 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ github.token }}
 
-      - name: Build and push
+      - name: Build and push by digest
+        id: build
         uses: docker/build-push-action@v6
         with:
-          context: .
+          context: runtime-context/${{ matrix.arch }}
-          platforms: linux/amd64,linux/arm64
+          file: runtime-context/${{ matrix.arch }}/Dockerfile
-          push: true
+          platforms: linux/${{ matrix.arch }}
-          tags: ${{ inputs.docker_tags }}
+          outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
-          cache-from: type=gha
+          cache-from: |
-          cache-to: type=gha,mode=max
+            type=gha,scope=${{ matrix.cache_scope }}
+            type=registry,ref=${{ matrix.cache_ref }}
+          cache-to: |
+            type=gha,scope=${{ matrix.cache_scope }},mode=max
+            type=registry,ref=${{ matrix.cache_ref }},mode=max,image-manifest=true,oci-mediatypes=true
           provenance: false
+          sbom: false
+
+      - name: Export digest
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "digests/${digest#sha256:}"
+
+      - name: Upload digest artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-digests-${{ matrix.arch }}
+          path: digests/*
+          retention-days: 1
+
+  publish-docker-manifest:
+    needs: [build-docker-image]
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    if: inputs.push_docker
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: docker-digests-*
+          merge-multiple: true
+          path: digests
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - name: Create and inspect manifests
+        env:
+          DOCKER_TAGS: ${{ inputs.docker_tags }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          mapfile -t refs < <(find digests -type f -printf "${IMAGE_NAME}@sha256:%f\n" | sort)
+          if [ "${#refs[@]}" -eq 0 ]; then
+            echo "No image digests were downloaded" >&2
+            exit 1
+          fi
+
+          while IFS= read -r tag; do
+            [ -n "${tag}" ] || continue
+            docker buildx imagetools create -t "${tag}" "${refs[@]}"
+            docker buildx imagetools inspect "${tag}"
+          done <<< "${DOCKER_TAGS}"
 
   publish-release:
     needs:
-      [build-docs, build-linux, build-windows, build-macos, build-docker]
+      [build-docs, build-linux, build-windows, build-macos, publish-docker-manifest]
     runs-on: ubuntu-latest
     timeout-minutes: 20
     if: inputs.publish_release

.github/workflows/ci.yml

@@ -26,6 +26,10 @@ jobs:
   rust-check:
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -37,10 +41,13 @@ jobs:
         with:
           components: rustfmt, clippy
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: ci-check
+          prefix-key: rust-linux-shared
 
       - name: Check formatting
         run: cargo fmt --all -- --check
@@ -54,6 +61,10 @@ jobs:
   rust-test:
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -63,6 +74,9 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Install FFmpeg
         shell: bash
         run: |
@@ -73,7 +87,7 @@ jobs:
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: ci-test
+          prefix-key: rust-linux-shared
 
       - name: Run tests
         run: cargo test --locked --all-targets -- --test-threads=4
@@ -103,12 +117,15 @@ jobs:
         with:
           bun-version: latest
 
-      - name: Restore Bun cache
+      - name: Restore Bun dependency cache
         uses: actions/cache@v4
         with:
-          path: web/node_modules
+          path: |
-          key: bun-ci-${{ hashFiles('web/bun.lock') }}
+            web/node_modules
-          restore-keys: bun-ci-
+            ~/.bun/install/cache
+          key: bun-web-${{ runner.os }}-${{ hashFiles('web/bun.lock') }}
+          restore-keys: |
+            bun-web-${{ runner.os }}-
 
       - name: Install dependencies
         run: bun install --frozen-lockfile
@@ -126,6 +143,10 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     needs: [frontend-check]
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     defaults:
       run:
         working-directory: web-e2e
@@ -138,19 +159,42 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Install FFmpeg
         run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: ci-e2e
+          prefix-key: rust-linux-shared
 
       - name: Install Bun
         uses: oven-sh/setup-bun@v2
         with:
           bun-version: latest
 
+      - name: Restore Bun dependency cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            web/node_modules
+            web-e2e/node_modules
+            ~/.bun/install/cache
+          key: bun-e2e-${{ runner.os }}-${{ hashFiles('web/bun.lock', 'web-e2e/bun.lock') }}
+          restore-keys: |
+            bun-e2e-${{ runner.os }}-
+            bun-web-${{ runner.os }}-
+
+      - name: Restore Playwright browser cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ hashFiles('web-e2e/package.json', 'web-e2e/bun.lock') }}
+          restore-keys: |
+            playwright-${{ runner.os }}-
+
       - name: Build frontend
         run: |
           cd ../web

.github/workflows/docker.yml

@@ -40,6 +40,7 @@ jobs:
             | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT"
 
       - name: Set up QEMU
+        if: github.event_name == 'workflow_dispatch'
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
@@ -60,8 +61,11 @@ jobs:
           context: .
           platforms: linux/amd64
           push: false
-          cache-from: type=gha
+          cache-from: |
-          cache-to: type=gha,mode=max
+            type=gha,scope=docker-amd64
+            type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-amd64
+          cache-to: |
+            type=gha,scope=docker-amd64,mode=max
           provenance: false
 
       - name: Build and push (manual dispatch)
@@ -72,6 +76,14 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.image.outputs.name }}:dev
-          cache-from: type=gha
+          cache-from: |
-          cache-to: type=gha,mode=max
+            type=gha,scope=docker-amd64
+            type=gha,scope=docker-arm64
+            type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-amd64
+            type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-arm64
+          cache-to: |
+            type=gha,scope=docker-amd64,mode=max
+            type=gha,scope=docker-arm64,mode=max
+            type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-amd64,mode=max,image-manifest=true,oci-mediatypes=true
+            type=registry,ref=ghcr.io/bybrooklyn/alchemist:buildcache-arm64,mode=max,image-manifest=true,oci-mediatypes=true
           provenance: false

.github/workflows/docs.yml

@@ -52,6 +52,8 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 20
+          cache: npm
+          cache-dependency-path: package-lock.json
 
       - name: Install dependencies
         run: npm ci

.github/workflows/nightly.yml

@@ -18,6 +18,10 @@ jobs:
   ci-gate:
     runs-on: ubuntu-latest
     timeout-minutes: 20
+    env:
+      RUSTC_WRAPPER: sccache
+      SCCACHE_GHA_ENABLED: "true"
+      SCCACHE_CACHE_SIZE: 2G
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -29,10 +33,13 @@ jobs:
         with:
           components: rustfmt, clippy
 
+      - name: Install sccache
+        uses: mozilla-actions/sccache-action@v0.0.9
+
       - name: Restore Rust cache
         uses: Swatinem/rust-cache@v2
         with:
-          prefix-key: nightly-gate
+          prefix-key: rust-linux-shared
 
       - name: Check formatting
         run: cargo fmt --all -- --check

Dockerfile.runtime

@@ -0,0 +1,55 @@
+FROM debian:testing-slim AS runtime
+
+ARG TARGETARCH
+
+WORKDIR /app
+
+RUN mkdir -p /app/config /app/data
+
+RUN apt-get update && \
+    sed -i 's/main/main contrib non-free non-free-firmware/g' /etc/apt/sources.list.d/debian.sources && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    xz-utils \
+    libva-drm2 \
+    libva2 \
+    va-driver-all \
+    libsqlite3-0 \
+    ca-certificates \
+    && if [ "${TARGETARCH}" = "amd64" ]; then \
+    apt-get install -y --no-install-recommends \
+    intel-media-va-driver-non-free \
+    i965-va-driver || true; \
+    fi \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN set -e; \
+    if [ "${TARGETARCH}" = "amd64" ]; then \
+      ARCHIVE="ffmpeg-release-amd64-static.tar.xz"; \
+      URL="https://johnvansickle.com/ffmpeg/releases/${ARCHIVE}"; \
+      SHA256="abda8d77ce8309141f83ab8edf0596834087c52467f6badf376a6a2a4c87cf67"; \
+    elif [ "${TARGETARCH}" = "arm64" ]; then \
+      ARCHIVE="ffmpeg-release-arm64-static.tar.xz"; \
+      URL="https://johnvansickle.com/ffmpeg/releases/${ARCHIVE}"; \
+      SHA256="f4149bb2b0784e30e99bdda85471c9b5930d3402014e934a5098b41d0f7201b1"; \
+    else \
+      echo "Unsupported architecture: ${TARGETARCH}" >&2; \
+      exit 1; \
+    fi; \
+    wget -O "${ARCHIVE}" "${URL}"; \
+    echo "${SHA256}  ${ARCHIVE}" | sha256sum -c -; \
+    tar xf "${ARCHIVE}"; \
+    mv ffmpeg-*-static/ffmpeg /usr/local/bin/; \
+    mv ffmpeg-*-static/ffprobe /usr/local/bin/; \
+    rm -rf "${ARCHIVE}" ffmpeg-*-static
+
+COPY alchemist /usr/local/bin/alchemist
+
+ENV LIBVA_DRIVER_NAME=iHD
+ENV RUST_LOG=info
+ENV ALCHEMIST_CONFIG_PATH=/app/config/config.toml
+ENV ALCHEMIST_DB_PATH=/app/data/alchemist.db
+
+EXPOSE 3000
+
+ENTRYPOINT ["alchemist"]

tests/integration_ffmpeg.rs

@@ -345,7 +345,9 @@ async fn test_audio_stream_handling() -> Result<()> {
     let output = temp_dir.join("output_with_audio.mp4");
 
     let (db, pipeline, db_path) = build_test_pipeline(|config| {
-        config.transcode.output_codec = OutputCodec::H264;
+        // Force a transcode so this test covers audio handling
+        // instead of the planner's same-codec skip path.
+        config.transcode.output_codec = OutputCodec::Hevc;
     })
     .await?;
 
@@ -391,7 +393,9 @@ async fn test_subtitle_extraction() -> Result<()> {
 
     let (db, pipeline, db_path) = build_test_pipeline(|config| {
         config.transcode.subtitle_mode = SubtitleMode::Extract;
-        config.transcode.output_codec = OutputCodec::H264;
+        // Force a transcode so subtitle extraction is exercised
+        // instead of skipping the already-H.264 fixture.
+        config.transcode.output_codec = OutputCodec::Hevc;
     })
     .await?;
 
@@ -459,9 +463,21 @@ async fn test_multiple_input_formats() -> Result<()> {
 
         let temp_dir = temp_output_dir(&format!("multi_format_{}", expected_input_codec))?;
         let output = temp_dir.join("output.mp4");
+        let target_codec = match expected_input_codec {
+            "h264" => OutputCodec::Hevc,
+            "hevc" => OutputCodec::H264,
+            other => anyhow::bail!("Unexpected fixture codec: {}", other),
+        };
+        let expected_output_codec = match target_codec {
+            OutputCodec::Hevc => "hevc",
+            OutputCodec::H264 => "h264",
+            OutputCodec::Av1 => "av1",
+        };
 
         let (db, pipeline, db_path) = build_test_pipeline(|config| {
-            config.transcode.output_codec = OutputCodec::H264;
+            // Pick the opposite codec so both fixtures exercise a
+            // completed transcode rather than a planner skip.
+            config.transcode.output_codec = target_codec;
         })
         .await?;
 
@@ -473,7 +489,7 @@ async fn test_multiple_input_formats() -> Result<()> {
             "Job should complete successfully for {}",
             filename
         );
-        verify_output_codec(&output, "h264")?;
+        verify_output_codec(&output, expected_output_codec)?;
 
         // Cleanup
         let _ = std::fs::remove_file(db_path);