From e50ca64e809edae76d2ad919457a3d118f26b669 Mon Sep 17 00:00:00 2001 From: bybrooklyn Date: Mon, 13 Apr 2026 16:02:11 -0400 Subject: [PATCH] Resolve audit findings + split db.rs into db/ module - P1: Fix cancel race in pipeline, fix VideoToolbox quality mapping - P2: SSRF protection, batch cancel N+1, archived filter fixes, metadata persistence, reverse proxy hardening, reprobe logging - TD: Remove AlchemistEvent legacy bridge, fix silent .ok() on DB writes, optimize sort-by-size query, split db.rs (3400 LOC) into 8 focused submodules under src/db/ - UX: Add queue position display for queued jobs - Docs: Update API docs, engine modes, library doctor, config ref - Plans: Add plans.md for remaining open items (UX-2/3, FG-4, RG-2) Co-Authored-By: Claude Opus 4.6 --- .claude/settings.local.json | 12 +- CLAUDE.md | 27 +- audit.md | 470 +++++-- docs/docs/api.md | 440 +------ docs/docs/configuration-reference.md | 2 +- docs/docs/engine-modes.md | 48 +- docs/docs/library-doctor.md | 45 +- docs/docs/overview.md | 10 +- docs/docusaurus.config.ts | 1 - docs/src/css/custom.css | 18 +- .../20260412000000_store_job_metadata.sql | 2 + plans.md | 191 +++ security_best_practices_report.md | 124 -- src/config.rs | 17 +- src/conversion.rs | 1 + src/db/config.rs | 584 +++++++++ src/db/conversion.rs | 152 +++ src/db/events.rs | 54 + src/db/jobs.rs | 1076 +++++++++++++++++ src/db/mod.rs | 148 +++ src/db/stats.rs | 422 +++++++ src/db/system.rs | 389 ++++++ src/db/types.rs | 640 ++++++++++ src/lib.rs | 1 - src/main.rs | 13 +- src/media/executor.rs | 48 +- src/media/ffmpeg/mod.rs | 4 +- src/media/ffmpeg/videotoolbox.rs | 29 +- src/media/pipeline.rs | 398 +++--- src/media/processor.rs | 17 +- src/notifications.rs | 243 ++-- src/orchestrator.rs | 14 + src/server/jobs.rs | 67 +- src/server/middleware.rs | 72 +- src/server/mod.rs | 42 +- src/server/scan.rs | 2 +- src/server/sse.rs | 4 + src/server/system.rs | 19 +- src/server/tests.rs | 9 +- tests/generated_media.rs | 1 - tests/integration_ffmpeg.rs | 1 - web/src/components/JobManager.tsx | 3 +- web/src/components/jobs/JobDetailModal.tsx | 7 +- web/src/components/jobs/types.ts | 1 + web/src/components/jobs/useJobSSE.ts | 11 +- 45 files changed, 4796 insertions(+), 1083 deletions(-) create mode 100644 migrations/20260412000000_store_job_metadata.sql create mode 100644 plans.md delete mode 100644 security_best_practices_report.md create mode 100644 src/db/config.rs create mode 100644 src/db/conversion.rs create mode 100644 src/db/events.rs create mode 100644 src/db/jobs.rs create mode 100644 src/db/mod.rs create mode 100644 src/db/stats.rs create mode 100644 src/db/system.rs create mode 100644 src/db/types.rs diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 3066837..82ffa85 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -27,7 +27,17 @@ "Bash(cargo fmt:*)", "Bash(cargo test:*)", "Bash(just check:*)", - "Bash(just test:*)" + "Bash(just test:*)", + "Bash(find /Users/brooklyn/data/alchemist -name *.sql -o -name *migration*)", + "Bash(grep -l \"DROP\\\\|RENAME\\\\|DELETE FROM\" /Users/brooklyn/data/alchemist/migrations/*.sql)", + "Bash(just test-filter:*)", + "Bash(npx tsc:*)", + "Bash(find /Users/brooklyn/data/alchemist -type f -name *.rs)", + "Bash(ls -la /Users/brooklyn/data/alchemist/src/*.rs)", + "Bash(grep -rn \"from_alchemist_event\\\\|AlchemistEvent.*JobEvent\\\\|JobEvent.*AlchemistEvent\" /Users/brooklyn/data/alchemist/src/ --include=*.rs)", + "Bash(grep -l AlchemistEvent /Users/brooklyn/data/alchemist/src/*.rs /Users/brooklyn/data/alchemist/src/**/*.rs)", + "Bash(/tmp/audit_report.txt:*)", + "Read(//tmp/**)" ] } } diff --git a/CLAUDE.md b/CLAUDE.md index 8e58871..e95831b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,6 +44,8 @@ just test-e2e-headed # E2e with browser visible Integration tests require FFmpeg and FFprobe installed locally. +Integration tests live in `tests/` — notably `integration_db_upgrade.rs` tests schema migrations against a v0.2.5 baseline database. Every migration must pass this. + ### Database ```bash just db-reset # Wipe dev database (keeps config) @@ -53,6 +55,10 @@ just db-shell # SQLite shell ## Architecture +### Clippy Strictness + +CI enforces `-D clippy::unwrap_used` and `-D clippy::expect_used`. Use `?` propagation or explicit match — no `.unwrap()` or `.expect()` in production code paths. + ### Rust Backend (`src/`) The backend is structured around a central `AppState` (holding SQLite pool, config, broadcast channels) passed to Axum handlers: @@ -77,15 +83,32 @@ The backend is structured around a central `AppState` (holding SQLite pool, conf - `pipeline.rs` — Orchestrates scan → analyze → plan → execute - `processor.rs` — Job queue controller (concurrency, pausing, draining) - `ffmpeg/` — FFmpeg command builder and progress parser, with platform-specific encoder modules -- **`orchestrator.rs`** — Spawns and monitors FFmpeg processes, streams progress back via channels +- **`orchestrator.rs`** — Spawns and monitors FFmpeg processes, streams progress back via channels. Uses `std::sync::Mutex` (not tokio) intentionally — critical sections never cross `.await` boundaries. - **`system/`** — Hardware detection (`hardware.rs`), file watcher (`watcher.rs`), library scanner (`scanner.rs`) - **`scheduler.rs`** — Off-peak cron scheduling - **`notifications.rs`** — Discord, Gotify, Webhook integrations - **`wizard.rs`** — First-run setup flow +#### Event Channel Architecture + +Three typed broadcast channels in `AppState` (defined in `db.rs`): +- `jobs` (capacity 1000) — high-frequency: progress, state changes, decisions, logs +- `config` (capacity 50) — watch folder changes, settings updates +- `system` (capacity 100) — scan lifecycle, hardware state changes + +`sse.rs` merges all three via `futures::stream::select_all`. SSE is capped at 50 concurrent connections (`MAX_SSE_CONNECTIONS`), enforced with a RAII guard that decrements on stream drop. + +`AlchemistEvent` still exists as a legacy bridge; `JobEvent` is the canonical type — new code uses `JobEvent`/`ConfigEvent`/`SystemEvent`. + +#### FFmpeg Command Builder + +`FFmpegCommandBuilder<'a>` in `src/media/ffmpeg/mod.rs` uses lifetime references to avoid cloning input/output paths. `.with_hardware(Option<&HardwareInfo>)` injects hardware flags; `.build_args()` returns `Vec` for unit testing without spawning a process. Each hardware platform is a submodule (amf, cpu, nvenc, qsv, vaapi, videotoolbox). `EncoderCapabilities` is detected once via live ffmpeg invocation and cached in `OnceLock`. + ### Frontend (`web/src/`) -Astro pages with React islands. UI reflects backend state via Server-Sent Events (SSE) — avoid optimistic UI unless reconciled with backend truth. +Astro pages (`web/src/pages/`) with React islands. UI reflects backend state via SSE — avoid optimistic UI unless reconciled with backend truth. + +Job management UI is split into focused subcomponents under `web/src/components/jobs/`: `JobsTable`, `JobDetailModal`, `JobsToolbar`, `JobExplanations`, `useJobSSE.ts` (SSE hook), and `types.ts` (shared types + pure data utilities). `JobManager.tsx` is the parent that owns state and wires them together. ### Database Schema diff --git a/audit.md b/audit.md index 62321a4..6bd26ce 100644 --- a/audit.md +++ b/audit.md @@ -1,136 +1,420 @@ # Audit Findings -Date: 2026-04-11 +Last updated: 2026-04-12 (second pass) -## Summary +--- -This audit focused on the highest-risk paths in Alchemist: +## P1 Issues -- queue claiming and cancellation -- media planning and execution -- conversion validation -- setup/auth exposure -- job detail and failure UX +--- -The current automated checks were green at audit time, but several real -correctness and behavior issues remain. +### [P1-1] Cancel during analysis can be overwritten by the pipeline -## Findings +**Status: RESOLVED** -### [P1] Canceling a job during analysis can be overwritten +**Files:** +- `src/server/jobs.rs:41–63` +- `src/media/pipeline.rs:1178–1221` +- `src/orchestrator.rs:84–90` -Relevant code: +**Severity:** P1 -- `src/server/jobs.rs:41` -- `src/media/pipeline.rs:927` -- `src/media/pipeline.rs:970` -- `src/orchestrator.rs:239` +**Problem:** -`request_job_cancel()` marks `analyzing` and `resuming` jobs as -`cancelled` immediately. But the analysis/planning path can still run to -completion and later overwrite that state to `skipped`, -`encoding`/`remuxing`, or another follow-on state. +`request_job_cancel()` in `jobs.rs` immediately writes `Cancelled` to the DB for jobs in `Analyzing` or `Resuming` state. The pipeline used to have race windows where it could overwrite this state with `Skipped`, `Encoding`, or `Remuxing` if it reached a checkpoint after the cancel was issued but before it could be processed. -The transcoder-side `pending_cancels` check only applies around FFmpeg -spawn, so a cancel issued during analysis is not guaranteed to stop the -pipeline before state transitions are persisted. +**Fix:** -Impact: +Implemented `cancel_requested: Arc>>` in `Transcoder` (orchestrator). The `update_job_state` wrapper in `pipeline.rs` now checks this set before any DB write for `Encoding`, `Remuxing`, `Skipped`, and `Completed` states. Terminal states (Completed, Failed, Cancelled, Skipped) also trigger removal from the set. -- a user-visible cancel can be lost -- the UI can report a cancelled job that later resumes or becomes skipped -- queue state becomes harder to trust +--- -### [P1] VideoToolbox quality controls are effectively a no-op +### [P1-2] VideoToolbox quality controls are effectively ignored -Relevant code: +**Status: RESOLVED** -- `src/config.rs:85` -- `src/media/planner.rs:633` -- `src/media/ffmpeg/videotoolbox.rs:3` -- `src/conversion.rs:424` +**Files:** +- `src/media/planner.rs:630–650` +- `src/media/ffmpeg/videotoolbox.rs:25–54` +- `src/config.rs:85–92` -The config still defines a VideoToolbox quality ladder, and the planner -still emits `RateControl::Cq` for VideoToolbox encoders. But the actual -VideoToolbox FFmpeg builder ignores rate-control input entirely. +**Severity:** P1 -The Convert workflow does the same thing by still generating `Cq` for -non-CPU/QSV encoders even though the VideoToolbox path does not consume -it. +**Problem:** -Impact: +The planner used to emit `RateControl::Cq` values that were incorrectly mapped for VideoToolbox, resulting in uncalibrated or inverted quality. -- quality profile does not meaningfully affect VideoToolbox jobs -- Convert quality values for VideoToolbox are misleading -- macOS throughput/quality tradeoffs are harder to reason about +**Fix:** -### [P2] Convert does not reuse subtitle/container compatibility checks +Fixed the mapping in `videotoolbox.rs` to use `-q:v` (1-100, lower is better) and clamped the input range to 1-51 to match user expectations from x264/x265. Updated `QualityProfile` in `config.rs` to provide sane default values (24, 28, 32) for VideoToolbox quality. -Relevant code: +--- -- `src/media/planner.rs:863` -- `src/media/planner.rs:904` -- `src/conversion.rs:272` -- `src/conversion.rs:366` +## P2 Issues -The main library planner explicitly rejects unsafe subtitle-copy -combinations, especially for MP4/MOV targets. The Convert flow has its -own normalization/build path and does not reuse that validation. +--- -Impact: +### [P2-1] Convert does not reuse subtitle/container compatibility checks -- the Convert UI can accept settings that are known to fail later in FFmpeg -- conversion behavior diverges from library-job behavior -- users can hit avoidable execution-time errors instead of fast validation +**Status: RESOLVED** -### [P2] Completed job details omit metadata at the API layer +**Files:** +- `src/conversion.rs:372–380` +- `src/media/planner.rs` -Relevant code: +**Severity:** P2 -- `src/server/jobs.rs:344` -- `web/src/components/JobManager.tsx:1774` +**Problem:** -The job detail endpoint explicitly returns `metadata = None` for -`completed` jobs, even though the Jobs modal is structured to display -input metadata when available. +The conversion path was not validating subtitle/container compatibility, leading to FFmpeg runtime failures instead of early validation errors. -Impact: +**Fix:** -- completed-job details are structurally incomplete -- the frontend needs special-case empty-state behavior -- operator confidence is lower when comparing completed jobs after the fact +Integrated `crate::media::planner::subtitle_copy_supported` into `src/conversion.rs:build_subtitle_plan`. The "copy" mode now returns an `AlchemistError::Config` if the combination is unsupported. -### [P2] LAN-only setup is easy to misconfigure behind a local reverse proxy +--- -Relevant code: +### [P2-2] Completed job metadata omitted at the API layer -- `src/server/middleware.rs:269` -- `src/server/middleware.rs:300` +**Status: RESOLVED** -The setup gate uses `request_ip()` and trusts forwarded headers only when -the direct peer is local/private. If Alchemist sits behind a loopback or -LAN reverse proxy that fails to forward the real client IP, the request -falls back to the proxy peer IP and is treated as LAN-local. +**Files:** +- `src/db.rs:254–263` +- `src/media/pipeline.rs:599` +- `src/server/jobs.rs:343` -Impact: +**Severity:** P2 -- public reverse-proxy deployments can accidentally expose setup -- behavior depends on correct proxy header forwarding -- the security model is sound in principle but fragile in deployment +**Problem:** -## What To Fix First +Job details required a live re-probe of the input file to show metadata, which failed if the file was moved or deleted after completion. -1. Fix the cancel-during-analysis race. -2. Fix or redesign VideoToolbox quality handling so the UI and planner do - not promise controls that the backend ignores. -3. Reuse planner validation in Convert for subtitle/container safety. -4. Decide whether completed jobs should persist and return metadata in the - detail API. +**Fix:** -## What To Investigate Next +Added `input_metadata_json` column to the `jobs` table (migration `20260412000000_store_job_metadata.sql`). The pipeline now stores the metadata string immediately after analysis. `get_job_detail_handler` reads this stored value, ensuring metadata is always available even if the source file is missing. -1. Use runtime diagnostics to confirm whether macOS slowness is true - hardware underperformance, silent fallback, or filter overhead. -2. Verify whether “only one job at a time” is caused by actual worker - serialization or by planner eligibility/skips. -3. Review dominant skip reasons before relaxing planner heuristics. +--- + +### [P2-3] LAN-only setup exposed to reverse proxy misconfig + +**Status: RESOLVED** + +**Files:** +- `src/config.rs` — `SystemConfig.trusted_proxies` +- `src/server/mod.rs` — `AppState.trusted_proxies`, `AppState.setup_token` +- `src/server/middleware.rs` — `is_trusted_peer`, `request_ip`, `auth_middleware` + +**Severity:** P2 + +**Problem:** + +The setup wizard gate trusts all private/loopback IPs for header forwarding. When running behind a misconfigured proxy that doesn't set headers, it falls back to the proxy's own IP (e.g. 127.0.0.1), making the setup endpoint accessible to external traffic. + +**Fix:** + +Added two independent security layers: +1. `trusted_proxies: Vec` to `SystemConfig`. When non-empty, only those exact IPs (plus loopback) are trusted for proxy header forwarding instead of all RFC-1918 ranges. Empty = previous behavior preserved. +2. `ALCHEMIST_SETUP_TOKEN` env var. When set, setup endpoints require `?token=` query param regardless of client IP. Token mode takes precedence over IP-based LAN check. + +--- + +### [P2-4] N+1 DB update in batch cancel + +**Status: RESOLVED** + +**Files:** +- `src/server/jobs.rs` — `batch_jobs_handler` + +**Severity:** P2 + +**Problem:** + +`batch_jobs_handler` for "cancel" action iterates over jobs and calls `request_job_cancel` which performs an individual `update_job_status` query per job. Cancelling a large number of jobs triggers N queries. + +**Fix:** + +Restructured the "cancel" branch in `batch_jobs_handler`. Orchestrator in-memory operations (add_cancel_request, cancel_job) still run per-job, but all DB status updates are batched into a single `db.batch_cancel_jobs(&ids)` call (which already existed at db.rs). Immediate-resolution jobs (Queued + successfully signalled Analyzing/Resuming) are collected and written in one UPDATE ... WHERE id IN (...) query. + +--- + +### [P2-5] Missing archived filter in health and stats queries + +**Status: RESOLVED** + +**Files:** +- `src/db.rs` — `get_aggregated_stats`, `get_job_stats`, `get_health_summary` + +**Severity:** P2 + +**Problem:** + +`get_health_summary` and `get_aggregated_stats` (total_jobs) do not include `AND archived = 0`. Archived (deleted) jobs are incorrectly included in library health metrics and total job counts. + +**Fix:** + +Added `AND archived = 0` to all three affected queries: `total_jobs` and `completed_jobs` subqueries in `get_aggregated_stats`, the `GROUP BY status` query in `get_job_stats`, and both subqueries in `get_health_summary`. Updated tests that were asserting the old (incorrect) behavior. + +--- + +### [P2-6] Daily summary notifications bypass SSRF protections + +**Status: RESOLVED** + +**Files:** +- `src/notifications.rs` — `build_safe_client()`, `send()`, `send_daily_summary_target()` + +**Severity:** P2 + +**Problem:** + +`send_daily_summary_target()` used `Client::new()` without any SSRF defences, while `send()` applied DNS timeout, private-IP blocking, no-redirect policy, and request timeout. + +**Fix:** + +Extracted all client-building logic into `build_safe_client(&self, target)` which applies the full SSRF defence stack. Both `send()` and `send_daily_summary_target()` now use this shared helper. + +--- + +### [P2-7] Silent reprobe failure corrupts saved encode stats + +**Status: RESOLVED** + +**Files:** +- `src/media/pipeline.rs` — `finalize_job()` duration reprobe + +**Severity:** P2 + +**Problem:** + +When a completed encode's metadata has `duration_secs <= 0.0`, the pipeline reprobes the output file to get the actual duration. If reprobe fails, the error was silently swallowed via `.ok()` and duration defaulted to 0.0, poisoning downstream stats. + +**Fix:** + +Replaced `.ok().and_then().unwrap_or(0.0)` chain with explicit `match` that logs the error via `tracing::warn!` and falls through to 0.0. Existing guards at the stats computation lines already handle `duration <= 0.0` correctly — operators now see *why* stats are zeroed. + +--- + +## Technical Debt + +--- + +### [TD-1] `db.rs` is a 3481-line monolith + +**Status: RESOLVED** + +**File:** `src/db/` (was `src/db.rs`) + +**Severity:** TD + +**Problem:** + +The database layer had grown to nearly 3500 lines. Every query, migration flag, and state enum was in one file, making navigation and maintenance difficult. + +**Fix:** + +Split into `src/db/` module with 8 submodules: `mod.rs` (Db struct, init, migrations, hash fns), `types.rs` (all type defs), `events.rs` (event enums + channels), `jobs.rs` (job CRUD/filtering/decisions), `stats.rs` (encode/aggregated/daily stats), `config.rs` (watch dirs/profiles/notifications/schedules/file settings/preferences), `conversion.rs` (ConversionJob CRUD), `system.rs` (auth/sessions/API tokens/logs/health). All tests moved alongside their methods. Public API unchanged — all types re-exported from `db/mod.rs`. + +--- + +### [TD-2] `AlchemistEvent` legacy bridge is dead weight + +**Status: RESOLVED** + +**Files:** +- `src/db.rs` — enum and From impls removed +- `src/media/pipeline.rs`, `src/media/executor.rs`, `src/media/processor.rs` — legacy `tx` channel removed +- `src/notifications.rs` — migrated to typed `EventChannels` (jobs + system) +- `src/server/mod.rs`, `src/main.rs` — legacy channel removed from AppState/RunServerArgs + +**Severity:** TD + +**Problem:** + +`AlchemistEvent` was a legacy event type duplicated by `JobEvent`, `ConfigEvent`, and `SystemEvent`. All senders were emitting events on both channels. + +**Fix:** + +Migrated the notification system (the sole consumer) to subscribe to `EventChannels.jobs` and `EventChannels.system` directly. Added `SystemEvent::EngineIdle` variant. Removed `AlchemistEvent` enum, its `From` impls, the legacy `tx` broadcast channel from all structs, and the `pub use` from `lib.rs`. + +--- + +### [TD-3] `pipeline.rs` legacy `AlchemistEvent::Progress` stub + +**Status: RESOLVED** + +**Files:** +- `src/media/pipeline.rs:1228` + +**Severity:** TD + +**Problem:** + +The pipeline used to emit zeroed progress events that could overwrite real stats from the executor. + +**Fix:** + +Emission removed. A comment at line 1228-1229 confirms that `AlchemistEvent::Progress` is no longer emitted from the pipeline wrapper. + +--- + +### [TD-4] Silent `.ok()` on pipeline decision and attempt DB writes + +**Status: RESOLVED** + +**Files:** +- `src/media/pipeline.rs` — all `add_decision`, `insert_encode_attempt`, `upsert_job_failure_explanation`, and `add_log` call sites + +**Severity:** TD + +**Problem:** + +Decision records, encode attempt records, failure explanations, and error logs were written with `.ok()` or `let _ =`, silently discarding DB write failures. These records are the only audit trail of *why* a job was skipped/transcoded/failed. + +**Fix:** + +Replaced all `.ok()` / `let _ =` patterns on `add_decision`, `insert_encode_attempt`, `upsert_job_failure_explanation`, and `add_log` calls with `if let Err(e) = ... { tracing::warn!(...) }`. Pipeline still continues on failure, but operators now see the error. + +--- + +### [TD-5] Correlated subquery for sort-by-size in job listing + +**Status: RESOLVED** + +**Files:** +- `src/db.rs` — `get_jobs_filtered()` query + +**Severity:** TD + +**Problem:** + +Sorting jobs by file size used a correlated subquery in ORDER BY, executing one subquery per row and producing NULL for jobs without encode_stats. + +**Fix:** + +Added `LEFT JOIN encode_stats es ON es.job_id = j.id` to the base query. Sort column changed to `COALESCE(es.input_size_bytes, 0)`, ensuring jobs without stats sort as 0 (smallest) instead of NULL. + +--- + +## Reliability Gaps + +--- + +### [RG-1] No encode resume after crash or restart + +**Status: PARTIALLY RESOLVED** + +**Files:** +- `src/main.rs:320` +- `src/media/processor.rs:255` + +**Severity:** RG + +**Problem:** + +Interrupted encodes were left in `Encoding` state and orphaned temp files remained on disk. + +**Fix:** + +Implemented `db.reset_interrupted_jobs()` in `main.rs` which resets `Encoding`, `Remuxing`, `Resuming`, and `Analyzing` jobs to `Queued` on startup. Orphaned temp files are also detected and removed. Full bitstream-level resume (resuming from the middle of a file) is still missing. + +--- + +### [RG-2] AMD VAAPI/AMF hardware paths unvalidated + +**Files:** +- `src/media/ffmpeg/vaapi.rs` +- `src/media/ffmpeg/amf.rs` + +**Severity:** RG + +**Problem:** + +Hardware paths for AMD (VAAPI on Linux, AMF on Windows) were implemented without real hardware validation. + +**Fix:** + +Verify exact flag compatibility on AMD hardware and add integration tests gated on GPU presence. + +--- + +## UX Gaps + +--- + +### [UX-1] Queued jobs show no position or estimated wait time + +**Status: RESOLVED** + +**Files:** +- `src/db.rs` — `get_queue_position` +- `src/server/jobs.rs` — `JobDetailResponse.queue_position` +- `web/src/components/jobs/JobDetailModal.tsx` +- `web/src/components/jobs/types.ts` — `JobDetail.queue_position` + +**Severity:** UX + +**Problem:** + +Queued jobs only show "Waiting" without indicating their position in the priority queue. + +**Fix:** + +Implemented `db.get_queue_position(job_id)` which counts jobs with higher priority or earlier `created_at` (matching the `priority DESC, created_at ASC` dequeue order). Added `queue_position: Option` to `JobDetailResponse` — populated only when `status == Queued`. Frontend shows `Queue position: #N` in the empty state card in `JobDetailModal`. + +--- + +### [UX-2] No way to add a single file to the queue via the UI + +**Severity:** UX + +**Problem:** + +Jobs only enter the queue via full library scans. No manual "Enqueue path" exists in the UI. + +**Fix:** + +Add `POST /api/jobs/enqueue` and a "Add file" action in the `JobsToolbar`. + +--- + +### [UX-3] Workers-blocked reason not surfaced for queued jobs + +**Severity:** UX + +**Problem:** + +Users cannot see why a job is stuck in Queued (paused, scheduled, or slots full). + +**Fix:** + +Add `GET /api/processor/status` and show the reason in the job detail. + +--- + +## Feature Gaps + +--- + +### [FG-4] Intelligence page content not actionable + +**Files:** +- `web/src/components/LibraryIntelligence.tsx` + +**Severity:** FG + +**Problem:** + +Intelligence page is informational only; recommendations cannot be acted upon directly from the page. + +**Fix:** + +Add "Queue all" for remux opportunities and "Review" actions for duplicates. + +--- + +## What To Fix Next + +1. **[UX-2]** Single file enqueue — New feature. +2. **[UX-3]** Workers-blocked reason — New feature. +3. **[FG-4]** Intelligence page actions — New feature. +4. **[RG-2]** AMD VAAPI/AMF validation — Needs real hardware. diff --git a/docs/docs/api.md b/docs/docs/api.md index e197f5d..2119764 100644 --- a/docs/docs/api.md +++ b/docs/docs/api.md @@ -1,54 +1,35 @@ --- -title: API +title: API Reference description: REST and SSE API reference for Alchemist. --- -All API routes require the `alchemist_session` auth cookie -except: - -- `/api/auth/*` -- `/api/health` -- `/api/ready` -- during first-time setup, the setup UI and setup-related - unauthenticated routes are only reachable from the local - network - -Authentication is established by `POST /api/auth/login`. -The backend also accepts `Authorization: Bearer `. -Bearer tokens now come in two classes: - -- `read_only` — observability-only routes -- `full_access` — same route access as an authenticated session - -The web UI still uses the session cookie. - -Machine-readable contract: - -- [OpenAPI spec](/openapi.yaml) - ## Authentication -### API tokens +All API routes require the `alchemist_session` auth cookie established via `/api/auth/login`, or an `Authorization: Bearer ` header. -API tokens are created in **Settings → API Tokens**. +Machine-readable contract: [OpenAPI spec](/openapi.yaml) -- token values are only shown once at creation time -- only hashed token material is stored server-side -- revoked tokens stop working immediately +### `POST /api/auth/login` +Establish a session. Returns a `Set-Cookie` header. -Read-only tokens are intentionally limited to observability -routes such as stats, jobs, logs history, SSE, system info, -hardware info, library intelligence, and health/readiness. +**Request:** +```json +{ + "username": "admin", + "password": "..." +} +``` + +### `POST /api/auth/logout` +Invalidate current session and clear cookie. ### `GET /api/settings/api-tokens` - -Lists token metadata only. Plaintext token values are never -returned after creation. +List metadata for configured API tokens. ### `POST /api/settings/api-tokens` +Create a new API token. The plaintext value is only returned once. -Request: - +**Request:** ```json { "name": "Prometheus", @@ -56,411 +37,114 @@ Request: } ``` -Response: - -```json -{ - "token": { - "id": 1, - "name": "Prometheus", - "access_level": "read_only" - }, - "plaintext_token": "alc_tok_..." -} -``` - ### `DELETE /api/settings/api-tokens/:id` +Revoke a token. -Revokes a token in place. Existing automations using it will -begin receiving `401` or `403` depending on route class. - -### `POST /api/auth/login` - -Request: - -```json -{ - "username": "admin", - "password": "secret" -} -``` - -Response: - -```http -HTTP/1.1 200 OK -Set-Cookie: alchemist_session=...; HttpOnly; SameSite=Lax; Path=/; Max-Age=2592000 -``` - -```json -{ - "status": "ok" -} -``` - -### `POST /api/auth/logout` - -Clears the session cookie and deletes the server-side -session if one exists. - -```json -{ - "status": "ok" -} -``` +--- ## Jobs ### `GET /api/jobs` +List jobs with filtering and pagination. -Canonical job listing endpoint. Supports query params such -as `limit`, `page`, `status`, `search`, `sort_by`, -`sort_desc`, and `archived`. - -Each returned job row still includes the legacy -`decision_reason` string when present, and now also includes -an optional `decision_explanation` object: - -- `category` -- `code` -- `summary` -- `detail` -- `operator_guidance` -- `measured` -- `legacy_reason` - -Example: - -```bash -curl -b cookie.txt \ - 'http://localhost:3000/api/jobs?status=queued,failed&limit=50&page=1' -``` +**Params:** `limit`, `page`, `status`, `search`, `sort_by`, `sort_desc`, `archived`. ### `GET /api/jobs/:id/details` - -Returns the job row, any available analyzed metadata, -encode stats for completed jobs, recent job logs, and a -failure summary for failed jobs. Structured explanation -fields are included when available: - -- `decision_explanation` -- `failure_explanation` -- `job_failure_summary` is retained as a compatibility field - -Example response shape: - -```json -{ - "job": { - "id": 42, - "input_path": "/media/movies/example.mkv", - "status": "completed" - }, - "metadata": { - "codec_name": "h264", - "width": 1920, - "height": 1080 - }, - "encode_stats": { - "input_size_bytes": 8011223344, - "output_size_bytes": 4112233445, - "compression_ratio": 1.95, - "encode_speed": 2.4, - "vmaf_score": 93.1 - }, - "job_logs": [], - "job_failure_summary": null, - "decision_explanation": { - "category": "decision", - "code": "transcode_recommended", - "summary": "Transcode recommended", - "detail": "Alchemist determined the file should be transcoded based on the current codec and measured efficiency.", - "operator_guidance": null, - "measured": { - "target_codec": "av1", - "current_codec": "h264", - "bpp": "0.1200" - }, - "legacy_reason": "transcode_recommended|target_codec=av1,current_codec=h264,bpp=0.1200" - }, - "failure_explanation": null -} -``` +Fetch full job state, metadata, logs, and stats. ### `POST /api/jobs/:id/cancel` - -Cancels a queued or active job if the current state allows -it. +Cancel a queued or active job. ### `POST /api/jobs/:id/restart` - -Restarts a non-active job by sending it back to `queued`. +Restart a terminal job (failed/cancelled/completed). ### `POST /api/jobs/:id/priority` +Update job priority. -Request: - -```json -{ - "priority": 100 -} -``` - -Response: - -```json -{ - "id": 42, - "priority": 100 -} -``` +**Request:** `{"priority": 100}` ### `POST /api/jobs/batch` +Bulk action on multiple jobs. -Supported `action` values: `cancel`, `restart`, `delete`. - +**Request:** ```json { - "action": "restart", - "ids": [41, 42, 43] -} -``` - -Response: - -```json -{ - "count": 3 + "action": "restart|cancel|delete", + "ids": [1, 2, 3] } ``` ### `POST /api/jobs/restart-failed` - -Response: - -```json -{ - "count": 2, - "message": "Queued 2 failed or cancelled jobs for retry." -} -``` +Restart all failed or cancelled jobs. ### `POST /api/jobs/clear-completed` +Archive all completed jobs from the active queue. -Archives completed jobs from the visible queue while -preserving historical encode stats. - -```json -{ - "count": 12, - "message": "Cleared 12 completed jobs from the queue. Historical stats were preserved." -} -``` +--- ## Engine -### `POST /api/engine/pause` +### `GET /api/engine/status` +Get current operational status and limits. -```json -{ - "status": "paused" -} -``` +### `POST /api/engine/pause` +Pause the engine (suspend active jobs). ### `POST /api/engine/resume` - -```json -{ - "status": "running" -} -``` +Resume the engine. ### `POST /api/engine/drain` - -```json -{ - "status": "draining" -} -``` - -### `POST /api/engine/stop-drain` - -```json -{ - "status": "running" -} -``` - -### `GET /api/engine/status` - -Response fields: - -- `status` -- `mode` -- `concurrent_limit` -- `manual_paused` -- `scheduler_paused` -- `draining` -- `is_manual_override` - -Example: - -```json -{ - "status": "paused", - "manual_paused": true, - "scheduler_paused": false, - "draining": false, - "mode": "balanced", - "concurrent_limit": 2, - "is_manual_override": false -} -``` - -### `GET /api/engine/mode` - -Returns current mode, whether a manual override is active, -the current concurrent limit, CPU count, and computed mode -limits. +Enter drain mode (finish active jobs, don't start new ones). ### `POST /api/engine/mode` +Switch engine mode or apply manual overrides. -Request: - +**Request:** ```json { - "mode": "balanced", + "mode": "background|balanced|throughput", "concurrent_jobs_override": 2, "threads_override": 0 } ``` -Response: +--- -```json -{ - "status": "ok", - "mode": "balanced", - "concurrent_limit": 2, - "is_manual_override": true -} -``` - -## Stats +## Statistics ### `GET /api/stats/aggregated` - -```json -{ - "total_input_bytes": 1234567890, - "total_output_bytes": 678901234, - "total_savings_bytes": 555666656, - "total_time_seconds": 81234.5, - "total_jobs": 87, - "avg_vmaf": 92.4 -} -``` +Total savings, job counts, and global efficiency. ### `GET /api/stats/daily` - -Returns the last 30 days of encode activity. - -### `GET /api/stats/detailed` - -Returns the most recent detailed encode stats rows. +Encode activity history for the last 30 days. ### `GET /api/stats/savings` +Detailed breakdown of storage savings. -Returns the storage-savings summary used by the statistics -dashboard. - -## Settings - -### `GET /api/settings/transcode` - -Returns the transcode settings payload currently loaded by -the backend. - -### `POST /api/settings/transcode` - -Request: - -```json -{ - "concurrent_jobs": 2, - "size_reduction_threshold": 0.3, - "min_bpp_threshold": 0.1, - "min_file_size_mb": 50, - "output_codec": "av1", - "quality_profile": "balanced", - "threads": 0, - "allow_fallback": true, - "hdr_mode": "preserve", - "tonemap_algorithm": "hable", - "tonemap_peak": 100.0, - "tonemap_desat": 0.2, - "subtitle_mode": "copy", - "stream_rules": { - "strip_audio_by_title": ["commentary"], - "keep_audio_languages": ["eng"], - "keep_only_default_audio": false - } -} -``` +--- ## System ### `GET /api/system/hardware` - -Returns the current detected hardware backend, supported -codecs, backends, selection reason, probe summary, and any -detection notes. +Detected hardware backend and codec support matrix. ### `GET /api/system/hardware/probe-log` - -Returns the per-encoder probe log with success/failure -status, selected-flag metadata, summary text, and stderr -excerpts. +Full logs from the startup hardware probe. ### `GET /api/system/resources` +Live telemetry: CPU, Memory, GPU utilization, and uptime. -Returns live resource data: +--- -- `cpu_percent` -- `memory_used_mb` -- `memory_total_mb` -- `memory_percent` -- `uptime_seconds` -- `active_jobs` -- `concurrent_limit` -- `cpu_count` -- `gpu_utilization` -- `gpu_memory_percent` - -## Server-Sent Events +## Events (SSE) ### `GET /api/events` +Real-time event stream. -Internal event types are `JobStateChanged`, `Progress`, -`Decision`, and `Log`. The SSE stream exposed to clients -emits lower-case event names: - -- `status` -- `progress` -- `decision` -- `log` - -Additional config/system events may also appear, including -`config_updated`, `scan_started`, `scan_completed`, -`engine_status_changed`, and `hardware_state_changed`. - -Example: - -```text -event: progress -data: {"job_id":42,"percentage":61.4,"time":"00:11:32"} -``` - -`decision` events include the legacy `reason` plus an -optional structured `explanation` object with the same shape -used by the jobs API. +**Emitted Events:** +- `status`: Job state changes. +- `progress`: Real-time encode statistics. +- `decision`: Skip/Transcode logic results. +- `log`: Engine and job logs. +- `config_updated`: Configuration hot-reload notification. +- `scan_started` / `scan_completed`: Library scan status. diff --git a/docs/docs/configuration-reference.md b/docs/docs/configuration-reference.md index 8f6e603..5fd52df 100644 --- a/docs/docs/configuration-reference.md +++ b/docs/docs/configuration-reference.md @@ -70,7 +70,7 @@ Default config file location: | `output_extension` | string | `"mkv"` | Output file extension | | `output_suffix` | string | `"-alchemist"` | Suffix added to the output filename | | `replace_strategy` | string | `"keep"` | Replace behavior for output collisions | -| `output_root` | string | optional | Mirror outputs into another root path instead of writing beside the source | +| `output_root` | string | optional | If set, Alchemist mirrors the source library directory structure under this root path instead of writing outputs alongside the source files | ## `[schedule]` diff --git a/docs/docs/engine-modes.md b/docs/docs/engine-modes.md index c293452..2a70e72 100644 --- a/docs/docs/engine-modes.md +++ b/docs/docs/engine-modes.md @@ -1,37 +1,39 @@ --- -title: Engine Modes -description: Background, Balanced, and Throughput — what they mean and when to use each. +title: Engine Modes & States +description: Background, Balanced, and Throughput — understanding concurrency and execution flow. --- -Engine modes set the concurrent job limit. +Alchemist uses **Modes** to dictate performance limits and **States** to control execution flow. -## Modes +## Engine Modes (Concurrency) -| Mode | Concurrent jobs | Use when | -|------|----------------|----------| -| Background | 1 | Server in active use | -| Balanced (default) | floor(cpu_count / 2), min 1, max 4 | General shared server | -| Throughput | floor(cpu_count / 2), min 1, no cap | Dedicated server, clear a backlog | +Modes define the maximum number of concurrent jobs the engine will attempt to run. -## Manual override +| Mode | Concurrent Jobs | Ideal For | +|------|----------------|-----------| +| **Background** | 1 | Server in active use by other applications. | +| **Balanced** | `floor(cpu_count / 2)` (min 1, max 4) | Default. Shared server usage. | +| **Throughput** | `floor(cpu_count / 2)` (min 1, no cap) | Dedicated server; clearing a large backlog. | -Override the computed limit in **Settings → Runtime**. Takes -effect immediately. A "manual" badge appears in engine -status. Switching modes clears the override. +:::tip Manual Override +You can override the computed limit in **Settings → Runtime**. A "Manual" badge will appear in the engine status. Switching modes clears manual overrides. +::: -## States vs. modes +--- -Modes determine *how many* jobs run. States determine -*whether* they run. +## Engine States (Execution) + +States determine whether the engine is actively processing the queue. | State | Behavior | |-------|----------| -| Running | Jobs start up to the mode's limit | -| Paused | No jobs start; active jobs freeze | -| Draining | Active jobs finish; no new jobs start | -| Scheduler paused | Paused by a schedule window | +| **Running** | Engine is active. Jobs start up to the current mode's limit. | +| **Paused** | Engine is suspended. No new jobs start; active jobs are frozen. | +| **Draining** | Engine is stopping. Active jobs finish, but no new jobs start. | +| **Scheduler Paused** | Engine is temporarily paused by a configured [Schedule Window](/scheduling). | -## Changing modes +--- -**Settings → Runtime**. Takes effect immediately; in-progress -jobs are not cancelled. +## Changing Engine Behavior + +Engine behavior can be adjusted in real-time via the **Runtime** dashboard or the [API](/api#engine). Changes take effect immediately without cancelling in-progress jobs. diff --git a/docs/docs/library-doctor.md b/docs/docs/library-doctor.md index 567e30f..3625c66 100644 --- a/docs/docs/library-doctor.md +++ b/docs/docs/library-doctor.md @@ -1,32 +1,35 @@ --- title: Library Doctor -description: Scan for corrupt, truncated, and unreadable media files. +description: Identifying corrupt, truncated, and unreadable media files in your library. --- -Library Doctor scans your configured directories for files -that are corrupt, truncated, or unreadable by FFprobe. +Library Doctor is a specialized diagnostic tool that scans your library for media files that are corrupt, truncated, or otherwise unreadable by the Alchemist analyzer. -Run from **Settings → Runtime → Library Doctor → Run Scan**. +Run a scan manually from **Settings → Runtime → Library Doctor**. -## What it checks +## Core Checks -| Check | What it detects | -|-------|-----------------| -| Probe failure | Files FFprobe cannot read at all | -| No video stream | Files with no detectable video track | -| Zero duration | Files reporting 0 seconds of content | -| Truncated file | Files that appear to end prematurely | -| Missing codec data | Files missing metadata needed to plan a transcode | +Library Doctor runs an intensive probe on every file in your watch directories to identify the following issues: -## What to do with results +| Check | Technical Detection | Action Recommended | +|-------|-----------------|--------------------| +| **Probe Failure** | `ffprobe` returns a non-zero exit code or cannot parse headers. | Re-download or Re-rip. | +| **No Video Stream** | File container is valid but contains no detectable video tracks. | Verify source; delete if unintended. | +| **Zero Duration** | File metadata reports a duration of 0 seconds. | Check for interrupted transfers. | +| **Truncated File** | File size is significantly smaller than expected for the reported bitrate/duration. | Check filesystem integrity. | +| **Missing Metadata** | Missing critical codec data (e.g., pixel format, profile) needed for planning. | Possible unsupported codec variant. | -Library Doctor reports issues — it does not repair or delete -files automatically. +--- -- **Re-download** — interrupted download -- **Re-rip** — disc read errors -- **Delete** — duplicate or unrecoverable -- **Ignore** — player handles it despite FFprobe failing +## Relationship to Jobs -Files that fail Library Doctor also fail the Analyzing -stage of a transcode job and appear as Failed in Jobs. +Files that fail Library Doctor checks will also fail the **Analyzing** stage of a standard transcode job. + +- **Pre-emptive detection**: Running Library Doctor helps you clear "broken" files from your library before they enter the processing queue. +- **Reporting**: Issues identified by the Doctor appear in the **Health** tab of the dashboard, separate from active transcode jobs. + +## Handling Results + +Library Doctor is read-only; it will **never delete or modify** your files automatically. + +If a file is flagged, you should manually verify it using a media player. If the file is indeed unplayable, we recommend replacing it from the source. Flags can be cleared by deleting the file or moving it out of a watched directory. diff --git a/docs/docs/overview.md b/docs/docs/overview.md index 470d9f1..894bff1 100644 --- a/docs/docs/overview.md +++ b/docs/docs/overview.md @@ -37,13 +37,9 @@ FFmpeg expert. ## Hardware support -| Vendor | AV1 | HEVC | H.264 | Notes | -|--------|-----|------|-------|-------| -| NVIDIA NVENC | RTX 30/40 | Maxwell+ | All | Best for speed | -| Intel QSV | 12th gen+ | 6th gen+ | All | Best for power efficiency | -| AMD VAAPI/AMF | RDNA 2+ on compatible driver/FFmpeg stacks | Polaris+ | All | Linux VAAPI / Windows AMF; HEVC/H.264 are the validated AMD paths for `0.3.0` | -| Apple VideoToolbox | M3+ | M1+ / T2 | All | Binary install recommended | -| CPU (SVT-AV1/x265/x264) | All | All | All | Always available | +Alchemist detects and selects the best available hardware encoder automatically (NVIDIA NVENC, Intel QSV, AMD VAAPI/AMF, Apple VideoToolbox, or CPU fallback). + +For detailed codec support matrices (AV1, HEVC, H.264) and vendor-specific setup guides, see the [Hardware Acceleration](/hardware) documentation. ## Where to start diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index eb43ff1..2a28c4b 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -103,7 +103,6 @@ const config: Config = { ], }, footer: { - style: 'dark', links: [ { title: 'Get Started', diff --git a/docs/src/css/custom.css b/docs/src/css/custom.css index 34f1590..8c3c892 100644 --- a/docs/src/css/custom.css +++ b/docs/src/css/custom.css @@ -94,8 +94,9 @@ html { } .footer { - border-top: 1px solid rgba(200, 155, 90, 0.22); - background: var(--ifm-footer-background-color); + border-top: 1px solid var(--doc-border); + background-color: var(--ifm-footer-background-color) !important; + color: #ddd0be; } .footer__links { @@ -118,13 +119,22 @@ html { } .footer__title { + color: #fdf6ee; font-weight: 700; } -.footer__bottom, +.footer__link-item { + color: #cfc0aa; +} + +.footer__link-item:hover { + color: var(--ifm-link-hover-color); + text-decoration: none; +} + .footer__copyright { - text-align: center; color: #b8a88e; + text-align: center; } .main-wrapper { diff --git a/migrations/20260412000000_store_job_metadata.sql b/migrations/20260412000000_store_job_metadata.sql new file mode 100644 index 0000000..15c0c42 --- /dev/null +++ b/migrations/20260412000000_store_job_metadata.sql @@ -0,0 +1,2 @@ +-- Store input metadata as JSON to avoid live re-probing completed jobs +ALTER TABLE jobs ADD COLUMN input_metadata_json TEXT; diff --git a/plans.md b/plans.md new file mode 100644 index 0000000..f4443f0 --- /dev/null +++ b/plans.md @@ -0,0 +1,191 @@ +# Open Item Plans + +--- + +## [UX-2] Single File Enqueue + +### Goal +`POST /api/jobs/enqueue` + "Add file" button in JobsToolbar. + +### Backend + +**New handler in `src/server/jobs.rs`:** +```rust +#[derive(Deserialize)] +struct EnqueueFilePayload { + input_path: String, + source_root: Option, +} + +async fn enqueue_file_handler(State(state), Json(payload)) -> impl IntoResponse +``` + +Logic: +1. Validate `input_path` exists on disk, is a file +2. Read `mtime` from filesystem metadata +3. Build `DiscoveredMedia { path, mtime, source_root }` +4. Call `enqueue_discovered_with_db(&db, discovered)` — reuses all existing skip checks, output path computation, file settings +5. If `Ok(true)` → fetch job via `db.get_job_by_input_path()`, return it +6. If `Ok(false)` → 409 "already tracked / output exists" +7. If `Err` → 400 with error + +**Route:** Add `.route("/api/jobs/enqueue", post(enqueue_file_handler))` in `src/server/mod.rs` + +### Frontend + +**`web/src/components/jobs/JobsToolbar.tsx`:** +- Add "Add File" button next to refresh +- Opens small modal/dialog with text input for path +- POST to `/api/jobs/enqueue`, toast result +- SSE handles job appearing in table automatically + +### Files to modify +- `src/server/jobs.rs` — new handler + payload struct +- `src/server/mod.rs` — route registration +- `web/src/components/jobs/JobsToolbar.tsx` — button + dialog +- `web/src/components/jobs/` — optional: new `EnqueueDialog.tsx` component + +### Verification +- `cargo check && cargo test && cargo clippy` +- Manual: POST valid path → job appears queued +- POST nonexistent path → 400 +- POST already-tracked path → 409 +- Frontend: click Add File, enter path, see job in table + +--- + +## [UX-3] Workers-Blocked Reason + +### Goal +Surface why queued jobs aren't being processed. Extend `/api/engine/status` → show reason in JobDetailModal. + +### Backend + +**Extend `engine_status_handler` response** (or create new endpoint) to include blocking state: + +```rust +struct EngineStatusResponse { + // existing fields... + blocked_reason: Option, // "paused", "scheduled", "draining", "boot_analysis", "slots_full", null + schedule_resume: Option, // next window open time if scheduler_paused +} +``` + +Derive from `Agent` state: +- `agent.is_manual_paused()` → `"paused"` +- `agent.is_scheduler_paused()` → `"scheduled"` +- `agent.is_draining()` → `"draining"` +- `agent.is_boot_analyzing()` → `"boot_analysis"` +- `agent.in_flight_jobs >= agent.concurrent_jobs_limit()` → `"slots_full"` +- else → `null` (processing normally) + +### Frontend + +**`web/src/components/jobs/JobDetailModal.tsx`:** +- Below queue position display, show blocked reason if present +- Fetch from engine status (already available via SSE `EngineStatusChanged` events, or poll `/api/engine/status`) +- Color-coded: yellow for schedule/pause, blue for boot analysis, gray for slots full + +### Files to modify +- `src/server/jobs.rs` or wherever `engine_status_handler` lives — extend response +- `web/src/components/jobs/JobDetailModal.tsx` — display blocked reason +- `web/src/components/jobs/useJobSSE.ts` — optionally track engine status via SSE + +### Verification +- Pause engine → queued job detail shows "Engine paused" +- Set schedule window outside current time → shows "Outside schedule window" +- Fill all slots → shows "All worker slots occupied" +- Resume → reason disappears + +--- + +## [FG-4] Intelligence Page Actions + +### Goal +Add actionable buttons to `LibraryIntelligence.tsx`: delete duplicates, queue remux opportunities. + +### Duplicate Group Actions + +**"Keep Latest, Delete Rest" button per group:** +- Each duplicate group card gets a "Clean Up" button +- Selects all jobs except the one with latest `updated_at` +- Calls `POST /api/jobs/batch` with `{ action: "delete", ids: [...] }` +- Confirmation modal: "Archive N duplicate jobs?" + +**"Clean All Duplicates" bulk button:** +- Top-level button in duplicates section header +- Same logic across all groups +- Shows total count in confirmation + +### Recommendation Actions + +**"Queue All Remux" button:** +- Gathers IDs of all remux opportunity jobs +- Calls `POST /api/jobs/batch` with `{ action: "restart", ids: [...] }` +- Jobs re-enter queue for remux processing + +**Per-recommendation "Queue" button:** +- Individual restart for single recommendation items + +### Backend +No new endpoints needed — existing `POST /api/jobs/batch` handles all actions (cancel/delete/restart). + +### Frontend + +**`web/src/components/LibraryIntelligence.tsx`:** +- Add "Clean Up" button to each duplicate group card +- Add "Clean All Duplicates" button to section header +- Add "Queue All" button to remux opportunities section +- Add confirmation modal component +- Add toast notifications for success/error +- Refresh data after action completes + +### Files to modify +- `web/src/components/LibraryIntelligence.tsx` — buttons, modals, action handlers + +### Verification +- Click "Clean Up" on duplicate group → archives all but latest +- Click "Queue All Remux" → remux jobs reset to queued +- Confirm counts in modal match actual +- Data refreshes after action + +--- + +## [RG-2] AMD VAAPI/AMF Validation + +### Goal +Verify AMD hardware encoder paths produce correct FFmpeg commands on real AMD hardware. + +### Problem +`src/media/ffmpeg/vaapi.rs` and `src/media/ffmpeg/amf.rs` were implemented without real hardware validation. Flag mappings, device paths, and quality controls may be incorrect. + +### Validation checklist + +**VAAPI (Linux):** +- [ ] Device path `/dev/dri/renderD128` detection works +- [ ] `hevc_vaapi` / `h264_vaapi` encoder selection +- [ ] CRF/quality mapping → `-rc_mode CQP -qp N` or `-rc_mode ICQ -quality N` +- [ ] HDR passthrough flags (if applicable) +- [ ] Container compatibility (MKV/MP4) + +**AMF (Windows):** +- [ ] `hevc_amf` / `h264_amf` encoder selection +- [ ] Quality mapping → `-quality quality -qp_i N -qp_p N` +- [ ] B-frame support detection +- [ ] HDR passthrough + +### Approach +1. Write unit tests for `build_args()` output — verify flag strings without hardware +2. Gate integration tests on `AMD_GPU_AVAILABLE` env var +3. Document known-good flag sets from AMD documentation +4. Add `EncoderCapabilities` detection for AMF/VAAPI (similar to existing NVENC/QSV detection) + +### Files to modify +- `src/media/ffmpeg/vaapi.rs` — flag corrections if needed +- `src/media/ffmpeg/amf.rs` — flag corrections if needed +- `tests/` — new integration test file gated on hardware + +### Verification +- Unit tests pass on CI (no hardware needed) +- Integration tests pass on AMD hardware (manual) +- Generated FFmpeg commands reviewed against AMD documentation diff --git a/security_best_practices_report.md b/security_best_practices_report.md deleted file mode 100644 index fef49b9..0000000 --- a/security_best_practices_report.md +++ /dev/null @@ -1,124 +0,0 @@ -# Security Best Practices Report - -## Executive Summary - -I found one critical security bug and one additional high-severity issue in the setup/bootstrap flow. - -The critical problem is that first-run setup is remotely accessible without authentication while the server listens on `0.0.0.0`. A network-reachable attacker can win the initial setup race, create the first admin account, and take over the instance. - -I did not find evidence of major client-side XSS sinks or obvious SQL injection paths during this audit. Most of the remaining concerns I saw were hardening-level issues rather than immediately exploitable major bugs. - -## Critical Findings - -### ALCH-SEC-001 - -- Severity: Critical -- Location: - - `src/server/middleware.rs:80-86` - - `src/server/wizard.rs:95-210` - - `src/server/mod.rs:176-197` - - `README.md:61-79` -- Impact: Any attacker who can reach the service before the legitimate operator completes setup can create the first admin account and fully compromise the instance. - -#### Evidence - -`auth_middleware` exempts the full `/api/setup` namespace from authentication: - -- `src/server/middleware.rs:80-86` - -`setup_complete_handler` only checks `setup_required` and then creates the user, session cookie, and persisted config: - -- `src/server/wizard.rs:95-210` - -The server binds to all interfaces by default: - -- `src/server/mod.rs:176-197` - -The documented Docker quick-start publishes port `3000` directly: - -- `README.md:61-79` - -#### Why This Is Exploitable - -On a fresh install, or any run where `setup_required == true`, the application accepts unauthenticated requests to `/api/setup/complete`. Because the listener binds `0.0.0.0`, that endpoint is reachable from any network that can reach the host unless an external firewall or reverse proxy blocks it. - -That lets a remote attacker: - -1. POST their own username and password to `/api/setup/complete` -2. Receive the initial authenticated session cookie -3. Persist attacker-controlled configuration and start operating as the admin user - -This is a full-authentication-bypass takeover of the instance during bootstrap. - -#### Recommended Fix - -Require setup completion to come only from a trusted local origin during bootstrap, matching the stricter treatment already used for `/api/fs/*` during setup. - -Minimal safe options: - -1. Restrict `/api/setup/*` and `/api/settings/bundle` to loopback-only while `setup_required == true`. -2. Alternatively require an explicit one-time bootstrap secret/token generated on startup and printed locally. -3. Consider binding to `127.0.0.1` by default until setup is complete, then allowing an explicit public bind only after bootstrap. - -#### Mitigation Until Fixed - -- Do not expose the service to any network before setup is completed. -- Do not publish the container port directly on untrusted networks. -- Complete setup only through a local-only tunnel or host firewall rule. - -## High Findings - -### ALCH-SEC-002 - -- Severity: High -- Location: - - `src/server/middleware.rs:116-117` - - `src/server/settings.rs:244-285` - - `src/config.rs:366-390` - - `src/main.rs:369-383` - - `src/db.rs:2566-2571` -- Impact: During setup mode, an unauthenticated remote attacker can read and overwrite the full runtime configuration; after `--reset-auth`, this can expose existing notification endpoints/tokens and let the attacker reconfigure the instance before the operator reclaims it. - -#### Evidence - -While `setup_required == true`, `auth_middleware` explicitly allows `/api/settings/bundle` without authentication: - -- `src/server/middleware.rs:116-117` - -`get_settings_bundle_handler` returns the full `Config`, and `update_settings_bundle_handler` writes an attacker-supplied `Config` back to disk and runtime state: - -- `src/server/settings.rs:244-285` - -The config structure includes notification targets and optional `auth_token` fields: - -- `src/config.rs:366-390` - -`--reset-auth` only clears users and sessions, then re-enters setup mode: - -- `src/main.rs:369-383` -- `src/db.rs:2566-2571` - -#### Why This Is Exploitable - -This endpoint is effectively a public config API whenever the app is in setup mode. On a brand-new install that broadens the same bootstrap attack surface as ALCH-SEC-001. On an existing deployment where an operator runs `--reset-auth`, the previous configuration remains on disk while authentication is removed, so a remote caller can: - -1. GET `/api/settings/bundle` and read the current config -2. Learn configured paths, schedules, webhook targets, and any stored notification bearer tokens -3. PUT a replacement config before the legitimate operator finishes recovery - -That creates both confidential-data exposure and unauthenticated remote reconfiguration during recovery/bootstrap windows. - -#### Recommended Fix - -Do not expose `/api/settings/bundle` anonymously. - -Safer options: - -1. Apply the same loopback-only setup restriction used for `/api/fs/*`. -2. Split bootstrap-safe fields from privileged configuration and expose only the minimal bootstrap payload anonymously. -3. Redact secret-bearing config fields such as notification tokens from any unauthenticated response path. - -## Notes - -- I did not find a major DOM-XSS path in `web/src`; there were no `dangerouslySetInnerHTML`, `innerHTML`, `insertAdjacentHTML`, `eval`, or similar high-risk sinks in the audited code paths. -- I also did not see obvious raw SQL string interpolation issues; the database code I reviewed uses parameter binding. diff --git a/src/config.rs b/src/config.rs index de48984..b2190d6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -82,12 +82,12 @@ impl QualityProfile { } } - /// Get FFmpeg quality value for Apple VideoToolbox + /// Get FFmpeg quality value for Apple VideoToolbox (-q:v 1-100, lower is better) pub fn videotoolbox_quality(&self) -> &'static str { match self { - Self::Quality => "55", - Self::Balanced => "65", - Self::Speed => "75", + Self::Quality => "24", + Self::Balanced => "28", + Self::Speed => "32", } } } @@ -684,6 +684,13 @@ pub struct SystemConfig { /// Enable HSTS header (only enable if running behind HTTPS) #[serde(default)] pub https_only: bool, + /// Explicit list of reverse proxy IPs (e.g. "192.168.1.1") whose + /// X-Forwarded-For / X-Real-IP headers are trusted. When non-empty, + /// only these IPs (plus loopback) are trusted as proxies; private + /// ranges are no longer trusted by default. Leave empty to preserve + /// the previous behaviour (trust all RFC-1918 private addresses). + #[serde(default)] + pub trusted_proxies: Vec, } fn default_true() -> bool { @@ -710,6 +717,7 @@ impl Default for SystemConfig { log_retention_days: default_log_retention_days(), engine_mode: EngineMode::default(), https_only: false, + trusted_proxies: Vec::new(), } } } @@ -825,6 +833,7 @@ impl Default for Config { log_retention_days: default_log_retention_days(), engine_mode: EngineMode::default(), https_only: false, + trusted_proxies: Vec::new(), }, } } diff --git a/src/conversion.rs b/src/conversion.rs index e152f61..c426c49 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -442,6 +442,7 @@ fn build_rate_control(mode: &str, value: Option, encoder: Encoder) -> Resul match encoder.backend() { EncoderBackend::Qsv => Ok(RateControl::QsvQuality { value: quality }), EncoderBackend::Cpu => Ok(RateControl::Crf { value: quality }), + EncoderBackend::Videotoolbox => Ok(RateControl::Cq { value: quality }), _ => Ok(RateControl::Cq { value: quality }), } } diff --git a/src/db/config.rs b/src/db/config.rs new file mode 100644 index 0000000..0e715ce --- /dev/null +++ b/src/db/config.rs @@ -0,0 +1,584 @@ +use crate::error::Result; +use sqlx::Row; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use super::types::*; +use super::Db; + +impl Db { + pub async fn get_watch_dirs(&self) -> Result> { + let has_is_recursive = self.watch_dir_flags.has_is_recursive; + let has_recursive = self.watch_dir_flags.has_recursive; + let has_enabled = self.watch_dir_flags.has_enabled; + let has_profile_id = self.watch_dir_flags.has_profile_id; + + let recursive_expr = if has_is_recursive { + "is_recursive" + } else if has_recursive { + "recursive" + } else { + "1" + }; + + let enabled_filter = if has_enabled { + "WHERE enabled = 1 " + } else { + "" + }; + let profile_expr = if has_profile_id { "profile_id" } else { "NULL" }; + let query = format!( + "SELECT id, path, {} as is_recursive, {} as profile_id, created_at + FROM watch_dirs {}ORDER BY path ASC", + recursive_expr, profile_expr, enabled_filter + ); + + let dirs = sqlx::query_as::<_, WatchDir>(&query) + .fetch_all(&self.pool) + .await?; + Ok(dirs) + } + + pub async fn add_watch_dir(&self, path: &str, is_recursive: bool) -> Result { + let has_is_recursive = self.watch_dir_flags.has_is_recursive; + let has_recursive = self.watch_dir_flags.has_recursive; + let has_profile_id = self.watch_dir_flags.has_profile_id; + + let row = if has_is_recursive && has_profile_id { + sqlx::query_as::<_, WatchDir>( + "INSERT INTO watch_dirs (path, is_recursive) VALUES (?, ?) + RETURNING id, path, is_recursive, profile_id, created_at", + ) + .bind(path) + .bind(is_recursive) + .fetch_one(&self.pool) + .await? + } else if has_is_recursive { + sqlx::query_as::<_, WatchDir>( + "INSERT INTO watch_dirs (path, is_recursive) VALUES (?, ?) + RETURNING id, path, is_recursive, NULL as profile_id, created_at", + ) + .bind(path) + .bind(is_recursive) + .fetch_one(&self.pool) + .await? + } else if has_recursive && has_profile_id { + sqlx::query_as::<_, WatchDir>( + "INSERT INTO watch_dirs (path, recursive) VALUES (?, ?) + RETURNING id, path, recursive as is_recursive, profile_id, created_at", + ) + .bind(path) + .bind(is_recursive) + .fetch_one(&self.pool) + .await? + } else if has_recursive { + sqlx::query_as::<_, WatchDir>( + "INSERT INTO watch_dirs (path, recursive) VALUES (?, ?) + RETURNING id, path, recursive as is_recursive, NULL as profile_id, created_at", + ) + .bind(path) + .bind(is_recursive) + .fetch_one(&self.pool) + .await? + } else { + sqlx::query_as::<_, WatchDir>( + "INSERT INTO watch_dirs (path) VALUES (?) + RETURNING id, path, 1 as is_recursive, NULL as profile_id, created_at", + ) + .bind(path) + .fetch_one(&self.pool) + .await? + }; + Ok(row) + } + + pub async fn replace_watch_dirs( + &self, + watch_dirs: &[crate::config::WatchDirConfig], + ) -> Result<()> { + let has_is_recursive = self.watch_dir_flags.has_is_recursive; + let has_recursive = self.watch_dir_flags.has_recursive; + let has_profile_id = self.watch_dir_flags.has_profile_id; + let preserved_profiles = if has_profile_id { + let rows = sqlx::query("SELECT path, profile_id FROM watch_dirs") + .fetch_all(&self.pool) + .await?; + rows.into_iter() + .map(|row| { + let path: String = row.get("path"); + let profile_id: Option = row.get("profile_id"); + (path, profile_id) + }) + .collect::>() + } else { + HashMap::new() + }; + let mut tx = self.pool.begin().await?; + sqlx::query("DELETE FROM watch_dirs") + .execute(&mut *tx) + .await?; + for watch_dir in watch_dirs { + let preserved_profile_id = preserved_profiles.get(&watch_dir.path).copied().flatten(); + if has_is_recursive && has_profile_id { + sqlx::query( + "INSERT INTO watch_dirs (path, is_recursive, profile_id) VALUES (?, ?, ?)", + ) + .bind(&watch_dir.path) + .bind(watch_dir.is_recursive) + .bind(preserved_profile_id) + .execute(&mut *tx) + .await?; + } else if has_recursive && has_profile_id { + sqlx::query( + "INSERT INTO watch_dirs (path, recursive, profile_id) VALUES (?, ?, ?)", + ) + .bind(&watch_dir.path) + .bind(watch_dir.is_recursive) + .bind(preserved_profile_id) + .execute(&mut *tx) + .await?; + } else if has_recursive { + sqlx::query("INSERT INTO watch_dirs (path, recursive) VALUES (?, ?)") + .bind(&watch_dir.path) + .bind(watch_dir.is_recursive) + .execute(&mut *tx) + .await?; + } else { + sqlx::query("INSERT INTO watch_dirs (path) VALUES (?)") + .bind(&watch_dir.path) + .execute(&mut *tx) + .await?; + } + } + tx.commit().await?; + Ok(()) + } + + pub async fn remove_watch_dir(&self, id: i64) -> Result<()> { + let res = sqlx::query("DELETE FROM watch_dirs WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + if res.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + Ok(()) + } + + pub async fn get_all_profiles(&self) -> Result> { + let profiles = sqlx::query_as::<_, LibraryProfile>( + "SELECT id, name, preset, codec, quality_profile, hdr_mode, audio_mode, + crf_override, notes, created_at, updated_at + FROM library_profiles + ORDER BY id ASC", + ) + .fetch_all(&self.pool) + .await?; + Ok(profiles) + } + + pub async fn get_profile(&self, id: i64) -> Result> { + let profile = sqlx::query_as::<_, LibraryProfile>( + "SELECT id, name, preset, codec, quality_profile, hdr_mode, audio_mode, + crf_override, notes, created_at, updated_at + FROM library_profiles + WHERE id = ?", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(profile) + } + + pub async fn create_profile(&self, profile: NewLibraryProfile) -> Result { + let id = sqlx::query( + "INSERT INTO library_profiles + (name, preset, codec, quality_profile, hdr_mode, audio_mode, crf_override, notes, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)", + ) + .bind(profile.name) + .bind(profile.preset) + .bind(profile.codec) + .bind(profile.quality_profile) + .bind(profile.hdr_mode) + .bind(profile.audio_mode) + .bind(profile.crf_override) + .bind(profile.notes) + .execute(&self.pool) + .await? + .last_insert_rowid(); + Ok(id) + } + + pub async fn update_profile(&self, id: i64, profile: NewLibraryProfile) -> Result<()> { + let result = sqlx::query( + "UPDATE library_profiles + SET name = ?, + preset = ?, + codec = ?, + quality_profile = ?, + hdr_mode = ?, + audio_mode = ?, + crf_override = ?, + notes = ?, + updated_at = CURRENT_TIMESTAMP + WHERE id = ?", + ) + .bind(profile.name) + .bind(profile.preset) + .bind(profile.codec) + .bind(profile.quality_profile) + .bind(profile.hdr_mode) + .bind(profile.audio_mode) + .bind(profile.crf_override) + .bind(profile.notes) + .bind(id) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + + Ok(()) + } + + pub async fn delete_profile(&self, id: i64) -> Result<()> { + let result = sqlx::query("DELETE FROM library_profiles WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + Ok(()) + } + + pub async fn assign_profile_to_watch_dir( + &self, + dir_id: i64, + profile_id: Option, + ) -> Result<()> { + let result = sqlx::query( + "UPDATE watch_dirs + SET profile_id = ? + WHERE id = ?", + ) + .bind(profile_id) + .bind(dir_id) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + + Ok(()) + } + + pub async fn get_profile_for_path(&self, path: &str) -> Result> { + let normalized = Path::new(path); + let candidate = sqlx::query_as::<_, LibraryProfile>( + "SELECT lp.id, lp.name, lp.preset, lp.codec, lp.quality_profile, lp.hdr_mode, + lp.audio_mode, lp.crf_override, lp.notes, lp.created_at, lp.updated_at + FROM watch_dirs wd + JOIN library_profiles lp ON lp.id = wd.profile_id + WHERE wd.profile_id IS NOT NULL + AND (? = wd.path OR ? LIKE wd.path || '/%' OR ? LIKE wd.path || '\\%') + ORDER BY LENGTH(wd.path) DESC + LIMIT 1", + ) + .bind(path) + .bind(path) + .bind(path) + .fetch_optional(&self.pool) + .await?; + + if candidate.is_some() { + return Ok(candidate); + } + + // SQLite prefix matching is a fast first pass; fall back to strict path ancestry + // if separators or normalization differ. + let rows = sqlx::query( + "SELECT wd.path, + lp.id, lp.name, lp.preset, lp.codec, lp.quality_profile, lp.hdr_mode, + lp.audio_mode, lp.crf_override, lp.notes, lp.created_at, lp.updated_at + FROM watch_dirs wd + JOIN library_profiles lp ON lp.id = wd.profile_id + WHERE wd.profile_id IS NOT NULL", + ) + .fetch_all(&self.pool) + .await?; + + let mut best: Option<(usize, LibraryProfile)> = None; + for row in rows { + let watch_path: String = row.get("path"); + let profile = LibraryProfile { + id: row.get("id"), + name: row.get("name"), + preset: row.get("preset"), + codec: row.get("codec"), + quality_profile: row.get("quality_profile"), + hdr_mode: row.get("hdr_mode"), + audio_mode: row.get("audio_mode"), + crf_override: row.get("crf_override"), + notes: row.get("notes"), + created_at: row.get("created_at"), + updated_at: row.get("updated_at"), + }; + let watch_path_buf = PathBuf::from(&watch_path); + if normalized == watch_path_buf || normalized.starts_with(&watch_path_buf) { + let score = watch_path.len(); + if best + .as_ref() + .is_none_or(|(best_score, _)| score > *best_score) + { + best = Some((score, profile)); + } + } + } + + Ok(best.map(|(_, profile)| profile)) + } + + pub async fn count_watch_dirs_using_profile(&self, profile_id: i64) -> Result { + let row: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM watch_dirs WHERE profile_id = ?") + .bind(profile_id) + .fetch_one(&self.pool) + .await?; + Ok(row.0) + } + + pub async fn get_notification_targets(&self) -> Result> { + let targets = sqlx::query_as::<_, NotificationTarget>( + "SELECT id, name, target_type, config_json, events, enabled, created_at FROM notification_targets", + ) + .fetch_all(&self.pool) + .await?; + Ok(targets) + } + + pub async fn add_notification_target( + &self, + name: &str, + target_type: &str, + config_json: &str, + events: &str, + enabled: bool, + ) -> Result { + let row = sqlx::query_as::<_, NotificationTarget>( + "INSERT INTO notification_targets (name, target_type, config_json, events, enabled) + VALUES (?, ?, ?, ?, ?) RETURNING *", + ) + .bind(name) + .bind(target_type) + .bind(config_json) + .bind(events) + .bind(enabled) + .fetch_one(&self.pool) + .await?; + Ok(row) + } + + pub async fn delete_notification_target(&self, id: i64) -> Result<()> { + let res = sqlx::query("DELETE FROM notification_targets WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + if res.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + Ok(()) + } + + pub async fn replace_notification_targets( + &self, + targets: &[crate::config::NotificationTargetConfig], + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query("DELETE FROM notification_targets") + .execute(&mut *tx) + .await?; + for target in targets { + sqlx::query( + "INSERT INTO notification_targets (name, target_type, config_json, events, enabled) VALUES (?, ?, ?, ?, ?)", + ) + .bind(&target.name) + .bind(&target.target_type) + .bind(target.config_json.to_string()) + .bind(serde_json::to_string(&target.events).unwrap_or_else(|_| "[]".to_string())) + .bind(target.enabled) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + pub async fn get_schedule_windows(&self) -> Result> { + let windows = sqlx::query_as::<_, ScheduleWindow>("SELECT * FROM schedule_windows") + .fetch_all(&self.pool) + .await?; + Ok(windows) + } + + pub async fn add_schedule_window( + &self, + start_time: &str, + end_time: &str, + days_of_week: &str, + enabled: bool, + ) -> Result { + let row = sqlx::query_as::<_, ScheduleWindow>( + "INSERT INTO schedule_windows (start_time, end_time, days_of_week, enabled) + VALUES (?, ?, ?, ?) + RETURNING *", + ) + .bind(start_time) + .bind(end_time) + .bind(days_of_week) + .bind(enabled) + .fetch_one(&self.pool) + .await?; + Ok(row) + } + + pub async fn delete_schedule_window(&self, id: i64) -> Result<()> { + let res = sqlx::query("DELETE FROM schedule_windows WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + + if res.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + Ok(()) + } + + pub async fn replace_schedule_windows( + &self, + windows: &[crate::config::ScheduleWindowConfig], + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query("DELETE FROM schedule_windows") + .execute(&mut *tx) + .await?; + for window in windows { + sqlx::query( + "INSERT INTO schedule_windows (start_time, end_time, days_of_week, enabled) VALUES (?, ?, ?, ?)", + ) + .bind(&window.start_time) + .bind(&window.end_time) + .bind(serde_json::to_string(&window.days_of_week).unwrap_or_else(|_| "[]".to_string())) + .bind(window.enabled) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; + Ok(()) + } + + pub async fn get_file_settings(&self) -> Result { + // Migration ensures row 1 exists, but we handle missing just in case + let row = sqlx::query_as::<_, FileSettings>("SELECT * FROM file_settings WHERE id = 1") + .fetch_optional(&self.pool) + .await?; + + match row { + Some(s) => Ok(s), + None => { + // If missing (shouldn't happen), return default + Ok(FileSettings { + id: 1, + delete_source: false, + output_extension: "mkv".to_string(), + output_suffix: "-alchemist".to_string(), + replace_strategy: "keep".to_string(), + output_root: None, + }) + } + } + } + + pub async fn update_file_settings( + &self, + delete_source: bool, + output_extension: &str, + output_suffix: &str, + replace_strategy: &str, + output_root: Option<&str>, + ) -> Result { + let row = sqlx::query_as::<_, FileSettings>( + "UPDATE file_settings + SET delete_source = ?, output_extension = ?, output_suffix = ?, replace_strategy = ?, output_root = ? + WHERE id = 1 + RETURNING *", + ) + .bind(delete_source) + .bind(output_extension) + .bind(output_suffix) + .bind(replace_strategy) + .bind(output_root) + .fetch_one(&self.pool) + .await?; + Ok(row) + } + + pub async fn replace_file_settings_projection( + &self, + settings: &crate::config::FileSettingsConfig, + ) -> Result { + self.update_file_settings( + settings.delete_source, + &settings.output_extension, + &settings.output_suffix, + &settings.replace_strategy, + settings.output_root.as_deref(), + ) + .await + } + + /// Set UI preference + pub async fn set_preference(&self, key: &str, value: &str) -> Result<()> { + sqlx::query( + "INSERT INTO ui_preferences (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP", + ) + .bind(key) + .bind(value) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Get UI preference + pub async fn get_preference(&self, key: &str) -> Result> { + let row: Option<(String,)> = + sqlx::query_as("SELECT value FROM ui_preferences WHERE key = ?") + .bind(key) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.0)) + } + + pub async fn delete_preference(&self, key: &str) -> Result<()> { + sqlx::query("DELETE FROM ui_preferences WHERE key = ?") + .bind(key) + .execute(&self.pool) + .await?; + Ok(()) + } +} diff --git a/src/db/conversion.rs b/src/db/conversion.rs new file mode 100644 index 0000000..1b891c2 --- /dev/null +++ b/src/db/conversion.rs @@ -0,0 +1,152 @@ +use crate::error::Result; + +use super::types::*; +use super::Db; + +impl Db { + pub async fn create_conversion_job( + &self, + upload_path: &str, + mode: &str, + settings_json: &str, + probe_json: Option<&str>, + expires_at: &str, + ) -> Result { + let row = sqlx::query_as::<_, ConversionJob>( + "INSERT INTO conversion_jobs (upload_path, mode, settings_json, probe_json, expires_at) + VALUES (?, ?, ?, ?, ?) + RETURNING *", + ) + .bind(upload_path) + .bind(mode) + .bind(settings_json) + .bind(probe_json) + .bind(expires_at) + .fetch_one(&self.pool) + .await?; + Ok(row) + } + + pub async fn get_conversion_job(&self, id: i64) -> Result> { + let row = sqlx::query_as::<_, ConversionJob>( + "SELECT id, upload_path, output_path, mode, settings_json, probe_json, linked_job_id, status, expires_at, downloaded_at, created_at, updated_at + FROM conversion_jobs + WHERE id = ?", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + Ok(row) + } + + pub async fn get_conversion_job_by_linked_job_id( + &self, + linked_job_id: i64, + ) -> Result> { + let row = sqlx::query_as::<_, ConversionJob>( + "SELECT id, upload_path, output_path, mode, settings_json, probe_json, linked_job_id, status, expires_at, downloaded_at, created_at, updated_at + FROM conversion_jobs + WHERE linked_job_id = ?", + ) + .bind(linked_job_id) + .fetch_optional(&self.pool) + .await?; + Ok(row) + } + + pub async fn update_conversion_job_probe(&self, id: i64, probe_json: &str) -> Result<()> { + sqlx::query( + "UPDATE conversion_jobs + SET probe_json = ?, updated_at = datetime('now') + WHERE id = ?", + ) + .bind(probe_json) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn update_conversion_job_settings( + &self, + id: i64, + settings_json: &str, + mode: &str, + ) -> Result<()> { + sqlx::query( + "UPDATE conversion_jobs + SET settings_json = ?, mode = ?, updated_at = datetime('now') + WHERE id = ?", + ) + .bind(settings_json) + .bind(mode) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn update_conversion_job_start( + &self, + id: i64, + output_path: &str, + linked_job_id: i64, + ) -> Result<()> { + sqlx::query( + "UPDATE conversion_jobs + SET output_path = ?, linked_job_id = ?, status = 'queued', updated_at = datetime('now') + WHERE id = ?", + ) + .bind(output_path) + .bind(linked_job_id) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn update_conversion_job_status(&self, id: i64, status: &str) -> Result<()> { + sqlx::query( + "UPDATE conversion_jobs + SET status = ?, updated_at = datetime('now') + WHERE id = ?", + ) + .bind(status) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn mark_conversion_job_downloaded(&self, id: i64) -> Result<()> { + sqlx::query( + "UPDATE conversion_jobs + SET downloaded_at = datetime('now'), status = 'downloaded', updated_at = datetime('now') + WHERE id = ?", + ) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn delete_conversion_job(&self, id: i64) -> Result<()> { + sqlx::query("DELETE FROM conversion_jobs WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_expired_conversion_jobs(&self, now: &str) -> Result> { + let rows = sqlx::query_as::<_, ConversionJob>( + "SELECT id, upload_path, output_path, mode, settings_json, probe_json, linked_job_id, status, expires_at, downloaded_at, created_at, updated_at + FROM conversion_jobs + WHERE expires_at <= ?", + ) + .bind(now) + .fetch_all(&self.pool) + .await?; + Ok(rows) + } +} diff --git a/src/db/events.rs b/src/db/events.rs new file mode 100644 index 0000000..677d297 --- /dev/null +++ b/src/db/events.rs @@ -0,0 +1,54 @@ +use crate::explanations::Explanation; +use serde::{Deserialize, Serialize}; + +use super::types::JobState; + +// Typed event channels for separating high-volume vs low-volume events +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", content = "data")] +pub enum JobEvent { + StateChanged { + job_id: i64, + status: JobState, + }, + Progress { + job_id: i64, + percentage: f64, + time: String, + }, + Decision { + job_id: i64, + action: String, + reason: String, + explanation: Option, + }, + Log { + level: String, + job_id: Option, + message: String, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", content = "data")] +pub enum ConfigEvent { + Updated(Box), + WatchFolderAdded(String), + WatchFolderRemoved(String), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", content = "data")] +pub enum SystemEvent { + ScanStarted, + ScanCompleted, + EngineIdle, + EngineStatusChanged, + HardwareStateChanged, +} + +pub struct EventChannels { + pub jobs: tokio::sync::broadcast::Sender, // 1000 capacity - high volume + pub config: tokio::sync::broadcast::Sender, // 50 capacity - rare + pub system: tokio::sync::broadcast::Sender, // 100 capacity - medium +} diff --git a/src/db/jobs.rs b/src/db/jobs.rs new file mode 100644 index 0000000..f0aecaf --- /dev/null +++ b/src/db/jobs.rs @@ -0,0 +1,1076 @@ +use crate::error::Result; +use crate::explanations::{ + Explanation, decision_from_legacy, explanation_from_json, explanation_to_json, + failure_from_summary, +}; +use sqlx::Row; +use std::collections::HashMap; +use std::path::Path; + +use super::timed_query; +use super::types::*; +use super::Db; + +impl Db { + pub async fn reset_interrupted_jobs(&self) -> Result { + let result = sqlx::query( + "UPDATE jobs + SET status = 'queued', + progress = 0.0, + updated_at = CURRENT_TIMESTAMP + WHERE status IN ('encoding', 'analyzing', 'remuxing', 'resuming') AND archived = 0", + ) + .execute(&self.pool) + .await?; + + Ok(result.rows_affected()) + } + + pub async fn enqueue_job( + &self, + input_path: &Path, + output_path: &Path, + mtime: std::time::SystemTime, + ) -> Result { + if input_path == output_path { + return Err(crate::error::AlchemistError::Config( + "Output path matches input path".into(), + )); + } + let input_str = input_path + .to_str() + .ok_or_else(|| crate::error::AlchemistError::Config("Invalid input path".into()))?; + let output_str = output_path + .to_str() + .ok_or_else(|| crate::error::AlchemistError::Config("Invalid output path".into()))?; + + // Stable mtime representation (seconds + nanos) + let mtime_hash = match mtime.duration_since(std::time::UNIX_EPOCH) { + Ok(d) => format!("{}.{:09}", d.as_secs(), d.subsec_nanos()), + Err(_) => "0.0".to_string(), // Fallback for very old files/clocks + }; + + let result = sqlx::query( + "INSERT INTO jobs (input_path, output_path, status, mtime_hash, updated_at) + VALUES (?, ?, 'queued', ?, CURRENT_TIMESTAMP) + ON CONFLICT(input_path) DO UPDATE SET + output_path = excluded.output_path, + status = CASE WHEN mtime_hash != excluded.mtime_hash THEN 'queued' ELSE status END, + archived = 0, + mtime_hash = excluded.mtime_hash, + updated_at = CURRENT_TIMESTAMP + WHERE mtime_hash != excluded.mtime_hash OR output_path != excluded.output_path", + ) + .bind(input_str) + .bind(output_str) + .bind(mtime_hash) + .execute(&self.pool) + .await?; + + Ok(result.rows_affected() > 0) + } + + pub async fn add_job(&self, job: Job) -> Result<()> { + sqlx::query( + "INSERT INTO jobs (input_path, output_path, status, mtime_hash, priority, progress, attempt_count, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(job.input_path) + .bind(job.output_path) + .bind(job.status) + .bind("0.0") + .bind(job.priority) + .bind(job.progress) + .bind(job.attempt_count) + .bind(job.created_at) + .bind(job.updated_at) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_next_job(&self) -> Result> { + let job = sqlx::query_as::<_, Job>( + "SELECT id, input_path, output_path, status, NULL as decision_reason, + COALESCE(priority, 0) as priority, COALESCE(CAST(progress AS REAL), 0.0) as progress, + COALESCE(attempt_count, 0) as attempt_count, + NULL as vmaf_score, + created_at, updated_at, + input_metadata_json + FROM jobs + WHERE status = 'queued' + AND archived = 0 + AND ( + COALESCE(attempt_count, 0) = 0 + OR CASE + WHEN COALESCE(attempt_count, 0) = 1 THEN datetime(updated_at, '+5 minutes') + WHEN COALESCE(attempt_count, 0) = 2 THEN datetime(updated_at, '+15 minutes') + WHEN COALESCE(attempt_count, 0) = 3 THEN datetime(updated_at, '+60 minutes') + ELSE datetime(updated_at, '+360 minutes') + END <= datetime('now') + ) + ORDER BY priority DESC, created_at ASC LIMIT 1", + ) + .fetch_optional(&self.pool) + .await?; + + Ok(job) + } + + pub async fn claim_next_job(&self) -> Result> { + let job = sqlx::query_as::<_, Job>( + "UPDATE jobs + SET status = 'analyzing', updated_at = CURRENT_TIMESTAMP + WHERE id = ( + SELECT id + FROM jobs + WHERE status = 'queued' + AND archived = 0 + AND ( + COALESCE(attempt_count, 0) = 0 + OR CASE + WHEN COALESCE(attempt_count, 0) = 1 THEN datetime(updated_at, '+5 minutes') + WHEN COALESCE(attempt_count, 0) = 2 THEN datetime(updated_at, '+15 minutes') + WHEN COALESCE(attempt_count, 0) = 3 THEN datetime(updated_at, '+60 minutes') + ELSE datetime(updated_at, '+360 minutes') + END <= datetime('now') + ) + ORDER BY priority DESC, created_at ASC LIMIT 1 + ) + RETURNING id, input_path, output_path, status, NULL as decision_reason, + COALESCE(priority, 0) as priority, COALESCE(CAST(progress AS REAL), 0.0) as progress, + COALESCE(attempt_count, 0) as attempt_count, + NULL as vmaf_score, + created_at, updated_at, + input_metadata_json", + ) + .fetch_optional(&self.pool) + .await?; + + Ok(job) + } + + pub async fn update_job_status(&self, id: i64, status: JobState) -> Result<()> { + let result = + sqlx::query("UPDATE jobs SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?") + .bind(status) + .bind(id) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + + Ok(()) + } + + pub async fn set_job_input_metadata( + &self, + id: i64, + metadata: &crate::media::pipeline::MediaMetadata, + ) -> Result<()> { + let json = serde_json::to_string(metadata) + .map_err(|e| crate::error::AlchemistError::Unknown(e.to_string()))?; + sqlx::query("UPDATE jobs SET input_metadata_json = ? WHERE id = ?") + .bind(json) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn add_decision_with_explanation( + &self, + job_id: i64, + action: &str, + explanation: &Explanation, + ) -> Result<()> { + sqlx::query( + "INSERT INTO decisions (job_id, action, reason, reason_code, reason_payload_json) + VALUES (?, ?, ?, ?, ?)", + ) + .bind(job_id) + .bind(action) + .bind(&explanation.legacy_reason) + .bind(&explanation.code) + .bind(explanation_to_json(explanation)) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn add_decision(&self, job_id: i64, action: &str, reason: &str) -> Result<()> { + let explanation = decision_from_legacy(action, reason); + self.add_decision_with_explanation(job_id, action, &explanation) + .await + } + + pub async fn get_all_jobs(&self) -> Result> { + let pool = &self.pool; + timed_query("get_all_jobs", || async { + let jobs = sqlx::query_as::<_, Job>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.archived = 0 + ORDER BY j.updated_at DESC", + ) + .fetch_all(pool) + .await?; + + Ok(jobs) + }) + .await + } + + pub async fn get_duplicate_candidates(&self) -> Result> { + timed_query("get_duplicate_candidates", || async { + let all_rows: Vec = sqlx::query_as( + "SELECT id, input_path, status + FROM jobs + WHERE status NOT IN ('cancelled') AND archived = 0 + ORDER BY input_path ASC", + ) + .fetch_all(&self.pool) + .await?; + + let mut filename_counts: std::collections::HashMap = + std::collections::HashMap::new(); + for row in &all_rows { + let filename = Path::new(&row.input_path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + if !filename.is_empty() { + *filename_counts.entry(filename).or_insert(0) += 1; + } + } + + let duplicates = all_rows + .into_iter() + .filter(|row| { + let filename = Path::new(&row.input_path) + .file_name() + .map(|n| n.to_string_lossy().to_string()) + .unwrap_or_default(); + filename_counts.get(&filename).copied().unwrap_or(0) > 1 + }) + .collect(); + + Ok(duplicates) + }) + .await + } + + pub async fn get_job_decision(&self, job_id: i64) -> Result> { + let decision = sqlx::query_as::<_, Decision>( + "SELECT id, job_id, action, reason, reason_code, reason_payload_json, created_at + FROM decisions + WHERE job_id = ? + ORDER BY created_at DESC, id DESC + LIMIT 1", + ) + .bind(job_id) + .fetch_optional(&self.pool) + .await?; + + Ok(decision) + } + + pub async fn get_job_decision_explanation(&self, job_id: i64) -> Result> { + let row = sqlx::query_as::<_, DecisionRecord>( + "SELECT job_id, action, reason, reason_payload_json + FROM decisions + WHERE job_id = ? + ORDER BY created_at DESC, id DESC + LIMIT 1", + ) + .bind(job_id) + .fetch_optional(&self.pool) + .await?; + + Ok(row.map(|row| { + row.reason_payload_json + .as_deref() + .and_then(explanation_from_json) + .unwrap_or_else(|| decision_from_legacy(&row.action, &row.reason)) + })) + } + + pub async fn get_job_decision_explanations( + &self, + job_ids: &[i64], + ) -> Result> { + if job_ids.is_empty() { + return Ok(HashMap::new()); + } + + let mut qb = sqlx::QueryBuilder::::new( + "SELECT d.job_id, d.action, d.reason, d.reason_payload_json + FROM decisions d + INNER JOIN (SELECT job_id, MAX(id) AS max_id FROM decisions WHERE job_id IN (", + ); + let mut separated = qb.separated(", "); + for job_id in job_ids { + separated.push_bind(job_id); + } + separated.push_unseparated(") GROUP BY job_id) latest ON latest.max_id = d.id"); + + let rows = qb + .build_query_as::() + .fetch_all(&self.pool) + .await?; + + Ok(rows + .into_iter() + .map(|row| { + let explanation = row + .reason_payload_json + .as_deref() + .and_then(explanation_from_json) + .unwrap_or_else(|| decision_from_legacy(&row.action, &row.reason)); + (row.job_id, explanation) + }) + .collect()) + } + + pub async fn upsert_job_failure_explanation( + &self, + job_id: i64, + explanation: &Explanation, + ) -> Result<()> { + sqlx::query( + "INSERT INTO job_failure_explanations (job_id, legacy_summary, code, payload_json, updated_at) + VALUES (?, ?, ?, ?, datetime('now')) + ON CONFLICT(job_id) DO UPDATE SET + legacy_summary = excluded.legacy_summary, + code = excluded.code, + payload_json = excluded.payload_json, + updated_at = datetime('now')", + ) + .bind(job_id) + .bind(&explanation.legacy_reason) + .bind(&explanation.code) + .bind(explanation_to_json(explanation)) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn get_job_failure_explanation(&self, job_id: i64) -> Result> { + let row = sqlx::query_as::<_, FailureExplanationRecord>( + "SELECT legacy_summary, code, payload_json + FROM job_failure_explanations + WHERE job_id = ?", + ) + .bind(job_id) + .fetch_optional(&self.pool) + .await?; + + Ok(row.map(|row| { + explanation_from_json(&row.payload_json).unwrap_or_else(|| { + failure_from_summary(row.legacy_summary.as_deref().unwrap_or(row.code.as_str())) + }) + })) + } + + /// Update job progress (for resume support) + pub async fn update_job_progress(&self, id: i64, progress: f64) -> Result<()> { + let result = sqlx::query( + "UPDATE jobs SET progress = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?", + ) + .bind(progress) + .bind(id) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + + Ok(()) + } + + /// Set job priority + pub async fn set_job_priority(&self, id: i64, priority: i32) -> Result<()> { + let result = sqlx::query( + "UPDATE jobs SET priority = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?", + ) + .bind(priority) + .bind(id) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + + Ok(()) + } + + /// Increment attempt count + pub async fn increment_attempt_count(&self, id: i64) -> Result<()> { + sqlx::query("UPDATE jobs SET attempt_count = attempt_count + 1 WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn restart_failed_jobs(&self) -> Result { + let result = sqlx::query( + "UPDATE jobs + SET status = 'queued', progress = 0.0, attempt_count = 0, updated_at = CURRENT_TIMESTAMP + WHERE status IN ('failed', 'cancelled') AND archived = 0", + ) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + + /// Get job by ID + pub async fn get_job(&self, id: i64) -> Result> { + let job = sqlx::query_as::<_, Job>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.id = ? AND j.archived = 0", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + + Ok(job) + } + + /// Get jobs by status + pub async fn get_jobs_by_status(&self, status: JobState) -> Result> { + let pool = &self.pool; + timed_query("get_jobs_by_status", || async { + let jobs = sqlx::query_as::<_, Job>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.status = ? AND j.archived = 0 + ORDER BY j.priority DESC, j.created_at ASC", + ) + .bind(status) + .fetch_all(pool) + .await?; + + Ok(jobs) + }) + .await + } + + /// Get jobs with filtering, sorting and pagination + pub async fn get_jobs_filtered(&self, query: JobFilterQuery) -> Result> { + let pool = &self.pool; + timed_query("get_jobs_filtered", || async { + let mut qb = sqlx::QueryBuilder::::new( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + LEFT JOIN encode_stats es ON es.job_id = j.id + WHERE 1 = 1 " + ); + + match query.archived { + Some(true) => { + qb.push(" AND j.archived = 1 "); + } + Some(false) => { + qb.push(" AND j.archived = 0 "); + } + None => {} + } + + if let Some(ref statuses) = query.statuses { + if !statuses.is_empty() { + qb.push(" AND j.status IN ("); + let mut separated = qb.separated(", "); + for status in statuses { + separated.push_bind(*status); + } + separated.push_unseparated(") "); + } + } + + if let Some(ref search) = query.search { + let escaped = search + .replace('\\', "\\\\") + .replace('%', "\\%") + .replace('_', "\\_"); + qb.push(" AND j.input_path LIKE "); + qb.push_bind(format!("%{}%", escaped)); + qb.push(" ESCAPE '\\'"); + } + + qb.push(" ORDER BY "); + let sort_col = match query.sort_by.as_deref() { + Some("created_at") => "j.created_at", + Some("updated_at") => "j.updated_at", + Some("input_path") => "j.input_path", + Some("size") => "COALESCE(es.input_size_bytes, 0)", + _ => "j.updated_at", + }; + qb.push(sort_col); + qb.push(if query.sort_desc { " DESC" } else { " ASC" }); + + qb.push(" LIMIT "); + qb.push_bind(query.limit); + qb.push(" OFFSET "); + qb.push_bind(query.offset); + + let jobs = qb.build_query_as::().fetch_all(pool).await?; + Ok(jobs) + }) + .await + } + + pub async fn batch_cancel_jobs(&self, ids: &[i64]) -> Result { + if ids.is_empty() { + return Ok(0); + } + let mut qb = sqlx::QueryBuilder::::new( + "UPDATE jobs SET status = 'cancelled', updated_at = CURRENT_TIMESTAMP WHERE status IN ('queued', 'analyzing', 'encoding', 'remuxing', 'resuming') AND id IN (", + ); + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + + let result = qb.build().execute(&self.pool).await?; + Ok(result.rows_affected()) + } + + pub async fn batch_delete_jobs(&self, ids: &[i64]) -> Result { + if ids.is_empty() { + return Ok(0); + } + let mut qb = sqlx::QueryBuilder::::new( + "UPDATE jobs SET archived = 1, updated_at = CURRENT_TIMESTAMP WHERE id IN (", + ); + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + + let result = qb.build().execute(&self.pool).await?; + Ok(result.rows_affected()) + } + + pub async fn batch_restart_jobs(&self, ids: &[i64]) -> Result { + if ids.is_empty() { + return Ok(0); + } + let mut qb = sqlx::QueryBuilder::::new( + "UPDATE jobs SET status = 'queued', progress = 0.0, attempt_count = 0, updated_at = CURRENT_TIMESTAMP WHERE id IN (", + ); + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + + let result = qb.build().execute(&self.pool).await?; + Ok(result.rows_affected()) + } + + pub async fn get_job_by_id(&self, id: i64) -> Result> { + let job = sqlx::query_as::<_, Job>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.id = ? AND j.archived = 0", + ) + .bind(id) + .fetch_optional(&self.pool) + .await?; + + Ok(job) + } + + /// Returns the 1-based position of a queued job in the priority queue, + /// or `None` if the job is not currently queued. + pub async fn get_queue_position(&self, job_id: i64) -> Result> { + let row = sqlx::query( + "SELECT priority, created_at FROM jobs WHERE id = ? AND status = 'queued' AND archived = 0", + ) + .bind(job_id) + .fetch_optional(&self.pool) + .await?; + + let Some(row) = row else { + return Ok(None); + }; + + let priority: i64 = row.get("priority"); + let created_at: String = row.get("created_at"); + + let pos: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM jobs + WHERE status = 'queued' + AND archived = 0 + AND ( + priority > ? + OR (priority = ? AND created_at < ?) + )", + ) + .bind(priority) + .bind(priority) + .bind(&created_at) + .fetch_one(&self.pool) + .await?; + + Ok(Some((pos + 1) as u32)) + } + + /// Returns all jobs in queued or failed state that need + /// analysis. Used by the startup auto-analyzer. + pub async fn get_jobs_for_analysis(&self) -> Result> { + timed_query("get_jobs_for_analysis", || async { + let rows: Vec = sqlx::query_as( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.status IN ('queued', 'failed') AND j.archived = 0 + ORDER BY j.priority DESC, j.created_at ASC", + ) + .fetch_all(&self.pool) + .await?; + Ok(rows) + }) + .await + } + + pub async fn get_jobs_for_analysis_batch(&self, offset: i64, limit: i64) -> Result> { + timed_query("get_jobs_for_analysis_batch", || async { + let rows: Vec = sqlx::query_as( + "SELECT j.id, j.input_path, j.output_path, + j.status, + (SELECT reason FROM decisions + WHERE job_id = j.id + ORDER BY created_at DESC LIMIT 1) + as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), + 0.0) as progress, + COALESCE(j.attempt_count, 0) + as attempt_count, + (SELECT vmaf_score FROM encode_stats + WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.status IN ('queued', 'failed') + AND j.archived = 0 + AND NOT EXISTS ( + SELECT 1 FROM decisions d + WHERE d.job_id = j.id + ) + ORDER BY j.priority DESC, j.created_at ASC + LIMIT ? OFFSET ?", + ) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; + Ok(rows) + }) + .await + } + + pub async fn get_jobs_by_ids(&self, ids: &[i64]) -> Result> { + if ids.is_empty() { + return Ok(Vec::new()); + } + + let mut qb = sqlx::QueryBuilder::::new( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.archived = 0 AND j.id IN (", + ); + let mut separated = qb.separated(", "); + for id in ids { + separated.push_bind(id); + } + separated.push_unseparated(")"); + qb.push(" ORDER BY j.updated_at DESC"); + + let jobs = qb.build_query_as::().fetch_all(&self.pool).await?; + Ok(jobs) + } + + pub async fn get_job_by_input_path(&self, path: &str) -> Result> { + let job = sqlx::query_as::<_, Job>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.input_path = ? AND j.archived = 0", + ) + .bind(path) + .fetch_optional(&self.pool) + .await?; + + Ok(job) + } + + pub async fn has_job_with_output_path(&self, path: &str) -> Result { + let row: Option<(i64,)> = + sqlx::query_as("SELECT 1 FROM jobs WHERE output_path = ? AND archived = 0 LIMIT 1") + .bind(path) + .fetch_optional(&self.pool) + .await?; + Ok(row.is_some()) + } + + pub async fn get_jobs_needing_health_check(&self) -> Result> { + let pool = &self.pool; + timed_query("get_jobs_needing_health_check", || async { + let jobs = sqlx::query_as::<_, Job>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, j.input_metadata_json + FROM jobs j + WHERE j.status = 'completed' + AND ( + j.last_health_check IS NULL + OR j.last_health_check < datetime('now', '-7 days') + ) + ORDER BY COALESCE(j.last_health_check, '1970-01-01') ASC, j.updated_at DESC", + ) + .fetch_all(pool) + .await?; + Ok(jobs) + }) + .await + } + + /// Batch update job statuses (for batch operations) + pub async fn batch_update_status( + &self, + status_from: JobState, + status_to: JobState, + ) -> Result { + let result = sqlx::query( + "UPDATE jobs SET status = ?, updated_at = CURRENT_TIMESTAMP WHERE status = ?", + ) + .bind(status_to) + .bind(status_from) + .execute(&self.pool) + .await?; + + Ok(result.rows_affected()) + } + + pub async fn delete_job(&self, id: i64) -> Result<()> { + let result = sqlx::query( + "UPDATE jobs + SET archived = 1, updated_at = CURRENT_TIMESTAMP + WHERE id = ?", + ) + .bind(id) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + use std::time::SystemTime; + + #[tokio::test] + async fn test_enqueue_job_reports_change_state() + -> std::result::Result<(), Box> { + let mut db_path = std::env::temp_dir(); + let token: u64 = rand::random(); + db_path.push(format!("alchemist_enqueue_test_{}.db", token)); + + let db = Db::new(db_path.to_string_lossy().as_ref()).await?; + + let input = Path::new("input.mkv"); + let output = Path::new("output.mkv"); + let changed = db + .enqueue_job(input, output, SystemTime::UNIX_EPOCH) + .await?; + assert!(changed); + + let unchanged = db + .enqueue_job(input, output, SystemTime::UNIX_EPOCH) + .await?; + assert!(!unchanged); + + drop(db); + let _ = std::fs::remove_file(db_path); + Ok(()) + } + + #[tokio::test] + async fn test_claim_next_job_marks_analyzing() + -> std::result::Result<(), Box> { + let mut db_path = std::env::temp_dir(); + let token: u64 = rand::random(); + db_path.push(format!("alchemist_test_{}.db", token)); + + let db = Db::new(db_path.to_string_lossy().as_ref()).await?; + + let input1 = Path::new("input1.mkv"); + let output1 = Path::new("output1.mkv"); + let _ = db + .enqueue_job(input1, output1, SystemTime::UNIX_EPOCH) + .await?; + + let input2 = Path::new("input2.mkv"); + let output2 = Path::new("output2.mkv"); + let _ = db + .enqueue_job(input2, output2, SystemTime::UNIX_EPOCH) + .await?; + + let first = db + .claim_next_job() + .await? + .ok_or_else(|| std::io::Error::other("missing job 1"))?; + assert_eq!(first.status, JobState::Analyzing); + + let second = db + .claim_next_job() + .await? + .ok_or_else(|| std::io::Error::other("missing job 2"))?; + assert_ne!(first.id, second.id); + + let none = db.claim_next_job().await?; + assert!(none.is_none()); + + drop(db); + let _ = std::fs::remove_file(db_path); + Ok(()) + } + + #[tokio::test] + async fn claim_next_job_respects_attempt_backoff() + -> std::result::Result<(), Box> { + let mut db_path = std::env::temp_dir(); + let token: u64 = rand::random(); + db_path.push(format!("alchemist_backoff_test_{}.db", token)); + + let db = Db::new(db_path.to_string_lossy().as_ref()).await?; + let input = Path::new("backoff-input.mkv"); + let output = Path::new("backoff-output.mkv"); + let _ = db + .enqueue_job(input, output, SystemTime::UNIX_EPOCH) + .await?; + + let job = db + .get_job_by_input_path("backoff-input.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing backoff job"))?; + + sqlx::query( + "UPDATE jobs + SET attempt_count = 1, + updated_at = datetime('now') + WHERE id = ?", + ) + .bind(job.id) + .execute(&db.pool) + .await?; + + assert!(db.claim_next_job().await?.is_none()); + + sqlx::query( + "UPDATE jobs + SET updated_at = datetime('now', '-6 minutes') + WHERE id = ?", + ) + .bind(job.id) + .execute(&db.pool) + .await?; + + let claimed = db.claim_next_job().await?; + assert!(claimed.is_some()); + + drop(db); + let _ = std::fs::remove_file(db_path); + Ok(()) + } + + #[tokio::test] + async fn reset_interrupted_jobs_requeues_only_interrupted_states() + -> std::result::Result<(), Box> { + let mut db_path = std::env::temp_dir(); + let token: u64 = rand::random(); + db_path.push(format!("alchemist_reset_interrupted_{}.db", token)); + + let db = Db::new(db_path.to_string_lossy().as_ref()).await?; + + let jobs = [ + ("queued.mkv", "queued-out.mkv", JobState::Queued), + ("analyzing.mkv", "analyzing-out.mkv", JobState::Analyzing), + ("encoding.mkv", "encoding-out.mkv", JobState::Encoding), + ("remuxing.mkv", "remuxing-out.mkv", JobState::Remuxing), + ("cancelled.mkv", "cancelled-out.mkv", JobState::Cancelled), + ("completed.mkv", "completed-out.mkv", JobState::Completed), + ]; + + for (input, output, status) in jobs { + let _ = db + .enqueue_job(Path::new(input), Path::new(output), SystemTime::UNIX_EPOCH) + .await?; + let job = db + .get_job_by_input_path(input) + .await? + .ok_or_else(|| std::io::Error::other("missing seeded job"))?; + db.update_job_status(job.id, status).await?; + } + + let reset = db.reset_interrupted_jobs().await?; + assert_eq!(reset, 3); + + assert_eq!( + db.get_job_by_input_path("analyzing.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing analyzing job"))? + .status, + JobState::Queued + ); + assert_eq!( + db.get_job_by_input_path("encoding.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing encoding job"))? + .status, + JobState::Queued + ); + assert_eq!( + db.get_job_by_input_path("remuxing.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing remuxing job"))? + .status, + JobState::Queued + ); + assert_eq!( + db.get_job_by_input_path("cancelled.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing cancelled job"))? + .status, + JobState::Cancelled + ); + assert_eq!( + db.get_job_by_input_path("completed.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing completed job"))? + .status, + JobState::Completed + ); + + drop(db); + let _ = std::fs::remove_file(db_path); + Ok(()) + } + + #[tokio::test] + async fn legacy_decision_rows_still_parse_into_structured_explanations() + -> std::result::Result<(), Box> { + let mut db_path = std::env::temp_dir(); + let token: u64 = rand::random(); + db_path.push(format!("alchemist_legacy_decision_test_{}.db", token)); + + let db = Db::new(db_path.to_string_lossy().as_ref()).await?; + let _ = db + .enqueue_job( + Path::new("legacy-input.mkv"), + Path::new("legacy-output.mkv"), + SystemTime::UNIX_EPOCH, + ) + .await?; + let job = db + .get_job_by_input_path("legacy-input.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing job"))?; + + sqlx::query( + "INSERT INTO decisions (job_id, action, reason, reason_code, reason_payload_json) + VALUES (?, 'skip', 'bpp_below_threshold|bpp=0.043,threshold=0.050', NULL, NULL)", + ) + .bind(job.id) + .execute(&db.pool) + .await?; + + let explanation = db + .get_job_decision_explanation(job.id) + .await? + .ok_or_else(|| std::io::Error::other("missing explanation"))?; + assert_eq!(explanation.code, "bpp_below_threshold"); + assert_eq!( + explanation.measured.get("bpp"), + Some(&serde_json::json!(0.043)) + ); + + drop(db); + let _ = std::fs::remove_file(db_path); + Ok(()) + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs new file mode 100644 index 0000000..1419d0f --- /dev/null +++ b/src/db/mod.rs @@ -0,0 +1,148 @@ +mod config; +mod conversion; +mod events; +mod jobs; +mod stats; +mod system; +mod types; + +pub use events::*; +pub use types::*; + +use crate::error::{AlchemistError, Result}; +use sha2::{Digest, Sha256}; +use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode}; +use sqlx::SqlitePool; +use std::time::Duration; +use tokio::time::timeout; +use tracing::info; + +/// Default timeout for potentially slow database queries +pub(crate) const QUERY_TIMEOUT: Duration = Duration::from_secs(5); + +/// Execute a query with a timeout to prevent blocking the job loop +pub(crate) async fn timed_query(operation: &str, f: F) -> Result +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + match timeout(QUERY_TIMEOUT, f()).await { + Ok(result) => result, + Err(_) => Err(AlchemistError::QueryTimeout( + QUERY_TIMEOUT.as_secs(), + operation.to_string(), + )), + } +} + +#[derive(Clone, Debug)] +pub(crate) struct WatchDirSchemaFlags { + has_is_recursive: bool, + has_recursive: bool, + has_enabled: bool, + has_profile_id: bool, +} + +#[derive(Clone, Debug)] +pub struct Db { + pub(crate) pool: SqlitePool, + pub(crate) watch_dir_flags: std::sync::Arc, +} + +impl Db { + pub async fn new(db_path: &str) -> Result { + let start = std::time::Instant::now(); + let options = SqliteConnectOptions::new() + .filename(db_path) + .create_if_missing(true) + .foreign_keys(true) + .journal_mode(SqliteJournalMode::Wal) + .busy_timeout(Duration::from_secs(5)); + + let pool = sqlx::sqlite::SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await?; + info!( + target: "startup", + "Database connection opened in {} ms", + start.elapsed().as_millis() + ); + + // Run migrations + let migrate_start = std::time::Instant::now(); + sqlx::migrate!("./migrations") + .run(&pool) + .await + .map_err(|e| crate::error::AlchemistError::Database(e.into()))?; + info!( + target: "startup", + "Database migrations completed in {} ms", + migrate_start.elapsed().as_millis() + ); + + // Cache watch_dirs schema flags once at startup to avoid repeated PRAGMA queries. + let check = |column: &str| { + let pool = pool.clone(); + let column = column.to_string(); + async move { + let row = + sqlx::query("SELECT name FROM pragma_table_info('watch_dirs') WHERE name = ?") + .bind(&column) + .fetch_optional(&pool) + .await + .unwrap_or(None); + row.is_some() + } + }; + let watch_dir_flags = WatchDirSchemaFlags { + has_is_recursive: check("is_recursive").await, + has_recursive: check("recursive").await, + has_enabled: check("enabled").await, + has_profile_id: check("profile_id").await, + }; + + Ok(Self { + pool, + watch_dir_flags: std::sync::Arc::new(watch_dir_flags), + }) + } +} + +/// Hash a session token using SHA256 for secure storage. +/// +/// # Security: Timing Attack Resistance +/// +/// Session tokens are hashed before storage and lookup. Token validation uses +/// SQL `WHERE token = ?` with the hashed value, so the comparison occurs in +/// SQLite rather than in Rust code. This is inherently constant-time from the +/// application's perspective because: +/// 1. The database performs the comparison, not our code +/// 2. Database query time doesn't leak information about partial matches +/// 3. No early-exit comparison in application code +/// +/// This design makes timing attacks infeasible without requiring the `subtle` +/// crate for constant-time comparison. +pub(crate) fn hash_session_token(token: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(token.as_bytes()); + let digest = hasher.finalize(); + let mut out = String::with_capacity(64); + for byte in digest { + use std::fmt::Write; + let _ = write!(&mut out, "{:02x}", byte); + } + out +} + +pub fn hash_api_token(token: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(token.as_bytes()); + let digest = hasher.finalize(); + let mut out = String::with_capacity(64); + for byte in digest { + use std::fmt::Write; + let _ = write!(&mut out, "{:02x}", byte); + } + out +} diff --git a/src/db/stats.rs b/src/db/stats.rs new file mode 100644 index 0000000..304712a --- /dev/null +++ b/src/db/stats.rs @@ -0,0 +1,422 @@ +use crate::error::Result; +use sqlx::Row; + +use super::timed_query; +use super::types::*; +use super::Db; + +impl Db { + pub async fn get_stats(&self) -> Result { + let pool = &self.pool; + timed_query("get_stats", || async { + let stats = sqlx::query("SELECT status, count(*) as count FROM jobs GROUP BY status") + .fetch_all(pool) + .await?; + + let mut map = serde_json::Map::new(); + for row in stats { + use sqlx::Row; + let status: String = row.get("status"); + let count: i64 = row.get("count"); + map.insert(status, serde_json::Value::Number(count.into())); + } + + Ok(serde_json::Value::Object(map)) + }) + .await + } + + /// Save encode statistics + pub async fn save_encode_stats(&self, stats: EncodeStatsInput) -> Result<()> { + let result = sqlx::query( + "INSERT INTO encode_stats + (job_id, input_size_bytes, output_size_bytes, compression_ratio, + encode_time_seconds, encode_speed, avg_bitrate_kbps, vmaf_score, output_codec) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(job_id) DO UPDATE SET + input_size_bytes = excluded.input_size_bytes, + output_size_bytes = excluded.output_size_bytes, + compression_ratio = excluded.compression_ratio, + encode_time_seconds = excluded.encode_time_seconds, + encode_speed = excluded.encode_speed, + avg_bitrate_kbps = excluded.avg_bitrate_kbps, + vmaf_score = excluded.vmaf_score, + output_codec = excluded.output_codec", + ) + .bind(stats.job_id) + .bind(stats.input_size as i64) + .bind(stats.output_size as i64) + .bind(stats.compression_ratio) + .bind(stats.encode_time) + .bind(stats.encode_speed) + .bind(stats.avg_bitrate) + .bind(stats.vmaf_score) + .bind(stats.output_codec) + .execute(&self.pool) + .await?; + + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + + Ok(()) + } + + /// Record a single encode attempt outcome + pub async fn insert_encode_attempt(&self, input: EncodeAttemptInput) -> Result<()> { + sqlx::query( + "INSERT INTO encode_attempts + (job_id, attempt_number, started_at, finished_at, outcome, + failure_code, failure_summary, input_size_bytes, output_size_bytes, + encode_time_seconds) + VALUES (?, ?, ?, datetime('now'), ?, ?, ?, ?, ?, ?)", + ) + .bind(input.job_id) + .bind(input.attempt_number) + .bind(input.started_at) + .bind(input.outcome) + .bind(input.failure_code) + .bind(input.failure_summary) + .bind(input.input_size_bytes) + .bind(input.output_size_bytes) + .bind(input.encode_time_seconds) + .execute(&self.pool) + .await?; + Ok(()) + } + + /// Get all encode attempts for a job, ordered by attempt_number + pub async fn get_encode_attempts_by_job(&self, job_id: i64) -> Result> { + let attempts = sqlx::query_as::<_, EncodeAttempt>( + "SELECT id, job_id, attempt_number, started_at, finished_at, outcome, + failure_code, failure_summary, input_size_bytes, output_size_bytes, + encode_time_seconds, created_at + FROM encode_attempts + WHERE job_id = ? + ORDER BY attempt_number ASC", + ) + .bind(job_id) + .fetch_all(&self.pool) + .await?; + Ok(attempts) + } + + pub async fn get_encode_stats_by_job_id(&self, job_id: i64) -> Result { + let stats = sqlx::query_as::<_, DetailedEncodeStats>( + "SELECT + e.job_id, + j.input_path, + e.input_size_bytes, + e.output_size_bytes, + e.compression_ratio, + e.encode_time_seconds, + e.encode_speed, + e.avg_bitrate_kbps, + e.vmaf_score, + e.created_at + FROM encode_stats e + JOIN jobs j ON e.job_id = j.id + WHERE e.job_id = ?", + ) + .bind(job_id) + .fetch_one(&self.pool) + .await?; + Ok(stats) + } + + pub async fn get_aggregated_stats(&self) -> Result { + let pool = &self.pool; + timed_query("get_aggregated_stats", || async { + let row = sqlx::query( + "SELECT + (SELECT COUNT(*) FROM jobs WHERE archived = 0) as total_jobs, + (SELECT COUNT(*) FROM jobs WHERE status = 'completed' AND archived = 0) as completed_jobs, + COALESCE(SUM(input_size_bytes), 0) as total_input_size, + COALESCE(SUM(output_size_bytes), 0) as total_output_size, + AVG(vmaf_score) as avg_vmaf, + COALESCE(SUM(encode_time_seconds), 0.0) as total_encode_time + FROM encode_stats", + ) + .fetch_one(pool) + .await?; + + Ok(AggregatedStats { + total_jobs: row.get("total_jobs"), + completed_jobs: row.get("completed_jobs"), + total_input_size: row.get("total_input_size"), + total_output_size: row.get("total_output_size"), + avg_vmaf: row.get("avg_vmaf"), + total_encode_time_seconds: row.get("total_encode_time"), + }) + }) + .await + } + + /// Get daily statistics for the last N days (for time-series charts) + pub async fn get_daily_stats(&self, days: i32) -> Result> { + let pool = &self.pool; + let days_str = format!("-{}", days); + timed_query("get_daily_stats", || async { + let rows = sqlx::query( + "SELECT + DATE(e.created_at) as date, + COUNT(*) as jobs_completed, + COALESCE(SUM(e.input_size_bytes - e.output_size_bytes), 0) as bytes_saved, + COALESCE(SUM(e.input_size_bytes), 0) as total_input_bytes, + COALESCE(SUM(e.output_size_bytes), 0) as total_output_bytes + FROM encode_stats e + WHERE e.created_at >= DATE('now', ? || ' days') + GROUP BY DATE(e.created_at) + ORDER BY date ASC", + ) + .bind(&days_str) + .fetch_all(pool) + .await?; + + let stats = rows + .iter() + .map(|row| DailyStats { + date: row.get("date"), + jobs_completed: row.get("jobs_completed"), + bytes_saved: row.get("bytes_saved"), + total_input_bytes: row.get("total_input_bytes"), + total_output_bytes: row.get("total_output_bytes"), + }) + .collect(); + + Ok(stats) + }) + .await + } + + /// Get detailed per-job encoding statistics (most recent first) + pub async fn get_detailed_encode_stats(&self, limit: i32) -> Result> { + let pool = &self.pool; + timed_query("get_detailed_encode_stats", || async { + let stats = sqlx::query_as::<_, DetailedEncodeStats>( + "SELECT + e.job_id, + j.input_path, + e.input_size_bytes, + e.output_size_bytes, + e.compression_ratio, + e.encode_time_seconds, + e.encode_speed, + e.avg_bitrate_kbps, + e.vmaf_score, + e.created_at + FROM encode_stats e + JOIN jobs j ON e.job_id = j.id + ORDER BY e.created_at DESC + LIMIT ?", + ) + .bind(limit) + .fetch_all(pool) + .await?; + + Ok(stats) + }) + .await + } + + pub async fn get_savings_summary(&self) -> Result { + let pool = &self.pool; + timed_query("get_savings_summary", || async { + let totals = sqlx::query( + "SELECT + COALESCE(SUM(input_size_bytes), 0) as total_input_bytes, + COALESCE(SUM(output_size_bytes), 0) as total_output_bytes, + COUNT(*) as job_count + FROM encode_stats + WHERE output_size_bytes IS NOT NULL", + ) + .fetch_one(pool) + .await?; + + let total_input_bytes: i64 = totals.get("total_input_bytes"); + let total_output_bytes: i64 = totals.get("total_output_bytes"); + let job_count: i64 = totals.get("job_count"); + let total_bytes_saved = (total_input_bytes - total_output_bytes).max(0); + let savings_percent = if total_input_bytes > 0 { + (total_bytes_saved as f64 / total_input_bytes as f64) * 100.0 + } else { + 0.0 + }; + + let savings_by_codec = sqlx::query( + "SELECT + COALESCE(NULLIF(TRIM(e.output_codec), ''), 'unknown') as codec, + COALESCE(SUM(e.input_size_bytes - e.output_size_bytes), 0) as bytes_saved, + COUNT(*) as job_count + FROM encode_stats e + JOIN jobs j ON j.id = e.job_id + WHERE e.output_size_bytes IS NOT NULL + GROUP BY codec + ORDER BY bytes_saved DESC, codec ASC", + ) + .fetch_all(pool) + .await? + .into_iter() + .map(|row| CodecSavings { + codec: row.get("codec"), + bytes_saved: row.get("bytes_saved"), + job_count: row.get("job_count"), + }) + .collect::>(); + + let savings_over_time = sqlx::query( + "SELECT + DATE(e.created_at) as date, + COALESCE(SUM(e.input_size_bytes - e.output_size_bytes), 0) as bytes_saved + FROM encode_stats e + WHERE e.output_size_bytes IS NOT NULL + AND e.created_at >= datetime('now', '-30 days') + GROUP BY DATE(e.created_at) + ORDER BY date ASC", + ) + .fetch_all(pool) + .await? + .into_iter() + .map(|row| DailySavings { + date: row.get("date"), + bytes_saved: row.get("bytes_saved"), + }) + .collect::>(); + + Ok(SavingsSummary { + total_input_bytes, + total_output_bytes, + total_bytes_saved, + savings_percent, + job_count, + savings_by_codec, + savings_over_time, + }) + }) + .await + } + + pub async fn get_job_stats(&self) -> Result { + let pool = &self.pool; + timed_query("get_job_stats", || async { + let rows = sqlx::query( + "SELECT status, COUNT(*) as count FROM jobs WHERE archived = 0 GROUP BY status", + ) + .fetch_all(pool) + .await?; + + let mut stats = JobStats::default(); + for row in rows { + let status_str: String = row.get("status"); + let count: i64 = row.get("count"); + + match status_str.as_str() { + "queued" => stats.queued += count, + "encoding" | "analyzing" | "remuxing" | "resuming" => stats.active += count, + "completed" => stats.completed += count, + "failed" | "cancelled" => stats.failed += count, + _ => {} + } + } + Ok(stats) + }) + .await + } + + pub async fn get_daily_summary_stats(&self) -> Result { + let pool = &self.pool; + timed_query("get_daily_summary_stats", || async { + let row = sqlx::query( + "SELECT + COALESCE(SUM(CASE WHEN status = 'completed' AND DATE(updated_at, 'localtime') = DATE('now', 'localtime') THEN 1 ELSE 0 END), 0) AS completed, + COALESCE(SUM(CASE WHEN status = 'failed' AND DATE(updated_at, 'localtime') = DATE('now', 'localtime') THEN 1 ELSE 0 END), 0) AS failed, + COALESCE(SUM(CASE WHEN status = 'skipped' AND DATE(updated_at, 'localtime') = DATE('now', 'localtime') THEN 1 ELSE 0 END), 0) AS skipped + FROM jobs", + ) + .fetch_one(pool) + .await?; + + let completed: i64 = row.get("completed"); + let failed: i64 = row.get("failed"); + let skipped: i64 = row.get("skipped"); + + let bytes_row = sqlx::query( + "SELECT COALESCE(SUM(input_size_bytes - output_size_bytes), 0) AS bytes_saved + FROM encode_stats + WHERE DATE(created_at, 'localtime') = DATE('now', 'localtime')", + ) + .fetch_one(pool) + .await?; + let bytes_saved: i64 = bytes_row.get("bytes_saved"); + + let failure_rows = sqlx::query( + "SELECT code, COUNT(*) AS count + FROM job_failure_explanations + WHERE DATE(updated_at, 'localtime') = DATE('now', 'localtime') + GROUP BY code + ORDER BY count DESC, code ASC + LIMIT 3", + ) + .fetch_all(pool) + .await?; + let top_failure_reasons = failure_rows + .into_iter() + .map(|row| row.get::("code")) + .collect::>(); + + let skip_rows = sqlx::query( + "SELECT COALESCE(reason_code, action) AS code, COUNT(*) AS count + FROM decisions + WHERE action = 'skip' + AND DATE(created_at, 'localtime') = DATE('now', 'localtime') + GROUP BY COALESCE(reason_code, action) + ORDER BY count DESC, code ASC + LIMIT 3", + ) + .fetch_all(pool) + .await?; + let top_skip_reasons = skip_rows + .into_iter() + .map(|row| row.get::("code")) + .collect::>(); + + Ok(DailySummaryStats { + completed, + failed, + skipped, + bytes_saved, + top_failure_reasons, + top_skip_reasons, + }) + }) + .await + } + + pub async fn get_skip_reason_counts(&self) -> Result> { + let pool = &self.pool; + timed_query("get_skip_reason_counts", || async { + let rows = sqlx::query( + "SELECT COALESCE(reason_code, action) AS code, COUNT(*) AS count + FROM decisions + WHERE action = 'skip' + AND DATE(created_at, 'localtime') = DATE('now', 'localtime') + GROUP BY COALESCE(reason_code, action) + ORDER BY count DESC, code ASC + LIMIT 20", + ) + .fetch_all(pool) + .await?; + Ok(rows + .into_iter() + .map(|row| { + let code: String = row.get("code"); + let count: i64 = row.get("count"); + (code, count) + }) + .collect()) + }) + .await + } +} diff --git a/src/db/system.rs b/src/db/system.rs new file mode 100644 index 0000000..d2bd67e --- /dev/null +++ b/src/db/system.rs @@ -0,0 +1,389 @@ +use chrono::{DateTime, Utc}; +use crate::error::Result; +use sqlx::Row; + +use super::timed_query; +use super::types::*; +use super::{hash_api_token, hash_session_token, Db}; + +impl Db { + pub async fn clear_completed_jobs(&self) -> Result { + let result = sqlx::query( + "UPDATE jobs + SET archived = 1, updated_at = CURRENT_TIMESTAMP + WHERE status = 'completed' AND archived = 0", + ) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + + pub async fn cleanup_sessions(&self) -> Result<()> { + sqlx::query("DELETE FROM sessions WHERE expires_at <= CURRENT_TIMESTAMP") + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn cleanup_expired_sessions(&self) -> Result { + let result = sqlx::query("DELETE FROM sessions WHERE expires_at <= CURRENT_TIMESTAMP") + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + + pub async fn add_log(&self, level: &str, job_id: Option, message: &str) -> Result<()> { + sqlx::query("INSERT INTO logs (level, job_id, message) VALUES (?, ?, ?)") + .bind(level) + .bind(job_id) + .bind(message) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_logs(&self, limit: i64, offset: i64) -> Result> { + let logs = sqlx::query_as::<_, LogEntry>( + "SELECT id, level, job_id, message, created_at FROM logs ORDER BY created_at DESC LIMIT ? OFFSET ?" + ) + .bind(limit) + .bind(offset) + .fetch_all(&self.pool) + .await?; + Ok(logs) + } + + pub async fn get_logs_for_job(&self, job_id: i64, limit: i64) -> Result> { + sqlx::query_as::<_, LogEntry>( + "SELECT id, level, job_id, message, created_at + FROM logs + WHERE job_id = ? + ORDER BY created_at ASC + LIMIT ?", + ) + .bind(job_id) + .bind(limit) + .fetch_all(&self.pool) + .await + .map_err(Into::into) + } + + pub async fn clear_logs(&self) -> Result<()> { + sqlx::query("DELETE FROM logs").execute(&self.pool).await?; + Ok(()) + } + + pub async fn prune_old_logs(&self, max_age_days: u32) -> Result { + let result = sqlx::query( + "DELETE FROM logs + WHERE created_at < datetime('now', '-' || ? || ' days')", + ) + .bind(max_age_days as i64) + .execute(&self.pool) + .await?; + Ok(result.rows_affected()) + } + + pub async fn create_user(&self, username: &str, password_hash: &str) -> Result { + let id = sqlx::query("INSERT INTO users (username, password_hash) VALUES (?, ?)") + .bind(username) + .bind(password_hash) + .execute(&self.pool) + .await? + .last_insert_rowid(); + Ok(id) + } + + pub async fn get_user_by_username(&self, username: &str) -> Result> { + let user = sqlx::query_as::<_, User>("SELECT * FROM users WHERE username = ?") + .bind(username) + .fetch_optional(&self.pool) + .await?; + Ok(user) + } + + pub async fn has_users(&self) -> Result { + let count: (i64,) = sqlx::query_as("SELECT COUNT(*) FROM users") + .fetch_one(&self.pool) + .await?; + Ok(count.0 > 0) + } + + pub async fn create_session( + &self, + user_id: i64, + token: &str, + expires_at: DateTime, + ) -> Result<()> { + let token_hash = hash_session_token(token); + sqlx::query("INSERT INTO sessions (token, user_id, expires_at) VALUES (?, ?, ?)") + .bind(token_hash) + .bind(user_id) + .bind(expires_at) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_session(&self, token: &str) -> Result> { + let token_hash = hash_session_token(token); + let session = sqlx::query_as::<_, Session>( + "SELECT * FROM sessions WHERE token = ? AND expires_at > CURRENT_TIMESTAMP", + ) + .bind(&token_hash) + .fetch_optional(&self.pool) + .await?; + Ok(session) + } + + pub async fn delete_session(&self, token: &str) -> Result<()> { + let token_hash = hash_session_token(token); + sqlx::query("DELETE FROM sessions WHERE token = ?") + .bind(&token_hash) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn list_api_tokens(&self) -> Result> { + let tokens = sqlx::query_as::<_, ApiToken>( + "SELECT id, name, access_level, created_at, last_used_at, revoked_at + FROM api_tokens + ORDER BY created_at DESC", + ) + .fetch_all(&self.pool) + .await?; + Ok(tokens) + } + + pub async fn create_api_token( + &self, + name: &str, + token: &str, + access_level: ApiTokenAccessLevel, + ) -> Result { + let token_hash = hash_api_token(token); + let row = sqlx::query_as::<_, ApiToken>( + "INSERT INTO api_tokens (name, token_hash, access_level) + VALUES (?, ?, ?) + RETURNING id, name, access_level, created_at, last_used_at, revoked_at", + ) + .bind(name) + .bind(token_hash) + .bind(access_level) + .fetch_one(&self.pool) + .await?; + Ok(row) + } + + pub async fn get_active_api_token(&self, token: &str) -> Result> { + let token_hash = hash_api_token(token); + let row = sqlx::query_as::<_, ApiTokenRecord>( + "SELECT id, name, token_hash, access_level, created_at, last_used_at, revoked_at + FROM api_tokens + WHERE token_hash = ? AND revoked_at IS NULL", + ) + .bind(token_hash) + .fetch_optional(&self.pool) + .await?; + Ok(row) + } + + pub async fn update_api_token_last_used(&self, id: i64) -> Result<()> { + sqlx::query("UPDATE api_tokens SET last_used_at = CURRENT_TIMESTAMP WHERE id = ?") + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn revoke_api_token(&self, id: i64) -> Result<()> { + let result = sqlx::query( + "UPDATE api_tokens + SET revoked_at = COALESCE(revoked_at, CURRENT_TIMESTAMP) + WHERE id = ?", + ) + .bind(id) + .execute(&self.pool) + .await?; + if result.rows_affected() == 0 { + return Err(crate::error::AlchemistError::Database( + sqlx::Error::RowNotFound, + )); + } + Ok(()) + } + + pub async fn record_health_check( + &self, + job_id: i64, + issues: Option<&crate::media::health::HealthIssueReport>, + ) -> Result<()> { + let serialized_issues = issues + .map(serde_json::to_string) + .transpose() + .map_err(|err| { + crate::error::AlchemistError::Unknown(format!( + "Failed to serialize health issue report: {}", + err + )) + })?; + + sqlx::query( + "UPDATE jobs + SET health_issues = ?, + last_health_check = datetime('now') + WHERE id = ?", + ) + .bind(serialized_issues) + .bind(job_id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_health_summary(&self) -> Result { + let pool = &self.pool; + timed_query("get_health_summary", || async { + let row = sqlx::query( + "SELECT + (SELECT COUNT(*) FROM jobs WHERE last_health_check IS NOT NULL AND archived = 0) as total_checked, + (SELECT COUNT(*) + FROM jobs + WHERE health_issues IS NOT NULL AND TRIM(health_issues) != '' AND archived = 0) as issues_found, + (SELECT MAX(started_at) FROM health_scan_runs) as last_run", + ) + .fetch_one(pool) + .await?; + + Ok(HealthSummary { + total_checked: row.get("total_checked"), + issues_found: row.get("issues_found"), + last_run: row.get("last_run"), + }) + }) + .await + } + + pub async fn create_health_scan_run(&self) -> Result { + let id = sqlx::query("INSERT INTO health_scan_runs DEFAULT VALUES") + .execute(&self.pool) + .await? + .last_insert_rowid(); + Ok(id) + } + + pub async fn complete_health_scan_run( + &self, + id: i64, + files_checked: i64, + issues_found: i64, + ) -> Result<()> { + sqlx::query( + "UPDATE health_scan_runs + SET completed_at = datetime('now'), + files_checked = ?, + issues_found = ? + WHERE id = ?", + ) + .bind(files_checked) + .bind(issues_found) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get_jobs_with_health_issues(&self) -> Result> { + let pool = &self.pool; + timed_query("get_jobs_with_health_issues", || async { + let jobs = sqlx::query_as::<_, JobWithHealthIssueRow>( + "SELECT j.id, j.input_path, j.output_path, j.status, + (SELECT reason FROM decisions WHERE job_id = j.id ORDER BY created_at DESC LIMIT 1) as decision_reason, + COALESCE(j.priority, 0) as priority, + COALESCE(CAST(j.progress AS REAL), 0.0) as progress, + COALESCE(j.attempt_count, 0) as attempt_count, + (SELECT vmaf_score FROM encode_stats WHERE job_id = j.id) as vmaf_score, + j.created_at, j.updated_at, + j.input_metadata_json, + j.health_issues + FROM jobs j + WHERE j.archived = 0 + AND j.health_issues IS NOT NULL + AND TRIM(j.health_issues) != '' + ORDER BY j.updated_at DESC", + ) + .fetch_all(pool) + .await?; + Ok(jobs) + }) + .await + } + + pub async fn reset_auth(&self) -> Result<()> { + sqlx::query("DELETE FROM sessions") + .execute(&self.pool) + .await?; + sqlx::query("DELETE FROM users").execute(&self.pool).await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + use std::time::SystemTime; + + #[tokio::test] + async fn clear_completed_archives_jobs_but_preserves_encode_stats() + -> std::result::Result<(), Box> { + let mut db_path = std::env::temp_dir(); + let token: u64 = rand::random(); + db_path.push(format!("alchemist_archive_completed_{}.db", token)); + + let db = Db::new(db_path.to_string_lossy().as_ref()).await?; + let input = Path::new("movie.mkv"); + let output = Path::new("movie-alchemist.mkv"); + let _ = db + .enqueue_job(input, output, SystemTime::UNIX_EPOCH) + .await?; + + let job = db + .get_job_by_input_path("movie.mkv") + .await? + .ok_or_else(|| std::io::Error::other("missing job"))?; + db.update_job_status(job.id, JobState::Completed).await?; + db.save_encode_stats(EncodeStatsInput { + job_id: job.id, + input_size: 2_000, + output_size: 1_000, + compression_ratio: 0.5, + encode_time: 42.0, + encode_speed: 1.2, + avg_bitrate: 800.0, + vmaf_score: Some(96.5), + output_codec: Some("av1".to_string()), + }) + .await?; + + let cleared = db.clear_completed_jobs().await?; + assert_eq!(cleared, 1); + assert!(db.get_job_by_id(job.id).await?.is_none()); + assert!(db.get_job_by_input_path("movie.mkv").await?.is_none()); + + let visible_completed = db.get_jobs_by_status(JobState::Completed).await?; + assert!(visible_completed.is_empty()); + + let aggregated = db.get_aggregated_stats().await?; + // Archived jobs are excluded from active stats. + assert_eq!(aggregated.completed_jobs, 0); + // encode_stats rows are preserved even after archiving. + assert_eq!(aggregated.total_input_size, 2_000); + assert_eq!(aggregated.total_output_size, 1_000); + + drop(db); + let _ = std::fs::remove_file(db_path); + Ok(()) + } +} diff --git a/src/db/types.rs b/src/db/types.rs new file mode 100644 index 0000000..fff64ed --- /dev/null +++ b/src/db/types.rs @@ -0,0 +1,640 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, sqlx::Type)] +#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum JobState { + Queued, + Analyzing, + Encoding, + Remuxing, + Completed, + Skipped, + Failed, + Cancelled, + Resuming, +} + +impl std::fmt::Display for JobState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + JobState::Queued => "queued", + JobState::Analyzing => "analyzing", + JobState::Encoding => "encoding", + JobState::Remuxing => "remuxing", + JobState::Completed => "completed", + JobState::Skipped => "skipped", + JobState::Failed => "failed", + JobState::Cancelled => "cancelled", + JobState::Resuming => "resuming", + }; + write!(f, "{}", s) + } +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[serde(default)] +pub struct JobStats { + pub active: i64, + pub queued: i64, + pub completed: i64, + pub failed: i64, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[serde(default)] +pub struct DailySummaryStats { + pub completed: i64, + pub failed: i64, + pub skipped: i64, + pub bytes_saved: i64, + pub top_failure_reasons: Vec, + pub top_skip_reasons: Vec, +} + +#[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] +pub struct LogEntry { + pub id: i64, + pub level: String, + pub job_id: Option, + pub message: String, + pub created_at: String, // SQLite datetime as string +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct Job { + pub id: i64, + pub input_path: String, + pub output_path: String, + pub status: JobState, + pub decision_reason: Option, + pub priority: i32, + pub progress: f64, + pub attempt_count: i32, + pub vmaf_score: Option, + pub created_at: DateTime, + pub updated_at: DateTime, + pub input_metadata_json: Option, +} + +impl Job { + pub fn input_metadata(&self) -> Option { + self.input_metadata_json + .as_ref() + .and_then(|json| serde_json::from_str(json).ok()) + } + + pub fn is_active(&self) -> bool { + matches!( + self.status, + JobState::Encoding | JobState::Analyzing | JobState::Remuxing | JobState::Resuming + ) + } + + pub fn can_retry(&self) -> bool { + matches!(self.status, JobState::Failed | JobState::Cancelled) + } + + pub fn status_class(&self) -> &'static str { + match self.status { + JobState::Completed => "badge-green", + JobState::Encoding | JobState::Remuxing | JobState::Resuming => "badge-yellow", + JobState::Analyzing => "badge-blue", + JobState::Failed | JobState::Cancelled => "badge-red", + _ => "badge-gray", + } + } + + pub fn progress_fixed(&self) -> String { + format!("{:.1}", self.progress) + } + + pub fn vmaf_fixed(&self) -> String { + self.vmaf_score + .map(|s| format!("{:.1}", s)) + .unwrap_or_else(|| "N/A".to_string()) + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct JobWithHealthIssueRow { + pub id: i64, + pub input_path: String, + pub output_path: String, + pub status: JobState, + pub decision_reason: Option, + pub priority: i32, + pub progress: f64, + pub attempt_count: i32, + pub vmaf_score: Option, + pub created_at: DateTime, + pub updated_at: DateTime, + pub input_metadata_json: Option, + pub health_issues: String, +} + +impl JobWithHealthIssueRow { + pub fn into_parts(self) -> (Job, String) { + ( + Job { + id: self.id, + input_path: self.input_path, + output_path: self.output_path, + status: self.status, + decision_reason: self.decision_reason, + priority: self.priority, + progress: self.progress, + attempt_count: self.attempt_count, + vmaf_score: self.vmaf_score, + created_at: self.created_at, + updated_at: self.updated_at, + input_metadata_json: self.input_metadata_json, + }, + self.health_issues, + ) + } +} + +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct DuplicateCandidate { + pub id: i64, + pub input_path: String, + pub status: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct WatchDir { + pub id: i64, + pub path: String, + pub is_recursive: bool, + pub profile_id: Option, + pub created_at: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct LibraryProfile { + pub id: i64, + pub name: String, + pub preset: String, + pub codec: String, + pub quality_profile: String, + pub hdr_mode: String, + pub audio_mode: String, + pub crf_override: Option, + pub notes: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct NewLibraryProfile { + pub name: String, + pub preset: String, + pub codec: String, + pub quality_profile: String, + pub hdr_mode: String, + pub audio_mode: String, + pub crf_override: Option, + pub notes: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct JobFilterQuery { + pub limit: i64, + pub offset: i64, + pub statuses: Option>, + pub search: Option, + pub sort_by: Option, + pub sort_desc: bool, + pub archived: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct NotificationTarget { + pub id: i64, + pub name: String, + pub target_type: String, + pub config_json: String, + pub events: String, + pub enabled: bool, + pub created_at: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct ConversionJob { + pub id: i64, + pub upload_path: String, + pub output_path: Option, + pub mode: String, + pub settings_json: String, + pub probe_json: Option, + pub linked_job_id: Option, + pub status: String, + pub expires_at: String, + pub downloaded_at: Option, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct ScheduleWindow { + pub id: i64, + pub start_time: String, + pub end_time: String, + pub days_of_week: String, // as JSON string + pub enabled: bool, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct FileSettings { + pub id: i64, + pub delete_source: bool, + pub output_extension: String, + pub output_suffix: String, + pub replace_strategy: String, + pub output_root: Option, +} + +impl FileSettings { + pub fn output_path_for(&self, input_path: &Path) -> PathBuf { + self.output_path_for_source(input_path, None) + } + + pub fn output_path_for_source(&self, input_path: &Path, source_root: Option<&Path>) -> PathBuf { + let mut output_path = self.output_base_path(input_path, source_root); + let stem = input_path.file_stem().unwrap_or_default().to_string_lossy(); + let extension = self.output_extension.trim_start_matches('.'); + let suffix = self.output_suffix.as_str(); + + let mut filename = String::new(); + filename.push_str(&stem); + filename.push_str(suffix); + if !extension.is_empty() { + filename.push('.'); + filename.push_str(extension); + } + if filename.is_empty() { + filename.push_str("output"); + } + output_path.set_file_name(filename); + + if output_path == input_path { + let safe_suffix = if suffix.is_empty() { + "-alchemist".to_string() + } else { + format!("{}-alchemist", suffix) + }; + let mut safe_name = String::new(); + safe_name.push_str(&stem); + safe_name.push_str(&safe_suffix); + if !extension.is_empty() { + safe_name.push('.'); + safe_name.push_str(extension); + } + output_path.set_file_name(safe_name); + } + + output_path + } + + fn output_base_path(&self, input_path: &Path, source_root: Option<&Path>) -> PathBuf { + let Some(output_root) = self + .output_root + .as_deref() + .filter(|value| !value.trim().is_empty()) + else { + return input_path.to_path_buf(); + }; + + let Some(root) = source_root else { + return input_path.to_path_buf(); + }; + + let Ok(relative_path) = input_path.strip_prefix(root) else { + return input_path.to_path_buf(); + }; + + let mut output_path = PathBuf::from(output_root); + if let Some(parent) = relative_path.parent() { + output_path.push(parent); + } + output_path.push(relative_path.file_name().unwrap_or_default()); + output_path + } + + pub fn should_replace_existing_output(&self) -> bool { + let strategy = self.replace_strategy.trim(); + strategy.eq_ignore_ascii_case("replace") || strategy.eq_ignore_ascii_case("overwrite") + } +} + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +#[serde(default)] +pub struct AggregatedStats { + pub total_jobs: i64, + pub completed_jobs: i64, + pub total_input_size: i64, + pub total_output_size: i64, + pub avg_vmaf: Option, + pub total_encode_time_seconds: f64, +} + +impl AggregatedStats { + pub fn total_savings_gb(&self) -> f64 { + self.total_input_size.saturating_sub(self.total_output_size) as f64 / 1_073_741_824.0 + } + + pub fn total_input_gb(&self) -> f64 { + self.total_input_size as f64 / 1_073_741_824.0 + } + + pub fn avg_reduction_percentage(&self) -> f64 { + if self.total_input_size == 0 { + 0.0 + } else { + (1.0 - (self.total_output_size as f64 / self.total_input_size as f64)) * 100.0 + } + } + + pub fn total_time_hours(&self) -> f64 { + self.total_encode_time_seconds / 3600.0 + } + + pub fn total_savings_fixed(&self) -> String { + format!("{:.1}", self.total_savings_gb()) + } + + pub fn total_input_fixed(&self) -> String { + format!("{:.1}", self.total_input_gb()) + } + + pub fn efficiency_fixed(&self) -> String { + format!("{:.1}", self.avg_reduction_percentage()) + } + + pub fn time_fixed(&self) -> String { + format!("{:.1}", self.total_time_hours()) + } + + pub fn avg_vmaf_fixed(&self) -> String { + self.avg_vmaf + .map(|v| format!("{:.1}", v)) + .unwrap_or_else(|| "N/A".to_string()) + } +} + +/// Daily aggregated statistics for time-series charts +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DailyStats { + pub date: String, + pub jobs_completed: i64, + pub bytes_saved: i64, + pub total_input_bytes: i64, + pub total_output_bytes: i64, +} + +/// Detailed per-job encoding statistics +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct DetailedEncodeStats { + pub job_id: i64, + pub input_path: String, + pub input_size_bytes: i64, + pub output_size_bytes: i64, + pub compression_ratio: f64, + pub encode_time_seconds: f64, + pub encode_speed: f64, + pub avg_bitrate_kbps: f64, + pub vmaf_score: Option, + pub created_at: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct EncodeAttempt { + pub id: i64, + pub job_id: i64, + pub attempt_number: i32, + pub started_at: Option, + pub finished_at: String, + pub outcome: String, + pub failure_code: Option, + pub failure_summary: Option, + pub input_size_bytes: Option, + pub output_size_bytes: Option, + pub encode_time_seconds: Option, + pub created_at: String, +} + +#[derive(Debug, Clone)] +pub struct EncodeAttemptInput { + pub job_id: i64, + pub attempt_number: i32, + pub started_at: Option, + pub outcome: String, + pub failure_code: Option, + pub failure_summary: Option, + pub input_size_bytes: Option, + pub output_size_bytes: Option, + pub encode_time_seconds: Option, +} + +#[derive(Debug, Clone)] +pub struct EncodeStatsInput { + pub job_id: i64, + pub input_size: u64, + pub output_size: u64, + pub compression_ratio: f64, + pub encode_time: f64, + pub encode_speed: f64, + pub avg_bitrate: f64, + pub vmaf_score: Option, + pub output_codec: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct CodecSavings { + pub codec: String, + pub bytes_saved: i64, + pub job_count: i64, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DailySavings { + pub date: String, + pub bytes_saved: i64, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SavingsSummary { + pub total_input_bytes: i64, + pub total_output_bytes: i64, + pub total_bytes_saved: i64, + pub savings_percent: f64, + pub job_count: i64, + pub savings_by_codec: Vec, + pub savings_over_time: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct HealthSummary { + pub total_checked: i64, + pub issues_found: i64, + pub last_run: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct Decision { + pub id: i64, + pub job_id: i64, + pub action: String, // "encode", "skip", "reject" + pub reason: String, + pub reason_code: Option, + pub reason_payload_json: Option, + pub created_at: DateTime, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +pub(crate) struct DecisionRecord { + pub(crate) job_id: i64, + pub(crate) action: String, + pub(crate) reason: String, + pub(crate) reason_payload_json: Option, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +pub(crate) struct FailureExplanationRecord { + pub(crate) legacy_summary: Option, + pub(crate) code: String, + pub(crate) payload_json: String, +} + +// Auth related structs +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct User { + pub id: i64, + pub username: String, + pub password_hash: String, + pub created_at: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct Session { + pub token: String, + pub user_id: i64, + pub expires_at: DateTime, + pub created_at: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq, sqlx::Type)] +#[sqlx(rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum ApiTokenAccessLevel { + ReadOnly, + FullAccess, +} + +#[derive(Debug, Serialize, Deserialize, Clone, sqlx::FromRow)] +pub struct ApiToken { + pub id: i64, + pub name: String, + pub access_level: ApiTokenAccessLevel, + pub created_at: DateTime, + pub last_used_at: Option>, + pub revoked_at: Option>, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct ApiTokenRecord { + pub id: i64, + pub name: String, + pub token_hash: String, + pub access_level: ApiTokenAccessLevel, + pub created_at: DateTime, + pub last_used_at: Option>, + pub revoked_at: Option>, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::{Path, PathBuf}; + + #[test] + fn test_output_path_for_suffix() { + let settings = FileSettings { + id: 1, + delete_source: false, + output_extension: "mkv".to_string(), + output_suffix: "-alchemist".to_string(), + replace_strategy: "keep".to_string(), + output_root: None, + }; + let input = Path::new("video.mp4"); + let output = settings.output_path_for(input); + assert_eq!(output, PathBuf::from("video-alchemist.mkv")); + } + + #[test] + fn test_output_path_avoids_inplace() { + let settings = FileSettings { + id: 1, + delete_source: false, + output_extension: "mkv".to_string(), + output_suffix: "".to_string(), + replace_strategy: "keep".to_string(), + output_root: None, + }; + let input = Path::new("video.mkv"); + let output = settings.output_path_for(input); + assert_ne!(output, input); + } + + #[test] + fn test_output_path_mirrors_source_root_under_output_root() { + let settings = FileSettings { + id: 1, + delete_source: false, + output_extension: "mkv".to_string(), + output_suffix: "-alchemist".to_string(), + replace_strategy: "keep".to_string(), + output_root: Some("/encoded".to_string()), + }; + let input = Path::new("/library/movies/action/video.mp4"); + let output = settings.output_path_for_source(input, Some(Path::new("/library"))); + assert_eq!( + output, + PathBuf::from("/encoded/movies/action/video-alchemist.mkv") + ); + } + + #[test] + fn test_output_path_falls_back_when_source_root_does_not_match() { + let settings = FileSettings { + id: 1, + delete_source: false, + output_extension: "mkv".to_string(), + output_suffix: "-alchemist".to_string(), + replace_strategy: "keep".to_string(), + output_root: Some("/encoded".to_string()), + }; + let input = Path::new("/library/movies/video.mp4"); + let output = settings.output_path_for_source(input, Some(Path::new("/other"))); + assert_eq!(output, PathBuf::from("/library/movies/video-alchemist.mkv")); + } + + #[test] + fn test_replace_strategy() { + let mut settings = FileSettings { + id: 1, + delete_source: false, + output_extension: "mkv".to_string(), + output_suffix: "-alchemist".to_string(), + replace_strategy: "keep".to_string(), + output_root: None, + }; + assert!(!settings.should_replace_existing_output()); + settings.replace_strategy = "replace".to_string(); + assert!(settings.should_replace_existing_output()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 19165b3..438a76f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,6 @@ pub mod version; pub mod wizard; pub use config::QualityProfile; -pub use db::AlchemistEvent; pub use media::ffmpeg::{EncodeStats, EncoderCapabilities, HardwareAccelerators}; pub use media::processor::Agent; pub use orchestrator::Transcoder; diff --git a/src/main.rs b/src/main.rs index 8b96cfb..131a0d1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -306,6 +306,10 @@ async fn run() -> Result<()> { Ok(mut remuxing_jobs) => jobs.append(&mut remuxing_jobs), Err(err) => error!("Failed to load interrupted remuxing jobs: {}", err), } + match db.get_jobs_by_status(db::JobState::Resuming).await { + Ok(mut resuming_jobs) => jobs.append(&mut resuming_jobs), + Err(err) => error!("Failed to load interrupted resuming jobs: {}", err), + } match db.get_jobs_by_status(db::JobState::Analyzing).await { Ok(mut analyzing_jobs) => jobs.append(&mut analyzing_jobs), Err(err) => error!("Failed to load interrupted analyzing jobs: {}", err), @@ -515,9 +519,6 @@ async fn run() -> Result<()> { system: system_tx, }); - // Keep legacy channel for transition compatibility - let (tx, _rx) = broadcast::channel(100); - let transcoder = Arc::new(Transcoder::new()); let hardware_state = hardware::HardwareState::new(Some(hw_info.clone())); let hardware_probe_log = Arc::new(RwLock::new(initial_probe_log)); @@ -528,7 +529,7 @@ async fn run() -> Result<()> { db.as_ref().clone(), config.clone(), )); - notification_manager.start_listener(tx.subscribe()); + notification_manager.start_listener(&event_channels); let maintenance_db = db.clone(); let maintenance_config = config.clone(); @@ -563,7 +564,6 @@ async fn run() -> Result<()> { transcoder.clone(), config.clone(), hardware_state.clone(), - tx.clone(), event_channels.clone(), matches!(args.command, Some(Commands::Run { dry_run: true, .. })), ) @@ -767,7 +767,6 @@ async fn run() -> Result<()> { transcoder, scheduler: scheduler_handle, event_channels, - tx, setup_required: setup_mode, config_path: config_path.clone(), config_mutable, @@ -1278,7 +1277,6 @@ mod tests { })); let hardware_probe_log = Arc::new(RwLock::new(hardware::HardwareProbeLog::default())); let transcoder = Arc::new(Transcoder::new()); - let (tx, _rx) = broadcast::channel(8); let (jobs_tx, _) = broadcast::channel(100); let (config_tx, _) = broadcast::channel(10); let (system_tx, _) = broadcast::channel(10); @@ -1293,7 +1291,6 @@ mod tests { transcoder, config_state.clone(), hardware_state.clone(), - tx, event_channels, true, ) diff --git a/src/media/executor.rs b/src/media/executor.rs index 4dd4a1b..9bebc66 100644 --- a/src/media/executor.rs +++ b/src/media/executor.rs @@ -1,8 +1,6 @@ -use crate::db::{AlchemistEvent, Db, EventChannels, Job, JobEvent}; +use crate::db::{Db, EventChannels, Job, JobEvent}; use crate::error::Result; -use crate::media::pipeline::{ - Encoder, ExecutionResult, ExecutionStats, Executor, MediaAnalysis, TranscodePlan, -}; +use crate::media::pipeline::{Encoder, ExecutionResult, Executor, MediaAnalysis, TranscodePlan}; use crate::orchestrator::{ AsyncExecutionObserver, ExecutionObserver, TranscodeRequest, Transcoder, }; @@ -10,13 +8,12 @@ use crate::system::hardware::HardwareInfo; use std::path::PathBuf; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::{Mutex, broadcast}; +use tokio::sync::Mutex; pub struct FfmpegExecutor { transcoder: Arc, db: Arc, hw_info: Option, - event_tx: Arc>, event_channels: Arc, dry_run: bool, } @@ -26,7 +23,6 @@ impl FfmpegExecutor { transcoder: Arc, db: Arc, hw_info: Option, - event_tx: Arc>, event_channels: Arc, dry_run: bool, ) -> Self { @@ -34,7 +30,6 @@ impl FfmpegExecutor { transcoder, db, hw_info, - event_tx, event_channels, dry_run, } @@ -44,7 +39,6 @@ impl FfmpegExecutor { struct JobExecutionObserver { job_id: i64, db: Arc, - event_tx: Arc>, event_channels: Arc, last_progress: Mutex>, } @@ -53,13 +47,11 @@ impl JobExecutionObserver { fn new( job_id: i64, db: Arc, - event_tx: Arc>, event_channels: Arc, ) -> Self { Self { job_id, db, - event_tx, event_channels, last_progress: Mutex::new(None), } @@ -68,18 +60,11 @@ impl JobExecutionObserver { impl AsyncExecutionObserver for JobExecutionObserver { async fn on_log(&self, message: String) { - // Send to typed channel let _ = self.event_channels.jobs.send(JobEvent::Log { level: "info".to_string(), job_id: Some(self.job_id), message: message.clone(), }); - // Also send to legacy channel for backwards compatibility - let _ = self.event_tx.send(AlchemistEvent::Log { - level: "info".to_string(), - job_id: Some(self.job_id), - message: message.clone(), - }); if let Err(err) = self.db.add_log("info", Some(self.job_id), &message).await { tracing::warn!( "Failed to persist transcode log for job {}: {}", @@ -117,14 +102,7 @@ impl AsyncExecutionObserver for JobExecutionObserver { } } - // Send to typed channel let _ = self.event_channels.jobs.send(JobEvent::Progress { - job_id: self.job_id, - percentage, - time: progress.time.clone(), - }); - // Also send to legacy channel for backwards compatibility - let _ = self.event_tx.send(AlchemistEvent::Progress { job_id: self.job_id, percentage, time: progress.time, @@ -155,7 +133,6 @@ impl Executor for FfmpegExecutor { let observer: Arc = Arc::new(JobExecutionObserver::new( job.id, self.db.clone(), - self.event_tx.clone(), self.event_channels.clone(), )); @@ -274,12 +251,6 @@ impl Executor for FfmpegExecutor { fallback_occurred: plan.fallback.is_some() || codec_mismatch || encoder_mismatch, actual_output_codec, actual_encoder_name, - stats: ExecutionStats { - encode_time_secs: 0.0, - input_size: 0, - output_size: 0, - vmaf: None, - }, }) } } @@ -392,8 +363,7 @@ mod tests { let Some(job) = db.get_job_by_input_path("input.mkv").await? else { panic!("expected seeded job"); }; - let (tx, mut rx) = broadcast::channel(8); - let (jobs_tx, _) = broadcast::channel(100); + let (jobs_tx, mut jobs_rx) = broadcast::channel(100); let (config_tx, _) = broadcast::channel(10); let (system_tx, _) = broadcast::channel(10); let event_channels = Arc::new(crate::db::EventChannels { @@ -401,7 +371,7 @@ mod tests { config: config_tx, system: system_tx, }); - let observer = JobExecutionObserver::new(job.id, db.clone(), Arc::new(tx), event_channels); + let observer = JobExecutionObserver::new(job.id, db.clone(), event_channels); LocalExecutionObserver::on_log(&observer, "ffmpeg line".to_string()).await; LocalExecutionObserver::on_progress( @@ -423,10 +393,10 @@ mod tests { }; assert!((updated.progress - 20.0).abs() < 0.01); - let first = rx.recv().await?; - assert!(matches!(first, AlchemistEvent::Log { .. })); - let second = rx.recv().await?; - assert!(matches!(second, AlchemistEvent::Progress { .. })); + let first = jobs_rx.recv().await?; + assert!(matches!(first, JobEvent::Log { .. })); + let second = jobs_rx.recv().await?; + assert!(matches!(second, JobEvent::Progress { .. })); drop(db); let _ = std::fs::remove_file(db_path); diff --git a/src/media/ffmpeg/mod.rs b/src/media/ffmpeg/mod.rs index d7c2f31..c98f6f9 100644 --- a/src/media/ffmpeg/mod.rs +++ b/src/media/ffmpeg/mod.rs @@ -1054,7 +1054,7 @@ mod tests { .unwrap_or_else(|err| panic!("failed to build videotoolbox args: {err}")); assert!(args.contains(&"hevc_videotoolbox".to_string())); assert!(!args.contains(&"hvc1".to_string())); - assert!(!args.contains(&"-q:v".to_string())); + assert!(args.contains(&"-q:v".to_string())); // P1-2 fix: Cq maps to -q:v assert!(!args.contains(&"-b:v".to_string())); } @@ -1074,7 +1074,7 @@ mod tests { .unwrap_or_else(|err| panic!("failed to build mp4 videotoolbox args: {err}")); assert!(args.contains(&"hevc_videotoolbox".to_string())); assert!(args.contains(&"hvc1".to_string())); - assert!(!args.contains(&"-q:v".to_string())); + assert!(args.contains(&"-q:v".to_string())); // P1-2 fix: Cq maps to -q:v } #[test] diff --git a/src/media/ffmpeg/videotoolbox.rs b/src/media/ffmpeg/videotoolbox.rs index 9aeb23b..8226c96 100644 --- a/src/media/ffmpeg/videotoolbox.rs +++ b/src/media/ffmpeg/videotoolbox.rs @@ -6,10 +6,6 @@ pub fn append_args( tag_hevc_as_hvc1: bool, rate_control: Option<&RateControl>, ) { - // VideoToolbox quality is controlled via -global_quality (0–100, 100=best). - // The config uses CQ-style semantics where lower value = better quality, - // so we invert: global_quality = 100 - cq_value. - // Bitrate mode is handled by the shared builder in mod.rs. match encoder { Encoder::Av1Videotoolbox => { args.extend(["-c:v".to_string(), "av1_videotoolbox".to_string()]); @@ -25,8 +21,27 @@ pub fn append_args( } _ => {} } - if let Some(RateControl::Cq { value }) = rate_control { - let global_quality = 100u8.saturating_sub(*value); - args.extend(["-global_quality".to_string(), global_quality.to_string()]); + + match rate_control { + Some(RateControl::Cq { value }) => { + // VideoToolbox -q:v: 1 (best) to 100 (worst). Config value is CRF-style + // where lower = better quality. Clamp to 1-51 range matching x264/x265. + let q = (*value).clamp(1, 51); + args.extend(["-q:v".to_string(), q.to_string()]); + } + Some(RateControl::Bitrate { kbps, .. }) => { + args.extend([ + "-b:v".to_string(), + format!("{}k", kbps), + "-maxrate".to_string(), + format!("{}k", kbps * 2), + "-bufsize".to_string(), + format!("{}k", kbps * 4), + ]); + } + _ => { + // Default: constant quality at 28 (HEVC-equivalent mid quality) + args.extend(["-q:v".to_string(), "28".to_string()]); + } } } diff --git a/src/media/pipeline.rs b/src/media/pipeline.rs index 1092a17..262b131 100644 --- a/src/media/pipeline.rs +++ b/src/media/pipeline.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::SystemTime; -use tokio::sync::{RwLock, broadcast}; +use tokio::sync::RwLock; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MediaMetadata { @@ -94,14 +94,6 @@ pub struct MediaAnalysis { pub confidence: AnalysisConfidence, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ExecutionStats { - pub encode_time_secs: f64, - pub input_size: u64, - pub output_size: u64, - pub vmaf: Option, -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum DynamicRange { @@ -390,7 +382,6 @@ pub struct ExecutionResult { pub fallback_occurred: bool, pub actual_output_codec: Option, pub actual_encoder_name: Option, - pub stats: ExecutionStats, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -432,7 +423,6 @@ pub struct Pipeline { orchestrator: Arc, config: Arc>, hardware_state: HardwareState, - tx: Arc>, event_channels: Arc, dry_run: bool, } @@ -466,7 +456,6 @@ impl Pipeline { orchestrator: Arc, config: Arc>, hardware_state: HardwareState, - tx: Arc>, event_channels: Arc, dry_run: bool, ) -> Self { @@ -475,7 +464,6 @@ impl Pipeline { orchestrator, config, hardware_state, - tx, event_channels, dry_run, } @@ -594,8 +582,7 @@ impl Pipeline { let job_id = job.id; // Update status to analyzing - self.db - .update_job_status(job_id, crate::db::JobState::Analyzing) + self.update_job_state(job_id, crate::db::JobState::Analyzing) .await?; // Run ffprobe analysis @@ -604,18 +591,24 @@ impl Pipeline { .analyze(std::path::Path::new(&job.input_path)) .await { - Ok(a) => a, + Ok(a) => { + // Store analyzed metadata for completed job detail retrieval + let _ = self.db.set_job_input_metadata(job_id, &a.metadata).await; + a + } Err(e) => { let reason = format!("analysis_failed|error={e}"); let failure_explanation = crate::explanations::failure_from_summary(&reason); - let _ = self.db.add_log("error", Some(job_id), &reason).await; - self.db.add_decision(job_id, "skip", &reason).await.ok(); - self.db - .upsert_job_failure_explanation(job_id, &failure_explanation) - .await - .ok(); - self.db - .update_job_status(job_id, crate::db::JobState::Failed) + if let Err(e) = self.db.add_log("error", Some(job_id), &reason).await { + tracing::warn!(job_id, "Failed to record log: {e}"); + } + if let Err(e) = self.db.add_decision(job_id, "skip", &reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } + if let Err(e) = self.db.upsert_job_failure_explanation(job_id, &failure_explanation).await { + tracing::warn!(job_id, "Failed to record failure explanation: {e}"); + } + self.update_job_state(job_id, crate::db::JobState::Failed) .await?; return Ok(()); } @@ -645,14 +638,16 @@ impl Pipeline { Err(e) => { let reason = format!("planning_failed|error={e}"); let failure_explanation = crate::explanations::failure_from_summary(&reason); - let _ = self.db.add_log("error", Some(job_id), &reason).await; - self.db.add_decision(job_id, "skip", &reason).await.ok(); - self.db - .upsert_job_failure_explanation(job_id, &failure_explanation) - .await - .ok(); - self.db - .update_job_status(job_id, crate::db::JobState::Failed) + if let Err(e) = self.db.add_log("error", Some(job_id), &reason).await { + tracing::warn!(job_id, "Failed to record log: {e}"); + } + if let Err(e) = self.db.add_decision(job_id, "skip", &reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } + if let Err(e) = self.db.upsert_job_failure_explanation(job_id, &failure_explanation).await { + tracing::warn!(job_id, "Failed to record failure explanation: {e}"); + } + self.update_job_state(job_id, crate::db::JobState::Failed) .await?; return Ok(()); } @@ -668,23 +663,26 @@ impl Pipeline { "Job skipped: {}", skip_code ); - self.db.add_decision(job_id, "skip", reason).await.ok(); - self.db - .update_job_status(job_id, crate::db::JobState::Skipped) + if let Err(e) = self.db.add_decision(job_id, "skip", reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } + self.update_job_state(job_id, crate::db::JobState::Skipped) .await?; } crate::media::pipeline::TranscodeDecision::Remux { reason } => { - self.db.add_decision(job_id, "transcode", reason).await.ok(); + if let Err(e) = self.db.add_decision(job_id, "transcode", reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } // Leave as queued — will be picked up for remux when engine starts - self.db - .update_job_status(job_id, crate::db::JobState::Queued) + self.update_job_state(job_id, crate::db::JobState::Queued) .await?; } crate::media::pipeline::TranscodeDecision::Transcode { reason } => { - self.db.add_decision(job_id, "transcode", reason).await.ok(); + if let Err(e) = self.db.add_decision(job_id, "transcode", reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } // Leave as queued — will be picked up for encoding when engine starts - self.db - .update_job_status(job_id, crate::db::JobState::Queued) + self.update_job_state(job_id, crate::db::JobState::Queued) .await?; } } @@ -775,15 +773,16 @@ impl Pipeline { Err(e) => { let msg = format!("Probing failed: {e}"); tracing::error!("Job {}: {}", job.id, msg); - let _ = self.db.add_log("error", Some(job.id), &msg).await; + if let Err(e) = self.db.add_log("error", Some(job.id), &msg).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } let explanation = crate::explanations::failure_from_summary(&msg); - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } return Err(JobFailure::MediaCorrupt); } }; @@ -829,15 +828,16 @@ impl Pipeline { Err(err) => { let msg = format!("Invalid conversion job settings: {err}"); tracing::error!("Job {}: {}", job.id, msg); - let _ = self.db.add_log("error", Some(job.id), &msg).await; + if let Err(e) = self.db.add_log("error", Some(job.id), &msg).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } let explanation = crate::explanations::failure_from_summary(&msg); - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } return Err(JobFailure::PlannerBug); } }; @@ -847,15 +847,16 @@ impl Pipeline { Err(err) => { let msg = format!("Conversion planning failed: {err}"); tracing::error!("Job {}: {}", job.id, msg); - let _ = self.db.add_log("error", Some(job.id), &msg).await; + if let Err(e) = self.db.add_log("error", Some(job.id), &msg).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } let explanation = crate::explanations::failure_from_summary(&msg); - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } return Err(JobFailure::PlannerBug); } } @@ -866,15 +867,16 @@ impl Pipeline { Err(err) => { let msg = format!("Failed to resolve library profile: {err}"); tracing::error!("Job {}: {}", job.id, msg); - let _ = self.db.add_log("error", Some(job.id), &msg).await; + if let Err(e) = self.db.add_log("error", Some(job.id), &msg).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } let explanation = crate::explanations::failure_from_summary(&msg); - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } return Err(JobFailure::Transient); } }; @@ -886,15 +888,16 @@ impl Pipeline { Err(e) => { let msg = format!("Planner failed: {e}"); tracing::error!("Job {}: {}", job.id, msg); - let _ = self.db.add_log("error", Some(job.id), &msg).await; + if let Err(e) = self.db.add_log("error", Some(job.id), &msg).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } let explanation = crate::explanations::failure_from_summary(&msg); - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } return Err(JobFailure::PlannerBug); } } @@ -954,10 +957,12 @@ impl Pipeline { explanation.code, explanation.summary ); - let _ = self.db.add_decision(job.id, "skip", &reason).await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Skipped) - .await; + if let Err(e) = self.db.add_decision(job.id, "skip", &reason).await { + tracing::warn!(job_id = job.id, "Failed to record decision: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Skipped).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } return Ok(()); } @@ -981,13 +986,6 @@ impl Pipeline { reason: explanation.legacy_reason.clone(), explanation: Some(explanation.clone()), }); - let _ = self.tx.send(crate::db::AlchemistEvent::Decision { - job_id: job.id, - action: action.to_string(), - reason: explanation.legacy_reason.clone(), - explanation: Some(explanation), - }); - if self.update_job_state(job.id, next_status).await.is_err() { return Err(JobFailure::Transient); } @@ -1022,7 +1020,6 @@ impl Pipeline { self.orchestrator.clone(), self.db.clone(), hw_info.clone(), - self.tx.clone(), self.event_channels.clone(), self.dry_run, ); @@ -1034,28 +1031,28 @@ impl Pipeline { tracing::error!("Job {}: Encoder fallback detected and not allowed.", job.id); let summary = "Encoder fallback detected and not allowed."; let explanation = crate::explanations::failure_from_summary(summary); - let _ = self.db.add_log("error", Some(job.id), summary).await; - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; - let _ = self - .db - .insert_encode_attempt(crate::db::EncodeAttemptInput { - job_id: job.id, - attempt_number: current_attempt_number, - started_at: Some(encode_started_at.to_rfc3339()), - outcome: "failed".to_string(), - failure_code: Some("fallback_blocked".to_string()), - failure_summary: Some(summary.to_string()), - input_size_bytes: Some(metadata.size_bytes as i64), - output_size_bytes: None, - encode_time_seconds: Some(start_time.elapsed().as_secs_f64()), - }) - .await; + if let Err(e) = self.db.add_log("error", Some(job.id), summary).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state: {e}"); + } + if let Err(e) = self.db.insert_encode_attempt(crate::db::EncodeAttemptInput { + job_id: job.id, + attempt_number: current_attempt_number, + started_at: Some(encode_started_at.to_rfc3339()), + outcome: "failed".to_string(), + failure_code: Some("fallback_blocked".to_string()), + failure_summary: Some(summary.to_string()), + input_size_bytes: Some(metadata.size_bytes as i64), + output_size_bytes: None, + encode_time_seconds: Some(start_time.elapsed().as_secs_f64()), + }).await { + tracing::warn!(job_id = job.id, "Failed to record encode attempt: {e}"); + } return Err(JobFailure::EncoderUnavailable); } @@ -1137,49 +1134,48 @@ impl Pipeline { .await; if let crate::error::AlchemistError::Cancelled = e { - let _ = self - .update_job_state(job.id, crate::db::JobState::Cancelled) - .await; - let _ = self - .db - .insert_encode_attempt(crate::db::EncodeAttemptInput { - job_id: job.id, - attempt_number: current_attempt_number, - started_at: Some(encode_started_at.to_rfc3339()), - outcome: "cancelled".to_string(), - failure_code: None, - failure_summary: None, - input_size_bytes: Some(metadata.size_bytes as i64), - output_size_bytes: None, - encode_time_seconds: Some(start_time.elapsed().as_secs_f64()), - }) - .await; + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Cancelled).await { + tracing::warn!(job_id = job.id, "Failed to update job state to cancelled: {e}"); + } + if let Err(e) = self.db.insert_encode_attempt(crate::db::EncodeAttemptInput { + job_id: job.id, + attempt_number: current_attempt_number, + started_at: Some(encode_started_at.to_rfc3339()), + outcome: "cancelled".to_string(), + failure_code: None, + failure_summary: None, + input_size_bytes: Some(metadata.size_bytes as i64), + output_size_bytes: None, + encode_time_seconds: Some(start_time.elapsed().as_secs_f64()), + }).await { + tracing::warn!(job_id = job.id, "Failed to record encode attempt: {e}"); + } } else { let msg = format!("Transcode failed: {e}"); tracing::error!("Job {}: {}", job.id, msg); - let _ = self.db.add_log("error", Some(job.id), &msg).await; + if let Err(e) = self.db.add_log("error", Some(job.id), &msg).await { + tracing::warn!(job_id = job.id, "Failed to record log: {e}"); + } let explanation = crate::explanations::failure_from_summary(&msg); - let _ = self - .db - .upsert_job_failure_explanation(job.id, &explanation) - .await; - let _ = self - .update_job_state(job.id, crate::db::JobState::Failed) - .await; - let _ = self - .db - .insert_encode_attempt(crate::db::EncodeAttemptInput { - job_id: job.id, - attempt_number: current_attempt_number, - started_at: Some(encode_started_at.to_rfc3339()), - outcome: "failed".to_string(), - failure_code: Some(explanation.code.clone()), - failure_summary: Some(msg), - input_size_bytes: Some(metadata.size_bytes as i64), - output_size_bytes: None, - encode_time_seconds: Some(start_time.elapsed().as_secs_f64()), - }) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job.id, &explanation).await { + tracing::warn!(job_id = job.id, "Failed to record failure explanation: {e}"); + } + if let Err(e) = self.update_job_state(job.id, crate::db::JobState::Failed).await { + tracing::warn!(job_id = job.id, "Failed to update job state to failed: {e}"); + } + if let Err(e) = self.db.insert_encode_attempt(crate::db::EncodeAttemptInput { + job_id: job.id, + attempt_number: current_attempt_number, + started_at: Some(encode_started_at.to_rfc3339()), + outcome: "failed".to_string(), + failure_code: Some(explanation.code.clone()), + failure_summary: Some(msg), + input_size_bytes: Some(metadata.size_bytes as i64), + output_size_bytes: None, + encode_time_seconds: Some(start_time.elapsed().as_secs_f64()), + }).await { + tracing::warn!(job_id = job.id, "Failed to record encode attempt: {e}"); + } } Err(map_failure(&e)) } @@ -1187,30 +1183,51 @@ impl Pipeline { } async fn update_job_state(&self, job_id: i64, status: crate::db::JobState) -> Result<()> { + if self.orchestrator.is_cancel_requested(job_id).await { + match status { + crate::db::JobState::Encoding + | crate::db::JobState::Remuxing + | crate::db::JobState::Skipped + | crate::db::JobState::Completed => { + tracing::info!( + "Ignoring state update to {:?} for job {} because it was cancelled", + status, + job_id + ); + self.orchestrator.remove_cancel_request(job_id).await; + return Ok(()); + } + _ => {} + } + } + if let Err(e) = self.db.update_job_status(job_id, status).await { tracing::error!("Failed to update job {} status {:?}: {}", job_id, status, e); return Err(e); } + + // Remove from cancel_requested if it's a terminal state + match status { + crate::db::JobState::Completed + | crate::db::JobState::Failed + | crate::db::JobState::Cancelled + | crate::db::JobState::Skipped => { + self.orchestrator.remove_cancel_request(job_id).await; + } + _ => {} + } + let _ = self .event_channels .jobs .send(crate::db::JobEvent::StateChanged { job_id, status }); - let _ = self - .tx - .send(crate::db::AlchemistEvent::JobStateChanged { job_id, status }); Ok(()) } async fn update_job_progress(&self, job_id: i64, progress: f64) { if let Err(e) = self.db.update_job_progress(job_id, progress).await { tracing::error!("Failed to update job progress: {}", e); - return; } - let _ = self.tx.send(crate::db::AlchemistEvent::Progress { - job_id, - percentage: progress, - time: String::new(), - }); } async fn should_stop_job(&self, job_id: i64) -> Result { @@ -1247,10 +1264,9 @@ impl Pipeline { tracing::error!("Job {}: Input file is empty. Finalizing as failed.", job_id); let _ = std::fs::remove_file(context.temp_output_path); cleanup_temp_subtitle_output(job_id, context.plan).await; - - self.update_job_state(job_id, crate::db::JobState::Failed) - .await?; - return Ok(()); + return Err(crate::error::AlchemistError::FFmpeg( + "Input file is empty".to_string(), + )); } let reduction = 1.0 - (output_size as f64 / input_size as f64); @@ -1282,7 +1298,9 @@ impl Pipeline { reduction, config.transcode.size_reduction_threshold, output_size ) }; - let _ = self.db.add_decision(job_id, "skip", &reason).await; + if let Err(e) = self.db.add_decision(job_id, "skip", &reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } self.update_job_state(job_id, crate::db::JobState::Skipped) .await?; return Ok(()); @@ -1353,11 +1371,14 @@ impl Pipeline { let mut media_duration = context.metadata.duration_secs; if media_duration <= 0.0 { - media_duration = crate::media::analyzer::Analyzer::probe_async(input_path) - .await - .ok() - .and_then(|meta| meta.format.duration.parse::().ok()) - .unwrap_or(0.0); + match crate::media::analyzer::Analyzer::probe_async(input_path).await { + Ok(meta) => { + media_duration = meta.format.duration.parse::().unwrap_or(0.0); + } + Err(e) => { + tracing::warn!(job_id, "Failed to reprobe output for duration: {e}"); + } + } } let encode_speed = if encode_duration > 0.0 && media_duration > 0.0 { @@ -1511,14 +1532,17 @@ impl Pipeline { tracing::error!("Job {}: Finalization failed: {}", job_id, err); let message = format!("Finalization failed: {err}"); - let _ = self.db.add_log("error", Some(job_id), &message).await; + if let Err(e) = self.db.add_log("error", Some(job_id), &message).await { + tracing::warn!(job_id, "Failed to record log: {e}"); + } let failure_explanation = crate::explanations::failure_from_summary(&message); - let _ = self - .db - .upsert_job_failure_explanation(job_id, &failure_explanation) - .await; + if let Err(e) = self.db.upsert_job_failure_explanation(job_id, &failure_explanation).await { + tracing::warn!(job_id, "Failed to record failure explanation: {e}"); + } if let crate::error::AlchemistError::QualityCheckFailed(reason) = err { - let _ = self.db.add_decision(job_id, "reject", reason).await; + if let Err(e) = self.db.add_decision(job_id, "reject", reason).await { + tracing::warn!(job_id, "Failed to record decision: {e}"); + } } if context.temp_output_path.exists() { @@ -1653,7 +1677,7 @@ mod tests { use crate::db::Db; use crate::system::hardware::{HardwareInfo, HardwareState, Vendor}; use std::sync::Arc; - use tokio::sync::{RwLock, broadcast}; + use tokio::sync::RwLock; #[test] fn generated_output_pattern_matches_default_suffix() { @@ -1789,10 +1813,9 @@ mod tests { selection_reason: String::new(), probe_summary: crate::system::hardware::ProbeSummary::default(), })); - let (tx, _rx) = broadcast::channel(8); - let (jobs_tx, _) = broadcast::channel(100); - let (config_tx, _) = broadcast::channel(10); - let (system_tx, _) = broadcast::channel(10); + let (jobs_tx, _) = tokio::sync::broadcast::channel(100); + let (config_tx, _) = tokio::sync::broadcast::channel(10); + let (system_tx, _) = tokio::sync::broadcast::channel(10); let event_channels = Arc::new(crate::db::EventChannels { jobs: jobs_tx, config: config_tx, @@ -1803,7 +1826,6 @@ mod tests { Arc::new(Transcoder::new()), config.clone(), hardware_state, - Arc::new(tx), event_channels, true, ); @@ -1864,12 +1886,6 @@ mod tests { fallback_occurred: false, actual_output_codec: Some(crate::config::OutputCodec::H264), actual_encoder_name: Some("libx264".to_string()), - stats: ExecutionStats { - encode_time_secs: 0.0, - input_size: 0, - output_size: 0, - vmaf: None, - }, }; let config_snapshot = config.read().await.clone(); diff --git a/src/media/processor.rs b/src/media/processor.rs index 5f4fa25..4fdd0c5 100644 --- a/src/media/processor.rs +++ b/src/media/processor.rs @@ -1,6 +1,6 @@ use crate::Transcoder; use crate::config::Config; -use crate::db::{AlchemistEvent, Db, EventChannels, JobEvent, SystemEvent}; +use crate::db::{Db, EventChannels, JobEvent, SystemEvent}; use crate::error::Result; use crate::media::pipeline::Pipeline; use crate::media::scanner::Scanner; @@ -8,7 +8,7 @@ use crate::system::hardware::HardwareState; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use tokio::sync::{Mutex, OwnedSemaphorePermit, RwLock, Semaphore, broadcast}; +use tokio::sync::{Mutex, OwnedSemaphorePermit, RwLock, Semaphore}; use tracing::{debug, error, info}; pub struct Agent { @@ -16,7 +16,6 @@ pub struct Agent { orchestrator: Arc, config: Arc>, hardware_state: HardwareState, - tx: Arc>, event_channels: Arc, semaphore: Arc, semaphore_limit: Arc, @@ -39,7 +38,6 @@ impl Agent { orchestrator: Arc, config: Arc>, hardware_state: HardwareState, - tx: broadcast::Sender, event_channels: Arc, dry_run: bool, ) -> Self { @@ -54,7 +52,6 @@ impl Agent { orchestrator, config, hardware_state, - tx: Arc::new(tx), event_channels, semaphore: Arc::new(Semaphore::new(concurrent_jobs)), semaphore_limit: Arc::new(AtomicUsize::new(concurrent_jobs)), @@ -99,15 +96,8 @@ impl Agent { job_id: 0, status: crate::db::JobState::Queued, }); - // Also send to legacy channel for backwards compatibility - let _ = self.tx.send(AlchemistEvent::JobStateChanged { - job_id: 0, - status: crate::db::JobState::Queued, - }); - // Notify scan completed let _ = self.event_channels.system.send(SystemEvent::ScanCompleted); - let _ = self.tx.send(AlchemistEvent::ScanCompleted); Ok(()) } @@ -479,7 +469,7 @@ impl Agent { if self.in_flight_jobs.load(Ordering::SeqCst) == 0 && !self.idle_notified.swap(true, Ordering::SeqCst) { - let _ = self.tx.send(crate::db::AlchemistEvent::EngineIdle); + let _ = self.event_channels.system.send(SystemEvent::EngineIdle); } drop(permit); tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; @@ -507,7 +497,6 @@ impl Agent { self.orchestrator.clone(), self.config.clone(), self.hardware_state.clone(), - self.tx.clone(), self.event_channels.clone(), self.dry_run, ) diff --git a/src/notifications.rs b/src/notifications.rs index de8dd40..e7aefaf 100644 --- a/src/notifications.rs +++ b/src/notifications.rs @@ -1,5 +1,5 @@ use crate::config::Config; -use crate::db::{AlchemistEvent, Db, NotificationTarget}; +use crate::db::{Db, EventChannels, JobEvent, NotificationTarget, SystemEvent}; use crate::explanations::Explanation; use chrono::Timelike; use lettre::message::{Mailbox, Message, SinglePart, header::ContentType}; @@ -12,7 +12,7 @@ use std::net::IpAddr; use std::sync::Arc; use std::time::Duration; use tokio::net::lookup_host; -use tokio::sync::{Mutex, RwLock, broadcast}; +use tokio::sync::{Mutex, RwLock}; use tracing::{error, warn}; type NotificationResult = Result>; @@ -86,9 +86,21 @@ fn endpoint_url_for_target(target: &NotificationTarget) -> NotificationResult Option<&'static str> { +/// Internal event type that unifies the events the notification system cares about. +#[derive(Debug, Clone, serde::Serialize)] +#[serde(tag = "type", content = "data")] +enum NotifiableEvent { + JobStateChanged { + job_id: i64, + status: crate::db::JobState, + }, + ScanCompleted, + EngineIdle, +} + +fn event_key(event: &NotifiableEvent) -> Option<&'static str> { match event { - AlchemistEvent::JobStateChanged { status, .. } => match status { + NotifiableEvent::JobStateChanged { status, .. } => match status { crate::db::JobState::Queued => Some(crate::config::NOTIFICATION_EVENT_ENCODE_QUEUED), crate::db::JobState::Encoding | crate::db::JobState::Remuxing => { Some(crate::config::NOTIFICATION_EVENT_ENCODE_STARTED) @@ -99,9 +111,8 @@ fn event_key_from_event(event: &AlchemistEvent) -> Option<&'static str> { crate::db::JobState::Failed => Some(crate::config::NOTIFICATION_EVENT_ENCODE_FAILED), _ => None, }, - AlchemistEvent::ScanCompleted => Some(crate::config::NOTIFICATION_EVENT_SCAN_COMPLETED), - AlchemistEvent::EngineIdle => Some(crate::config::NOTIFICATION_EVENT_ENGINE_IDLE), - _ => None, + NotifiableEvent::ScanCompleted => Some(crate::config::NOTIFICATION_EVENT_SCAN_COMPLETED), + NotifiableEvent::EngineIdle => Some(crate::config::NOTIFICATION_EVENT_ENGINE_IDLE), } } @@ -114,22 +125,104 @@ impl NotificationManager { } } - pub fn start_listener(&self, mut rx: broadcast::Receiver) { + /// Build an HTTP client with SSRF protections: DNS resolution timeout, + /// private-IP blocking (unless allow_local_notifications), no redirects, + /// and a 10-second request timeout. + async fn build_safe_client( + &self, + target: &NotificationTarget, + ) -> NotificationResult { + if let Some(endpoint_url) = endpoint_url_for_target(target)? { + let url = Url::parse(&endpoint_url)?; + let host = url + .host_str() + .ok_or("notification endpoint host is missing")?; + let port = url.port_or_known_default().ok_or("invalid port")?; + + let allow_local = self + .config + .read() + .await + .notifications + .allow_local_notifications; + + if !allow_local && host.eq_ignore_ascii_case("localhost") { + return Err("localhost is not allowed as a notification endpoint".into()); + } + + let addr = format!("{}:{}", host, port); + let ips = tokio::time::timeout(Duration::from_secs(3), lookup_host(&addr)).await??; + + let target_ip = if allow_local { + ips.into_iter() + .map(|a| a.ip()) + .next() + .ok_or("no IP address found for notification endpoint")? + } else { + ips.into_iter() + .map(|a| a.ip()) + .find(|ip| !is_private_ip(*ip)) + .ok_or("no public IP address found for notification endpoint")? + }; + + Ok(Client::builder() + .timeout(Duration::from_secs(10)) + .redirect(Policy::none()) + .resolve(host, std::net::SocketAddr::new(target_ip, port)) + .build()?) + } else { + Ok(Client::builder() + .timeout(Duration::from_secs(10)) + .redirect(Policy::none()) + .build()?) + } + } + + pub fn start_listener(&self, event_channels: &EventChannels) { let manager_clone = self.clone(); let summary_manager = self.clone(); + // Listen for job events (state changes are the only ones we notify on) + let mut jobs_rx = event_channels.jobs.subscribe(); + let job_manager = self.clone(); tokio::spawn(async move { loop { - match rx.recv().await { - Ok(event) => { - if let Err(e) = manager_clone.handle_event(event).await { + match jobs_rx.recv().await { + Ok(JobEvent::StateChanged { job_id, status }) => { + let event = NotifiableEvent::JobStateChanged { job_id, status }; + if let Err(e) = job_manager.handle_event(event).await { error!("Notification error: {}", e); } } - Err(broadcast::error::RecvError::Lagged(_)) => { - warn!("Notification listener lagged") + Ok(_) => {} // Ignore Progress, Decision, Log + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + warn!("Notification job listener lagged") } - Err(broadcast::error::RecvError::Closed) => break, + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, + } + } + }); + + // Listen for system events (scan completed, engine idle) + let mut system_rx = event_channels.system.subscribe(); + tokio::spawn(async move { + loop { + match system_rx.recv().await { + Ok(SystemEvent::ScanCompleted) => { + if let Err(e) = manager_clone.handle_event(NotifiableEvent::ScanCompleted).await { + error!("Notification error: {}", e); + } + } + Ok(SystemEvent::EngineIdle) => { + if let Err(e) = manager_clone.handle_event(NotifiableEvent::EngineIdle).await { + error!("Notification error: {}", e); + } + } + Ok(_) => {} // Ignore ScanStarted, EngineStatusChanged, HardwareStateChanged + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { + warn!("Notification system listener lagged") + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => break, } } }); @@ -145,14 +238,14 @@ impl NotificationManager { } pub async fn send_test(&self, target: &NotificationTarget) -> NotificationResult<()> { - let event = AlchemistEvent::JobStateChanged { + let event = NotifiableEvent::JobStateChanged { job_id: 0, status: crate::db::JobState::Completed, }; self.send(target, &event).await } - async fn handle_event(&self, event: AlchemistEvent) -> NotificationResult<()> { + async fn handle_event(&self, event: NotifiableEvent) -> NotificationResult<()> { let targets = match self.db.get_notification_targets().await { Ok(t) => t, Err(e) => { @@ -165,7 +258,7 @@ impl NotificationManager { return Ok(()); } - let event_key = match event_key_from_event(&event) { + let event_key = match event_key(&event) { Some(event_key) => event_key, None => return Ok(()), }; @@ -224,10 +317,13 @@ impl NotificationManager { let summary_key = now.format("%Y-%m-%d").to_string(); { - let last_sent = self.daily_summary_last_sent.lock().await; + let mut last_sent = self.daily_summary_last_sent.lock().await; if last_sent.as_deref() == Some(summary_key.as_str()) { return Ok(()); } + // Mark sent before releasing lock to prevent duplicate sends + // if the scheduler fires twice in the same minute. + *last_sent = Some(summary_key.clone()); } let summary = self.db.get_daily_summary_stats().await?; @@ -252,63 +348,19 @@ impl NotificationManager { } } - *self.daily_summary_last_sent.lock().await = Some(summary_key); Ok(()) } async fn send( &self, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, ) -> NotificationResult<()> { - let event_key = event_key_from_event(event).unwrap_or("unknown"); - let client = if let Some(endpoint_url) = endpoint_url_for_target(target)? { - let url = Url::parse(&endpoint_url)?; - let host = url - .host_str() - .ok_or("notification endpoint host is missing")?; - let port = url.port_or_known_default().ok_or("invalid port")?; - - let allow_local = self - .config - .read() - .await - .notifications - .allow_local_notifications; - - if !allow_local && host.eq_ignore_ascii_case("localhost") { - return Err("localhost is not allowed as a notification endpoint".into()); - } - - let addr = format!("{}:{}", host, port); - let ips = tokio::time::timeout(Duration::from_secs(3), lookup_host(&addr)).await??; - - let target_ip = if allow_local { - ips.into_iter() - .map(|a| a.ip()) - .next() - .ok_or("no IP address found for notification endpoint")? - } else { - ips.into_iter() - .map(|a| a.ip()) - .find(|ip| !is_private_ip(*ip)) - .ok_or("no public IP address found for notification endpoint")? - }; - - Client::builder() - .timeout(Duration::from_secs(10)) - .redirect(Policy::none()) - .resolve(host, std::net::SocketAddr::new(target_ip, port)) - .build()? - } else { - Client::builder() - .timeout(Duration::from_secs(10)) - .redirect(Policy::none()) - .build()? - }; + let event_key = event_key(event).unwrap_or("unknown"); + let client = self.build_safe_client(target).await?; let (decision_explanation, failure_explanation) = match event { - AlchemistEvent::JobStateChanged { job_id, status } => { + NotifiableEvent::JobStateChanged { job_id, status } => { let decision_explanation = self .db .get_job_decision_explanation(*job_id) @@ -423,25 +475,24 @@ impl NotificationManager { fn message_for_event( &self, - event: &AlchemistEvent, + event: &NotifiableEvent, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, ) -> String { match event { - AlchemistEvent::JobStateChanged { job_id, status } => self.notification_message( + NotifiableEvent::JobStateChanged { job_id, status } => self.notification_message( *job_id, &status.to_string(), decision_explanation, failure_explanation, ), - AlchemistEvent::ScanCompleted => { + NotifiableEvent::ScanCompleted => { "Library scan completed. Review the queue for newly discovered work.".to_string() } - AlchemistEvent::EngineIdle => { + NotifiableEvent::EngineIdle => { "The engine is idle. There are no active jobs and no queued work ready to run." .to_string() } - _ => "Event occurred".to_string(), } } @@ -472,7 +523,7 @@ impl NotificationManager { &self, client: &Client, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, event_key: &str, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, @@ -511,7 +562,7 @@ impl NotificationManager { &self, client: &Client, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, _event_key: &str, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, @@ -536,7 +587,7 @@ impl NotificationManager { &self, client: &Client, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, event_key: &str, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, @@ -550,16 +601,21 @@ impl NotificationManager { _ => 2, }; - let req = client.post(&config.server_url).json(&json!({ - "title": "Alchemist", - "message": message, - "priority": priority, - "extras": { - "client::display": { - "contentType": "text/plain" + let req = client + .post(format!( + "{}/message", + config.server_url.trim_end_matches('/') + )) + .json(&json!({ + "title": "Alchemist", + "message": message, + "priority": priority, + "extras": { + "client::display": { + "contentType": "text/plain" + } } - } - })); + })); req.header("X-Gotify-Key", config.app_token) .send() .await? @@ -571,7 +627,7 @@ impl NotificationManager { &self, client: &Client, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, event_key: &str, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, @@ -601,7 +657,7 @@ impl NotificationManager { &self, client: &Client, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, _event_key: &str, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, @@ -627,7 +683,7 @@ impl NotificationManager { async fn send_email( &self, target: &NotificationTarget, - event: &AlchemistEvent, + event: &NotifiableEvent, _event_key: &str, decision_explanation: Option<&Explanation>, failure_explanation: Option<&Explanation>, @@ -677,10 +733,11 @@ impl NotificationManager { summary: &crate::db::DailySummaryStats, ) -> NotificationResult<()> { let message = self.daily_summary_message(summary); + let client = self.build_safe_client(target).await?; match target.target_type.as_str() { "discord_webhook" => { let config = parse_target_config::(target)?; - Client::new() + client .post(config.webhook_url) .json(&json!({ "embeds": [{ @@ -696,7 +753,7 @@ impl NotificationManager { } "discord_bot" => { let config = parse_target_config::(target)?; - Client::new() + client .post(format!( "https://discord.com/api/v10/channels/{}/messages", config.channel_id @@ -709,7 +766,7 @@ impl NotificationManager { } "gotify" => { let config = parse_target_config::(target)?; - Client::new() + client .post(config.server_url) .header("X-Gotify-Key", config.app_token) .json(&json!({ @@ -723,7 +780,7 @@ impl NotificationManager { } "webhook" => { let config = parse_target_config::(target)?; - let mut req = Client::new().post(config.url).json(&json!({ + let mut req = client.post(config.url).json(&json!({ "event": crate::config::NOTIFICATION_EVENT_DAILY_SUMMARY, "summary": summary, "message": message, @@ -736,7 +793,7 @@ impl NotificationManager { } "telegram" => { let config = parse_target_config::(target)?; - Client::new() + client .post(format!( "https://api.telegram.org/bot{}/sendMessage", config.bot_token @@ -896,7 +953,7 @@ mod tests { enabled: true, created_at: chrono::Utc::now(), }; - let event = AlchemistEvent::JobStateChanged { + let event = NotifiableEvent::JobStateChanged { job_id: 1, status: crate::db::JobState::Failed, }; @@ -976,7 +1033,7 @@ mod tests { enabled: true, created_at: chrono::Utc::now(), }; - let event = AlchemistEvent::JobStateChanged { + let event = NotifiableEvent::JobStateChanged { job_id: job.id, status: JobState::Failed, }; diff --git a/src/orchestrator.rs b/src/orchestrator.rs index f33278a..44b48ef 100644 --- a/src/orchestrator.rs +++ b/src/orchestrator.rs @@ -17,6 +17,7 @@ pub struct Transcoder { // so there is no deadlock risk. Contention is negligible (≤ concurrent_jobs entries). cancel_channels: Arc>>>, pending_cancels: Arc>>, + pub(crate) cancel_requested: Arc>>, } pub struct TranscodeRequest<'a> { @@ -80,9 +81,22 @@ impl Transcoder { Self { cancel_channels: Arc::new(Mutex::new(HashMap::new())), pending_cancels: Arc::new(Mutex::new(HashSet::new())), + cancel_requested: Arc::new(tokio::sync::RwLock::new(HashSet::new())), } } + pub async fn is_cancel_requested(&self, job_id: i64) -> bool { + self.cancel_requested.read().await.contains(&job_id) + } + + pub async fn remove_cancel_request(&self, job_id: i64) { + self.cancel_requested.write().await.remove(&job_id); + } + + pub async fn add_cancel_request(&self, job_id: i64) { + self.cancel_requested.write().await.insert(job_id); + } + pub fn cancel_job(&self, job_id: i64) -> bool { let mut channels = match self.cancel_channels.lock() { Ok(channels) => channels, diff --git a/src/server/jobs.rs b/src/server/jobs.rs index 3e912a2..6287fe6 100644 --- a/src/server/jobs.rs +++ b/src/server/jobs.rs @@ -39,12 +39,14 @@ pub(crate) fn blocked_jobs_response(message: impl Into, blocked: &[Job]) } pub(crate) async fn request_job_cancel(state: &AppState, job: &Job) -> Result { + state.transcoder.add_cancel_request(job.id).await; match job.status { JobState::Queued => { state .db .update_job_status(job.id, JobState::Cancelled) .await?; + state.transcoder.remove_cancel_request(job.id).await; Ok(true) } JobState::Analyzing | JobState::Resuming => { @@ -55,6 +57,7 @@ pub(crate) async fn request_job_cancel(state: &AppState, job: &Job) -> Result Ok(state.transcoder.cancel_job(job.id)), @@ -162,17 +165,49 @@ pub(crate) async fn batch_jobs_handler( match payload.action.as_str() { "cancel" => { - let mut count = 0_u64; + // Add all cancel requests first (in-memory, cheap). for job in &jobs { - match request_job_cancel(&state, job).await { - Ok(true) => count += 1, - Ok(false) => {} - Err(e) if is_row_not_found(&e) => {} + state.transcoder.add_cancel_request(job.id).await; + } + + // Collect IDs that can be immediately set to Cancelled in the DB. + let mut immediate_ids: Vec = Vec::new(); + let mut active_count: u64 = 0; + + for job in &jobs { + match job.status { + JobState::Queued => { + immediate_ids.push(job.id); + } + JobState::Analyzing | JobState::Resuming => { + if state.transcoder.cancel_job(job.id) { + immediate_ids.push(job.id); + } + } + JobState::Encoding | JobState::Remuxing => { + if state.transcoder.cancel_job(job.id) { + active_count += 1; + } + } + _ => {} + } + } + + // Single batch DB update instead of N individual queries. + if !immediate_ids.is_empty() { + match state.db.batch_cancel_jobs(&immediate_ids).await { + Ok(_) => {} Err(e) => { return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(); } } + // Remove cancel requests for jobs already resolved in DB. + for id in &immediate_ids { + state.transcoder.remove_cancel_request(*id).await; + } } + + let count = immediate_ids.len() as u64 + active_count; axum::Json(serde_json::json!({ "count": count })).into_response() } "delete" | "restart" => { @@ -330,6 +365,7 @@ pub(crate) struct JobDetailResponse { job_failure_summary: Option, decision_explanation: Option, failure_explanation: Option, + queue_position: Option, } pub(crate) async fn get_job_detail_handler( @@ -342,19 +378,7 @@ pub(crate) async fn get_job_detail_handler( Err(e) => return (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()).into_response(), }; - // Avoid long probes while the job is still active. - let metadata = match job.status { - JobState::Queued | JobState::Analyzing => None, - _ => { - let analyzer = crate::media::analyzer::FfmpegAnalyzer; - use crate::media::pipeline::Analyzer; - analyzer - .analyze(std::path::Path::new(&job.input_path)) - .await - .ok() - .map(|analysis| analysis.metadata) - } - }; + let metadata = job.input_metadata(); // Try to get encode stats (using the subquery result or a specific query) // For now we'll just query the encode_stats table if completed @@ -406,6 +430,12 @@ pub(crate) async fn get_job_detail_handler( .await .unwrap_or_default(); + let queue_position = if job.status == JobState::Queued { + state.db.get_queue_position(id).await.unwrap_or(None) + } else { + None + }; + axum::Json(JobDetailResponse { job, metadata, @@ -415,6 +445,7 @@ pub(crate) async fn get_job_detail_handler( job_failure_summary, decision_explanation, failure_explanation, + queue_position, }) .into_response() } diff --git a/src/server/middleware.rs b/src/server/middleware.rs index bc04e33..7bee9fc 100644 --- a/src/server/middleware.rs +++ b/src/server/middleware.rs @@ -76,16 +76,26 @@ pub(crate) async fn auth_middleware( let path = req.uri().path(); let method = req.method().clone(); - if state.setup_required.load(Ordering::Relaxed) - && path != "/api/health" - && path != "/api/ready" - && !request_is_lan(&req) + if state.setup_required.load(Ordering::Relaxed) && path != "/api/health" && path != "/api/ready" { - return ( - StatusCode::FORBIDDEN, - "Alchemist setup is only available from the local network", - ) - .into_response(); + let allowed = if let Some(expected_token) = &state.setup_token { + // Token mode: require `?token=` regardless of client IP. + req.uri() + .query() + .and_then(|q| q.split('&').find_map(|pair| pair.strip_prefix("token="))) + .map(|t| t == expected_token.as_str()) + .unwrap_or(false) + } else { + request_is_lan(&req, &state.trusted_proxies) + }; + + if !allowed { + return ( + StatusCode::FORBIDDEN, + "Alchemist setup is only available from the local network", + ) + .into_response(); + } } // 1. API Protection: Only lock down /api routes @@ -148,12 +158,12 @@ pub(crate) async fn auth_middleware( next.run(req).await } -fn request_is_lan(req: &Request) -> bool { +fn request_is_lan(req: &Request, trusted_proxies: &[IpAddr]) -> bool { let direct_peer = req .extensions() .get::>() .map(|info| info.0.ip()); - let resolved = request_ip(req); + let resolved = request_ip(req, trusted_proxies); // If resolved IP differs from direct peer, forwarded headers were used. // Warn operators so misconfigured proxies surface in logs. @@ -216,7 +226,7 @@ pub(crate) async fn rate_limit_middleware( return next.run(req).await; } - let ip = request_ip(&req).unwrap_or(IpAddr::from([0, 0, 0, 0])); + let ip = request_ip(&req, &state.trusted_proxies).unwrap_or(IpAddr::from([0, 0, 0, 0])); if !allow_global_request(&state, ip).await { return (StatusCode::TOO_MANY_REQUESTS, "Too many requests").into_response(); } @@ -287,18 +297,18 @@ pub(crate) fn get_cookie_value(headers: &axum::http::HeaderMap, name: &str) -> O None } -pub(crate) fn request_ip(req: &Request) -> Option { +pub(crate) fn request_ip(req: &Request, trusted_proxies: &[IpAddr]) -> Option { let peer_ip = req .extensions() .get::>() .map(|info| info.0.ip()); // Only trust proxy headers (X-Forwarded-For, X-Real-IP) when the direct - // TCP peer is a loopback or private IP — i.e., a trusted reverse proxy. - // This prevents external attackers from spoofing these headers to bypass - // rate limiting. + // TCP peer is a trusted reverse proxy. When trusted_proxies is non-empty, + // only those exact IPs (plus loopback) are trusted. Otherwise, fall back + // to trusting all RFC-1918 private ranges (legacy behaviour). if let Some(peer) = peer_ip { - if is_trusted_peer(peer) { + if is_trusted_peer(peer, trusted_proxies) { if let Some(xff) = req.headers().get("X-Forwarded-For") { if let Ok(xff_str) = xff.to_str() { if let Some(ip_str) = xff_str.split(',').next() { @@ -321,13 +331,27 @@ pub(crate) fn request_ip(req: &Request) -> Option { peer_ip } -/// Returns true if the peer IP is a loopback or private address, -/// meaning it is likely a local reverse proxy that can be trusted -/// to set forwarded headers. -fn is_trusted_peer(ip: IpAddr) -> bool { - match ip { - IpAddr::V4(v4) => v4.is_loopback() || v4.is_private() || v4.is_link_local(), - IpAddr::V6(v6) => v6.is_loopback() || v6.is_unique_local() || v6.is_unicast_link_local(), +/// Returns true if the peer IP may be trusted to set forwarded headers. +/// +/// When `trusted_proxies` is non-empty, only loopback addresses and the +/// explicitly configured IPs are trusted, tightening the default which +/// previously trusted all RFC-1918 private ranges. +fn is_trusted_peer(ip: IpAddr, trusted_proxies: &[IpAddr]) -> bool { + let is_loopback = match ip { + IpAddr::V4(v4) => v4.is_loopback(), + IpAddr::V6(v6) => v6.is_loopback(), + }; + if is_loopback { + return true; + } + if trusted_proxies.is_empty() { + // Legacy: trust all private ranges when no explicit list is configured. + match ip { + IpAddr::V4(v4) => v4.is_private() || v4.is_link_local(), + IpAddr::V6(v6) => v6.is_unique_local() || v6.is_unicast_link_local(), + } + } else { + trusted_proxies.contains(&ip) } } diff --git a/src/server/mod.rs b/src/server/mod.rs index fdca9ba..c8871f0 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -17,7 +17,7 @@ mod tests; use crate::Agent; use crate::Transcoder; use crate::config::Config; -use crate::db::{AlchemistEvent, Db, EventChannels}; +use crate::db::{Db, EventChannels}; use crate::error::{AlchemistError, Result}; use crate::system::hardware::{HardwareInfo, HardwareProbeLog, HardwareState}; use axum::{ @@ -38,7 +38,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; use tokio::net::lookup_host; -use tokio::sync::{Mutex, RwLock, broadcast}; +use tokio::sync::{Mutex, RwLock}; use tokio::time::Duration; #[cfg(not(feature = "embed-web"))] use tracing::warn; @@ -71,7 +71,6 @@ pub struct AppState { pub transcoder: Arc, pub scheduler: crate::scheduler::SchedulerHandle, pub event_channels: Arc, - pub tx: broadcast::Sender, // Legacy channel for transition pub setup_required: Arc, pub start_time: Instant, pub telemetry_runtime_id: String, @@ -87,6 +86,10 @@ pub struct AppState { pub(crate) login_rate_limiter: Mutex>, pub(crate) global_rate_limiter: Mutex>, pub(crate) sse_connections: Arc, + /// IPs whose proxy headers are trusted. Empty = trust all private ranges. + pub(crate) trusted_proxies: Vec, + /// If set, setup endpoints require `?token=` query parameter. + pub(crate) setup_token: Option, } pub struct RunServerArgs { @@ -96,7 +99,6 @@ pub struct RunServerArgs { pub transcoder: Arc, pub scheduler: crate::scheduler::SchedulerHandle, pub event_channels: Arc, - pub tx: broadcast::Sender, // Legacy channel for transition pub setup_required: bool, pub config_path: PathBuf, pub config_mutable: bool, @@ -115,7 +117,6 @@ pub async fn run_server(args: RunServerArgs) -> Result<()> { transcoder, scheduler, event_channels, - tx, setup_required, config_path, config_mutable, @@ -145,6 +146,34 @@ pub async fn run_server(args: RunServerArgs) -> Result<()> { sys.refresh_cpu_usage(); sys.refresh_memory(); + // Read setup token from environment (opt-in security layer). + let setup_token = std::env::var("ALCHEMIST_SETUP_TOKEN").ok(); + if setup_token.is_some() { + info!("ALCHEMIST_SETUP_TOKEN is set — setup endpoints require token query param"); + } + + // Parse trusted proxy IPs from config. Unparseable entries are logged and skipped. + let trusted_proxies: Vec = { + let cfg = config.read().await; + cfg.system + .trusted_proxies + .iter() + .filter_map(|s| { + s.parse::() + .map_err(|_| { + error!("Invalid trusted_proxy entry (not a valid IP address): {s}"); + }) + .ok() + }) + .collect() + }; + if !trusted_proxies.is_empty() { + info!( + "Trusted proxies configured ({}): only these IPs will be trusted for X-Forwarded-For headers", + trusted_proxies.len() + ); + } + let state = Arc::new(AppState { db, config, @@ -152,7 +181,6 @@ pub async fn run_server(args: RunServerArgs) -> Result<()> { transcoder, scheduler, event_channels, - tx, setup_required: Arc::new(AtomicBool::new(setup_required)), start_time: std::time::Instant::now(), telemetry_runtime_id: Uuid::new_v4().to_string(), @@ -168,6 +196,8 @@ pub async fn run_server(args: RunServerArgs) -> Result<()> { login_rate_limiter: Mutex::new(HashMap::new()), global_rate_limiter: Mutex::new(HashMap::new()), sse_connections: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + trusted_proxies, + setup_token, }); // Clone agent for shutdown handler before moving state into router diff --git a/src/server/scan.rs b/src/server/scan.rs index ff36f61..cc11613 100644 --- a/src/server/scan.rs +++ b/src/server/scan.rs @@ -126,7 +126,7 @@ async fn run_library_health_scan(db: Arc) { let semaphore = Arc::new(tokio::sync::Semaphore::new(2)); stream::iter(jobs) - .for_each_concurrent(Some(10), { + .for_each_concurrent(None, { let db = db.clone(); let counters = counters.clone(); let semaphore = semaphore.clone(); diff --git a/src/server/sse.rs b/src/server/sse.rs index 188b196..9073cfb 100644 --- a/src/server/sse.rs +++ b/src/server/sse.rs @@ -108,6 +108,10 @@ pub(crate) fn sse_message_for_system_event(event: &SystemEvent) -> SseMessage { event_name: "scan_completed", data: "{}".to_string(), }, + SystemEvent::EngineIdle => SseMessage { + event_name: "engine_idle", + data: "{}".to_string(), + }, SystemEvent::EngineStatusChanged => SseMessage { event_name: "engine_status_changed", data: "{}".to_string(), diff --git a/src/server/system.rs b/src/server/system.rs index 08b5ef4..8ace029 100644 --- a/src/server/system.rs +++ b/src/server/system.rs @@ -1,7 +1,7 @@ //! System information, hardware info, resources, health handlers. use super::{AppState, config_read_error_response}; -use crate::media::pipeline::{Analyzer as _, Planner as _, TranscodeDecision}; +use crate::media::pipeline::{Planner as _, TranscodeDecision}; use axum::{ extract::State, http::StatusCode, @@ -195,7 +195,6 @@ pub(crate) async fn library_intelligence_handler(State(state): State analysis, - Err(_) => continue, + // Use stored metadata only — no live ffprobe spawning per job. + let metadata = match job.input_metadata() { + Some(m) => m, + None => continue, + }; + let analysis = crate::media::pipeline::MediaAnalysis { + metadata, + warnings: vec![], + confidence: crate::media::pipeline::AnalysisConfidence::High, }; let profile: Option = state diff --git a/src/server/tests.rs b/src/server/tests.rs index d4941fc..0340142 100644 --- a/src/server/tests.rs +++ b/src/server/tests.rs @@ -61,7 +61,6 @@ where probe_summary: crate::system::hardware::ProbeSummary::default(), })); let hardware_probe_log = Arc::new(RwLock::new(HardwareProbeLog::default())); - let (tx, _rx) = broadcast::channel(tx_capacity); let transcoder = Arc::new(Transcoder::new()); // Create event channels before Agent @@ -81,7 +80,6 @@ where transcoder.clone(), config.clone(), hardware_state.clone(), - tx.clone(), event_channels.clone(), true, ) @@ -101,7 +99,6 @@ where transcoder, scheduler: scheduler.handle(), event_channels, - tx, setup_required: Arc::new(AtomicBool::new(setup_required)), start_time: Instant::now(), telemetry_runtime_id: "test-runtime".to_string(), @@ -120,6 +117,8 @@ where login_rate_limiter: Mutex::new(HashMap::new()), global_rate_limiter: Mutex::new(HashMap::new()), sse_connections: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + trusted_proxies: Vec::new(), + setup_token: None, }); Ok((state.clone(), app_router(state), config_path, db_path)) @@ -1719,7 +1718,9 @@ async fn clear_completed_archives_jobs_and_preserves_stats() assert!(state.db.get_job_by_id(job.id).await?.is_none()); let aggregated = state.db.get_aggregated_stats().await?; - assert_eq!(aggregated.completed_jobs, 1); + // Archived jobs are excluded from active stats. + assert_eq!(aggregated.completed_jobs, 0); + // encode_stats rows are preserved even after archiving. assert_eq!(aggregated.total_input_size, 2_000); assert_eq!(aggregated.total_output_size, 1_000); diff --git a/tests/generated_media.rs b/tests/generated_media.rs index 8e38fb6..7bcc4b5 100644 --- a/tests/generated_media.rs +++ b/tests/generated_media.rs @@ -317,7 +317,6 @@ where selection_reason: String::new(), probe_summary: alchemist::system::hardware::ProbeSummary::default(), })), - Arc::new(broadcast::channel(16).0), event_channels, false, ); diff --git a/tests/integration_ffmpeg.rs b/tests/integration_ffmpeg.rs index 2752f91..7b0f7e4 100644 --- a/tests/integration_ffmpeg.rs +++ b/tests/integration_ffmpeg.rs @@ -120,7 +120,6 @@ where selection_reason: String::new(), probe_summary: alchemist::system::hardware::ProbeSummary::default(), })), - Arc::new(broadcast::channel(16).0), event_channels, false, ); diff --git a/web/src/components/JobManager.tsx b/web/src/components/JobManager.tsx index fdbcf4f..557c05c 100644 --- a/web/src/components/JobManager.tsx +++ b/web/src/components/JobManager.tsx @@ -182,6 +182,7 @@ function JobManager() { const serverIsTerminal = terminal.includes(serverJob.status); if ( local && + local.status === serverJob.status && terminal.includes(local.status) && serverIsTerminal ) { @@ -232,7 +233,7 @@ function JobManager() { }; }, []); - useJobSSE({ setJobs, fetchJobsRef, encodeStartTimes }); + useJobSSE({ setJobs, setFocusedJob, fetchJobsRef, encodeStartTimes }); useEffect(() => { const encodingJobIds = new Set(); diff --git a/web/src/components/jobs/JobDetailModal.tsx b/web/src/components/jobs/JobDetailModal.tsx index 8b88ddc..281fa9e 100644 --- a/web/src/components/jobs/JobDetailModal.tsx +++ b/web/src/components/jobs/JobDetailModal.tsx @@ -179,7 +179,7 @@ export function JobDetailModal({
Reduction - {((1 - focusedJob.encode_stats.compression_ratio) * 100).toFixed(1)}% Saved + {(focusedJob.encode_stats.compression_ratio * 100).toFixed(1)}% Saved
@@ -262,6 +262,11 @@ export function JobDetailModal({

{focusedEmptyState.detail}

+ {focusedJob.job.status === "queued" && focusedJob.queue_position != null && ( +

+ Queue position: #{focusedJob.queue_position} +

+ )}
) : null} diff --git a/web/src/components/jobs/types.ts b/web/src/components/jobs/types.ts index 5956311..8cd10e2 100644 --- a/web/src/components/jobs/types.ts +++ b/web/src/components/jobs/types.ts @@ -91,6 +91,7 @@ export interface JobDetail { job_failure_summary: string | null; decision_explanation: ExplanationPayload | null; failure_explanation: ExplanationPayload | null; + queue_position: number | null; } export interface CountMessageResponse { diff --git a/web/src/components/jobs/useJobSSE.ts b/web/src/components/jobs/useJobSSE.ts index 983927f..24015f3 100644 --- a/web/src/components/jobs/useJobSSE.ts +++ b/web/src/components/jobs/useJobSSE.ts @@ -1,14 +1,15 @@ import { useEffect } from "react"; import type { MutableRefObject, Dispatch, SetStateAction } from "react"; -import type { Job } from "./types"; +import type { Job, JobDetail } from "./types"; interface UseJobSSEOptions { setJobs: Dispatch>; + setFocusedJob: Dispatch>; fetchJobsRef: MutableRefObject<() => Promise>; encodeStartTimes: MutableRefObject>; } -export function useJobSSE({ setJobs, fetchJobsRef, encodeStartTimes }: UseJobSSEOptions): void { +export function useJobSSE({ setJobs, setFocusedJob, fetchJobsRef, encodeStartTimes }: UseJobSSEOptions): void { useEffect(() => { let eventSource: EventSource | null = null; let cancelled = false; @@ -38,14 +39,18 @@ export function useJobSSE({ setJobs, fetchJobsRef, encodeStartTimes }: UseJobSSE job_id: number; status: string; }; + const terminalStatuses = ["completed", "failed", "cancelled", "skipped"]; if (status === "encoding") { encodeStartTimes.current.set(job_id, Date.now()); - } else { + } else if (terminalStatuses.includes(status)) { encodeStartTimes.current.delete(job_id); } setJobs((prev) => prev.map((job) => job.id === job_id ? { ...job, status } : job) ); + setFocusedJob((prev) => + prev?.job.id === job_id ? { ...prev, job: { ...prev.job, status } } : prev + ); } catch { /* ignore malformed */ }