From 769b05d2017c01022b94d9deee0b1cc2083496bf Mon Sep 17 00:00:00 2001 From: Christian Benincasa Date: Fri, 28 Feb 2025 12:53:29 -0800 Subject: [PATCH] refactor: add media_source_id to relevant entities (#1106) We should be referencing media_sources by their ID on programs, external_ids, etc. This enables us to use proper foreign keys for referential integrity at the DB level, not worry about unique names for media sources, and simplifies a lot of the code relating to media source deletion and the cleanup thereafter. This change also introduces the DBContext, which should allow for arbitrarily calling other DB accessor functions when within transactions and not deadlocking the connection to the DB. --- package.json | 3 +- patches/kysely.patch | 467 ++++++++++++++++++ pnpm-lock.yaml | 19 +- server/src/Server.ts | 2 + server/src/bootstrap.ts | 24 +- server/src/cli/RunServerCommand.ts | 9 +- server/src/db/ChannelDB.ts | 33 ++ server/src/db/DBAccess.ts | 238 ++++++--- server/src/db/ProgramDB.ts | 109 ++-- .../db/converters/ProgramGroupingMinter.ts | 90 +--- server/src/db/converters/ProgramMinter.ts | 66 ++- server/src/db/interfaces/IChannelDB.ts | 2 + server/src/db/mediaSourceDB.ts | 88 ++-- server/src/db/programExternalIdHelpers.ts | 42 +- server/src/db/schema/Program.ts | 11 +- server/src/db/schema/ProgramExternalId.ts | 28 +- .../db/schema/ProgramGroupingExternalId.ts | 22 + .../external/jellyfin/JellyfinItemFinder.ts | 34 +- server/src/external/plex/PlexApiClient.ts | 4 + .../src/migration/DirectMigrationProvider.ts | 2 + .../src/migration/db/DatabaseCopyMigrator.ts | 21 +- ...igration1740691984_ProgramMediaSourceId.ts | 137 +++++ .../legacy_migration/LegacyChannelMigrator.ts | 14 +- .../legacy_migration/libraryMigrator.ts | 14 +- .../legacy_migration/migrationUtil.ts | 19 +- .../fixers/BackfillMediaSourceIdFixer.ts | 76 +++ .../fixers/BackfillProgramExternalIds.ts | 15 +- server/src/tasks/fixers/FixerModule.ts | 2 + .../tasks/fixers/backfillProgramGroupings.ts | 1 + .../SaveJellyfinProgramExternalIdsTask.ts | 9 +- .../plex/SavePlexProgramExternalIdsTask.ts | 19 +- server/src/types/util.ts | 11 +- server/src/util/externalIds.ts | 5 +- shared/src/services/ApiProgramMinter.ts | 10 +- 34 files changed, 1315 insertions(+), 331 deletions(-) create mode 100644 patches/kysely.patch create mode 100644 server/src/migration/db/Migration1740691984_ProgramMediaSourceId.ts create mode 100644 server/src/tasks/fixers/BackfillMediaSourceIdFixer.ts diff --git a/package.json b/package.json index 50686d7e..9512e54c 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,8 @@ "packageManager": "pnpm@9.12.3", "pnpm": { "patchedDependencies": { - "ts-essentials@9.4.1": "patches/ts-essentials@9.4.1.patch" + "ts-essentials@9.4.1": "patches/ts-essentials@9.4.1.patch", + "kysely": "patches/kysely.patch" }, "overrides": { "eslint": "9.17.0", diff --git a/patches/kysely.patch b/patches/kysely.patch new file mode 100644 index 00000000..92bb1301 --- /dev/null +++ b/patches/kysely.patch @@ -0,0 +1,467 @@ +diff --git a/dist/esm/index.d.ts b/dist/esm/index.d.ts +index d556ad76e2a23d600e3ee3024bad9d7dd8beccf7..5f25e02adfed39f2cbebb7db31d04e9615b1a544 100644 +--- a/dist/esm/index.d.ts ++++ b/dist/esm/index.d.ts +@@ -1,95 +1,48 @@ +-export * from './kysely.js'; +-export * from './query-creator.js'; +-export * from './expression/expression.js'; +-export { ExpressionBuilder, expressionBuilder, } from './expression/expression-builder.js'; +-export * from './expression/expression-wrapper.js'; +-export * from './query-builder/where-interface.js'; +-export * from './query-builder/returning-interface.js'; +-export * from './query-builder/output-interface.js'; +-export * from './query-builder/having-interface.js'; +-export * from './query-builder/select-query-builder.js'; +-export * from './query-builder/insert-query-builder.js'; +-export * from './query-builder/update-query-builder.js'; +-export * from './query-builder/delete-query-builder.js'; +-export * from './query-builder/no-result-error.js'; +-export * from './query-builder/join-builder.js'; +-export * from './query-builder/function-module.js'; +-export * from './query-builder/insert-result.js'; +-export * from './query-builder/delete-result.js'; +-export * from './query-builder/update-result.js'; +-export * from './query-builder/on-conflict-builder.js'; +-export * from './query-builder/aggregate-function-builder.js'; +-export * from './query-builder/case-builder.js'; +-export * from './query-builder/json-path-builder.js'; +-export * from './query-builder/merge-query-builder.js'; +-export * from './query-builder/merge-result.js'; +-export * from './raw-builder/raw-builder.js'; +-export * from './raw-builder/sql.js'; +-export * from './query-executor/query-executor.js'; +-export * from './query-executor/default-query-executor.js'; +-export * from './query-executor/noop-query-executor.js'; +-export * from './query-executor/query-executor-provider.js'; +-export * from './query-compiler/default-query-compiler.js'; +-export * from './query-compiler/compiled-query.js'; +-export * from './schema/schema.js'; +-export * from './schema/create-table-builder.js'; +-export * from './schema/create-type-builder.js'; +-export * from './schema/drop-table-builder.js'; +-export * from './schema/drop-type-builder.js'; +-export * from './schema/create-index-builder.js'; +-export * from './schema/drop-index-builder.js'; +-export * from './schema/create-schema-builder.js'; +-export * from './schema/drop-schema-builder.js'; +-export * from './schema/column-definition-builder.js'; +-export * from './schema/foreign-key-constraint-builder.js'; +-export * from './schema/alter-table-builder.js'; +-export * from './schema/create-view-builder.js'; +-export * from './schema/drop-view-builder.js'; +-export * from './schema/alter-column-builder.js'; +-export * from './dynamic/dynamic.js'; +-export * from './driver/driver.js'; +-export * from './driver/database-connection.js'; +-export * from './driver/connection-provider.js'; +-export * from './driver/default-connection-provider.js'; +-export * from './driver/single-connection-provider.js'; +-export * from './driver/dummy-driver.js'; +-export * from './dialect/dialect.js'; +-export * from './dialect/dialect-adapter.js'; +-export * from './dialect/dialect-adapter-base.js'; + export * from './dialect/database-introspector.js'; +-export * from './dialect/sqlite/sqlite-dialect.js'; +-export * from './dialect/sqlite/sqlite-dialect-config.js'; +-export * from './dialect/sqlite/sqlite-driver.js'; +-export * from './dialect/postgres/postgres-query-compiler.js'; +-export * from './dialect/postgres/postgres-introspector.js'; +-export * from './dialect/postgres/postgres-adapter.js'; +-export * from './dialect/mysql/mysql-dialect.js'; +-export * from './dialect/mysql/mysql-dialect-config.js'; +-export * from './dialect/mysql/mysql-driver.js'; +-export * from './dialect/mysql/mysql-query-compiler.js'; +-export * from './dialect/mysql/mysql-introspector.js'; +-export * from './dialect/mysql/mysql-adapter.js'; +-export * from './dialect/postgres/postgres-driver.js'; +-export * from './dialect/postgres/postgres-dialect-config.js'; +-export * from './dialect/postgres/postgres-dialect.js'; +-export * from './dialect/sqlite/sqlite-query-compiler.js'; +-export * from './dialect/sqlite/sqlite-introspector.js'; +-export * from './dialect/sqlite/sqlite-adapter.js'; ++export * from './dialect/dialect-adapter-base.js'; ++export * from './dialect/dialect-adapter.js'; ++export * from './dialect/dialect.js'; + export * from './dialect/mssql/mssql-adapter.js'; + export * from './dialect/mssql/mssql-dialect-config.js'; + export * from './dialect/mssql/mssql-dialect.js'; + export * from './dialect/mssql/mssql-driver.js'; + export * from './dialect/mssql/mssql-introspector.js'; + export * from './dialect/mssql/mssql-query-compiler.js'; +-export * from './query-compiler/default-query-compiler.js'; +-export * from './query-compiler/query-compiler.js'; +-export * from './migration/migrator.js'; ++export * from './dialect/mysql/mysql-adapter.js'; ++export * from './dialect/mysql/mysql-dialect-config.js'; ++export * from './dialect/mysql/mysql-dialect.js'; ++export * from './dialect/mysql/mysql-driver.js'; ++export * from './dialect/mysql/mysql-introspector.js'; ++export * from './dialect/mysql/mysql-query-compiler.js'; ++export * from './dialect/postgres/postgres-adapter.js'; ++export * from './dialect/postgres/postgres-dialect-config.js'; ++export * from './dialect/postgres/postgres-dialect.js'; ++export * from './dialect/postgres/postgres-driver.js'; ++export * from './dialect/postgres/postgres-introspector.js'; ++export * from './dialect/postgres/postgres-query-compiler.js'; ++export * from './dialect/sqlite/sqlite-adapter.js'; ++export * from './dialect/sqlite/sqlite-dialect-config.js'; ++export * from './dialect/sqlite/sqlite-dialect.js'; ++export * from './dialect/sqlite/sqlite-driver.js'; ++export * from './dialect/sqlite/sqlite-introspector.js'; ++export * from './dialect/sqlite/sqlite-query-compiler.js'; ++export * from './driver/connection-provider.js'; ++export * from './driver/database-connection.js'; ++export * from './driver/default-connection-provider.js'; ++export * from './driver/driver.js'; ++export * from './driver/dummy-driver.js'; ++export * from './driver/runtime-driver.js'; ++export * from './driver/single-connection-provider.js'; ++export * from './dynamic/dynamic.js'; ++export { ++ ExpressionBuilder, ++ expressionBuilder, ++} from './expression/expression-builder.js'; ++export * from './expression/expression-wrapper.js'; ++export * from './expression/expression.js'; ++export * from './kysely.js'; + export * from './migration/file-migration-provider.js'; +-export * from './plugin/kysely-plugin.js'; +-export * from './plugin/camel-case/camel-case-plugin.js'; +-export * from './plugin/deduplicate-joins/deduplicate-joins-plugin.js'; +-export * from './plugin/with-schema/with-schema-plugin.js'; +-export * from './plugin/parse-json-results/parse-json-results-plugin.js'; ++export * from './migration/migrator.js'; + export * from './operation-node/add-column-node.js'; + export * from './operation-node/add-constraint-node.js'; + export * from './operation-node/add-index-node.js'; +@@ -190,22 +143,117 @@ export * from './operation-node/values-node.js'; + export * from './operation-node/when-node.js'; + export * from './operation-node/where-node.js'; + export * from './operation-node/with-node.js'; ++export { ++ ComparisonOperatorExpression, ++ FilterObject, ++ OperandValueExpression, ++ OperandValueExpressionOrList, ++} from './parser/binary-operation-parser.js'; ++export { ++ ExpressionOrFactory, ++ OperandExpression, ++} from './parser/expression-parser.js'; ++export { InsertObject } from './parser/insert-values-parser.js'; ++export { ++ JoinCallbackExpression, ++ JoinReferenceExpression, ++} from './parser/join-parser.js'; ++export { ++ OrderByDirectionExpression, ++ OrderByExpression, ++} from './parser/order-by-parser.js'; ++export { ++ ExtractTypeFromReferenceExpression, ++ ExtractTypeFromStringReference, ++ ReferenceExpression, ++ ReferenceExpressionOrList, ++ SimpleReferenceExpression, ++ StringReference, ++} from './parser/reference-parser.js'; ++export { ++ CallbackSelection, ++ SelectArg, ++ SelectCallback, ++ SelectExpression, ++ Selection, ++} from './parser/select-parser.js'; ++export { ++ SimpleTableReference, ++ TableExpression, ++ TableExpressionOrList, ++} from './parser/table-parser.js'; ++export { ExistsExpression } from './parser/unary-operation-parser.js'; ++export { UpdateObject } from './parser/update-set-parser.js'; ++export { ++ ValueExpression, ++ ValueExpressionOrList, ++} from './parser/value-parser.js'; ++export * from './plugin/camel-case/camel-case-plugin.js'; ++export * from './plugin/deduplicate-joins/deduplicate-joins-plugin.js'; ++export * from './plugin/kysely-plugin.js'; ++export * from './plugin/parse-json-results/parse-json-results-plugin.js'; ++export * from './plugin/with-schema/with-schema-plugin.js'; ++export * from './query-builder/aggregate-function-builder.js'; ++export * from './query-builder/case-builder.js'; ++export * from './query-builder/delete-query-builder.js'; ++export * from './query-builder/delete-result.js'; ++export * from './query-builder/function-module.js'; ++export * from './query-builder/having-interface.js'; ++export * from './query-builder/insert-query-builder.js'; ++export * from './query-builder/insert-result.js'; ++export * from './query-builder/join-builder.js'; ++export * from './query-builder/json-path-builder.js'; ++export * from './query-builder/merge-query-builder.js'; ++export * from './query-builder/merge-result.js'; ++export * from './query-builder/no-result-error.js'; ++export * from './query-builder/on-conflict-builder.js'; ++export * from './query-builder/output-interface.js'; ++export * from './query-builder/returning-interface.js'; ++export * from './query-builder/select-query-builder.js'; ++export * from './query-builder/update-query-builder.js'; ++export * from './query-builder/update-result.js'; ++export * from './query-builder/where-interface.js'; ++export * from './query-compiler/compiled-query.js'; ++export * from './query-compiler/default-query-compiler.js'; ++export * from './query-compiler/query-compiler.js'; ++export * from './query-creator.js'; ++export * from './query-executor/default-query-executor.js'; ++export * from './query-executor/noop-query-executor.js'; ++export * from './query-executor/query-executor-provider.js'; ++export * from './query-executor/query-executor.js'; ++export * from './raw-builder/raw-builder.js'; ++export * from './raw-builder/sql.js'; ++export * from './schema/alter-column-builder.js'; ++export * from './schema/alter-table-builder.js'; ++export * from './schema/column-definition-builder.js'; ++export * from './schema/create-index-builder.js'; ++export * from './schema/create-schema-builder.js'; ++export * from './schema/create-table-builder.js'; ++export * from './schema/create-type-builder.js'; ++export * from './schema/create-view-builder.js'; ++export * from './schema/drop-index-builder.js'; ++export * from './schema/drop-schema-builder.js'; ++export * from './schema/drop-table-builder.js'; ++export * from './schema/drop-type-builder.js'; ++export * from './schema/drop-view-builder.js'; ++export * from './schema/foreign-key-constraint-builder.js'; ++export * from './schema/schema.js'; + export * from './util/column-type.js'; + export * from './util/compilable.js'; + export * from './util/explainable.js'; +-export * from './util/streamable.js'; +-export * from './util/log.js'; +-export { AnyAliasedColumn, AnyAliasedColumnWithTable, AnyColumn, AnyColumnWithTable, Equals, UnknownRow, Simplify, SqlBool, Nullable, NotNull, } from './util/type-utils.js'; + export * from './util/infer-result.js'; + export { logOnce } from './util/log-once.js'; +-export { SelectExpression, SelectCallback, SelectArg, Selection, CallbackSelection, } from './parser/select-parser.js'; +-export { ReferenceExpression, ReferenceExpressionOrList, SimpleReferenceExpression, StringReference, ExtractTypeFromStringReference, ExtractTypeFromReferenceExpression, } from './parser/reference-parser.js'; +-export { ValueExpression, ValueExpressionOrList, } from './parser/value-parser.js'; +-export { SimpleTableReference, TableExpression, TableExpressionOrList, } from './parser/table-parser.js'; +-export { JoinReferenceExpression, JoinCallbackExpression, } from './parser/join-parser.js'; +-export { InsertObject } from './parser/insert-values-parser.js'; +-export { UpdateObject } from './parser/update-set-parser.js'; +-export { OrderByExpression, OrderByDirectionExpression, } from './parser/order-by-parser.js'; +-export { ComparisonOperatorExpression, OperandValueExpression, OperandValueExpressionOrList, FilterObject, } from './parser/binary-operation-parser.js'; +-export { ExistsExpression } from './parser/unary-operation-parser.js'; +-export { OperandExpression, ExpressionOrFactory, } from './parser/expression-parser.js'; ++export * from './util/log.js'; ++export * from './util/streamable.js'; ++export { ++ AnyAliasedColumn, ++ AnyAliasedColumnWithTable, ++ AnyColumn, ++ AnyColumnWithTable, ++ Equals, ++ NotNull, ++ Nullable, ++ Simplify, ++ SqlBool, ++ UnknownRow, ++} from './util/type-utils.js'; +diff --git a/dist/esm/index.js b/dist/esm/index.js +index c85949e9080a6e78f1aef5561ed050cb6d924b15..6ddadd439b6f5aa4b45749e3297035a7ad0b3496 100644 +--- a/dist/esm/index.js ++++ b/dist/esm/index.js +@@ -1,96 +1,46 @@ + /// +-export * from './kysely.js'; +-export * from './query-creator.js'; +-export * from './expression/expression.js'; +-export { expressionBuilder, } from './expression/expression-builder.js'; +-export * from './expression/expression-wrapper.js'; +-export * from './query-builder/where-interface.js'; +-export * from './query-builder/returning-interface.js'; +-export * from './query-builder/output-interface.js'; +-export * from './query-builder/having-interface.js'; +-export * from './query-builder/select-query-builder.js'; +-export * from './query-builder/insert-query-builder.js'; +-export * from './query-builder/update-query-builder.js'; +-export * from './query-builder/delete-query-builder.js'; +-export * from './query-builder/no-result-error.js'; +-export * from './query-builder/join-builder.js'; +-export * from './query-builder/function-module.js'; +-export * from './query-builder/insert-result.js'; +-export * from './query-builder/delete-result.js'; +-export * from './query-builder/update-result.js'; +-export * from './query-builder/on-conflict-builder.js'; +-export * from './query-builder/aggregate-function-builder.js'; +-export * from './query-builder/case-builder.js'; +-export * from './query-builder/json-path-builder.js'; +-export * from './query-builder/merge-query-builder.js'; +-export * from './query-builder/merge-result.js'; +-export * from './raw-builder/raw-builder.js'; +-export * from './raw-builder/sql.js'; +-export * from './query-executor/query-executor.js'; +-export * from './query-executor/default-query-executor.js'; +-export * from './query-executor/noop-query-executor.js'; +-export * from './query-executor/query-executor-provider.js'; +-export * from './query-compiler/default-query-compiler.js'; +-export * from './query-compiler/compiled-query.js'; +-export * from './schema/schema.js'; +-export * from './schema/create-table-builder.js'; +-export * from './schema/create-type-builder.js'; +-export * from './schema/drop-table-builder.js'; +-export * from './schema/drop-type-builder.js'; +-export * from './schema/create-index-builder.js'; +-export * from './schema/drop-index-builder.js'; +-export * from './schema/create-schema-builder.js'; +-export * from './schema/drop-schema-builder.js'; +-export * from './schema/column-definition-builder.js'; +-export * from './schema/foreign-key-constraint-builder.js'; +-export * from './schema/alter-table-builder.js'; +-export * from './schema/create-view-builder.js'; +-export * from './schema/drop-view-builder.js'; +-export * from './schema/alter-column-builder.js'; +-export * from './dynamic/dynamic.js'; +-export * from './driver/driver.js'; +-export * from './driver/database-connection.js'; +-export * from './driver/connection-provider.js'; +-export * from './driver/default-connection-provider.js'; +-export * from './driver/single-connection-provider.js'; +-export * from './driver/dummy-driver.js'; +-export * from './dialect/dialect.js'; +-export * from './dialect/dialect-adapter.js'; +-export * from './dialect/dialect-adapter-base.js'; + export * from './dialect/database-introspector.js'; +-export * from './dialect/sqlite/sqlite-dialect.js'; +-export * from './dialect/sqlite/sqlite-dialect-config.js'; +-export * from './dialect/sqlite/sqlite-driver.js'; +-export * from './dialect/postgres/postgres-query-compiler.js'; +-export * from './dialect/postgres/postgres-introspector.js'; +-export * from './dialect/postgres/postgres-adapter.js'; +-export * from './dialect/mysql/mysql-dialect.js'; +-export * from './dialect/mysql/mysql-dialect-config.js'; +-export * from './dialect/mysql/mysql-driver.js'; +-export * from './dialect/mysql/mysql-query-compiler.js'; +-export * from './dialect/mysql/mysql-introspector.js'; +-export * from './dialect/mysql/mysql-adapter.js'; +-export * from './dialect/postgres/postgres-driver.js'; +-export * from './dialect/postgres/postgres-dialect-config.js'; +-export * from './dialect/postgres/postgres-dialect.js'; +-export * from './dialect/sqlite/sqlite-query-compiler.js'; +-export * from './dialect/sqlite/sqlite-introspector.js'; +-export * from './dialect/sqlite/sqlite-adapter.js'; ++export * from './dialect/dialect-adapter-base.js'; ++export * from './dialect/dialect-adapter.js'; ++export * from './dialect/dialect.js'; + export * from './dialect/mssql/mssql-adapter.js'; + export * from './dialect/mssql/mssql-dialect-config.js'; + export * from './dialect/mssql/mssql-dialect.js'; + export * from './dialect/mssql/mssql-driver.js'; + export * from './dialect/mssql/mssql-introspector.js'; + export * from './dialect/mssql/mssql-query-compiler.js'; +-export * from './query-compiler/default-query-compiler.js'; +-export * from './query-compiler/query-compiler.js'; +-export * from './migration/migrator.js'; ++export * from './dialect/mysql/mysql-adapter.js'; ++export * from './dialect/mysql/mysql-dialect-config.js'; ++export * from './dialect/mysql/mysql-dialect.js'; ++export * from './dialect/mysql/mysql-driver.js'; ++export * from './dialect/mysql/mysql-introspector.js'; ++export * from './dialect/mysql/mysql-query-compiler.js'; ++export * from './dialect/postgres/postgres-adapter.js'; ++export * from './dialect/postgres/postgres-dialect-config.js'; ++export * from './dialect/postgres/postgres-dialect.js'; ++export * from './dialect/postgres/postgres-driver.js'; ++export * from './dialect/postgres/postgres-introspector.js'; ++export * from './dialect/postgres/postgres-query-compiler.js'; ++export * from './dialect/sqlite/sqlite-adapter.js'; ++export * from './dialect/sqlite/sqlite-dialect-config.js'; ++export * from './dialect/sqlite/sqlite-dialect.js'; ++export * from './dialect/sqlite/sqlite-driver.js'; ++export * from './dialect/sqlite/sqlite-introspector.js'; ++export * from './dialect/sqlite/sqlite-query-compiler.js'; ++export * from './driver/connection-provider.js'; ++export * from './driver/database-connection.js'; ++export * from './driver/default-connection-provider.js'; ++export * from './driver/driver.js'; ++export * from './driver/dummy-driver.js'; ++export * from './driver/runtime-driver.js'; ++export * from './driver/single-connection-provider.js'; ++export * from './dynamic/dynamic.js'; ++export { expressionBuilder } from './expression/expression-builder.js'; ++export * from './expression/expression-wrapper.js'; ++export * from './expression/expression.js'; ++export * from './kysely.js'; + export * from './migration/file-migration-provider.js'; +-export * from './plugin/kysely-plugin.js'; +-export * from './plugin/camel-case/camel-case-plugin.js'; +-export * from './plugin/deduplicate-joins/deduplicate-joins-plugin.js'; +-export * from './plugin/with-schema/with-schema-plugin.js'; +-export * from './plugin/parse-json-results/parse-json-results-plugin.js'; ++export * from './migration/migrator.js'; + export * from './operation-node/add-column-node.js'; + export * from './operation-node/add-constraint-node.js'; + export * from './operation-node/add-index-node.js'; +@@ -191,10 +141,60 @@ export * from './operation-node/values-node.js'; + export * from './operation-node/when-node.js'; + export * from './operation-node/where-node.js'; + export * from './operation-node/with-node.js'; ++export * from './plugin/camel-case/camel-case-plugin.js'; ++export * from './plugin/deduplicate-joins/deduplicate-joins-plugin.js'; ++export * from './plugin/kysely-plugin.js'; ++export * from './plugin/parse-json-results/parse-json-results-plugin.js'; ++export * from './plugin/with-schema/with-schema-plugin.js'; ++export * from './query-builder/aggregate-function-builder.js'; ++export * from './query-builder/case-builder.js'; ++export * from './query-builder/delete-query-builder.js'; ++export * from './query-builder/delete-result.js'; ++export * from './query-builder/function-module.js'; ++export * from './query-builder/having-interface.js'; ++export * from './query-builder/insert-query-builder.js'; ++export * from './query-builder/insert-result.js'; ++export * from './query-builder/join-builder.js'; ++export * from './query-builder/json-path-builder.js'; ++export * from './query-builder/merge-query-builder.js'; ++export * from './query-builder/merge-result.js'; ++export * from './query-builder/no-result-error.js'; ++export * from './query-builder/on-conflict-builder.js'; ++export * from './query-builder/output-interface.js'; ++export * from './query-builder/returning-interface.js'; ++export * from './query-builder/select-query-builder.js'; ++export * from './query-builder/update-query-builder.js'; ++export * from './query-builder/update-result.js'; ++export * from './query-builder/where-interface.js'; ++export * from './query-compiler/compiled-query.js'; ++export * from './query-compiler/default-query-compiler.js'; ++export * from './query-compiler/query-compiler.js'; ++export * from './query-creator.js'; ++export * from './query-executor/default-query-executor.js'; ++export * from './query-executor/noop-query-executor.js'; ++export * from './query-executor/query-executor-provider.js'; ++export * from './query-executor/query-executor.js'; ++export * from './raw-builder/raw-builder.js'; ++export * from './raw-builder/sql.js'; ++export * from './schema/alter-column-builder.js'; ++export * from './schema/alter-table-builder.js'; ++export * from './schema/column-definition-builder.js'; ++export * from './schema/create-index-builder.js'; ++export * from './schema/create-schema-builder.js'; ++export * from './schema/create-table-builder.js'; ++export * from './schema/create-type-builder.js'; ++export * from './schema/create-view-builder.js'; ++export * from './schema/drop-index-builder.js'; ++export * from './schema/drop-schema-builder.js'; ++export * from './schema/drop-table-builder.js'; ++export * from './schema/drop-type-builder.js'; ++export * from './schema/drop-view-builder.js'; ++export * from './schema/foreign-key-constraint-builder.js'; ++export * from './schema/schema.js'; + export * from './util/column-type.js'; + export * from './util/compilable.js'; + export * from './util/explainable.js'; +-export * from './util/streamable.js'; +-export * from './util/log.js'; + export * from './util/infer-result.js'; + export { logOnce } from './util/log-once.js'; ++export * from './util/log.js'; ++export * from './util/streamable.js'; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 283f10e2..b51ad54b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,6 +9,9 @@ overrides: '@types/node': 22.10.7 patchedDependencies: + kysely: + hash: 5ewjqngd4oyjgaqa3fcufrviwq + path: patches/kysely.patch ts-essentials@9.4.1: hash: 254pvnpgpcwoswa4ncfq4tq6su path: patches/ts-essentials@9.4.1.patch @@ -145,7 +148,7 @@ importers: version: 1.11.10 drizzle-orm: specifier: ^0.39.3 - version: 0.39.3(@opentelemetry/api@1.4.1)(@types/better-sqlite3@7.6.12)(better-sqlite3@11.8.1)(bun-types@1.2.1)(kysely@0.27.4) + version: 0.39.3(@opentelemetry/api@1.4.1)(@types/better-sqlite3@7.6.12)(better-sqlite3@11.8.1)(bun-types@1.2.1)(kysely@0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq)) fast-xml-parser: specifier: ^4.3.5 version: 4.3.5 @@ -172,7 +175,7 @@ importers: version: 6.2.1(reflect-metadata@0.2.2) kysely: specifier: ^0.27.4 - version: 0.27.4 + version: 0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq) lodash-es: specifier: ^4.17.21 version: 4.17.21 @@ -293,7 +296,7 @@ importers: version: 15.0.0 kysely-ctl: specifier: ^0.9.0 - version: 0.9.0(kysely@0.27.4)(magicast@0.3.5) + version: 0.9.0(kysely@0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq))(magicast@0.3.5) make-vfs: specifier: ^1.0.15 version: 1.0.15 @@ -11087,13 +11090,13 @@ snapshots: transitivePeerDependencies: - supports-color - drizzle-orm@0.39.3(@opentelemetry/api@1.4.1)(@types/better-sqlite3@7.6.12)(better-sqlite3@11.8.1)(bun-types@1.2.1)(kysely@0.27.4): + drizzle-orm@0.39.3(@opentelemetry/api@1.4.1)(@types/better-sqlite3@7.6.12)(better-sqlite3@11.8.1)(bun-types@1.2.1)(kysely@0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq)): optionalDependencies: '@opentelemetry/api': 1.4.1 '@types/better-sqlite3': 7.6.12 better-sqlite3: 11.8.1 bun-types: 1.2.1 - kysely: 0.27.4 + kysely: 0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq) dunder-proto@1.0.1: dependencies: @@ -12565,12 +12568,12 @@ snapshots: dependencies: json-buffer: 3.0.1 - kysely-ctl@0.9.0(kysely@0.27.4)(magicast@0.3.5): + kysely-ctl@0.9.0(kysely@0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq))(magicast@0.3.5): dependencies: c12: 1.11.2(magicast@0.3.5) citty: 0.1.6 consola: 3.2.3 - kysely: 0.27.4 + kysely: 0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq) nypm: 0.3.12 ofetch: 1.4.1 pathe: 1.1.2 @@ -12580,7 +12583,7 @@ snapshots: transitivePeerDependencies: - magicast - kysely@0.27.4: {} + kysely@0.27.4(patch_hash=5ewjqngd4oyjgaqa3fcufrviwq): {} lazystream@1.0.1: dependencies: diff --git a/server/src/Server.ts b/server/src/Server.ts index 7d6735d6..4babfe01 100644 --- a/server/src/Server.ts +++ b/server/src/Server.ts @@ -35,6 +35,7 @@ import { HdhrApiRouter } from './api/hdhrApi.js'; import { apiRouter } from './api/index.js'; import { streamApi } from './api/streamApi.js'; import { videoApiRouter } from './api/videoApi.js'; +import { DBContext, makeDatabaseConnection } from './db/DBAccess.ts'; import { FfmpegInfo } from './ffmpeg/ffmpegInfo.js'; import { type ServerOptions, @@ -439,6 +440,7 @@ export class Server { this.app.after(() => { this.app.gracefulShutdown(async (signal) => { + DBContext.enter(makeDatabaseConnection()); this.logger.info( 'Received exit signal %s, attempting graceful shutdown', signal, diff --git a/server/src/bootstrap.ts b/server/src/bootstrap.ts index 501e2ca2..8ec07165 100644 --- a/server/src/bootstrap.ts +++ b/server/src/bootstrap.ts @@ -4,7 +4,9 @@ import path from 'node:path'; import type { DeepPartial } from 'ts-essentials'; import { databaseNeedsMigration, + DBContext, getDatabase, + makeDatabaseConnection, migrateExistingDatabase, runDBMigrations, syncMigrationTablesIfNecessary, @@ -74,18 +76,20 @@ export async function bootstrapTunarr( } await initDbDirectories(opts); - const db = getDatabase(); // Initialize the DB + await DBContext.create(makeDatabaseConnection(), async () => { + const db = getDatabase(); // Initialize the DB - // not the first run, use the copy migrator - if (hasTunarrDb) { - const migrationNecessary = await databaseNeedsMigration(db); - if (migrationNecessary) { - await migrateExistingDatabase(getDefaultDatabaseName()); + // not the first run, use the copy migrator + if (hasTunarrDb) { + const migrationNecessary = await databaseNeedsMigration(db); + if (migrationNecessary) { + await migrateExistingDatabase(getDefaultDatabaseName()); + } + } else { + await syncMigrationTablesIfNecessary(db); + await runDBMigrations(db); } - } else { - await syncMigrationTablesIfNecessary(db); - await runDBMigrations(db); - } + }); LoggerFactory.initialize(settingsDb); } diff --git a/server/src/cli/RunServerCommand.ts b/server/src/cli/RunServerCommand.ts index 31aac963..2d016342 100644 --- a/server/src/cli/RunServerCommand.ts +++ b/server/src/cli/RunServerCommand.ts @@ -2,11 +2,13 @@ import { container } from '@/container.js'; import { isProduction } from '@/util/index.js'; import { type MarkOptional } from 'ts-essentials'; import type { ArgumentsCamelCase, CommandModule } from 'yargs'; +import { DBContext } from '../db/DBAccess.ts'; import { type ISettingsDB } from '../db/interfaces/ISettingsDB.ts'; import { setServerOptions } from '../globals.ts'; import { Server } from '../Server.ts'; import { StartupService } from '../services/StartupService.ts'; import { KEYS } from '../types/inject.ts'; +import { getDefaultDatabaseName } from '../util/defaults.ts'; import { getBooleanEnvVar, getNumericEnvVar, @@ -52,7 +54,10 @@ export const RunServerCommand: CommandModule = { portSetting; // port precedence - env var -> argument -> settings setServerOptions({ ...opts, port: portToUse }); - await container.get(StartupService).runStartupServices(); - await container.get(Server).initAndRun(); + + await DBContext.createForName(getDefaultDatabaseName(), async () => { + await container.get(StartupService).runStartupServices(); + await container.get(Server).initAndRun(); + }); }, }; diff --git a/server/src/db/ChannelDB.ts b/server/src/db/ChannelDB.ts index 7caf2cff..46340852 100644 --- a/server/src/db/ChannelDB.ts +++ b/server/src/db/ChannelDB.ts @@ -1172,6 +1172,39 @@ export class ChannelDB implements IChannelDB { await this.saveLineup(channelId, lineup); } + async removeProgramsFromAllLineups(programIds: string[]): Promise { + if (isEmpty(programIds)) { + return; + } + + const lineups = await this.loadAllLineups(); + + const programsToRemove = new Set(programIds); + for (const [channelId, { lineup }] of Object.entries(lineups)) { + const newLineupItems: LineupItem[] = lineup.items.map((item) => { + switch (item.type) { + case 'content': { + if (programsToRemove.has(item.id)) { + return { + type: 'offline', + durationMs: item.durationMs, + }; + } + return item; + } + case 'offline': + case 'redirect': + return item; + } + }); + + await this.saveLineup(channelId, { + ...lineup, + items: newLineupItems, + }); + } + } + private async createLineup(channelId: string) { const db = await this.getFileDb(channelId); await db.write(); diff --git a/server/src/db/DBAccess.ts b/server/src/db/DBAccess.ts index 32e1372a..d78bebe5 100644 --- a/server/src/db/DBAccess.ts +++ b/server/src/db/DBAccess.ts @@ -7,19 +7,32 @@ import { drizzle, type BetterSQLite3Database, } from 'drizzle-orm/better-sqlite3'; +import type { + IsolationLevel, + KyselyConfig, + KyselyProps, + Transaction, +} from 'kysely'; import { CamelCasePlugin, + DefaultConnectionProvider, + DefaultQueryExecutor, Kysely, + Log, Migrator, ParseJSONResultsPlugin, + RuntimeDriver, SqliteDialect, + TransactionBuilder, } from 'kysely'; import { findIndex, has, isError, last, map, once, slice } from 'lodash-es'; +import { AsyncLocalStorage } from 'node:async_hooks'; import { DatabaseCopyMigrator } from '../migration/db/DatabaseCopyMigrator.ts'; import { DirectMigrationProvider, LegacyMigrationNameToNewMigrationName, } from '../migration/DirectMigrationProvider.ts'; +import type { Maybe } from '../types/util.ts'; import { getDefaultDatabaseName } from '../util/defaults.ts'; import type { DB } from './schema/db.ts'; @@ -30,6 +43,7 @@ export const MigrationLockTableName = 'migration_lock'; // let _directDbAccess: Kysely; type Conn = { + name: string; kysely: Kysely; drizzle: BetterSQLite3Database; }; @@ -38,55 +52,176 @@ const connections = new Map(); const logger = once(() => LoggerFactory.child({ className: 'DirectDBAccess' })); +export class DBContext { + private static storage = new AsyncLocalStorage(); + + constructor(private connections: Map = new Map()) {} + + get db(): Maybe> { + return this.getKyselyDatabase(); + } + + getConnection(name: string = getDefaultDatabaseName()): Maybe { + return this.connections.get(name); + } + + getKyselyDatabase( + name: string = getDefaultDatabaseName(), + ): Maybe> { + return this.getConnection(name)?.kysely; + } + + getOrCreateKyselyDatabase(name: string): Kysely { + return this.getOrCreateConnection(name)?.kysely; + } + + getOrCreateConnection(name: string): Conn { + const existing = this.connections.get(name); + if (existing) { + return existing; + } + const conn = makeDatabaseConnection(name); + this.connections.set(name, conn); + return conn; + } + + setConnection(name: string) { + this.connections.set(name, makeDatabaseConnection(name)); + } + + static create(context: Conn, next: (...args: unknown[]) => T) { + const m = new Map([[context.name, context]]); + return this.storage.run(new DBContext(m), next); + } + + static createForName(name: string, next: (...args: unknown[]) => T) { + if (connections.has(name)) { + return this.create(connections.get(name)!, next); + } + return this.create(makeDatabaseConnection(name), next); + } + + static enter(context: Conn) { + const m = new Map([[context.name, context]]); + this.storage.enterWith(new DBContext(m)); + } + + static currentDBContext(): Maybe { + return this.storage.getStore(); + } +} + +class TransactionBuilderWrapper extends TransactionBuilder { + constructor( + private dbName: string, + props: KyselyProps & { isolationLevel?: IsolationLevel }, + ) { + super(props); + } + + execute(callback: (trx: Transaction) => Promise): Promise { + return super.execute((tx) => { + const curr = DBContext.currentDBContext()?.getConnection(this.dbName); + if (!curr) { + throw new Error('no DB context'); + } + + return DBContext.create({ ...curr, kysely: tx }, () => callback(tx)); + }); + } +} + +class KyselyWrapper extends Kysely { + constructor( + private dbName: string, + private config: KyselyConfig, + ) { + super(config); + } + + transaction(): TransactionBuilder { + const driver = new RuntimeDriver( + this.config.dialect.createDriver(), + new Log(this.config.log ?? []), + ); + return new TransactionBuilderWrapper(this.dbName, { + config: this.config, + dialect: this.config.dialect, + driver, + executor: new DefaultQueryExecutor( + this.config.dialect.createQueryCompiler(), + this.config.dialect.createAdapter(), + new DefaultConnectionProvider(driver), + this.config.plugins ?? [], + ), + }); + } +} + +export function makeDatabaseConnection( + dbName: string = getDefaultDatabaseName(), +): Conn { + const dbConn = new Sqlite(dbName, { + timeout: 5000, + }); + + const kysely = new KyselyWrapper(dbName, { + dialect: new SqliteDialect({ + database: dbConn, + }), + log: (event) => { + switch (event.level) { + case 'query': + if ( + process.env['DATABASE_DEBUG_LOGGING'] || + process.env['DIRECT_DATABASE_DEBUG_LOGGING'] + ) { + logger().setBindings({ db: dbName }); + logger().debug( + 'Query: %O (%d ms)', + event.query.sql, + event.queryDurationMillis, + ); + } + return; + case 'error': + logger().setBindings({ db: dbName }); + logger().error( + event.error, + 'Query error: %O\n%O', + event.query.sql, + event.query.parameters, + ); + return; + } + }, + plugins: [new ParseJSONResultsPlugin(), new CamelCasePlugin()], + }); + + const drizzleConn = drizzle({ + client: dbConn, + casing: 'snake_case', + }); + + const connection = { name: dbName, kysely, drizzle: drizzleConn }; + + connections.set(dbName, connection); + + return connection; +} + +export function getDatabaseContext() { + return DBContext.currentDBContext(); +} + export const getDatabase = ( dbName: string = getDefaultDatabaseName(), forceInit: boolean = false, ) => { - let conn = connections.get(dbName); - if (!conn || forceInit) { - const dbConn = new Sqlite(dbName, { - timeout: 5000, - }); - const kysely = new Kysely({ - dialect: new SqliteDialect({ - database: dbConn, - }), - log: (event) => { - switch (event.level) { - case 'query': - if ( - process.env['DATABASE_DEBUG_LOGGING'] || - process.env['DIRECT_DATABASE_DEBUG_LOGGING'] - ) { - logger().debug( - 'Query: %O (%d ms)', - event.query.sql, - event.queryDurationMillis, - ); - } - return; - case 'error': - logger().error( - event.error, - 'Query error: %O\n%O', - event.query.sql, - event.query.parameters, - ); - return; - } - }, - plugins: [new ParseJSONResultsPlugin(), new CamelCasePlugin()], - }); - - const drizzleConn = drizzle({ - client: dbConn, - casing: 'snake_case', - }); - - conn = { kysely, drizzle: drizzleConn }; + if (forceInit) { + DBContext.enter(makeDatabaseConnection(dbName)); } - connections.set(dbName, conn); - return conn.kysely; + return DBContext.currentDBContext()!.getKyselyDatabase(dbName)!; }; export function getMigrator(db: Kysely = getDatabase()) { @@ -98,9 +233,7 @@ export function getMigrator(db: Kysely = getDatabase()) { }); } -export async function pendingDatabaseMigrations( - db: Kysely = getDatabase(), -) { +export async function pendingDatabaseMigrations(db: Kysely) { return lock.runExclusive(async () => { const tables = await db.introspection.getTables({ withInternalKyselyTables: true, @@ -127,7 +260,7 @@ export async function pendingDatabaseMigrations( }); } -export async function databaseNeedsMigration(db: Kysely = getDatabase()) { +export async function databaseNeedsMigration(db: Kysely) { return (await pendingDatabaseMigrations(db)).length > 0; } @@ -221,10 +354,7 @@ export async function syncMigrationTablesIfNecessary( } } -export async function runDBMigrations( - db: Kysely = getDatabase(), - migrateTo?: string, -) { +export async function runDBMigrations(db: Kysely, migrateTo?: string) { const _logger = logger(); const migrator = getMigrator(db); const { error, results } = await (isNonEmptyString(migrateTo) @@ -245,9 +375,7 @@ export async function runDBMigrations( } // Runs through pending migrations, using the DB copier if necessary -export async function migrateExistingDatabase( - dbPath: string = getDefaultDatabaseName(), -) { +export async function migrateExistingDatabase(dbPath: string) { const db = getDatabase(dbPath); const pendingMigrations = await pendingDatabaseMigrations(db); diff --git a/server/src/db/ProgramDB.ts b/server/src/db/ProgramDB.ts index 2972f42d..44a5bb86 100644 --- a/server/src/db/ProgramDB.ts +++ b/server/src/db/ProgramDB.ts @@ -12,7 +12,7 @@ import { type SavePlexProgramExternalIdsTaskFactory, } from '@/tasks/plex/SavePlexProgramExternalIdsTask.js'; import { KEYS } from '@/types/inject.js'; -import { Maybe } from '@/types/util.js'; +import { MarkNonNullable, Maybe } from '@/types/util.js'; import { Timer } from '@/util/Timer.js'; import { devAssert } from '@/util/debug.js'; import { type Logger } from '@/util/logging/LoggerFactory.js'; @@ -25,7 +25,7 @@ import { } from '@tunarr/types'; import dayjs from 'dayjs'; import { inject, injectable } from 'inversify'; -import { CaseWhenBuilder, UpdateResult } from 'kysely'; +import { CaseWhenBuilder, NotNull, UpdateResult } from 'kysely'; import { chunk, concat, @@ -35,6 +35,7 @@ import { flatMap, forEach, groupBy, + head, isEmpty, isNil, isNull, @@ -52,6 +53,7 @@ import { } from 'lodash-es'; import { MarkOptional, MarkRequired } from 'ts-essentials'; import { v4 } from 'uuid'; +import { typedProperty } from '../types/path.ts'; import { getNumericEnvVar, TUNARR_ENV_VARS } from '../util/env.ts'; import { flatMapAsyncSeq, @@ -70,7 +72,7 @@ import { ProgramSourceType, programSourceTypeFromString, } from './custom_types/ProgramSourceType.ts'; -import { upsertRawProgramExternalIds } from './programExternalIdHelpers.ts'; +import { upsertProgramExternalIds } from './programExternalIdHelpers.ts'; import { AllProgramJoins, ProgramUpsertFields, @@ -84,7 +86,8 @@ import { withTvShow, } from './programQueryHelpers.ts'; import { - NewProgramDao as NewRawProgram, + NewProgramDao, + ProgramDao, programExternalIdString, ProgramType, ProgramDao as RawProgram, @@ -92,7 +95,7 @@ import { import { MinimalProgramExternalId, NewProgramExternalId, - NewProgramExternalId as NewRawProgramExternalId, + NewSingleOrMultiExternalId, ProgramExternalId, ProgramExternalIdKeys, } from './schema/ProgramExternalId.ts'; @@ -100,7 +103,10 @@ import { NewProgramGrouping, ProgramGroupingType, } from './schema/ProgramGrouping.ts'; -import { NewProgramGroupingExternalId } from './schema/ProgramGroupingExternalId.ts'; +import { + NewProgramGroupingExternalId, + toInsertableProgramGroupingExternalId, +} from './schema/ProgramGroupingExternalId.ts'; import { DB } from './schema/db.ts'; import type { ProgramGroupingWithExternalIds, @@ -112,9 +118,9 @@ type ValidatedContentProgram = MarkRequired< 'externalSourceName' | 'externalSourceType' >; -type MintedRawProgramInfo = { - program: NewRawProgram; - externalIds: NewRawProgramExternalId[]; +type MintedNewProgramInfo = { + program: NewProgramDao; + externalIds: NewSingleOrMultiExternalId[]; apiProgram: ValidatedContentProgram; }; @@ -549,12 +555,13 @@ export class ProgramDB implements IProgramDB { // ); // TODO: handle custom shows - const programsToPersist: MintedRawProgramInfo[] = map( + const programsToPersist: MintedNewProgramInfo[] = map( contentPrograms, (p) => { const program = minter.contentProgramDtoToDao(p); const externalIds = minter.mintExternalIds( p.externalSourceName, + p.externalSourceId, program.uuid, p, ); @@ -572,7 +579,7 @@ export class ProgramDB implements IProgramDB { // TODO: The way we deal with uniqueness right now makes a Program entity // exist 1:1 with its "external" entity, i.e. the same logical movie will // have duplicate entries in the DB across different servers and sources. - const upsertedPrograms: RawProgram[] = []; + const upsertedPrograms: MarkNonNullable[] = []; await this.timer.timeAsync('programUpsert', async () => { for (const c of chunk(programsToPersist, programUpsertBatchSize)) { upsertedPrograms.push( @@ -591,7 +598,17 @@ export class ProgramDB implements IProgramDB { })), ), ) + .onConflict((oc) => + oc + .columns(['sourceType', 'mediaSourceId', 'externalKey']) + .doUpdateSet((eb) => + mapToObj(ProgramUpsertFields, (f) => ({ + [f.replace('excluded.', '')]: eb.ref(f), + })), + ), + ) .returningAll() + .$narrowType<{ mediaSourceId: NotNull }>() .execute(), )), ); @@ -624,7 +641,7 @@ export class ProgramDB implements IProgramDB { // TODO: We could optimize further here by only saving IDs necessary for streaming await this.timer.timeAsync( `upsert ${requiredExternalIds.length} external ids`, - () => upsertRawProgramExternalIds(requiredExternalIds, 200), + () => upsertProgramExternalIds(requiredExternalIds, 200), // upsertProgramExternalIds_deprecated(requiredExternalIds), ); @@ -650,7 +667,7 @@ export class ProgramDB implements IProgramDB { AnonymousTask('UpsertExternalIds', () => this.timer.timeAsync( `background external ID upsert (${backgroundExternalIds.length} ids)`, - () => upsertRawProgramExternalIds(backgroundExternalIds), + () => upsertProgramExternalIds(backgroundExternalIds), ), ), ); @@ -667,18 +684,21 @@ export class ProgramDB implements IProgramDB { } private async handleProgramGroupings( - upsertedPrograms: RawProgram[], - programInfos: Record, + upsertedPrograms: MarkNonNullable[], + programInfos: Record, ) { const programsBySourceAndServer = mapValues( groupBy(upsertedPrograms, 'sourceType'), - (ps) => groupBy(ps, 'externalSourceId'), + (ps) => groupBy(ps, typedProperty('mediaSourceId')), ); - for (const [sourceType, byServerName] of Object.entries( + for (const [sourceType, byServerId] of Object.entries( programsBySourceAndServer, )) { - for (const [serverName, programs] of Object.entries(byServerName)) { + for (const [serverId, programs] of Object.entries(byServerId)) { + // Making an assumption that these are all the same... this field will + // go away soon anyway + const serverName = head(programs)!.externalSourceId; // This is just extra safety because lodash erases the type in groupBy const typ = programSourceTypeFromString(sourceType); if (!typ) { @@ -690,6 +710,7 @@ export class ProgramDB implements IProgramDB { programInfos, typ, serverName, + serverId, ); } } @@ -697,8 +718,9 @@ export class ProgramDB implements IProgramDB { private async handleSingleSourceProgramGroupings( upsertedPrograms: RawProgram[], - programInfos: Record, + programInfos: Record, mediaSourceType: ProgramSourceType, + mediaSourceName: string, mediaSourceId: string, ) { const grandparentRatingKeyToParentRatingKey: Record< @@ -790,7 +812,7 @@ export class ProgramDB implements IProgramDB { eb( 'programGroupingExternalId.externalSourceId', '=', - mediaSourceId, + mediaSourceName, ), eb( 'programGroupingExternalId.externalKey', @@ -954,6 +976,7 @@ export class ProgramDB implements IProgramDB { ...ProgramGroupingMinter.mintGroupingExternalIds( programs[0][1], parentGrouping.uuid, + mediaSourceName, mediaSourceId, 'parent', ), @@ -965,6 +988,7 @@ export class ProgramDB implements IProgramDB { ...ProgramGroupingMinter.mintGroupingExternalIds( matchingPrograms[0][1], grandparentGrouping.uuid, + mediaSourceName, mediaSourceId, 'grandparent', ), @@ -986,14 +1010,41 @@ export class ProgramDB implements IProgramDB { if (!isEmpty(externalIds)) { await this.timer.timeAsync('upsert program_grouping external ids', () => - getDatabase() - .transaction() - .execute((tx) => - tx - .insertInto('programGroupingExternalId') - .values(externalIds) - .executeTakeFirstOrThrow(), + Promise.all( + chunk( + externalIds.map(toInsertableProgramGroupingExternalId), + 100, + ).map((externalIds) => + getDatabase() + .transaction() + .execute((tx) => + tx + .insertInto('programGroupingExternalId') + .values(externalIds) + .onConflict((oc) => + oc + .columns(['groupUuid', 'sourceType']) + .where('mediaSourceId', 'is', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + groupUuid: eb.ref('excluded.groupUuid'), + })), + ) + .onConflict((oc) => + oc + .columns(['groupUuid', 'sourceType', 'mediaSourceId']) + .where('mediaSourceId', 'is not', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + groupUuid: eb.ref('excluded.groupUuid'), + })), + ) + .executeTakeFirstOrThrow(), + ), ), + ), ); } @@ -1138,7 +1189,7 @@ export class ProgramDB implements IProgramDB { } } - private schedulePlexExternalIdsTask(upsertedPrograms: NewRawProgram[]) { + private schedulePlexExternalIdsTask(upsertedPrograms: ProgramDao[]) { PlexTaskQueue.pause(); this.timer.timeSync('schedule Plex external IDs tasks', () => { forEach( @@ -1168,7 +1219,7 @@ export class ProgramDB implements IProgramDB { }); } - private scheduleJellyfinExternalIdsTask(upsertedPrograms: NewRawProgram[]) { + private scheduleJellyfinExternalIdsTask(upsertedPrograms: ProgramDao[]) { JellyfinTaskQueue.pause(); this.timer.timeSync('Schedule Jellyfin external IDs tasks', () => { forEach( diff --git a/server/src/db/converters/ProgramGroupingMinter.ts b/server/src/db/converters/ProgramGroupingMinter.ts index 7372c29a..7048bdc2 100644 --- a/server/src/db/converters/ProgramGroupingMinter.ts +++ b/server/src/db/converters/ProgramGroupingMinter.ts @@ -1,13 +1,12 @@ import { ProgramExternalIdType } from '@/db/custom_types/ProgramExternalIdType.js'; -import type { NewProgramGroupingExternalId } from '@/db/schema/ProgramGroupingExternalId.js'; +import type { NewSingleOrMultiProgramGroupingExternalId } from '@/db/schema/ProgramGroupingExternalId.js'; import { isNonEmptyString } from '@/util/index.js'; import type { ContentProgram } from '@tunarr/types'; import type { JellyfinItem } from '@tunarr/types/jellyfin'; import type { PlexEpisode, PlexMusicTrack } from '@tunarr/types/plex'; import dayjs from 'dayjs'; -import { find, first } from 'lodash-es'; +import { first } from 'lodash-es'; import type { MarkRequired } from 'ts-essentials'; -import { P, match } from 'ts-pattern'; import { v4 } from 'uuid'; import type { Nullable } from '../../types/util.ts'; import { @@ -68,14 +67,15 @@ export class ProgramGroupingMinter { program: ContentProgram, groupingId: string, externalSourceId: string, + mediaSourceId: string, relationType: 'parent' | 'grandparent', - ): NewProgramGroupingExternalId[] { + ): NewSingleOrMultiProgramGroupingExternalId[] { if (program.subtype === 'movie') { return []; } const now = +dayjs(); - const parentExternalIds: NewProgramGroupingExternalId[] = []; + const parentExternalIds: NewSingleOrMultiProgramGroupingExternalId[] = []; const ratingKey = relationType === 'grandparent' @@ -83,6 +83,7 @@ export class ProgramGroupingMinter { : program.parent?.externalKey; if (isNonEmptyString(ratingKey)) { parentExternalIds.push({ + type: 'multi', uuid: v4(), createdAt: now, updatedAt: now, @@ -90,6 +91,7 @@ export class ProgramGroupingMinter { externalKey: ratingKey, sourceType: ProgramExternalIdType.PLEX, externalSourceId, + mediaSourceId, groupUuid: groupingId, }); } @@ -101,89 +103,13 @@ export class ProgramGroupingMinter { ); if (isNonEmptyString(guid)) { parentExternalIds.push({ + type: 'single', uuid: v4(), createdAt: now, updatedAt: now, externalFilePath: null, externalKey: guid, sourceType: ProgramExternalIdType.PLEX_GUID, - externalSourceId: null, - groupUuid: groupingId, - }); - } - - return parentExternalIds; - } - - static mintGroupingExternalIdsForPlex( - plexItem: PlexEpisode | PlexMusicTrack, - groupingId: string, - externalSourceId: string, - relationType: 'parent' | 'grandparent', - ): NewProgramGroupingExternalId[] { - const now = +dayjs(); - const parentExternalIds: NewProgramGroupingExternalId[] = []; - - const ratingKey = plexItem[`${relationType}RatingKey`]; - if (isNonEmptyString(ratingKey)) { - parentExternalIds.push({ - uuid: v4(), - createdAt: now, - updatedAt: now, - externalFilePath: null, - externalKey: ratingKey, - sourceType: ProgramExternalIdType.PLEX, - externalSourceId, - groupUuid: groupingId, - }); - } - - const guid = plexItem[`${relationType}Guid`]; - if (isNonEmptyString(guid)) { - parentExternalIds.push({ - uuid: v4(), - createdAt: now, - updatedAt: now, - externalFilePath: null, - externalKey: guid, - sourceType: ProgramExternalIdType.PLEX_GUID, - externalSourceId: null, - groupUuid: groupingId, - }); - } - - return parentExternalIds; - } - - static mintGroupingExternalIdsForJellyfin( - jellyfinItem: JellyfinItem, - groupingId: string, - externalSourceId: string, - relationType: 'parent' | 'grandparent', - ): NewProgramGroupingExternalId[] { - const now = +dayjs(); - const parentExternalIds: NewProgramGroupingExternalId[] = []; - - const jellyfinId = match([jellyfinItem, relationType] as const) - .with([{ Type: 'Episode' }, 'grandparent'], () => jellyfinItem.SeriesId) - .with( - [{ Type: 'Audio' }, 'parent'], - () => - find(jellyfinItem.AlbumArtists, { Name: jellyfinItem.AlbumArtist }) - ?.Id, - ) - .with([P._, 'parent'], () => jellyfinItem.ParentId) - .otherwise(() => null); - - if (isNonEmptyString(jellyfinId)) { - parentExternalIds.push({ - uuid: v4(), - createdAt: now, - updatedAt: now, - externalFilePath: null, - externalKey: jellyfinId, - sourceType: ProgramExternalIdType.JELLYFIN, - externalSourceId, groupUuid: groupingId, }); } diff --git a/server/src/db/converters/ProgramMinter.ts b/server/src/db/converters/ProgramMinter.ts index c8ff7314..bf6a954b 100644 --- a/server/src/db/converters/ProgramMinter.ts +++ b/server/src/db/converters/ProgramMinter.ts @@ -1,6 +1,9 @@ import { ProgramExternalIdType } from '@/db/custom_types/ProgramExternalIdType.js'; import { ProgramSourceType } from '@/db/custom_types/ProgramSourceType.js'; -import type { NewProgramExternalId } from '@/db/schema/ProgramExternalId.js'; +import type { + NewProgramExternalId, + NewSingleOrMultiExternalId, +} from '@/db/schema/ProgramExternalId.js'; import { seq } from '@tunarr/shared/util'; import type { ContentProgram } from '@tunarr/types'; import type { JellyfinItem } from '@tunarr/types/jellyfin'; @@ -26,7 +29,9 @@ class ProgramDaoMinter { return { uuid: v4(), sourceType: program.externalSourceType, - externalSourceId: program.externalSourceId ?? program.externalSourceName, + // Deprecated + externalSourceId: program.externalSourceName, + mediaSourceId: program.externalSourceId, externalKey: program.externalKey, originalAirDate: program.date ?? null, duration: program.duration, @@ -50,21 +55,24 @@ class ProgramDaoMinter { mint( serverName: string, + serverId: string, program: ContentProgramOriginalProgram, ): NewRawProgram { const ret = match(program) .with( { sourceType: 'plex', program: { type: 'movie' } }, - ({ program: movie }) => this.mintProgramForPlexMovie(serverName, movie), + ({ program: movie }) => + this.mintProgramForPlexMovie(serverName, serverId, movie), ) .with( { sourceType: 'plex', program: { type: 'episode' } }, ({ program: episode }) => - this.mintProgramForPlexEpisode(serverName, episode), + this.mintProgramForPlexEpisode(serverName, serverId, episode), ) .with( { sourceType: 'plex', program: { type: 'track' } }, - ({ program: track }) => this.mintProgramForPlexTrack(serverName, track), + ({ program: track }) => + this.mintProgramForPlexTrack(serverName, serverId, track), ) .with( { @@ -80,7 +88,8 @@ class ProgramDaoMinter { ), }, }, - ({ program }) => this.mintProgramForJellyfinItem(serverName, program), + ({ program }) => + this.mintProgramForJellyfinItem(serverName, serverId, program), ) .otherwise(() => new Error('Unexpected program type')); if (isError(ret)) { @@ -91,6 +100,7 @@ class ProgramDaoMinter { private mintProgramForPlexMovie( serverName: string, + serverId: string, plexMovie: PlexMovie, ): NewRawProgram { const file = first(first(plexMovie.Media)?.Part ?? []); @@ -101,6 +111,7 @@ class ProgramDaoMinter { duration: plexMovie.duration ?? 0, filePath: file?.file ?? null, externalSourceId: serverName, + mediaSourceId: serverId, externalKey: plexMovie.ratingKey, plexRatingKey: plexMovie.ratingKey, plexFilePath: file?.key ?? null, @@ -116,6 +127,7 @@ class ProgramDaoMinter { private mintProgramForJellyfinItem( serverName: string, + serverId: string, item: Omit & { Type: 'Movie' | 'Episode' | 'Audio' | 'Video' | 'MusicVideo' | 'Trailer'; }, @@ -128,6 +140,7 @@ class ProgramDaoMinter { originalAirDate: item.PremiereDate, duration: (item.RunTimeTicks ?? 0) / 10_000, externalSourceId: serverName, + mediaSourceId: serverId, externalKey: item.Id, rating: item.OfficialRating, summary: item.Overview, @@ -154,6 +167,7 @@ class ProgramDaoMinter { private mintProgramForPlexEpisode( serverName: string, + serverId: string, plexEpisode: PlexEpisode, ): NewRawProgram { const file = first(first(plexEpisode.Media)?.Part ?? []); @@ -166,6 +180,7 @@ class ProgramDaoMinter { duration: plexEpisode.duration ?? 0, filePath: file?.file, externalSourceId: serverName, + mediaSourceId: serverId, externalKey: plexEpisode.ratingKey, plexRatingKey: plexEpisode.ratingKey, plexFilePath: file?.key, @@ -185,6 +200,7 @@ class ProgramDaoMinter { private mintProgramForPlexTrack( serverName: string, + serverId: string, plexTrack: PlexMusicTrack, ): NewRawProgram { const file = first(first(plexTrack.Media)?.Part ?? []); @@ -196,6 +212,7 @@ class ProgramDaoMinter { duration: plexTrack.duration ?? 0, filePath: file?.file, externalSourceId: serverName, + mediaSourceId: serverId, externalKey: plexTrack.ratingKey, plexRatingKey: plexTrack.ratingKey, plexFilePath: file?.key, @@ -216,32 +233,34 @@ class ProgramDaoMinter { mintExternalIds( serverName: string, + serverId: string, programId: string, program: ContentProgram, - // originalProgram: ContentProgramOriginalProgram, - ) { + ): NewSingleOrMultiExternalId[] { return match(program) .with({ externalSourceType: 'plex' }, () => - this.mintPlexExternalIds(serverName, programId, program), + this.mintPlexExternalIds(serverName, serverId, programId, program), ) .with({ externalSourceType: 'jellyfin' }, () => - this.mintJellyfinExternalIds(serverName, programId, program), + this.mintJellyfinExternalIds(serverName, serverId, programId, program), ) .with({ externalSourceType: 'emby' }, () => - this.mintEmbyExternalIds(serverName, programId, program), + this.mintEmbyExternalIds(serverName, serverId, programId, program), ) .exhaustive(); } mintPlexExternalIds( serverName: string, + serverId: string, programId: string, program: ContentProgram, - ): NewProgramExternalId[] { + ): NewSingleOrMultiExternalId[] { const now = +dayjs(); - const ids: NewProgramExternalId[] = [ + const ids: NewSingleOrMultiExternalId[] = [ { + type: 'multi', uuid: v4(), createdAt: now, updatedAt: now, @@ -249,6 +268,7 @@ class ProgramDaoMinter { sourceType: ProgramExternalIdType.PLEX, programUuid: programId, externalSourceId: serverName, + mediaSourceId: serverId, externalFilePath: program.serverFileKey, directFilePath: program.serverFilePath, }, @@ -257,6 +277,7 @@ class ProgramDaoMinter { const plexGuid = find(program.externalIds, { source: 'plex-guid' }); if (plexGuid) { ids.push({ + type: 'single', uuid: v4(), createdAt: now, updatedAt: now, @@ -273,13 +294,14 @@ class ProgramDaoMinter { case 'imdb': case 'tvdb': return { + type: 'single', uuid: v4(), createdAt: now, updatedAt: now, externalKey: eid.id, sourceType: eid.source, programUuid: programId, - } satisfies NewProgramExternalId; + } satisfies NewSingleOrMultiExternalId; default: return null; } @@ -307,12 +329,14 @@ class ProgramDaoMinter { mintJellyfinExternalIds( serverName: string, + serverId: string, programId: string, program: ContentProgram, ) { const now = +dayjs(); - const ids: NewProgramExternalId[] = [ + const ids: NewSingleOrMultiExternalId[] = [ { + type: 'multi', uuid: v4(), createdAt: now, updatedAt: now, @@ -320,6 +344,7 @@ class ProgramDaoMinter { sourceType: ProgramExternalIdType.JELLYFIN, programUuid: programId, externalSourceId: serverName, + mediaSourceId: serverId, externalFilePath: program.serverFileKey, directFilePath: program.serverFilePath, }, @@ -332,13 +357,14 @@ class ProgramDaoMinter { case 'imdb': case 'tvdb': return { + type: 'single', uuid: v4(), createdAt: now, updatedAt: now, externalKey: eid.id, sourceType: eid.source, programUuid: programId, - } satisfies NewProgramExternalId; + } satisfies NewSingleOrMultiExternalId; default: return null; } @@ -350,12 +376,14 @@ class ProgramDaoMinter { mintEmbyExternalIds( serverName: string, + serverId: string, programId: string, program: ContentProgram, ) { const now = +dayjs(); - const ids: NewProgramExternalId[] = [ + const ids: NewSingleOrMultiExternalId[] = [ { + type: 'multi', uuid: v4(), createdAt: now, updatedAt: now, @@ -363,6 +391,7 @@ class ProgramDaoMinter { sourceType: ProgramExternalIdType.EMBY, programUuid: programId, externalSourceId: serverName, + mediaSourceId: serverId, externalFilePath: program.serverFileKey, directFilePath: program.serverFilePath, }, @@ -375,13 +404,14 @@ class ProgramDaoMinter { case 'imdb': case 'tvdb': return { + type: 'single', uuid: v4(), createdAt: now, updatedAt: now, externalKey: eid.id, sourceType: eid.source, programUuid: programId, - } satisfies NewProgramExternalId; + } satisfies NewSingleOrMultiExternalId; default: return null; } diff --git a/server/src/db/interfaces/IChannelDB.ts b/server/src/db/interfaces/IChannelDB.ts index 47a8314c..f31ed45c 100644 --- a/server/src/db/interfaces/IChannelDB.ts +++ b/server/src/db/interfaces/IChannelDB.ts @@ -93,6 +93,8 @@ export interface IChannelDB { programIds: string[], ): Promise; + removeProgramsFromAllLineups(programIds: string[]): Promise; + loadAllLineupConfigs( forceRead?: boolean, ): Promise>; diff --git a/server/src/db/mediaSourceDB.ts b/server/src/db/mediaSourceDB.ts index cbd4b084..887139af 100644 --- a/server/src/db/mediaSourceDB.ts +++ b/server/src/db/mediaSourceDB.ts @@ -117,32 +117,36 @@ export class MediaSourceDB { .executeTakeFirst(); } - async deleteMediaSource(id: string, removePrograms: boolean = true) { + async deleteMediaSource(id: string) { const deletedServer = await this.getById(id); if (isNil(deletedServer)) { throw new Error(`MediaSource not found: ${id}`); } + // This should cascade all relevant deletes across the DB await getDatabase() - .deleteFrom('mediaSource') - .where('uuid', '=', id) - // TODO: Blocked on https://github.com/oven-sh/bun/issues/16909 - // .limit(1) - .execute(); + .transaction() + .execute(async (tx) => { + const relatedProgramIds = await tx + .selectFrom('program') + .where('program.mediaSourceId', '=', id) + .select('uuid') + .execute() + .then((_) => _.map(({ uuid }) => uuid)); - let reports: Report[]; - if (!removePrograms) { - reports = []; - } else { - reports = await this.fixupProgramReferences( - deletedServer.name, - deletedServer.type, - ); - } + await tx + .deleteFrom('mediaSource') + .where('uuid', '=', id) + .limit(1) + .execute(); + // TODO: Update lineups + + await this.channelDb.removeProgramsFromAllLineups(relatedProgramIds); + }); this.mediaSourceApiFactory().deleteCachedClient(deletedServer); - return { deletedServer, reports }; + return { deletedServer }; } async updateMediaSource(server: UpdateMediaSourceRequest) { @@ -155,9 +159,9 @@ export class MediaSourceDB { } const sendGuideUpdates = - server.type === 'plex' ? server.sendGuideUpdates ?? false : false; + server.type === 'plex' ? (server.sendGuideUpdates ?? false) : false; const sendChannelUpdates = - server.type === 'plex' ? server.sendChannelUpdates ?? false : false; + server.type === 'plex' ? (server.sendChannelUpdates ?? false) : false; await getDatabase() .updateTable('mediaSource') @@ -188,9 +192,9 @@ export class MediaSourceDB { async addMediaSource(server: InsertMediaSourceRequest): Promise { const name = isUndefined(server.name) ? 'plex' : server.name; const sendGuideUpdates = - server.type === 'plex' ? server.sendGuideUpdates ?? false : false; + server.type === 'plex' ? (server.sendGuideUpdates ?? false) : false; const sendChannelUpdates = - server.type === 'plex' ? server.sendChannelUpdates ?? false : false; + server.type === 'plex' ? (server.sendChannelUpdates ?? false) : false; const index = await getDatabase() .selectFrom('mediaSource') .select((eb) => eb.fn.count('uuid').as('count')) @@ -218,38 +222,6 @@ export class MediaSourceDB { return newServer?.uuid; } - // private async removeDanglingPrograms(mediaSource: MediaSource) { - // const knownProgramIds = await directDbAccess() - // .selectFrom('programExternalId as p1') - // .where(({ eb, and }) => - // and([ - // eb('p1.externalSourceId', '=', mediaSource.name), - // eb('p1.sourceType', '=', mediaSource.type), - // ]), - // ) - // .selectAll('p1') - // .select((eb) => - // jsonArrayFrom( - // eb - // .selectFrom('programExternalId as p2') - // .whereRef('p2.programUuid', '=', 'p1.programUuid') - // .whereRef('p2.uuid', '!=', 'p1.uuid') - // .select(['p2.sourceType', 'p2.externalSourceId', 'p2.externalKey']), - // ).as('otherExternalIds'), - // ) - // .groupBy('p1.uuid') - // .execute(); - - // const mediaSourceTypes = map(enumValues(MediaSourceType), (typ) => - // typ.toString(), - // ); - // const danglingPrograms = reject(knownProgramIds, (program) => { - // some(program.otherExternalIds, (eid) => - // mediaSourceTypes.includes(eid.sourceType), - // ); - // }); - // } - private async fixupProgramReferences( serverName: string, serverType: MediaSourceType, @@ -350,8 +322,8 @@ export class MediaSourceDB { id, channelNumber: number, channelName: name, - destroyedPrograms: isUpdate ? 0 : channelToProgramCount[id] ?? 0, - modifiedPrograms: isUpdate ? channelToProgramCount[id] ?? 0 : 0, + destroyedPrograms: isUpdate ? 0 : (channelToProgramCount[id] ?? 0), + modifiedPrograms: isUpdate ? (channelToProgramCount[id] ?? 0) : 0, } as Report; }, ); @@ -359,15 +331,15 @@ export class MediaSourceDB { const fillerReports: Report[] = map(fillersById, ({ uuid }) => ({ type: 'filler', id: uuid, - destroyedPrograms: isUpdate ? 0 : fillerToProgramCount[uuid] ?? 0, - modifiedPrograms: isUpdate ? fillerToProgramCount[uuid] ?? 0 : 0, + destroyedPrograms: isUpdate ? 0 : (fillerToProgramCount[uuid] ?? 0), + modifiedPrograms: isUpdate ? (fillerToProgramCount[uuid] ?? 0) : 0, })); const customShowReports: Report[] = map(customShowById, ({ uuid }) => ({ type: 'custom-show', id: uuid, - destroyedPrograms: isUpdate ? 0 : customShowToProgramCount[uuid] ?? 0, - modifiedPrograms: isUpdate ? customShowToProgramCount[uuid] ?? 0 : 0, + destroyedPrograms: isUpdate ? 0 : (customShowToProgramCount[uuid] ?? 0), + modifiedPrograms: isUpdate ? (customShowToProgramCount[uuid] ?? 0) : 0, })); return [...channelReports, ...fillerReports, ...customShowReports]; diff --git a/server/src/db/programExternalIdHelpers.ts b/server/src/db/programExternalIdHelpers.ts index 5fac52f7..cc667fc7 100644 --- a/server/src/db/programExternalIdHelpers.ts +++ b/server/src/db/programExternalIdHelpers.ts @@ -1,12 +1,14 @@ import { mapAsyncSeq } from '@/util/index.js'; import { LoggerFactory } from '@/util/logging/LoggerFactory.js'; -import { isValidSingleExternalIdType } from '@tunarr/types/schemas'; -import { chunk, flatten, isEmpty, isUndefined, partition } from 'lodash-es'; +import { chunk, flatten, isEmpty, partition } from 'lodash-es'; import { getDatabase } from './DBAccess.ts'; -import type { NewProgramExternalId as NewRawProgramExternalId } from './schema/ProgramExternalId.ts'; +import { + toInsertableProgramExternalId, + type NewSingleOrMultiExternalId, +} from './schema/ProgramExternalId.ts'; -export const upsertRawProgramExternalIds = async ( - externalIds: NewRawProgramExternalId[], +export const upsertProgramExternalIds = async ( + externalIds: NewSingleOrMultiExternalId[], chunkSize: number = 100, ) => { if (isEmpty(externalIds)) { @@ -17,9 +19,7 @@ export const upsertRawProgramExternalIds = async ( const [singles, multiples] = partition( externalIds, - (id) => - isValidSingleExternalIdType(id.sourceType) && - isUndefined(id.externalSourceId), + (id) => id.type === 'single', ); let singleIdPromise: Promise<{ uuid: string }[]>; @@ -30,7 +30,7 @@ export const upsertRawProgramExternalIds = async ( .execute((tx) => tx .insertInto('programExternalId') - .values(singleChunk) + .values(singleChunk.map(toInsertableProgramExternalId)) .onConflict((oc) => oc .columns(['programUuid', 'sourceType']) @@ -42,6 +42,17 @@ export const upsertRawProgramExternalIds = async ( programUuid: eb.ref('excluded.programUuid'), })), ) + .onConflict((oc) => + oc + .columns(['programUuid', 'sourceType']) + .where('mediaSourceId', 'is', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + directFilePath: eb.ref('excluded.directFilePath'), + programUuid: eb.ref('excluded.programUuid'), + })), + ) .returning('uuid as uuid') .execute(), ); @@ -58,7 +69,7 @@ export const upsertRawProgramExternalIds = async ( .execute((tx) => tx .insertInto('programExternalId') - .values(multiChunk) + .values(multiChunk.map(toInsertableProgramExternalId)) .onConflict((oc) => oc .columns(['programUuid', 'sourceType', 'externalSourceId']) @@ -70,6 +81,17 @@ export const upsertRawProgramExternalIds = async ( programUuid: eb.ref('excluded.programUuid'), })), ) + .onConflict((oc) => + oc + .columns(['programUuid', 'sourceType', 'mediaSourceId']) + .where('mediaSourceId', 'is not', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + directFilePath: eb.ref('excluded.directFilePath'), + programUuid: eb.ref('excluded.programUuid'), + })), + ) .returning('uuid as uuid') .execute(), ); diff --git a/server/src/db/schema/Program.ts b/server/src/db/schema/Program.ts index 5692bc14..270e1b11 100644 --- a/server/src/db/schema/Program.ts +++ b/server/src/db/schema/Program.ts @@ -10,8 +10,9 @@ import { uniqueIndex, } from 'drizzle-orm/sqlite-core'; import type { Insertable, Selectable, Updateable } from 'kysely'; +import type { MarkNotNilable } from '../../types/util.ts'; import { type KyselifyBetter } from './KyselifyBetter.ts'; -import { MediaSourceTypes } from './MediaSource.ts'; +import { MediaSource, MediaSourceTypes } from './MediaSource.ts'; import { ProgramGrouping } from './ProgramGrouping.ts'; export const ProgramTypes = ['movie', 'episode', 'track'] as const; @@ -37,6 +38,9 @@ export const Program = sqliteTable( episodeIcon: text(), externalKey: text().notNull(), externalSourceId: text().notNull(), + mediaSourceId: text().references(() => MediaSource.uuid, { + onDelete: 'cascade', + }), filePath: text(), grandparentExternalKey: text(), icon: text(), @@ -78,7 +82,10 @@ export const Program = sqliteTable( export type ProgramTable = KyselifyBetter; export type ProgramDao = Selectable; -export type NewProgramDao = Insertable; +export type NewProgramDao = MarkNotNilable< + Insertable, + 'mediaSourceId' +>; export type ProgramDaoUpdate = Updateable; export function programExternalIdString(p: ProgramDao | NewProgramDao) { diff --git a/server/src/db/schema/ProgramExternalId.ts b/server/src/db/schema/ProgramExternalId.ts index c8f52130..f2525009 100644 --- a/server/src/db/schema/ProgramExternalId.ts +++ b/server/src/db/schema/ProgramExternalId.ts @@ -8,9 +8,12 @@ import { uniqueIndex, } from 'drizzle-orm/sqlite-core'; import type { Insertable, Selectable } from 'kysely'; -import type { MarkRequired } from 'ts-essentials'; +import { omit } from 'lodash-es'; +import type { MarkRequired, StrictOmit } from 'ts-essentials'; +import type { MarkNotNilable } from '../../types/util.ts'; import { ProgramExternalIdSourceTypes } from './base.ts'; import { type KyselifyBetter } from './KyselifyBetter.ts'; +import { MediaSource } from './MediaSource.ts'; import { Program } from './Program.ts'; export const ProgramExternalId = sqliteTable( @@ -23,9 +26,12 @@ export const ProgramExternalId = sqliteTable( externalFilePath: text(), externalKey: text().notNull(), externalSourceId: text(), + mediaSourceId: text().references(() => MediaSource.uuid, { + onDelete: 'cascade', + }), programUuid: text() .notNull() - .references(() => Program.uuid), + .references(() => Program.uuid, { onDelete: 'cascade' }), sourceType: text({ enum: ProgramExternalIdSourceTypes }).notNull(), }, (table) => [ @@ -46,6 +52,24 @@ export const ProgramExternalId = sqliteTable( export type ProgramExternalIdTable = KyselifyBetter; export type ProgramExternalId = Selectable; export type NewProgramExternalId = Insertable; +export type NewSingleOrMultiExternalId = + | (StrictOmit< + Insertable, + 'mediaSourceId' | 'externalSourceId' + > & { type: 'single' }) + | (MarkNotNilable< + Insertable, + 'mediaSourceId' | 'externalSourceId' + > & { type: 'multi' }); + +export function toInsertableProgramExternalId( + extraInfo: NewSingleOrMultiExternalId, +): NewProgramExternalId { + return omit(extraInfo, 'type') satisfies Omit< + NewSingleOrMultiExternalId, + 'type' + >; +} export type MinimalProgramExternalId = MarkRequired< Partial, diff --git a/server/src/db/schema/ProgramGroupingExternalId.ts b/server/src/db/schema/ProgramGroupingExternalId.ts index 67cea68b..821ecb7b 100644 --- a/server/src/db/schema/ProgramGroupingExternalId.ts +++ b/server/src/db/schema/ProgramGroupingExternalId.ts @@ -7,8 +7,12 @@ import { text, } from 'drizzle-orm/sqlite-core'; import type { Insertable, Selectable } from 'kysely'; +import { omit } from 'lodash-es'; +import type { StrictOmit } from 'ts-essentials'; +import type { MarkNotNilable } from '../../types/util.ts'; import { ProgramExternalIdSourceTypes } from './base.ts'; import { type KyselifyBetter } from './KyselifyBetter.ts'; +import { MediaSource } from './MediaSource.ts'; import { ProgramGrouping } from './ProgramGrouping.ts'; export const ProgramGroupingExternalId = sqliteTable( @@ -20,6 +24,9 @@ export const ProgramGroupingExternalId = sqliteTable( externalFilePath: text(), externalKey: text().notNull(), externalSourceId: text(), + mediaSourceId: text().references(() => MediaSource.uuid, { + onDelete: 'cascade', + }), groupUuid: text() .notNull() .references(() => ProgramGrouping.uuid, { @@ -45,6 +52,21 @@ export type ProgramGroupingExternalId = Selectable; export type NewProgramGroupingExternalId = Insertable; +export type NewSingleOrMultiProgramGroupingExternalId = + | (StrictOmit< + Insertable, + 'externalSourceId' | 'mediaSourceId' + > & { type: 'single' }) + | (MarkNotNilable< + Insertable, + 'externalSourceId' | 'mediaSourceId' + > & { type: 'multi' }); + +export function toInsertableProgramGroupingExternalId( + eid: NewSingleOrMultiProgramGroupingExternalId, +): NewProgramGroupingExternalId { + return omit(eid, 'type') satisfies NewProgramGroupingExternalId; +} export type ProgramGroupingExternalIdFields< Alias extends string = 'ProgramGroupingExternalId', diff --git a/server/src/external/jellyfin/JellyfinItemFinder.ts b/server/src/external/jellyfin/JellyfinItemFinder.ts index 6c854455..18202d23 100644 --- a/server/src/external/jellyfin/JellyfinItemFinder.ts +++ b/server/src/external/jellyfin/JellyfinItemFinder.ts @@ -8,7 +8,7 @@ import { GlobalScheduler } from '@/services/Scheduler.js'; import { ReconcileProgramDurationsTask } from '@/tasks/ReconcileProgramDurationsTask.js'; import { KEYS } from '@/types/inject.js'; import { Maybe } from '@/types/util.js'; -import { groupByUniq, isDefined } from '@/util/index.js'; +import { groupByUniq, isDefined, run } from '@/util/index.js'; import { type Logger } from '@/util/logging/LoggerFactory.js'; import { JellyfinItem, JellyfinItemKind } from '@tunarr/types/jellyfin'; import dayjs from 'dayjs'; @@ -19,6 +19,8 @@ import { ProgramExternalIdType, programExternalIdTypeFromJellyfinProvider, } from '../../db/custom_types/ProgramExternalIdType.ts'; +import { MediaSourceDB } from '../../db/mediaSourceDB.ts'; +import { MediaSourceType } from '../../db/schema/MediaSource.ts'; import { JellyfinGetItemsQuery } from './JellyfinApiClient.ts'; @injectable() @@ -28,6 +30,7 @@ export class JellyfinItemFinder { @inject(KEYS.Logger) private logger: Logger, @inject(MediaSourceApiFactory) private mediaSourceApiFactory: MediaSourceApiFactory, + @inject(MediaSourceDB) private mediaSourceDB: MediaSourceDB, ) {} async findForProgramAndUpdate(programId: string) { @@ -65,10 +68,24 @@ export class JellyfinItemFinder { // Right now just check if the durations are different. // otherwise we might blow away details we already have, since // Jellyfin collects metadata asynchronously (sometimes) - const updatedProgram = minter.mint(program.externalSourceId, { - sourceType: 'jellyfin', - program: potentialApiMatch, - }); + const mediaSourceId = + program.mediaSourceId ?? + (await run(async () => { + const ms = await this.findMediaSource(program.externalSourceId); + if (!ms) + throw new Error( + `Could not find media source by name: ${program.externalSourceId}`, + ); + return ms.uuid; + })); + const updatedProgram = minter.mint( + program.externalSourceId, + mediaSourceId, + { + sourceType: 'jellyfin', + program: potentialApiMatch, + }, + ); if (updatedProgram.duration !== program.duration) { await this.programDB.updateProgramDuration( @@ -207,4 +224,11 @@ export class JellyfinItemFinder { return possibleMatch; } + + private findMediaSource(mediaSourceName: string) { + return this.mediaSourceDB.findByType( + MediaSourceType.Jellyfin, + mediaSourceName, + ); + } } diff --git a/server/src/external/plex/PlexApiClient.ts b/server/src/external/plex/PlexApiClient.ts index 3a9d8d5f..4da14df5 100644 --- a/server/src/external/plex/PlexApiClient.ts +++ b/server/src/external/plex/PlexApiClient.ts @@ -69,6 +69,10 @@ export class PlexApiClient extends BaseApiClient { return this.opts.name; } + get serverId() { + return this.opts.uuid; + } + getFullUrl(path: string): string { const url = super.getFullUrl(path); const parsed = new URL(url); diff --git a/server/src/migration/DirectMigrationProvider.ts b/server/src/migration/DirectMigrationProvider.ts index 9154b80f..1d7a63a3 100644 --- a/server/src/migration/DirectMigrationProvider.ts +++ b/server/src/migration/DirectMigrationProvider.ts @@ -23,6 +23,7 @@ import Migration1730806741 from './db/Migration1730806741.ts'; import Migration1731982492 from './db/Migration1731982492.ts'; import Migration1732969335_AddTranscodeConfig from './db/Migration1732969335_AddTranscodeConfig.ts'; import Migration1738604866_AddEmby from './db/Migration1738604866_AddEmby.ts'; +import Migration1740691984_ProgramMediaSourceId from './db/Migration1740691984_ProgramMediaSourceId.ts'; export const LegacyMigrationNameToNewMigrationName = [ ['Migration20240124115044', '_Legacy_Migration00'], @@ -90,6 +91,7 @@ export class DirectMigrationProvider implements MigrationProvider { migration1732969335: Migration1732969335_AddTranscodeConfig, migration1735044379: Migration1735044379_AddHlsDirect, migration1738604866: Migration1738604866_AddEmby, + migration1740691984: Migration1740691984_ProgramMediaSourceId, }, wrapWithTransaction, ), diff --git a/server/src/migration/db/DatabaseCopyMigrator.ts b/server/src/migration/db/DatabaseCopyMigrator.ts index fc1ebad4..0121f285 100644 --- a/server/src/migration/db/DatabaseCopyMigrator.ts +++ b/server/src/migration/db/DatabaseCopyMigrator.ts @@ -1,10 +1,10 @@ import dayjs from 'dayjs'; import { type Kysely, sql } from 'kysely'; -import { trimEnd } from 'lodash-es'; +import { replace } from 'lodash-es'; import fs from 'node:fs/promises'; import tmp from 'tmp-promise'; import { - getDatabase, + getDatabaseContext, MigrationLockTableName, MigrationTableName, runDBMigrations, @@ -23,10 +23,11 @@ export class DatabaseCopyMigrator { async migrate(currentDbPath: string, migrateTo?: string) { const { path: tmpPath } = await tmp.file({ keep: false }); this.logger.debug('Migrating to temp DB %s', tmpPath); - const tempDB = getDatabase(tmpPath); + const tempDB = getDatabaseContext()!.getOrCreateKyselyDatabase(tmpPath); await runDBMigrations(tempDB, migrateTo); - const oldDB = getDatabase(currentDbPath); + const oldDB = + getDatabaseContext()!.getOrCreateKyselyDatabase(currentDbPath); const oldTables = await this.getTables(oldDB); const newTables = await this.getTables(tempDB); // Prepare for copy. @@ -48,11 +49,11 @@ export class DatabaseCopyMigrator { continue; } - const columnUnion = new Set(table.columns.map((col) => col.name)).union( - new Set(newTable.columns.map((col) => col.name)), - ); + const columnIntersection = new Set( + table.columns.map((col) => col.name), + ).intersection(new Set(newTable.columns.map((col) => col.name))); - const colNames = [...columnUnion].sort(); + const colNames = [...columnIntersection].sort(); await sql`INSERT INTO ${sql.table(table.name)} (${sql.join(colNames.map((n) => sql.ref(n)))}) SELECT ${sql.join(colNames.map((n) => sql.ref(n)))} FROM ${sql.ref('old')}.${sql.table(table.name)} WHERE true ON CONFLICT DO NOTHING;`.execute( tempDB, ); @@ -63,11 +64,11 @@ export class DatabaseCopyMigrator { await fs.copyFile( currentDbPath, - `${trimEnd(currentDbPath, '.db')}-${+dayjs()}.bak`, + `${replace(currentDbPath, '.db', '')}-${+dayjs()}.bak`, ); await fs.cp(tmpPath, currentDbPath); // Force reinit at the new path - getDatabase(currentDbPath, true); + getDatabaseContext()!.setConnection(currentDbPath); } // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/server/src/migration/db/Migration1740691984_ProgramMediaSourceId.ts b/server/src/migration/db/Migration1740691984_ProgramMediaSourceId.ts new file mode 100644 index 00000000..8082d731 --- /dev/null +++ b/server/src/migration/db/Migration1740691984_ProgramMediaSourceId.ts @@ -0,0 +1,137 @@ +import { type Kysely, CompiledQuery, sql } from 'kysely'; + +export default { + fullCopy: true, + async up(db: Kysely) { + await db.executeQuery(CompiledQuery.raw('PRAGMA foreign_keys = OFF')); + await db.executeQuery(CompiledQuery.raw('PRAGMA defer_foreign_keys = ON')); + + const createProgramTableTemp = sql` +CREATE TABLE IF NOT EXISTS "program_tmp" ( + "uuid" text not null primary key, + "created_at" datetime, + "updated_at" datetime, + "source_type" text not null check ((\`source_type\` in ('plex', 'jellyfin', 'emby', 'local'))), + "original_air_date" text, + "duration" integer not null, + "episode" integer, + "episode_icon" text, + "file_path" text, + "icon" text, + "external_source_id" text not null, + "media_source_id" text references "media_source" ("uuid") on delete cascade, + "external_key" text not null, + "plex_rating_key" text, + "plex_file_path" text, + "parent_external_key" text, + "grandparent_external_key" text, + "rating" text, + "season_number" integer, + "season_icon" text, + "show_icon" text, + "show_title" text, + "summary" text, + "title" text not null, + "type" text not null check ((\`type\` in ('movie', 'episode', 'track'))), + "year" integer, + "artist_name" text, + "album_name" text, + "season_uuid" text, + "album_uuid" text, + "artist_uuid" text, + "tv_show_uuid" text, + constraint "program_season_uuid_foreign" foreign key ("season_uuid") references "program_grouping" ("uuid") on update cascade, + constraint "program_album_uuid_foreign" foreign key ("album_uuid") references "program_grouping" ("uuid") on update cascade, + constraint "program_artist_uuid_foreign" foreign key ("artist_uuid") references "program_grouping" ("uuid") on update cascade, + constraint "program_tv_show_uuid_foreign" foreign key ("tv_show_uuid") references "program_grouping" ("uuid") on update cascade); +`; + + const createProgramGroupingExternalIdTemp = sql` +CREATE TABLE IF NOT EXISTS "program_external_id_temp" ( + "uuid" text not null primary key, + "created_at" datetime, + "updated_at" datetime, + "source_type" text not null check ((\`source_type\` in ('plex', 'plex-guid', 'tmdb', 'imdb', 'tvdb', 'jellyfin', 'emby', 'local'))), + "external_source_id" text, + "media_source_id" text references "media_source" ("uuid") on delete cascade, + "external_key" text not null, + "external_file_path" text, + "direct_file_path" text, + "program_uuid" text not null, + constraint "program_external_id_program_uuid_foreign" foreign key ("program_uuid") references "program" ("uuid") on delete cascade + ); +`; + + await db.executeQuery(createProgramGroupingExternalIdTemp.compile(db)); + + const createProgramGroupExternalIdTemp = sql` +CREATE TABLE IF NOT EXISTS "program_grouping_external_id_tmp" ( + "uuid" text not null primary key, + "created_at" datetime, + "updated_at" datetime, + "source_type" text not null check ((\`source_type\` in ('plex', 'plex-guid', 'tmdb', 'imdb', 'tvdb', 'jellyfin', 'emby', 'local'))), + "external_source_id" text, + "media_source_id" text references "media_source" ("uuid") on delete cascade, + "external_key" text not null, + "group_uuid" text not null, + "external_file_path" text, + constraint "program_grouping_external_id_group_uuid_foreign" foreign key ("group_uuid") references "program_grouping" ("uuid") on delete cascade on update cascade +); +`; + await db.executeQuery(createProgramGroupExternalIdTemp.compile(db)); + + const indexes = [ + // Programs + 'DROP INDEX IF EXISTS "program_season_uuid_index"', + 'CREATE INDEX "program_season_uuid_index" on "program_tmp" ("season_uuid")', + 'DROP INDEX IF EXISTS "program_tv_show_uuid_index"', + 'CREATE INDEX "program_tv_show_uuid_index" on "program_tmp" ("tv_show_uuid")', + 'DROP INDEX IF EXISTS "program_album_uuid_index"', + 'CREATE INDEX "program_album_uuid_index" on "program_tmp" ("album_uuid")', + 'DROP INDEX IF EXISTS "program_artist_uuid_index"', + 'CREATE INDEX "program_artist_uuid_index" on "program_tmp" ("artist_uuid")', + 'DROP INDEX IF EXISTS "program_source_type_external_source_id_external_key_unique"', + 'CREATE UNIQUE INDEX "program_source_type_external_source_id_external_key_unique" on "program_tmp" ("source_type", "external_source_id", "external_key")', + // New one + 'CREATE UNIQUE INDEX "program_media_source_uniq" on "program_tmp" ("source_type", "media_source_id", "external_key")', + // Program external IDs + 'DROP INDEX IF EXISTS "unique_program_multiple_external_id"', + 'DROP INDEX IF EXISTS "unique_program_single_external_id"', + `CREATE UNIQUE INDEX "unique_program_multiple_external_id" on "program_external_id_temp" ("program_uuid", "source_type", "external_source_id") where \`external_source_id\` is not null`, + `CREATE UNIQUE INDEX "unique_program_single_external_id" on "program_external_id_temp" ("program_uuid", "source_type") where \`external_source_id\` is null`, + // New indexes on program external ids + `CREATE UNIQUE INDEX "unique_program_multiple_external_id_media_source" on "program_external_id_temp" ("program_uuid", "source_type", "media_source_id") where \`media_source_id\` is not null`, + `CREATE UNIQUE INDEX "unique_program_single_external_id_media_source" on "program_external_id_temp" ("program_uuid", "source_type") where \`media_source_id\` is null`, + // Program grouping external IDs + // 'DROP INDEX IF EXISTS "unique_program_multiple_external_id"', + // 'DROP INDEX IF EXISTS "unique_program_single_external_id"', + // `CREATE UNIQUE INDEX "unique_program_multiple_external_id" on "program_grouping_external_id_tmp" ("program_uuid", "source_type", "external_source_id") where \`external_source_id\` is not null`, + // `CREATE UNIQUE INDEX "unique_program_single_external_id" on "program_grouping_external_id_tmp" ("program_uuid", "source_type") where \`external_source_id\` is null`, + // New indexes on program external ids + `CREATE UNIQUE INDEX "unique_program_grouping_multiple_external_id_media_source" on "program_grouping_external_id_tmp" ("group_uuid", "source_type", "media_source_id") where \`media_source_id\` is not null`, + `CREATE UNIQUE INDEX "unique_program_grouping_single_external_id_media_source" on "program_grouping_external_id_tmp" ("group_uuid", "source_type") where \`media_source_id\` is null`, + ]; + + await db.executeQuery(createProgramTableTemp.compile(db)); + + for (const idx of indexes) { + await db.executeQuery(CompiledQuery.raw(idx)); + } + + await db.schema.dropTable('program').execute(); + await db.schema.alterTable('program_tmp').renameTo('program').execute(); + await db.schema.dropTable('program_external_id').execute(); + await db.schema + .alterTable('program_external_id_temp') + .renameTo('program_external_id') + .execute(); + await db.schema.dropTable('program_grouping_external_id').execute(); + await db.schema + .alterTable('program_grouping_external_id_tmp') + .renameTo('program_grouping_external_id') + .execute(); + + await db.executeQuery(CompiledQuery.raw('PRAGMA foreign_keys = ON')); + await db.executeQuery(CompiledQuery.raw('PRAGMA defer_foreign_keys = OFF')); + }, +}; diff --git a/server/src/migration/legacy_migration/LegacyChannelMigrator.ts b/server/src/migration/legacy_migration/LegacyChannelMigrator.ts index fa6638bf..330c6553 100644 --- a/server/src/migration/legacy_migration/LegacyChannelMigrator.ts +++ b/server/src/migration/legacy_migration/LegacyChannelMigrator.ts @@ -176,9 +176,19 @@ export class LegacyChannelMigrator { isNonEmptyString(p.key), ); + const mediaSources = await getDatabase() + .selectFrom('mediaSource') + .selectAll() + .execute(); + const mediaSourcesByName = groupByUniqPropAndMap( + mediaSources, + 'name', + (ms) => ms.uuid, + ); + const programEntities = seq.collect( - uniqBy(programs, uniqueProgramId), - createProgramEntity, + uniqBy(programs, uniqueProgramId), + (program) => createProgramEntity(program, mediaSourcesByName), ); this.logger.debug( diff --git a/server/src/migration/legacy_migration/libraryMigrator.ts b/server/src/migration/legacy_migration/libraryMigrator.ts index 2a662b88..8b6bfd53 100644 --- a/server/src/migration/legacy_migration/libraryMigrator.ts +++ b/server/src/migration/legacy_migration/libraryMigrator.ts @@ -27,10 +27,12 @@ import { import { groupByUniq, groupByUniqProp, + groupByUniqPropAndMap, isNonEmptyString, mapAsyncSeq, mapToObj, } from '../../util/index.ts'; +import type { LegacyProgram } from './LegacyChannelMigrator.ts'; import type { CustomShow } from './legacyDbMigration.ts'; import type { JSONArray, JSONObject } from './migrationUtil.ts'; import { @@ -88,7 +90,7 @@ export class LegacyLibraryMigrator { Promise.resolve([] as CustomShow[]), ); - const uniquePrograms = uniqBy( + const uniquePrograms = uniqBy( filter( flatMap(newCustomShows, (cs) => cs.content), (p) => @@ -99,7 +101,15 @@ export class LegacyLibraryMigrator { uniqueProgramId, ); - const programEntities = seq.collect(uniquePrograms, createProgramEntity); + const mediaSourcesByName = await getDatabase() + .selectFrom('mediaSource') + .selectAll() + .execute() + .then((_) => groupByUniqPropAndMap(_, 'name', (ms) => ms.uuid)); + + const programEntities = seq.collect(uniquePrograms, (program) => + createProgramEntity(program, mediaSourcesByName), + ); this.logger.debug( 'Upserting %d programs from legacy DB', diff --git a/server/src/migration/legacy_migration/migrationUtil.ts b/server/src/migration/legacy_migration/migrationUtil.ts index 527a5122..5da9b362 100644 --- a/server/src/migration/legacy_migration/migrationUtil.ts +++ b/server/src/migration/legacy_migration/migrationUtil.ts @@ -12,8 +12,11 @@ import type { import type { LegacyProgram } from './LegacyChannelMigrator.ts'; // JSON representation for easier parsing of legacy db files -export interface JSONArray extends Array {} -export interface JSONObject extends Record {} +export type JSONArray = Array; +export interface JSONObject { + [x: string]: JSONValue; +} + export type JSONValue = | string | number @@ -68,7 +71,7 @@ export function convertRawProgram(program: JSONObject): LegacyProgram { const programType = program['type'] as string | undefined; const isMovie = programType === 'movie'; const id = v4(); - const outProgram: LegacyProgram = { + return { id, duration: program['duration'] as number, episodeIcon: program['episodeIcon'] as Maybe, @@ -96,15 +99,18 @@ export function convertRawProgram(program: JSONObject): LegacyProgram { customShowId: program['customShowId'] as Maybe, customShowName: program['customShowName'] as Maybe, sourceType: 'plex', - }; - - return outProgram; + } satisfies LegacyProgram; } export function createProgramEntity( program: LegacyProgram, + mediaSourceIdByName: Record, ): NewProgramDao | undefined { const now = +dayjs(); + if (!mediaSourceIdByName[program.serverKey ?? '']) { + return; + } + if ( ['movie', 'episode', 'track'].includes(program.type ?? '') && every([program.ratingKey, program.serverKey, program.key], isNonEmptyString) @@ -122,6 +128,7 @@ export function createProgramEntity( plexRatingKey: program.key!, plexFilePath: program.plexFile, externalSourceId: program.serverKey!, + mediaSourceId: mediaSourceIdByName[program.serverKey ?? ''], showTitle: program.showTitle, summary: program.summary, title: program.title!, diff --git a/server/src/tasks/fixers/BackfillMediaSourceIdFixer.ts b/server/src/tasks/fixers/BackfillMediaSourceIdFixer.ts new file mode 100644 index 00000000..588ad27c --- /dev/null +++ b/server/src/tasks/fixers/BackfillMediaSourceIdFixer.ts @@ -0,0 +1,76 @@ +import { inject, injectable } from 'inversify'; +import { getDatabase } from '../../db/DBAccess.ts'; +import { KEYS } from '../../types/inject.ts'; +import { Logger } from '../../util/logging/LoggerFactory.ts'; +import Fixer from './fixer.ts'; + +@injectable() +export class BackfillMediaSourceIdFixer extends Fixer { + constructor(@inject(KEYS.Logger) protected logger: Logger) { + super(); + } + + protected async runInternal(): Promise { + const db = getDatabase(); + + await db + .updateTable('program') + .set({ + mediaSourceId: (eb) => + eb + .selectFrom('mediaSource') + .whereRef('mediaSource.name', '=', 'program.externalSourceId') + .whereRef('mediaSource.type', '=', 'program.sourceType') + .select('mediaSource.uuid') + .limit(1), + }) + .where('program.mediaSourceId', 'is', null) + .execute(); + + await db + .updateTable('programExternalId') + .set({ + mediaSourceId: (eb) => + eb + .selectFrom('mediaSource') + .whereRef( + 'mediaSource.name', + '=', + 'programExternalId.externalSourceId', + ) + .whereRef('mediaSource.type', '=', 'programExternalId.sourceType') + .select('mediaSource.uuid') + .limit(1), + }) + .where('programExternalId.mediaSourceId', 'is', null) + .where('programExternalId.sourceType', 'in', ['plex', 'emby', 'jellyfin']) + .execute(); + + await db + .updateTable('programGroupingExternalId') + .set({ + mediaSourceId: (eb) => + eb + .selectFrom('mediaSource') + .whereRef( + 'mediaSource.name', + '=', + 'programGroupingExternalId.externalSourceId', + ) + .whereRef( + 'mediaSource.type', + '=', + 'programGroupingExternalId.sourceType', + ) + .select('mediaSource.uuid') + .limit(1), + }) + .where('programGroupingExternalId.mediaSourceId', 'is', null) + .where('programGroupingExternalId.sourceType', 'in', [ + 'plex', + 'emby', + 'jellyfin', + ]) + .execute(); + } +} diff --git a/server/src/tasks/fixers/BackfillProgramExternalIds.ts b/server/src/tasks/fixers/BackfillProgramExternalIds.ts index e0dd3f72..28dcb5bf 100644 --- a/server/src/tasks/fixers/BackfillProgramExternalIds.ts +++ b/server/src/tasks/fixers/BackfillProgramExternalIds.ts @@ -1,10 +1,10 @@ import { getDatabase } from '@/db/DBAccess.js'; import { ProgramExternalIdType } from '@/db/custom_types/ProgramExternalIdType.js'; import { ProgramSourceType } from '@/db/custom_types/ProgramSourceType.js'; -import { upsertRawProgramExternalIds } from '@/db/programExternalIdHelpers.js'; +import { upsertProgramExternalIds } from '@/db/programExternalIdHelpers.js'; import { withProgramExternalIds } from '@/db/programQueryHelpers.js'; import { ProgramDao } from '@/db/schema/Program.js'; -import { NewProgramExternalId } from '@/db/schema/ProgramExternalId.js'; +import { NewSingleOrMultiExternalId } from '@/db/schema/ProgramExternalId.js'; import { isQueryError } from '@/external/BaseApiClient.js'; import { MediaSourceApiFactory } from '@/external/MediaSourceApiFactory.js'; import { PlexApiClient } from '@/external/plex/PlexApiClient.js'; @@ -118,7 +118,7 @@ export class BackfillProgramExternalIds extends Fixer { ); } else { const upsertResult = await attempt(() => - upsertRawProgramExternalIds(result.result), + upsertProgramExternalIds(result.result), ); if (isError(upsertResult)) { this.logger.warn( @@ -147,6 +147,10 @@ export class BackfillProgramExternalIds extends Fixer { ); } + if (isUndefined(plex.serverId)) { + throw new Error('Plex server is not a saved media source'); + } + const metadataResult = await plex.getItemMetadata(program.externalKey); if (isQueryError(metadataResult)) { @@ -160,12 +164,14 @@ export class BackfillProgramExternalIds extends Fixer { // We're here, might as well use the real thing. const firstPart = first(first(metadata.Media)?.Part); - const entities: NewProgramExternalId[] = [ + const entities: NewSingleOrMultiExternalId[] = [ { + type: 'multi', externalFilePath: firstPart?.key ?? program.plexFilePath, directFilePath: firstPart?.file ?? program.filePath, externalKey: metadata.ratingKey, externalSourceId: plex.serverName, + mediaSourceId: plex.serverId, programUuid: program.uuid, sourceType: ProgramExternalIdType.PLEX, uuid: v4(), @@ -183,6 +189,7 @@ export class BackfillProgramExternalIds extends Fixer { } entities.push({ + type: 'single', uuid: v4(), createdAt: +dayjs(), updatedAt: +dayjs(), diff --git a/server/src/tasks/fixers/FixerModule.ts b/server/src/tasks/fixers/FixerModule.ts index 0c4fe52f..a3e0df5f 100644 --- a/server/src/tasks/fixers/FixerModule.ts +++ b/server/src/tasks/fixers/FixerModule.ts @@ -6,6 +6,7 @@ import type Fixer from '@/tasks/fixers/fixer.js'; import { MissingSeasonNumbersFixer } from '@/tasks/fixers/missingSeasonNumbersFixer.js'; import { KEYS } from '@/types/inject.js'; import { ContainerModule } from 'inversify'; +import { BackfillMediaSourceIdFixer } from './BackfillMediaSourceIdFixer.ts'; const FixerModule = new ContainerModule((bind) => { bind(KEYS.Fixer).to(BackfillProgramExternalIds); @@ -13,6 +14,7 @@ const FixerModule = new ContainerModule((bind) => { bind(KEYS.Fixer).to(AddPlexServerIdsFixer); bind(KEYS.Fixer).to(BackfillProgramGroupings); bind(KEYS.Fixer).to(MissingSeasonNumbersFixer); + bind(KEYS.Fixer).to(BackfillMediaSourceIdFixer); }); export { FixerModule }; diff --git a/server/src/tasks/fixers/backfillProgramGroupings.ts b/server/src/tasks/fixers/backfillProgramGroupings.ts index cad75d4e..ce0402ea 100644 --- a/server/src/tasks/fixers/backfillProgramGroupings.ts +++ b/server/src/tasks/fixers/backfillProgramGroupings.ts @@ -18,6 +18,7 @@ export class BackfillProgramGroupings extends Fixer { // This clears out mismatches that might have happened on bugged earlier versions // There was a bug where we were setting the season ID to the show ID. // This should only affect seasons since the music album stuff had the fix + console.log('backfill', getDatabase().transaction()); const clearedSeasons = await getDatabase() .transaction() .execute((tx) => diff --git a/server/src/tasks/jellyfin/SaveJellyfinProgramExternalIdsTask.ts b/server/src/tasks/jellyfin/SaveJellyfinProgramExternalIdsTask.ts index 141483d1..8364e5b9 100644 --- a/server/src/tasks/jellyfin/SaveJellyfinProgramExternalIdsTask.ts +++ b/server/src/tasks/jellyfin/SaveJellyfinProgramExternalIdsTask.ts @@ -1,5 +1,5 @@ import type { IProgramDB } from '@/db/interfaces/IProgramDB.js'; -import { upsertRawProgramExternalIds } from '@/db/programExternalIdHelpers.js'; +import { upsertProgramExternalIds } from '@/db/programExternalIdHelpers.js'; import { isQueryError } from '@/external/BaseApiClient.js'; import { type MediaSourceApiFactory } from '@/external/MediaSourceApiFactory.js'; import type { JellyfinApiClient } from '@/external/jellyfin/JellyfinApiClient.js'; @@ -15,7 +15,7 @@ import { } from '../../db/custom_types/ProgramExternalIdType.ts'; import type { MinimalProgramExternalId, - NewProgramExternalId, + NewSingleOrMultiExternalId, } from '../../db/schema/ProgramExternalId.ts'; export type SaveJellyfinProgramExternalIdsTaskFactory = ( @@ -96,17 +96,18 @@ export class SaveJellyfinProgramExternalIdsTask extends Task { } return { + type: 'single', uuid: v4(), createdAt: +dayjs(), updatedAt: +dayjs(), externalKey: id, sourceType: type, programUuid: program.uuid, - } satisfies NewProgramExternalId; + } satisfies NewSingleOrMultiExternalId; }), ); - return await upsertRawProgramExternalIds(eids); + return await upsertProgramExternalIds(eids); } get taskName() { diff --git a/server/src/tasks/plex/SavePlexProgramExternalIdsTask.ts b/server/src/tasks/plex/SavePlexProgramExternalIdsTask.ts index fb24d0cd..256bdeae 100644 --- a/server/src/tasks/plex/SavePlexProgramExternalIdsTask.ts +++ b/server/src/tasks/plex/SavePlexProgramExternalIdsTask.ts @@ -1,5 +1,5 @@ import { ProgramExternalIdType } from '@/db/custom_types/ProgramExternalIdType.js'; -import { upsertRawProgramExternalIds } from '@/db/programExternalIdHelpers.js'; +import { upsertProgramExternalIds } from '@/db/programExternalIdHelpers.js'; import type { MinimalProgramExternalId } from '@/db/schema/ProgramExternalId.js'; import { isQueryError } from '@/external/BaseApiClient.js'; import { type MediaSourceApiFactory } from '@/external/MediaSourceApiFactory.js'; @@ -8,8 +8,9 @@ import { Task } from '@/tasks/Task.js'; import type { Maybe } from '@/types/util.js'; import { mintExternalIdForPlexGuid } from '@/util/externalIds.js'; import { isDefined, isNonEmptyString } from '@/util/index.js'; +import { seq } from '@tunarr/shared/util'; import type { PlexTerminalMedia } from '@tunarr/types/plex'; -import { compact, isEmpty, isNil, isUndefined, map } from 'lodash-es'; +import { isEmpty, isNil, isUndefined } from 'lodash-es'; import type { IProgramDB } from '../../db/interfaces/IProgramDB.ts'; export type SavePlexProgramExternalIdsTaskFactory = ( @@ -78,19 +79,11 @@ export class SavePlexProgramExternalIdsTask extends Task { const metadata = metadataResult.data as PlexTerminalMedia; - const eids = compact( - map(metadata.Guid, (guid) => { - const parsed = mintExternalIdForPlexGuid(guid.id, program.uuid); - if (parsed) { - parsed.externalSourceId = undefined; - return parsed; - } - - return; - }), + const eids = seq.collect(metadata.Guid, (guid) => + mintExternalIdForPlexGuid(guid.id, program.uuid), ); - return await upsertRawProgramExternalIds(eids); + return await upsertProgramExternalIds(eids); } get taskName() { diff --git a/server/src/types/util.ts b/server/src/types/util.ts index e61fd03a..fa828e2f 100644 --- a/server/src/types/util.ts +++ b/server/src/types/util.ts @@ -1,4 +1,8 @@ -import type { DeepNonNullable, StrictExclude } from 'ts-essentials'; +import type { + DeepNonNullable, + MarkRequired, + StrictExclude, +} from 'ts-essentials'; export type Maybe = T | undefined; @@ -31,3 +35,8 @@ export type MarkNullable = { }; export type NonEmptyArray = [T, ...T[]]; + +export type MarkNotNilable = MarkNonNullable< + MarkRequired, + Keys +>; diff --git a/server/src/util/externalIds.ts b/server/src/util/externalIds.ts index babc3f1c..22dafd95 100644 --- a/server/src/util/externalIds.ts +++ b/server/src/util/externalIds.ts @@ -1,5 +1,5 @@ import { programExternalIdTypeFromExternalIdType } from '@/db/custom_types/ProgramExternalIdType.js'; -import type { NewProgramExternalId } from '@/db/schema/ProgramExternalId.js'; +import type { NewSingleOrMultiExternalId } from '@/db/schema/ProgramExternalId.js'; import type { Nullable } from '@/types/util.js'; import type { MultiExternalId } from '@tunarr/types'; import { isValidSingleExternalIdType } from '@tunarr/types/schemas'; @@ -29,10 +29,11 @@ export const createPlexExternalId = ( export const mintExternalIdForPlexGuid = ( guid: string, programId: string, -): Nullable => { +): Nullable => { const parsed = parsePlexGuid(guid); if (parsed) { return { + type: 'single', uuid: v4(), createdAt: +dayjs(), updatedAt: +dayjs(), diff --git a/shared/src/services/ApiProgramMinter.ts b/shared/src/services/ApiProgramMinter.ts index 63e31790..f3a84dee 100644 --- a/shared/src/services/ApiProgramMinter.ts +++ b/shared/src/services/ApiProgramMinter.ts @@ -83,6 +83,7 @@ export class ApiProgramMinter { return { type: 'content', externalSourceType: 'plex', + externalSourceId: server.id, externalSourceName: server.name, date: plexMovie.originallyAvailableAt, duration: plexMovie.duration ?? 0, @@ -95,7 +96,6 @@ export class ApiProgramMinter { subtype: 'movie', persisted: false, externalIds: this.mintExternalIdsForPlex(server.name, plexMovie), - externalSourceId: server.name, uniqueId: id, id, }; @@ -113,7 +113,7 @@ export class ApiProgramMinter { index: plexEpisode.index, externalKey: plexEpisode.ratingKey, externalSourceName: server.name, - externalSourceId: server.name, + externalSourceId: server.id, externalSourceType: ExternalSourceTypeSchema.enum.plex, parent: { title: plexEpisode.parentTitle, @@ -242,7 +242,7 @@ export class ApiProgramMinter { persisted: false, uniqueId: id, id, - externalSourceId: server.name, + externalSourceId: server.id, }; } @@ -257,7 +257,7 @@ export class ApiProgramMinter { externalSourceType: ExternalSourceTypeSchema.enum.jellyfin, date: nullToUndefined(item.PremiereDate), duration: (item.RunTimeTicks ?? 0) / 10_000, - externalSourceId: server.name, + externalSourceId: server.id, externalKey: item.Id, rating: nullToUndefined(item.OfficialRating), summary: nullToUndefined(item.Overview), @@ -322,7 +322,7 @@ export class ApiProgramMinter { externalSourceType: ExternalSourceTypeSchema.enum.emby, date: nullToUndefined(item.PremiereDate), duration: (item.RunTimeTicks ?? 0) / 10_000, - externalSourceId: server.name, + externalSourceId: server.id, externalKey: item.Id, rating: nullToUndefined(item.OfficialRating), summary: nullToUndefined(item.Overview),