From c95d8028a631f2498d9f09f7eab615ea24a7b053 Mon Sep 17 00:00:00 2001 From: Christian Benincasa Date: Tue, 31 Mar 2026 12:43:48 -0400 Subject: [PATCH] refactor: break up DB classes into focused repositories --- server/src/db/ChannelDB.test.ts | 54 +- server/src/db/ChannelDB.ts | 2270 +---------- server/src/db/CustomShowDB.test.ts | 52 +- server/src/db/DBModule.ts | 30 + server/src/db/ProgramDB.test.ts | 40 +- server/src/db/ProgramDB.ts | 3317 +---------------- .../src/db/channel/BasicChannelRepository.ts | 532 +++ .../src/db/channel/ChannelConfigRepository.ts | 23 + .../db/channel/ChannelProgramRepository.ts | 478 +++ server/src/db/channel/LineupRepository.ts | 1225 ++++++ .../src/db/program/BasicProgramRepository.ts | 114 + .../db/program/ProgramExternalIdRepository.ts | 375 ++ .../db/program/ProgramGroupingRepository.ts | 610 +++ .../ProgramGroupingUpsertRepository.ts | 494 +++ .../db/program/ProgramMetadataRepository.ts | 444 +++ .../src/db/program/ProgramSearchRepository.ts | 229 ++ .../src/db/program/ProgramStateRepository.ts | 57 + .../src/db/program/ProgramUpsertRepository.ts | 1080 ++++++ server/src/types/inject.ts | 16 + 19 files changed, 6238 insertions(+), 5202 deletions(-) create mode 100644 server/src/db/channel/BasicChannelRepository.ts create mode 100644 server/src/db/channel/ChannelConfigRepository.ts create mode 100644 server/src/db/channel/ChannelProgramRepository.ts create mode 100644 server/src/db/channel/LineupRepository.ts create mode 100644 server/src/db/program/BasicProgramRepository.ts create mode 100644 server/src/db/program/ProgramExternalIdRepository.ts create mode 100644 server/src/db/program/ProgramGroupingRepository.ts create mode 100644 server/src/db/program/ProgramGroupingUpsertRepository.ts create mode 100644 server/src/db/program/ProgramMetadataRepository.ts create mode 100644 server/src/db/program/ProgramSearchRepository.ts create mode 100644 server/src/db/program/ProgramStateRepository.ts create mode 100644 server/src/db/program/ProgramUpsertRepository.ts diff --git a/server/src/db/ChannelDB.test.ts b/server/src/db/ChannelDB.test.ts index a4cffc59..c87b0a30 100644 --- a/server/src/db/ChannelDB.test.ts +++ b/server/src/db/ChannelDB.test.ts @@ -17,6 +17,11 @@ import type { IChannelDB } from './interfaces/IChannelDB.ts'; import { ChannelPrograms } from './schema/ChannelPrograms.ts'; import { Program } from './schema/Program.ts'; import { IProgramDB } from './interfaces/IProgramDB.ts'; +import { BasicChannelRepository } from './channel/BasicChannelRepository.ts'; +import { ChannelProgramRepository } from './channel/ChannelProgramRepository.ts'; +import { LineupRepository } from './channel/LineupRepository.ts'; +import { ChannelConfigRepository } from './channel/ChannelConfigRepository.ts'; +import { MaterializeLineupCommand } from '../commands/MaterializeLineupCommand.ts'; type Fixture = { db: string; @@ -51,24 +56,51 @@ const test = baseTest.extend({ } return task; }, + queueTask: async (task: any) => { + return { result: task }; + }, }) as any; const mockMaterializeLineupCommand = { - execute: async () => {}, + execute: async () => ({}), } as any; - const channelDb = new ChannelDB( - new ProgramConverter( - LoggerFactory.child({ className: ProgramConverter.name }), - dbAccess.db!, - ), - mock(), - mock(CacheImageService), - dbAccess.db!, // Kysely instance + const fileSystemService = new FileSystemService(globalOptions()); + + const programConverter = new ProgramConverter( + LoggerFactory.child({ className: ProgramConverter.name }), + dbAccess.db!, + ); + + const lineupRepo = new LineupRepository( + dbAccess.db!, + dbAccess.drizzle!, + fileSystemService, mockWorkerPoolFactory, - new FileSystemService(globalOptions()), - dbAccess.drizzle!, // Drizzle instance mockMaterializeLineupCommand, + mock(), + programConverter, + ); + + const basicChannelRepo = new BasicChannelRepository( + dbAccess.db!, + dbAccess.drizzle!, + mock(CacheImageService), + lineupRepo, + ); + + const channelProgramRepo = new ChannelProgramRepository( + dbAccess.db!, + dbAccess.drizzle!, + ); + + const channelConfigRepo = new ChannelConfigRepository(dbAccess.db!); + + const channelDb = new ChannelDB( + basicChannelRepo, + channelProgramRepo, + lineupRepo, + channelConfigRepo, ); await use(channelDb); diff --git a/server/src/db/ChannelDB.ts b/server/src/db/ChannelDB.ts index e3d8b57f..8f494120 100644 --- a/server/src/db/ChannelDB.ts +++ b/server/src/db/ChannelDB.ts @@ -1,281 +1,68 @@ -import { ChannelQueryBuilder } from '@/db/ChannelQueryBuilder.js'; -import { ProgramConverter } from '@/db/converters/ProgramConverter.js'; -import { +import type { ChannelQueryBuilder } from '@/db/ChannelQueryBuilder.js'; +import type { ChannelAndLineup, ChannelAndRawLineup, - type IChannelDB, + IChannelDB, + UpdateChannelLineupRequest, } from '@/db/interfaces/IChannelDB.js'; -import type { IProgramDB } from '@/db/interfaces/IProgramDB.js'; -import { globalOptions } from '@/globals.js'; -import { FileSystemService } from '@/services/FileSystemService.js'; -import { CacheImageService } from '@/services/cacheImageService.js'; -import { ChannelNotFoundError } from '@/types/errors.js'; import { KEYS } from '@/types/inject.js'; -import { typedProperty } from '@/types/path.js'; -import { Result } from '@/types/result.js'; -import { jsonSchema } from '@/types/schemas.js'; -import { Maybe, Nullable, PagedResult } from '@/types/util.js'; -import { Timer } from '@/util/Timer.js'; -import { asyncPool } from '@/util/asyncPool.js'; -import dayjs from '@/util/dayjs.js'; -import { fileExists } from '@/util/fsUtil.js'; -import { LoggerFactory } from '@/util/logging/LoggerFactory.js'; -import { MutexMap } from '@/util/mutexMap.js'; -import { booleanToNumber } from '@/util/sqliteUtil.js'; -import { seq } from '@tunarr/shared/util'; -import { - ChannelProgram, +import type { Maybe, Nullable, PagedResult } from '@/types/util.js'; +import type { ChannelProgramming, - CondensedChannelProgram, CondensedChannelProgramming, - ContentProgram, SaveableChannel, - Watermark, } from '@tunarr/types'; -import { UpdateChannelProgrammingRequest } from '@tunarr/types/api'; -import { ContentProgramType } from '@tunarr/types/schemas'; -import { and, asc, count, countDistinct, eq, isNotNull } from 'drizzle-orm'; -import { inject, injectable, interfaces } from 'inversify'; -import { Kysely } from 'kysely'; -import { jsonArrayFrom } from 'kysely/helpers/sqlite'; -import { - chunk, - drop, - entries, - filter, - flatten, - forEach, - groupBy, - head, - isEmpty, - isNil, - isNull, - isNumber, - isString, - isUndefined, - map, - mapValues, - nth, - omit, - omitBy, - partition, - reject, - sum, - sumBy, - take, - uniq, - uniqBy, -} from 'lodash-es'; -import { Low } from 'lowdb'; -import fs from 'node:fs/promises'; -import { join } from 'node:path'; -import { MarkRequired } from 'ts-essentials'; -import { match } from 'ts-pattern'; -import { v4 } from 'uuid'; -import { MaterializeLineupCommand } from '../commands/MaterializeLineupCommand.ts'; -import { IWorkerPool } from '../interfaces/IWorkerPool.ts'; -import { - createManyRelationAgg, - mapRawJsonRelationResult, -} from '../util/drizzleUtil.ts'; -import { - asyncMapToRecord, - groupByFunc, - groupByUniqProp, - isDefined, - isNonEmptyString, - mapReduceAsyncSeq, - programExternalIdString, - run, -} from '../util/index.ts'; -import { - ContentItem, - CurrentLineupSchemaVersion, - isContentItem, - isOfflineItem, - isRedirectItem, +import type { UpdateChannelProgrammingRequest } from '@tunarr/types/api'; +import type { ContentProgramType } from '@tunarr/types/schemas'; +import { inject, injectable } from 'inversify'; +import type { MarkRequired } from 'ts-essentials'; +import type { Lineup, LineupItem, - LineupSchema, PendingProgram, } from './derived_types/Lineup.ts'; -import { - PageParams, - UpdateChannelLineupRequest, -} from './interfaces/IChannelDB.ts'; -import { SchemaBackedDbAdapter } from './json/SchemaBackedJsonDBAdapter.ts'; -import { calculateStartTimeOffsets } from './lineupUtil.ts'; -import { - AllProgramGroupingFields, - withFallbackPrograms, - withPrograms, - withTrackAlbum, - withTrackArtist, - withTvSeason, - withTvShow, -} from './programQueryHelpers.ts'; -import { Artwork } from './schema/Artwork.ts'; -import { - Channel, - ChannelOrm, - ChannelUpdate, - NewChannel, -} from './schema/Channel.ts'; -import { NewChannelFillerShow } from './schema/ChannelFillerShow.ts'; -import { - ChannelPrograms, - NewChannelProgram, -} from './schema/ChannelPrograms.ts'; -import { Program, ProgramType } from './schema/Program.ts'; -import { - ProgramGrouping, - ProgramGroupingType, -} from './schema/ProgramGrouping.ts'; -import { ProgramGroupingExternalIdOrm } from './schema/ProgramGroupingExternalId.ts'; -import { - ChannelSubtitlePreferences, - NewChannelSubtitlePreference, -} from './schema/SubtitlePreferences.ts'; -import { DB } from './schema/db.ts'; -import { +import type { Channel, ChannelOrm } from './schema/Channel.ts'; +import type { ProgramDao } from './schema/Program.ts'; +import type { ProgramExternalId } from './schema/ProgramExternalId.ts'; +import type { ChannelSubtitlePreferences } from './schema/SubtitlePreferences.ts'; +import type { ChannelOrmWithPrograms, ChannelOrmWithRelations, ChannelOrmWithTranscodeConfig, - ChannelWithPrograms, ChannelWithRelations, - MusicAlbumOrm, MusicArtistOrm, - MusicArtistWithExternalIds, - ProgramGroupingOrmWithRelations, ProgramWithRelationsOrm, - TvSeasonOrm, TvShowOrm, -} from './schema/derivedTypes.js'; -import { DrizzleDBAccess } from './schema/index.ts'; - -// We use this to chunk super huge channel / program relation updates because -// of the way that mikro-orm generates these (e.g. "delete from XYZ where () or () ..."). -// When updating a _huge_ channel, we hit internal sqlite limits, so we must chunk these -// operations ourselves. -const SqliteMaxDepthLimit = 1000; - -type ProgramRelationOperation = { operation: 'add' | 'remove'; id: string }; - -function sanitizeChannelWatermark( - watermark: Maybe, -): Maybe { - if (isUndefined(watermark)) { - return; - } - - const validFadePoints = filter( - watermark.fadeConfig, - (conf) => conf.periodMins > 0, - ); - - return { - ...watermark, - fadeConfig: isEmpty(validFadePoints) ? undefined : validFadePoints, - }; -} - -function updateRequestToChannel(updateReq: SaveableChannel): ChannelUpdate { - const sanitizedWatermark = sanitizeChannelWatermark(updateReq.watermark); - - return { - number: updateReq.number, - watermark: sanitizedWatermark - ? JSON.stringify(sanitizedWatermark) - : undefined, - icon: JSON.stringify(updateReq.icon), - guideMinimumDuration: updateReq.guideMinimumDuration, - groupTitle: updateReq.groupTitle, - disableFillerOverlay: booleanToNumber(updateReq.disableFillerOverlay), - startTime: +dayjs(updateReq.startTime).second(0).millisecond(0), - offline: JSON.stringify(updateReq.offline), - name: updateReq.name, - duration: updateReq.duration, - stealth: booleanToNumber(updateReq.stealth), - fillerRepeatCooldown: updateReq.fillerRepeatCooldown, - guideFlexTitle: updateReq.guideFlexTitle, - transcodeConfigId: updateReq.transcodeConfigId, - streamMode: updateReq.streamMode, - subtitlesEnabled: booleanToNumber(updateReq.subtitlesEnabled), - } satisfies ChannelUpdate; -} - -function createRequestToChannel(saveReq: SaveableChannel): NewChannel { - const now = +dayjs(); - - return { - uuid: v4(), - createdAt: now, - updatedAt: now, - number: saveReq.number, - watermark: saveReq.watermark ? JSON.stringify(saveReq.watermark) : null, - icon: JSON.stringify(saveReq.icon), - guideMinimumDuration: saveReq.guideMinimumDuration, - groupTitle: saveReq.groupTitle, - disableFillerOverlay: saveReq.disableFillerOverlay ? 1 : 0, - startTime: saveReq.startTime, - offline: JSON.stringify(saveReq.offline), - name: saveReq.name, - duration: saveReq.duration, - stealth: saveReq.stealth ? 1 : 0, - fillerRepeatCooldown: saveReq.fillerRepeatCooldown, - guideFlexTitle: saveReq.guideFlexTitle, - streamMode: saveReq.streamMode, - transcodeConfigId: saveReq.transcodeConfigId, - subtitlesEnabled: booleanToNumber(saveReq.subtitlesEnabled), - } satisfies NewChannel; -} - -// Let's see if this works... in so we can have many ChannelDb objects flying around. -const fileDbCache: Record> = {}; -const fileDbLocks = new MutexMap(); +} from './schema/derivedTypes.ts'; +import { BasicChannelRepository } from './channel/BasicChannelRepository.ts'; +import { ChannelProgramRepository } from './channel/ChannelProgramRepository.ts'; +import { LineupRepository } from './channel/LineupRepository.ts'; +import { ChannelConfigRepository } from './channel/ChannelConfigRepository.ts'; +import type { PageParams } from './interfaces/IChannelDB.ts'; @injectable() export class ChannelDB implements IChannelDB { - private logger = LoggerFactory.child({ - caller: import.meta, - className: this.constructor.name, - }); - - private timer = new Timer(this.logger, 'trace'); - constructor( - @inject(ProgramConverter) private programConverter: ProgramConverter, - @inject(KEYS.ProgramDB) private programDB: IProgramDB, - @inject(CacheImageService) private cacheImageService: CacheImageService, - @inject(KEYS.Database) private db: Kysely, - @inject(KEYS.WorkerPoolFactory) - private workerPoolProvider: interfaces.AutoFactory, - @inject(FileSystemService) private fileSystemService: FileSystemService, - @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, - @inject(MaterializeLineupCommand) - private materializeLineupCommand: MaterializeLineupCommand, + @inject(KEYS.BasicChannelRepository) + private readonly basicChannel: BasicChannelRepository, + @inject(KEYS.ChannelProgramRepository) + private readonly channelProgram: ChannelProgramRepository, + @inject(KEYS.LineupRepository) + private readonly lineup: LineupRepository, + @inject(KEYS.ChannelConfigRepository) + private readonly channelConfig: ChannelConfigRepository, ) {} - async channelExists(channelId: string) { - const channel = await this.db - .selectFrom('channel') - .where('channel.uuid', '=', channelId) - .select('uuid') - .executeTakeFirst(); - return !isNil(channel); + // --- BasicChannelRepository delegation --- + + channelExists(channelId: string): Promise { + return this.basicChannel.channelExists(channelId); } getChannelOrm( id: string | number, ): Promise> { - return this.drizzleDB.query.channels.findFirst({ - where: (channel, { eq }) => { - return isString(id) ? eq(channel.uuid, id) : eq(channel.number, id); - }, - with: { - transcodeConfig: true, - }, - }); + return this.basicChannel.getChannelOrm(id); } getChannel(id: string | number): Promise>; @@ -283,1888 +70,239 @@ export class ChannelDB implements IChannelDB { id: string | number, includeFiller: true, ): Promise>>; - async getChannel( + getChannel( + id: string | number, + includeFiller: boolean, + ): Promise>; + getChannel( id: string | number, includeFiller: boolean = false, ): Promise> { - // return this.drizzleDB.query.channels.findFirst({ - // where: (fields, { eq }) => { - // if (isString(id)) { - // return eq(fields.uuid, id); - // } else { - // return eq(fields.number, id); - // } - // }, - // with: { - // channelFillerShow: includeFiller - // ? { - // with: { - // filler: true, - // }, - // } - // : undefined, - // }, - // }); - return this.db - .selectFrom('channel') - .$if(isString(id), (eb) => eb.where('channel.uuid', '=', id as string)) - .$if(isNumber(id), (eb) => eb.where('channel.number', '=', id as number)) - .$if(includeFiller, (eb) => - eb.select((qb) => - jsonArrayFrom( - qb - .selectFrom('channelFillerShow') - .whereRef('channel.uuid', '=', 'channelFillerShow.channelUuid') - .select([ - 'channelFillerShow.channelUuid', - 'channelFillerShow.fillerShowUuid', - 'channelFillerShow.cooldown', - 'channelFillerShow.weight', - ]), - ).as('fillerShows'), - ), - ) - .selectAll() - .executeTakeFirst(); - } - - getChannelBuilder(id: string | number) { - return ChannelQueryBuilder.createForIdOrNumber(this.db, id); - } - - async getChannelAndPrograms( - uuid: string, - ): Promise>> { - const channelsAndPrograms = await this.drizzleDB.query.channels.findFirst({ - where: (fields, { eq }) => eq(fields.uuid, uuid), - with: { - channelPrograms: { - with: { - program: { - with: { - show: true, - season: true, - artist: true, - album: true, - externalIds: true, - }, - }, - }, - }, - }, - orderBy: (fields, { asc }) => asc(fields.number), - }); - - if (channelsAndPrograms) { - return { - ...channelsAndPrograms, - programs: channelsAndPrograms.channelPrograms.map( - ({ program }) => program, - ), - } satisfies MarkRequired; + if (includeFiller) { + return this.basicChannel.getChannel(id, true); } - - return; + return this.basicChannel.getChannel(id); } - async getChannelAndProgramsOld( + getChannelBuilder( + id: string | number, + ): ChannelQueryBuilder { + return this.basicChannel.getChannelBuilder(id); + } + + getAllChannels(): Promise { + return this.basicChannel.getAllChannels(); + } + + saveChannel(createReq: SaveableChannel): Promise> { + return this.basicChannel.saveChannel(createReq); + } + + updateChannel( + id: string, + updateReq: SaveableChannel, + ): Promise> { + return this.basicChannel.updateChannel(id, updateReq); + } + + updateChannelDuration(id: string, duration: number): Promise { + return this.basicChannel.updateChannelDuration(id, duration); + } + + updateChannelStartTime(id: string, newTime: number): Promise { + return this.basicChannel.updateChannelStartTime(id, newTime); + } + + syncChannelDuration(id: string): Promise { + return this.basicChannel.syncChannelDuration(id); + } + + copyChannel(id: string): Promise> { + return this.basicChannel.copyChannel(id); + } + + deleteChannel( + channelId: string, + blockOnLineupUpdates?: boolean, + ): Promise { + return this.basicChannel.deleteChannel(channelId, blockOnLineupUpdates); + } + + // --- ChannelProgramRepository delegation --- + + getChannelAndPrograms( uuid: string, - ): Promise { - return this.db - .selectFrom('channel') - .selectAll(['channel']) - .where('channel.uuid', '=', uuid) - .leftJoin( - 'channelPrograms', - 'channel.uuid', - 'channelPrograms.channelUuid', - ) - .select((eb) => - withPrograms(eb, { - joins: { - customShows: true, - tvShow: [ - 'programGrouping.uuid', - 'programGrouping.title', - 'programGrouping.summary', - 'programGrouping.type', - ], - tvSeason: [ - 'programGrouping.uuid', - 'programGrouping.title', - 'programGrouping.summary', - 'programGrouping.type', - ], - trackArtist: [ - 'programGrouping.uuid', - 'programGrouping.title', - 'programGrouping.summary', - 'programGrouping.type', - ], - trackAlbum: [ - 'programGrouping.uuid', - 'programGrouping.title', - 'programGrouping.summary', - 'programGrouping.type', - ], - }, - }), - ) - .groupBy('channel.uuid') - .orderBy('channel.number asc') - .executeTakeFirst(); + typeFilter?: ContentProgramType, + ): Promise>> { + return this.channelProgram.getChannelAndPrograms(uuid, typeFilter); } - async getChannelTvShows( + getChannelAndProgramsOld(uuid: string) { + return this.channelProgram.getChannelAndProgramsOld(uuid); + } + + getChannelTvShows( id: string, pageParams?: PageParams, ): Promise> { - const groups = await this.drizzleDB - .select({ - programGrouping: ProgramGrouping, - artwork: createManyRelationAgg( - this.drizzleDB - .select() - .from(Artwork) - .where(eq(ProgramGrouping.uuid, Artwork.groupingId)) - .as('artwork'), - 'artwork', - ), - }) - .from(ChannelPrograms) - .where( - and( - eq(ChannelPrograms.channelUuid, id), - eq(Program.type, ProgramType.Episode), - isNotNull(Program.tvShowUuid), - eq(ProgramGrouping.type, ProgramGroupingType.Show), - ), - ) - .groupBy(Program.tvShowUuid) - .orderBy(asc(ProgramGrouping.uuid)) - .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) - .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.tvShowUuid)) - .offset(pageParams?.offset ?? 0) - .limit(pageParams?.limit ?? 1_000_000); - - const countPromise = this.drizzleDB - .select({ - count: countDistinct(ProgramGrouping.uuid), - }) - .from(ChannelPrograms) - .where( - and( - eq(ChannelPrograms.channelUuid, id), - eq(Program.type, ProgramType.Episode), - isNotNull(Program.tvShowUuid), - eq(ProgramGrouping.type, ProgramGroupingType.Show), - ), - ) - .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) - .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.tvShowUuid)); - - // Populate external ids - const externalIdQueries: Promise[] = []; - const seasonQueries: Promise[] = []; - for (const groupChunk of chunk(groups, 100)) { - const ids = groupChunk.map(({ programGrouping }) => programGrouping.uuid); - externalIdQueries.push( - this.drizzleDB.query.programGroupingExternalId.findMany({ - where: (fields, { inArray }) => inArray(fields.groupUuid, ids), - }), - ); - seasonQueries.push( - this.drizzleDB.query.programGrouping.findMany({ - where: (fields, { eq, and, inArray }) => - and( - eq(fields.type, ProgramGroupingType.Season), - inArray(fields.showUuid, ids), - ), - with: { - externalIds: true, - }, - }), - ); - } - - const [externalIdResults, seasonResults] = await Promise.all([ - Promise.all(externalIdQueries).then(flatten), - Promise.all(seasonQueries).then(flatten), - ]); - - const externalIdsByGroupId = groupBy( - externalIdResults, - (id) => id.groupUuid, - ); - const seasonByGroupId = groupBy(seasonResults, (season) => season.showUuid); - - const shows: TvShowOrm[] = []; - for (const { programGrouping, artwork } of groups) { - if (programGrouping.type === 'show') { - const seasons = - seasonByGroupId[programGrouping.uuid]?.filter( - (group): group is TvSeasonOrm => group.type === 'season', - ) ?? []; - shows.push({ - ...programGrouping, - type: 'show', - externalIds: externalIdsByGroupId[programGrouping.uuid] ?? [], - seasons, - artwork: mapRawJsonRelationResult(artwork, Artwork), - }); - } - } - - return { - total: sum((await countPromise).map(({ count }) => count)), - results: shows, - }; + return this.channelProgram.getChannelTvShows(id, pageParams); } - async getChannelMusicArtists( + getChannelMusicArtists( id: string, pageParams?: PageParams, - ): Promise> { - const groups = await this.drizzleDB - .select({ - programGrouping: ProgramGrouping, - }) - .from(ChannelPrograms) - .where( - and( - eq(ChannelPrograms.channelUuid, id), - eq(Program.type, ProgramType.Track), - isNotNull(Program.artistUuid), - eq(ProgramGrouping.type, ProgramGroupingType.Artist), - ), - ) - .groupBy(Program.artistUuid) - .orderBy(asc(ProgramGrouping.uuid)) - .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) - .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.artistUuid)) - .offset(pageParams?.offset ?? 0) - .limit(pageParams?.limit ?? 1_000_000); - - const countPromise = this.drizzleDB - .select({ - count: count(), - }) - .from(ChannelPrograms) - .where( - and( - eq(ChannelPrograms.channelUuid, id), - eq(Program.type, ProgramType.Episode), - isNotNull(Program.tvShowUuid), - eq(ProgramGrouping.type, ProgramGroupingType.Show), - ), - ) - .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) - .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.tvShowUuid)); - - // Populate external ids - const externalIdQueries: Promise[] = []; - const seasonQueries: Promise[] = []; - for (const groupChunk of chunk(groups, 100)) { - const ids = groupChunk.map(({ programGrouping }) => programGrouping.uuid); - externalIdQueries.push( - this.drizzleDB.query.programGroupingExternalId.findMany({ - where: (fields, { inArray }) => inArray(fields.groupUuid, ids), - }), - ); - seasonQueries.push( - this.drizzleDB.query.programGrouping.findMany({ - where: (fields, { eq, and, inArray }) => - and( - eq(fields.type, ProgramGroupingType.Season), - inArray(fields.showUuid, ids), - ), - with: { - externalIds: true, - }, - }), - ); - } - - const [externalIdResults, seasonResults] = await Promise.all([ - Promise.all(externalIdQueries).then(flatten), - Promise.all(seasonQueries).then(flatten), - ]); - - const externalIdsByGroupId = groupBy( - externalIdResults, - (id) => id.groupUuid, - ); - const seasonByGroupId = groupBy(seasonResults, (season) => season.showUuid); - - const artists: MusicArtistOrm[] = []; - for (const { programGrouping } of groups) { - if (programGrouping.type === 'artist') { - const albums = - seasonByGroupId[programGrouping.uuid]?.filter( - (group): group is MusicAlbumOrm => group.type === 'album', - ) ?? []; - artists.push({ - ...programGrouping, - type: 'artist', - externalIds: externalIdsByGroupId[programGrouping.uuid] ?? [], - albums, - }); - } - } - - return { - total: sum((await countPromise).map(({ count }) => count)), - results: artists, - }; + ): Promise> { + return this.channelProgram.getChannelMusicArtists(id, pageParams); } - async getChannelPrograms( + getChannelPrograms( id: string, pageParams?: PageParams, typeFilter?: ContentProgramType, ): Promise> { - let query = this.drizzleDB - .select({ programId: ChannelPrograms.programUuid, count: count() }) - .from(ChannelPrograms) - .where( - and( - eq(ChannelPrograms.channelUuid, id), - typeFilter ? eq(Program.type, typeFilter) : undefined, - ), - ) - .innerJoin(Program, eq(ChannelPrograms.programUuid, Program.uuid)) - .$dynamic(); - - const countResult = head(await query.execute())?.count ?? 0; - - if (pageParams) { - query = query - .groupBy(Program.uuid) - .orderBy(asc(Program.title)) - .offset(pageParams.offset) - .limit(pageParams.limit); - } - - const results = await query.execute(); - - const materialized: ProgramWithRelationsOrm[] = []; - for (const idChunk of chunk( - results.map(({ programId }) => programId), - 100, - )) { - materialized.push( - ...(await this.drizzleDB.query.program.findMany({ - where: (fields, { inArray }) => inArray(fields.uuid, idChunk), - with: { - externalIds: true, - album: true, - artist: true, - season: true, - show: true, - artwork: true, - subtitles: true, - credits: true, - versions: { - with: { - mediaStreams: true, - mediaFiles: true, - chapters: true, - }, - }, - }, - orderBy: (fields, { asc }) => asc(fields.uuid), - })), - ); - } - - return { results: materialized, total: countResult }; + return this.channelProgram.getChannelPrograms(id, pageParams, typeFilter); } - getChannelProgramExternalIds(uuid: string) { - return this.db - .selectFrom('channelPrograms') - .where('channelUuid', '=', uuid) - .innerJoin( - 'programExternalId', - 'channelPrograms.programUuid', - 'programExternalId.programUuid', - ) - .selectAll('programExternalId') - .execute(); + getChannelProgramExternalIds(uuid: string): Promise { + return this.channelProgram.getChannelProgramExternalIds(uuid); } - async getChannelFallbackPrograms(uuid: string) { - const result = await this.db - .selectFrom('channelFallback') - .where('channelFallback.channelUuid', '=', uuid) - .select(withFallbackPrograms) - .groupBy('channelFallback.channelUuid') - .executeTakeFirst(); - return result?.programs ?? []; + getChannelFallbackPrograms(uuid: string): Promise { + return this.channelProgram.getChannelFallbackPrograms(uuid); } - async saveChannel(createReq: SaveableChannel) { - const existing = await this.getChannel(createReq.number); - if (!isNil(existing)) { - throw new Error( - `Channel with number ${createReq.number} already exists: ${existing.name}`, - ); - } - - const channel = await this.db.transaction().execute(async (tx) => { - const channel = await tx - .insertInto('channel') - .values(createRequestToChannel(createReq)) - .returningAll() - .executeTakeFirst(); - - if (!channel) { - throw new Error('Error while saving new channel.'); - } - - if (!isEmpty(createReq.fillerCollections)) { - await tx - .insertInto('channelFillerShow') - .values( - map( - createReq.fillerCollections, - (fc) => - ({ - channelUuid: channel.uuid, - cooldown: fc.cooldownSeconds, - fillerShowUuid: fc.id, - weight: fc.weight, - }) satisfies NewChannelFillerShow, - ), - ) - .execute(); - } - - const subtitlePreferences = createReq.subtitlePreferences?.map( - (pref) => - ({ - channelId: channel.uuid, - uuid: v4(), - languageCode: pref.langugeCode, - allowExternal: booleanToNumber(pref.allowExternal), - allowImageBased: booleanToNumber(pref.allowImageBased), - filterType: pref.filter, - priority: pref.priority, - }) satisfies NewChannelSubtitlePreference, - ); - if (subtitlePreferences) { - await tx - .insertInto('channelSubtitlePreferences') - .values(subtitlePreferences) - .executeTakeFirstOrThrow(); - } - - return channel; - }); - - await this.createLineup(channel.uuid); - - if (isDefined(createReq.onDemand) && createReq.onDemand.enabled) { - const db = await this.getFileDb(channel.uuid); - await db.update((lineup) => { - lineup.onDemandConfig = { - state: 'paused', - cursor: 0, - }; - }); - } - - // TODO: convert everything to use kysely - return { - channel, - lineup: (await this.getFileDb(channel.uuid)).data, - }; - } - - async updateChannel(id: string, updateReq: SaveableChannel) { - const channel = await this.getChannel(id); - - if (isNil(channel)) { - throw new ChannelNotFoundError(id); - } - - const update = updateRequestToChannel(updateReq); - - if ( - isNonEmptyString(updateReq.watermark?.url) && - URL.canParse(updateReq.watermark.url) - ) { - const url = updateReq.watermark?.url; - const parsed = new URL(url); - if (!parsed.hostname.includes('localhost')) { - // Attempt to download the watermark and cache it. - const cacheWatermarkResult = await Result.attemptAsync(() => - this.cacheImageService.getOrDownloadImageUrl(url), - ); - if (cacheWatermarkResult.isFailure()) { - this.logger.warn( - cacheWatermarkResult.error, - 'Was unable to cache watermark URL at %s', - url, - ); - } - } - } - - await this.db.transaction().execute(async (tx) => { - await tx - .updateTable('channel') - .where('channel.uuid', '=', id) - // TODO: Blocked on https://github.com/oven-sh/bun/issues/16909 - // .limit(1) - .set(update) - .executeTakeFirstOrThrow(); - - if (!isEmpty(updateReq.fillerCollections)) { - const channelFillerShows = map( - updateReq.fillerCollections, - (filler) => - ({ - cooldown: filler.cooldownSeconds, - channelUuid: channel.uuid, - fillerShowUuid: filler.id, - weight: filler.weight, - }) satisfies NewChannelFillerShow, - ); - - await tx - .deleteFrom('channelFillerShow') - .where('channelFillerShow.channelUuid', '=', channel.uuid) - .executeTakeFirstOrThrow(); - await tx - .insertInto('channelFillerShow') - .values(channelFillerShows) - .executeTakeFirstOrThrow(); - } - const subtitlePreferences = updateReq.subtitlePreferences?.map( - (pref) => - ({ - channelId: channel.uuid, - uuid: v4(), - languageCode: pref.langugeCode, - allowExternal: booleanToNumber(pref.allowExternal), - allowImageBased: booleanToNumber(pref.allowImageBased), - filterType: pref.filter, - priority: pref.priority, - }) satisfies NewChannelSubtitlePreference, - ); - await tx - .deleteFrom('channelSubtitlePreferences') - .where('channelSubtitlePreferences.channelId', '=', channel.uuid) - .executeTakeFirstOrThrow(); - if (subtitlePreferences) { - await tx - .insertInto('channelSubtitlePreferences') - .values(subtitlePreferences) - .executeTakeFirstOrThrow(); - } - }); - - if (isDefined(updateReq.onDemand)) { - const db = await this.getFileDb(id); - await db.update((lineup) => { - if (updateReq.onDemand?.enabled ?? false) { - lineup.onDemandConfig = { - state: 'paused', - cursor: 0, - }; - } else { - delete lineup['onDemandConfig']; - } - }); - } - - return { - channel: (await this.getChannel(id, true))!, - lineup: await this.loadLineup(id), - }; - } - - updateChannelDuration(id: string, newDur: number): Promise { - return this.drizzleDB - .update(Channel) - .set({ - duration: newDur, - }) - .where(eq(Channel.uuid, id)) - .limit(1) - .execute() - .then((_) => _.changes); - } - - async copyChannel(id: string): Promise> { - const channel = await this.getChannel(id); - if (!channel) { - throw new Error(`Cannot copy channel: channel ID: ${id} not found`); - } - - const lineup = await this.loadLineup(id); - - const newChannelId = v4(); - const now = +dayjs(); - // have to get all channel relations... - const newChannel = await this.db.transaction().execute(async (tx) => { - const { number: maxId } = await tx - .selectFrom('channel') - .select('number') - .orderBy('number desc') - .limit(1) - .executeTakeFirstOrThrow(); - const newChannel = await tx - .insertInto('channel') - .values({ - ...channel, - uuid: newChannelId, - name: `${channel.name} - Copy`, - number: maxId + 1, - icon: JSON.stringify(channel.icon), - offline: JSON.stringify(channel.offline), - watermark: JSON.stringify(channel.watermark), - createdAt: now, - updatedAt: now, - transcoding: null, // Deprecated - }) - .returningAll() - .executeTakeFirstOrThrow(); - - // Copy filler shows - await tx - .insertInto('channelFillerShow') - .columns(['channelUuid', 'cooldown', 'fillerShowUuid', 'weight']) - .expression((eb) => - eb - .selectFrom('channelFillerShow') - .select([ - eb.val(newChannelId).as('channelUuid'), - 'channelFillerShow.cooldown', - 'channelFillerShow.fillerShowUuid', - 'channelFillerShow.weight', - ]) - .where('channelFillerShow.channelUuid', '=', channel.uuid), - ) - .executeTakeFirstOrThrow(); - - // Copy programs - await tx - .insertInto('channelPrograms') - .columns(['channelUuid', 'programUuid']) - .expression((eb) => - eb - .selectFrom('channelPrograms') - .select([ - eb.val(newChannelId).as('channelUuid'), - 'channelPrograms.programUuid', - ]) - .where('channelPrograms.channelUuid', '=', channel.uuid), - ) - .executeTakeFirstOrThrow(); - - // Copy custom shows - await tx - .insertInto('channelCustomShows') - .columns(['channelUuid', 'customShowUuid']) - .expression((eb) => - eb - .selectFrom('channelCustomShows') - .select([ - eb.val(newChannelId).as('channelUuid'), - 'channelCustomShows.customShowUuid', - ]) - .where('channelCustomShows.channelUuid', '=', channel.uuid), - ) - .executeTakeFirstOrThrow(); - - return newChannel; - }); - - const newLineup = await this.saveLineup(newChannel.uuid, lineup); - - return { - channel: newChannel, - lineup: newLineup, - }; - } - - async updateChannelStartTime(id: string, newTime: number): Promise { - return this.db - .updateTable('channel') - .where('channel.uuid', '=', id) - .set('startTime', newTime) - .executeTakeFirst() - .then(() => {}); - } - - async syncChannelDuration(id: string) { - const channelAndLineup = await this.loadChannelAndLineup(id); - if (!channelAndLineup) { - return false; - } - const { channel, lineup } = channelAndLineup; - const lineupDuration = sum(map(lineup.items, (item) => item.durationMs)); - if (lineupDuration !== channel.duration) { - await this.db - .updateTable('channel') - .where('channel.uuid', '=', id) - .set('duration', lineupDuration) - .executeTakeFirst(); - return true; - } - return false; - } - - async deleteChannel( - channelId: string, - blockOnLineupUpdates: boolean = false, - ) { - let marked = false; - try { - await this.markLineupFileForDeletion(channelId); - marked = true; - - await this.db.transaction().execute(async (tx) => { - await tx - .deleteFrom('channelSubtitlePreferences') - .where('channelId', '=', channelId) - .executeTakeFirstOrThrow(); - await tx - .deleteFrom('channel') - .where('uuid', '=', channelId) - .limit(1) - .executeTakeFirstOrThrow(); - }); - - // Best effort remove references to this channel - const removeRefs = () => - this.removeRedirectReferences(channelId).catch((e) => { - this.logger.error(e, 'Error while removing redirect references'); - }); - - if (blockOnLineupUpdates) { - await removeRefs(); - } else { - setTimeout(() => { - removeRefs().catch((e) => { - this.logger.error( - e, - 'Error while removing channel references in background.', - ); - }); - }); - } - } catch (e) { - // If we failed at the DB level for some reason, - // try to restore the lineup file. - if (marked) { - await this.restoreLineupFile(channelId); - } - - this.logger.error( - e, - 'Error while attempting to delete channel %s', - channelId, - ); - - throw e; - } - } - - getAllChannels(): Promise { - return this.drizzleDB.query.channels - .findMany({ - orderBy: (fields, { asc }) => asc(fields.number), - }) - .execute(); - // return this.db - // .selectFrom('channel') - // .selectAll() - // .orderBy('channel.number asc') - // .$if(isDefined(pageParams) && pageParams.offset >= 0, (qb) => - // qb - // .offset(pageParams!.offset) - // .$if(pageParams!.limit >= 0, (qb) => qb.limit(pageParams!.limit)), - // ) - // .execute(); - } - - async getAllChannelsAndPrograms(): Promise { - return await this.drizzleDB.query.channels - .findMany({ - with: { - channelPrograms: { - with: { - program: { - with: { - album: true, - artist: true, - show: true, - season: true, - externalIds: true, - }, - }, - }, - }, - }, - orderBy: (fields, { asc }) => asc(fields.number), - }) - .then((result) => { - return result.map((channel) => { - const withoutJoinTable = omit(channel, 'channelPrograms'); - return { - ...withoutJoinTable, - programs: channel.channelPrograms.map((cp) => cp.program), - } satisfies ChannelOrmWithPrograms; - }); - }); - // return await this.db - // .selectFrom('channel') - // .selectAll(['channel']) - // .leftJoin( - // 'channelPrograms', - // 'channelPrograms.channelUuid', - // 'channel.uuid', - // ) - // .select((eb) => [ - // withPrograms(eb, { - // joins: { - // trackAlbum: MinimalProgramGroupingFields, - // trackArtist: MinimalProgramGroupingFields, - // tvShow: MinimalProgramGroupingFields, - // tvSeason: MinimalProgramGroupingFields, - // }, - // }), - // ]) - // .groupBy('channel.uuid') - // .orderBy('channel.number asc') - // .execute(); - } - - async replaceChannelPrograms( + replaceChannelPrograms( channelId: string, programIds: string[], ): Promise { - const uniqueIds = uniq(programIds); - await this.drizzleDB.transaction(async (tx) => { - await tx - .delete(ChannelPrograms) - .where(eq(ChannelPrograms.channelUuid, channelId)); - for (const c of chunk(uniqueIds, 250)) { - await tx - .insert(ChannelPrograms) - .values(c.map((id) => ({ channelUuid: channelId, programUuid: id }))); - } - }); + return this.channelProgram.replaceChannelPrograms(channelId, programIds); } - async updateLineup( + findChannelsForProgramId(programId: string): Promise { + return this.channelProgram.findChannelsForProgramId(programId); + } + + getAllChannelsAndPrograms(): Promise { + return this.channelProgram.getAllChannelsAndPrograms(); + } + + // --- LineupRepository delegation --- + + loadLineup(channelId: string, forceRead?: boolean): Promise { + return this.lineup.loadLineup(channelId, forceRead); + } + + loadCondensedLineup( + channelId: string, + offset?: number, + limit?: number, + ): Promise { + return this.lineup.loadCondensedLineup(channelId, offset, limit); + } + + loadAndMaterializeLineup( + channelId: string, + offset?: number, + limit?: number, + ): Promise { + return this.lineup.loadAndMaterializeLineup(channelId, offset, limit); + } + + loadChannelAndLineup( + channelId: string, + ): Promise | null> { + return this.lineup.loadChannelAndLineup(channelId); + } + + loadChannelAndLineupOrm( + channelId: string, + ): Promise | null> { + return this.lineup.loadChannelAndLineupOrm(channelId); + } + + loadChannelWithProgamsAndLineup( + channelId: string, + ): Promise<{ channel: ChannelOrmWithPrograms; lineup: Lineup } | null> { + return this.lineup.loadChannelWithProgamsAndLineup(channelId); + } + + loadAllLineups(): Promise> { + return this.lineup.loadAllLineups(); + } + + loadAllLineupConfigs( + forceRead?: boolean, + ): Promise> { + return this.lineup.loadAllLineupConfigs(forceRead); + } + + loadAllRawLineups(): Promise> { + return this.lineup.loadAllRawLineups(); + } + + saveLineup( + channelId: string, + newLineup: UpdateChannelLineupRequest, + ): Promise { + return this.lineup.saveLineup(channelId, newLineup); + } + + updateLineup( id: string, req: UpdateChannelProgrammingRequest, ): Promise> { - const channel = await this.drizzleDB.query.channels.findFirst({ - where: (fields, { eq }) => eq(fields.uuid, id), - with: { - channelPrograms: { - columns: { - programUuid: true, - }, - }, - }, - }); - - const lineup = await this.loadLineup(id); - - if (isNil(channel)) { - return null; - } - - const updateChannel = async (lineup: readonly LineupItem[]) => { - return await this.db.transaction().execute(async (tx) => { - await tx - .updateTable('channel') - .where('channel.uuid', '=', id) - .set({ - duration: sumBy(lineup, typedProperty('durationMs')), - }) - .limit(1) - .executeTakeFirstOrThrow(); - - const allNewIds = new Set([ - ...uniq(map(filter(lineup, isContentItem), (p) => p.id)), - ]); - - const existingIds = new Set([ - ...channel.channelPrograms.map((cp) => cp.programUuid), - ]); - - // Create our remove operations - const removeOperations: ProgramRelationOperation[] = map( - reject([...existingIds], (existingId) => allNewIds.has(existingId)), - (removalId) => ({ - operation: 'remove' as const, - id: removalId, - }), - ); - - // Create addition operations - const addOperations: ProgramRelationOperation[] = map( - reject([...allNewIds], (newId) => existingIds.has(newId)), - (addId) => ({ - operation: 'add' as const, - id: addId, - }), - ); - - // TODO: See if this is still necessary w/ kysely building - // This is busted....wtf - for (const ops of chunk( - [...addOperations, ...removeOperations], - SqliteMaxDepthLimit / 2, - )) { - const [adds, removes] = partition( - ops, - ({ operation }) => operation === 'add', - ); - - if (!isEmpty(removes)) { - await tx - .deleteFrom('channelPrograms') - .where('channelPrograms.programUuid', 'in', map(removes, 'id')) - .where('channelPrograms.channelUuid', '=', id) - .execute(); - } - - if (!isEmpty(adds)) { - await tx - .insertInto('channelPrograms') - .values( - map( - adds, - ({ id }) => - ({ - channelUuid: channel.uuid, - programUuid: id, - }) satisfies NewChannelProgram, - ), - ) - .execute(); - } - } - return channel; - }); - }; - - const createNewLineup = async ( - programs: ChannelProgram[], - lineup: ChannelProgram[] = programs, - ) => { - const upsertedPrograms = - await this.programDB.upsertContentPrograms(programs); - const dbIdByUniqueId = groupByFunc( - upsertedPrograms, - programExternalIdString, - (p) => p.uuid, - ); - return map(lineup, channelProgramToLineupItemFunc(dbIdByUniqueId)); - }; - - const upsertPrograms = async (programs: ChannelProgram[]) => { - const upsertedPrograms = - await this.programDB.upsertContentPrograms(programs); - return groupByFunc( - upsertedPrograms, - programExternalIdString, - (p) => p.uuid, - ); - }; - - if (req.type === 'manual') { - const newLineupItems = await run(async () => { - const newItems = await this.timer.timeAsync( - 'createNewLineup', - async () => { - const programs = req.programs; - const dbIdByUniqueId = await upsertPrograms(programs); - const convertFunc = channelProgramToLineupItemFunc(dbIdByUniqueId); - return seq.collect(req.lineup, (lineupItem) => { - switch (lineupItem.type) { - // Lookup the item in the program lookup list - case 'index': { - const program = nth(programs, lineupItem.index); - if (program) { - return convertFunc({ - ...program, - duration: lineupItem.duration ?? program.duration, - }); - } - return null; - } - case 'persisted': { - return { - type: 'content', - id: lineupItem.programId, - customShowId: lineupItem.customShowId, - durationMs: lineupItem.duration, - } satisfies ContentItem; - } - } - }); - }, - ); - if (req.append) { - const existingLineup = await this.loadLineup(channel.uuid); - return [...existingLineup.items, ...newItems]; - } else { - return newItems; - } - }); - - const updatedChannel = await this.timer.timeAsync('updateChannel', () => - updateChannel(newLineupItems), - ); - - await this.timer.timeAsync('saveLineup', () => - this.saveLineup(id, { - items: newLineupItems, - onDemandConfig: isDefined(lineup.onDemandConfig) - ? { - ...lineup.onDemandConfig, - cursor: 0, - } - : undefined, - }), - ); - - return { - channel: updatedChannel, - newLineup: newLineupItems, - }; - } else if (req.type === 'time' || req.type === 'random') { - let programs: ChannelProgram[]; - if (req.type === 'time') { - const { result } = await this.workerPoolProvider().queueTask({ - type: 'time-slots', - request: { - type: 'programs', - programIds: req.programs, - schedule: req.schedule, - seed: req.seed, - startTime: channel.startTime, - }, - }); - - programs = MaterializeLineupCommand.expandLineup( - result.lineup, - await this.materializeLineupCommand.execute({ - lineup: result.lineup, - }), - ); - } else { - const { result } = await this.workerPoolProvider().queueTask({ - type: 'schedule-slots', - request: { - type: 'programs', - programIds: req.programs, - startTime: channel.startTime, - schedule: req.schedule, - seed: req.seed, - }, - }); - programs = MaterializeLineupCommand.expandLineup( - result.lineup, - await this.materializeLineupCommand.execute({ - lineup: result.lineup, - }), - ); - } - - const newLineup = await createNewLineup(programs); - - const updatedChannel = await updateChannel(newLineup); - await this.saveLineup(id, { - items: newLineup, - schedule: req.schedule, - }); - - return { - channel: updatedChannel, - newLineup, - }; - } - - return null; + return this.lineup.updateLineup(id, req); } - /** - * Like {@link ChannelDB#saveLineup} but only allows updating config-based information in the lineup - */ - async updateLineupConfig< + updateLineupConfig< Key extends keyof Omit< Lineup, 'items' | 'startTimeOffsets' | 'pendingPrograms' >, - >(id: string, key: Key, conf: Lineup[Key]) { - const lineupDb = await this.getFileDb(id); - return await lineupDb.update((existing) => { - existing[key] = conf; - }); + >(id: string, key: Key, conf: Lineup[Key]): Promise { + return this.lineup.updateLineupConfig(id, key, conf); } - async setChannelPrograms( + setChannelPrograms( channel: Channel, lineup: readonly LineupItem[], ): Promise; - async setChannelPrograms( + setChannelPrograms( + channel: string | Channel, + lineup: readonly LineupItem[], + startTime?: number, + ): Promise; + setChannelPrograms( channel: string | Channel, lineup: readonly LineupItem[], startTime?: number, ): Promise { - const loadedChannel = await run(async () => { - if (isString(channel)) { - return await this.getChannel(channel); - } else { - return channel; - } - }); - - if (isNil(loadedChannel)) { - return null; - } - - const allIds = uniq(map(filter(lineup, isContentItem), 'id')); - - return await this.db.transaction().execute(async (tx) => { - // await tx - if (!isUndefined(startTime)) { - loadedChannel.startTime = startTime; - } - loadedChannel.duration = sumBy(lineup, typedProperty('durationMs')); - const updatedChannel = await tx - .updateTable('channel') - .where('channel.uuid', '=', loadedChannel.uuid) - .set('duration', sumBy(lineup, typedProperty('durationMs'))) - .$if(isDefined(startTime), (_) => _.set('startTime', startTime!)) - .returningAll() - .executeTakeFirst(); - - for (const idChunk of chunk(allIds, 500)) { - await tx - .deleteFrom('channelPrograms') - .where('channelUuid', '=', loadedChannel.uuid) - .where('programUuid', 'not in', idChunk) - .execute(); - } - - for (const idChunk of chunk(allIds, 500)) { - await tx - .insertInto('channelPrograms') - .values( - map(idChunk, (id) => ({ - programUuid: id, - channelUuid: loadedChannel.uuid, - })), - ) - .onConflict((oc) => oc.doNothing()) - .executeTakeFirst(); - } - - return updatedChannel ?? null; - }); + return this.lineup.setChannelPrograms(channel, lineup, startTime); } - async addPendingPrograms( + addPendingPrograms( channelId: string, pendingPrograms: PendingProgram[], - ) { - if (pendingPrograms.length === 0) { - return; - } - - const db = await this.getFileDb(channelId); - return await db.update((data) => { - if (isUndefined(data.pendingPrograms)) { - data.pendingPrograms = [...pendingPrograms]; - } else { - data.pendingPrograms.push(...pendingPrograms); - } - }); + ): Promise { + return this.lineup.addPendingPrograms(channelId, pendingPrograms); } - async loadAllLineups() { - return mapReduceAsyncSeq( - await this.getAllChannels(), - async (channel) => { - return { - channel, - lineup: await this.loadLineup(channel.uuid), - }; - }, - (prev, { channel, lineup }) => { - prev[channel.uuid] = { channel, lineup }; - return prev; - }, - {} as Record, - ); - } - - async loadAllLineupConfigs(forceRead: boolean = false) { - return asyncMapToRecord( - await this.getAllChannels(), - async (channel) => ({ - channel, - lineup: await this.loadLineup(channel.uuid, forceRead), - }), - ({ channel }) => channel.uuid, - ); - } - - async loadAllRawLineups(): Promise> { - return asyncMapToRecord( - await this.getAllChannels(), - async (channel) => { - if ( - !(await fileExists( - this.fileSystemService.getChannelLineupPath(channel.uuid), - )) - ) { - await this.createLineup(channel.uuid); - } - - return { - channel, - lineup: jsonSchema.parse( - JSON.parse( - ( - await fs.readFile( - this.fileSystemService.getChannelLineupPath(channel.uuid), - ) - ).toString('utf-8'), - ), - ), - }; - }, - ({ channel }) => channel.uuid, - ); - } - - async loadChannelAndLineup( + removeProgramsFromLineup( channelId: string, - ): Promise | null> { - const channel = await this.getChannel(channelId); - if (isNil(channel)) { - return null; - } - - return { - channel, - lineup: await this.loadLineup(channelId), - }; + programIds: string[], + ): Promise { + return this.lineup.removeProgramsFromLineup(channelId, programIds); } - async loadChannelAndLineupOrm( - channelId: string, - ): Promise | null> { - const channel = await this.getChannelOrm(channelId); - if (isNil(channel)) { - return null; - } - - return { - channel, - lineup: await this.loadLineup(channelId), - }; + removeProgramsFromAllLineups(programIds: string[]): Promise { + return this.lineup.removeProgramsFromAllLineups(programIds); } - async loadChannelWithProgamsAndLineup( - channelId: string, - ): Promise<{ channel: ChannelOrmWithPrograms; lineup: Lineup } | null> { - const channel = await this.getChannelAndPrograms(channelId); - if (isNil(channel)) { - return null; - } + // --- ChannelConfigRepository delegation --- - return { - channel, - lineup: await this.loadLineup(channelId), - }; - } - - async loadLineup(channelId: string, forceRead: boolean = false) { - const db = await this.getFileDb(channelId, forceRead); - return db.data; - } - - async loadAndMaterializeLineup( - channelId: string, - offset: number = 0, - limit: number = -1, - ): Promise { - const channel = await this.getChannelAndProgramsOld(channelId); - if (isNil(channel)) { - return null; - } - - const lineup = await this.loadLineup(channelId); - const len = lineup.items.length; - const cleanOffset = offset < 0 ? 0 : offset; - const cleanLimit = limit < 0 ? len : limit; - - const { lineup: apiLineup, offsets } = await this.buildApiLineup( - channel, - take(drop(lineup.items, cleanOffset), cleanLimit), - ); - - return { - icon: channel.icon, - name: channel.name, - number: channel.number, - totalPrograms: len, - programs: apiLineup, - startTimeOffsets: offsets, - }; - } - - async loadCondensedLineup( - channelId: string, - offset: number = 0, - limit: number = -1, - ): Promise { - const lineup = await this.timer.timeAsync('loadLineup', () => - this.loadLineup(channelId), - ); - - const len = lineup.items.length; - const cleanOffset = offset < 0 ? 0 : offset; - const cleanLimit = limit < 0 ? len : limit; - const pagedLineup = take(drop(lineup.items, cleanOffset), cleanLimit); - - const channel = await this.timer.timeAsync('select channel', () => - this.getChannel(channelId), - ); - - if (isNil(channel)) { - return null; - } - - const contentItems = filter(pagedLineup, isContentItem); - - const directPrograms = await this.timer.timeAsync('direct', () => - this.db - .selectFrom('channelPrograms') - .where('channelUuid', '=', channelId) - .innerJoin('program', 'channelPrograms.programUuid', 'program.uuid') - .selectAll('program') - .select((eb) => [ - withTvShow(eb, AllProgramGroupingFields, true), - withTvSeason(eb, AllProgramGroupingFields, true), - withTrackAlbum(eb, AllProgramGroupingFields, true), - withTrackArtist(eb, AllProgramGroupingFields, true), - ]) - .execute(), - ); - - const externalIds = await this.timer.timeAsync('eids', () => - this.getChannelProgramExternalIds(channelId), - ); - - const externalIdsByProgramId = groupBy( - externalIds, - (eid) => eid.programUuid, - ); - - const programsById = groupByUniqProp(directPrograms, 'uuid'); - - const materializedPrograms = this.timer.timeSync('program convert', () => { - const ret: Record = {}; - forEach(uniqBy(contentItems, 'id'), (item) => { - const program = programsById[item.id]; - if (!program) { - return; - } - - const converted = this.programConverter.programDaoToContentProgram( - program, - externalIdsByProgramId[program.uuid] ?? [], - ); - - if (converted) { - ret[converted.id] = converted; - } - }); - - return ret; - }); - - const { lineup: condensedLineup, offsets } = await this.timer.timeAsync( - 'build condensed lineup', - () => - this.buildCondensedLineup( - channel, - new Set([...seq.collect(directPrograms, (p) => p.uuid)]), - pagedLineup, - ), - ); - - let apiOffsets: number[]; - if (lineup.startTimeOffsets) { - apiOffsets = take(drop(lineup.startTimeOffsets, cleanOffset), cleanLimit); - } else { - const scale = sumBy( - take(lineup.items, cleanOffset - 1), - (i) => i.durationMs, - ); - apiOffsets = map(offsets, (o) => o + scale); - } - - return { - icon: channel.icon, - name: channel.name, - number: channel.number, - totalPrograms: len, - programs: omitBy(materializedPrograms, isNil), - lineup: condensedLineup, - startTimeOffsets: apiOffsets, - schedule: lineup.schedule, - }; - } - - /** - * Updates the lineup config for a channel - * Some values accept 'null', which will clear their value - * Other values can be left undefined, which will leave them untouched - */ - async saveLineup( - channelId: string, - newLineup: UpdateChannelLineupRequest, - ): Promise { - const db = await this.getFileDb(channelId); - await db.update((data) => { - if (isDefined(newLineup.items)) { - data.items = newLineup.items; - data.startTimeOffsets = - newLineup.startTimeOffsets ?? - calculateStartTimeOffsets(newLineup.items); - } - - if (isDefined(newLineup.schedule)) { - if (isNull(newLineup.schedule)) { - data.schedule = undefined; - } else { - data.schedule = newLineup.schedule; - } - } - - if (isDefined(newLineup.pendingPrograms)) { - data.pendingPrograms = - newLineup.pendingPrograms === null - ? undefined - : newLineup.pendingPrograms; - } - - if (isDefined(newLineup.onDemandConfig)) { - data.onDemandConfig = - newLineup.onDemandConfig === null - ? undefined - : newLineup.onDemandConfig; - } - - data.version = newLineup?.version ?? data.version; - - data.lastUpdated = dayjs().valueOf(); - }); - - if (isDefined(newLineup.items)) { - const newDur = sum(newLineup.items.map((item) => item.durationMs)); - await this.updateChannelDuration(channelId, newDur); - } - return db.data; - } - - async removeProgramsFromLineup(channelId: string, programIds: string[]) { - if (programIds.length === 0) { - return; - } - - const idSet = new Set(programIds); - const lineup = await this.loadLineup(channelId); - lineup.items = map(lineup.items, (item) => { - if (isContentItem(item) && idSet.has(item.id)) { - return { - type: 'offline', - durationMs: item.durationMs, - }; - } else { - return item; - } - }); - await this.saveLineup(channelId, lineup); - } - - async removeProgramsFromAllLineups(programIds: string[]): Promise { - if (isEmpty(programIds)) { - return; - } - - const lineups = await this.loadAllLineups(); - - const programsToRemove = new Set(programIds); - for (const [channelId, { lineup }] of Object.entries(lineups)) { - const newLineupItems: LineupItem[] = lineup.items.map((item) => { - switch (item.type) { - case 'content': { - if (programsToRemove.has(item.id)) { - return { - type: 'offline', - durationMs: item.durationMs, - }; - } - return item; - } - case 'offline': - case 'redirect': - return item; - } - }); - - await this.saveLineup(channelId, { - ...lineup, - items: newLineupItems, - }); - - const duration = sum(newLineupItems.map((item) => item.durationMs)); - - await this.db - .updateTable('channel') - .set({ - duration, - }) - .where('uuid', '=', channelId) - .limit(1) - .executeTakeFirst(); - } - } - - async getChannelSubtitlePreferences( + getChannelSubtitlePreferences( id: string, ): Promise { - return this.db - .selectFrom('channelSubtitlePreferences') - .selectAll() - .where('channelId', '=', id) - .orderBy('priority asc') - .execute(); - } - - findChannelsForProgramId(programId: string) { - return this.drizzleDB.query.channelPrograms - .findMany({ - where: (cp, { eq }) => eq(cp.programUuid, programId), - with: { - channel: true, - }, - }) - .then((result) => result.map((row) => row.channel)); - } - - private async createLineup(channelId: string) { - const db = await this.getFileDb(channelId); - await db.write(); - } - - private async getFileDb(channelId: string, forceRead: boolean = false) { - return await fileDbLocks.getOrCreateLock(channelId).then((lock) => - lock.runExclusive(async () => { - const existing = fileDbCache[channelId]; - if (isDefined(existing)) { - if (forceRead) { - await existing.read(); - } - return existing; - } - - const defaultValue = { - items: [], - startTimeOffsets: [], - lastUpdated: dayjs().valueOf(), - version: CurrentLineupSchemaVersion, - }; - const db = new Low( - new SchemaBackedDbAdapter( - LineupSchema, - this.fileSystemService.getChannelLineupPath(channelId), - defaultValue, - ), - defaultValue, - ); - await db.read(); - fileDbCache[channelId] = db; - return db; - }), - ); - } - - private async restoreLineupFile(channelId: string) { - return this.markLineupFileForDeletion(channelId, false); - } - - private async markLineupFileForDeletion( - channelId: string, - isDelete: boolean = true, - ) { - const path = join( - globalOptions().databaseDirectory, - `channel-lineups/${channelId}.json${isDelete ? '' : '.bak'}`, - ); - try { - if (await fileExists(path)) { - const newPath = isDelete ? `${path}.bak` : path.replace('.bak', ''); - await fs.rename(path, newPath); - } - if (isDelete) { - delete fileDbCache[channelId]; - } else { - // Reload the file into the DB cache - await this.getFileDb(channelId); - } - } catch (e) { - this.logger.error( - e, - `Error while trying to ${ - isDelete ? 'mark' : 'unmark' - } Channel %s lineup json for deletion`, - channelId, - ); - } - } - - private async buildApiLineup( - channel: ChannelWithPrograms, - lineup: LineupItem[], - ): Promise<{ lineup: ChannelProgram[]; offsets: number[] }> { - const allChannels = await this.db - .selectFrom('channel') - .select(['channel.uuid', 'channel.number', 'channel.name']) - .execute(); - let lastOffset = 0; - const offsets: number[] = []; - - const programsById = groupByUniqProp(channel.programs, 'uuid'); - - const programs: ChannelProgram[] = []; - - for (const item of lineup) { - const apiItem = match(item) - .with({ type: 'content' }, (contentItem) => { - const fullProgram = programsById[contentItem.id]; - if (!fullProgram) { - return null; - } - return this.programConverter.programDaoToContentProgram( - fullProgram, - fullProgram.externalIds ?? [], - ); - }) - .otherwise((item) => - this.programConverter.lineupItemToChannelProgram( - channel, - item, - allChannels, - ), - ); - - if (apiItem) { - offsets.push(lastOffset); - lastOffset += item.durationMs; - programs.push(apiItem); - } - } - - return { lineup: programs, offsets }; - } - - private async buildCondensedLineup( - channel: Channel, - dbProgramIds: Set, - lineup: LineupItem[], - ): Promise<{ lineup: CondensedChannelProgram[]; offsets: number[] }> { - let lastOffset = 0; - const offsets: number[] = []; - - const customShowLineupItemsByShowId = mapValues( - groupBy( - filter( - lineup, - (l): l is MarkRequired => - l.type === 'content' && isNonEmptyString(l.customShowId), - ), - (i) => i.customShowId, - ), - (items) => uniqBy(items, 'id'), - ); - - const customShowIndexes: Record> = {}; - for (const [customShowId, items] of entries( - customShowLineupItemsByShowId, - )) { - customShowIndexes[customShowId] = {}; - - const results = await this.db - .selectFrom('customShowContent') - .select(['customShowContent.contentUuid', 'customShowContent.index']) - .where('customShowContent.contentUuid', 'in', map(items, 'id')) - .where('customShowContent.customShowUuid', '=', customShowId) - .groupBy('customShowContent.contentUuid') - .execute(); - - const byItemId: Record = {}; - for (const { contentUuid, index } of results) { - byItemId[contentUuid] = index; - } - - customShowIndexes[customShowId] = byItemId; - } - - const allChannels = await this.db - .selectFrom('channel') - .select(['uuid', 'name', 'number']) - .execute(); - - const channelsById = groupByUniqProp(allChannels, 'uuid'); - - const programs = seq.collect(lineup, (item) => { - let p: CondensedChannelProgram | null = null; - if (isOfflineItem(item)) { - p = this.programConverter.offlineLineupItemToProgram(channel, item); - } else if (isRedirectItem(item)) { - if (channelsById[item.channel]) { - p = this.programConverter.redirectLineupItemToProgram( - item, - channelsById[item.channel]!, - ); - } else { - this.logger.warn( - 'Found dangling redirect program. Bad ID = %s', - item.channel, - ); - p = { - persisted: true, - type: 'flex', - duration: item.durationMs, - }; - } - } else if (item.customShowId) { - p = { - persisted: true, - type: 'custom', - customShowId: item.customShowId, - duration: item.durationMs, - index: customShowIndexes[item.customShowId]![item.id] ?? -1, - id: item.id, - }; - } else if (item.fillerListId) { - p = { - persisted: true, - type: 'filler', - fillerListId: item.fillerListId, - fillerType: item.fillerType, - id: item.id, - duration: item.durationMs, - }; - } else { - if (dbProgramIds.has(item.id)) { - p = { - persisted: true, - type: 'content', - id: item.id, - duration: item.durationMs, - }; - } - } - - if (p) { - offsets.push(lastOffset); - lastOffset += item.durationMs; - } - - return p; - }); - return { lineup: programs, offsets }; - } - - private async removeRedirectReferences(toChannel: string) { - const allChannels = await this.getAllChannels(); - - const ops = asyncPool( - reject(allChannels, { uuid: toChannel }), - async (channel) => { - const lineup = await this.loadLineup(channel.uuid); - let changed = false; - const newLineup: LineupItem[] = map(lineup.items, (item) => { - if (item.type === 'redirect' && item.channel === toChannel) { - changed = true; - return { - type: 'offline', - durationMs: item.durationMs, - }; - } else { - return item; - } - }); - if (changed) { - return this.saveLineup(channel.uuid, { ...lineup, items: newLineup }); - } - return; - }, - { concurrency: 2 }, - ); - - for await (const updateResult of ops) { - if (updateResult.isFailure()) { - this.logger.error( - 'Error removing redirect references for channel %s from channel %s', - toChannel, - updateResult.error.input.uuid, - ); - } - } + return this.channelConfig.getChannelSubtitlePreferences(id); } } - -function channelProgramToLineupItemFunc( - dbIdByUniqueId: Record, -): (p: ChannelProgram) => LineupItem { - return (p) => - match(p) - .returnType() - .with({ type: 'content' }, (program) => ({ - type: 'content', - id: program.persisted ? program.id! : dbIdByUniqueId[program.uniqueId]!, - durationMs: program.duration, - })) - .with({ type: 'custom' }, (program) => ({ - type: 'content', // Custom program - durationMs: program.duration, - id: program.id, - customShowId: program.customShowId, - })) - .with({ type: 'filler' }, (program) => ({ - type: 'content', - durationMs: program.duration, - id: program.id, - fillerListId: program.fillerListId, - fillerType: program.fillerType, - })) - .with({ type: 'redirect' }, (program) => ({ - type: 'redirect', - channel: program.channel, - durationMs: program.duration, - })) - .with({ type: 'flex' }, (program) => ({ - type: 'offline', - durationMs: program.duration, - })) - .exhaustive(); -} diff --git a/server/src/db/CustomShowDB.test.ts b/server/src/db/CustomShowDB.test.ts index b45e84cc..337cc4ee 100644 --- a/server/src/db/CustomShowDB.test.ts +++ b/server/src/db/CustomShowDB.test.ts @@ -11,6 +11,14 @@ import { LoggerFactory } from '../util/logging/LoggerFactory.ts'; import { CustomShowDB } from './CustomShowDB.ts'; import { DBAccess } from './DBAccess.ts'; import { ProgramDB } from './ProgramDB.ts'; +import { BasicProgramRepository } from './program/BasicProgramRepository.ts'; +import { ProgramExternalIdRepository } from './program/ProgramExternalIdRepository.ts'; +import { ProgramGroupingRepository } from './program/ProgramGroupingRepository.ts'; +import { ProgramGroupingUpsertRepository } from './program/ProgramGroupingUpsertRepository.ts'; +import { ProgramMetadataRepository } from './program/ProgramMetadataRepository.ts'; +import { ProgramSearchRepository } from './program/ProgramSearchRepository.ts'; +import { ProgramStateRepository } from './program/ProgramStateRepository.ts'; +import { ProgramUpsertRepository } from './program/ProgramUpsertRepository.ts'; import type { MediaSourceId, MediaSourceName } from './schema/base.ts'; import { CustomShow } from './schema/CustomShow.ts'; import type { NewCustomShowContent } from './schema/CustomShowContent.ts'; @@ -122,13 +130,49 @@ const test = baseTest.extend({ }, }); - const programDb = new ProgramDB( + const metadataRepo = new ProgramMetadataRepository( + dbAccess.getConnection(':memory:')!.drizzle!, + ); + const externalIdRepo = new ProgramExternalIdRepository( logger, - mockTaskFactory, - mockTaskFactory, dbAccess.getKyselyDatabase(':memory:')!, + dbAccess.getConnection(':memory:')!.drizzle!, + ); + const groupingUpsertRepo = new ProgramGroupingUpsertRepository( + dbAccess.getKyselyDatabase(':memory:')!, + dbAccess.getConnection(':memory:')!.drizzle!, + metadataRepo, + ); + const upsertRepo = new ProgramUpsertRepository( + logger, + dbAccess.getKyselyDatabase(':memory:')!, + dbAccess.getConnection(':memory:')!.drizzle!, + mockTaskFactory, + mockTaskFactory, mockMinterFactory as any, - dbAccess.getConnection(':memory:')?.drizzle!, + externalIdRepo, + metadataRepo, + groupingUpsertRepo, + ); + const programDb = new ProgramDB( + new BasicProgramRepository( + dbAccess.getKyselyDatabase(':memory:')!, + dbAccess.getConnection(':memory:')!.drizzle!, + ), + new ProgramGroupingRepository( + logger, + dbAccess.getKyselyDatabase(':memory:')!, + dbAccess.getConnection(':memory:')!.drizzle!, + ), + externalIdRepo, + upsertRepo, + metadataRepo, + groupingUpsertRepo, + new ProgramSearchRepository( + dbAccess.getKyselyDatabase(':memory:')!, + dbAccess.getConnection(':memory:')!.drizzle!, + ), + new ProgramStateRepository(dbAccess.getConnection(':memory:')!.drizzle!), ); const customShowDb = new CustomShowDB( diff --git a/server/src/db/DBModule.ts b/server/src/db/DBModule.ts index e815605a..b6f884ce 100644 --- a/server/src/db/DBModule.ts +++ b/server/src/db/DBModule.ts @@ -12,8 +12,38 @@ import { ProgramPlayHistoryDB } from './ProgramPlayHistoryDB.ts'; import { ProgramDaoMinter } from './converters/ProgramMinter.ts'; import type { DB } from './schema/db.ts'; import type { DrizzleDBAccess } from './schema/index.ts'; +import { BasicProgramRepository } from './program/BasicProgramRepository.ts'; +import { ProgramGroupingRepository } from './program/ProgramGroupingRepository.ts'; +import { ProgramExternalIdRepository } from './program/ProgramExternalIdRepository.ts'; +import { ProgramUpsertRepository } from './program/ProgramUpsertRepository.ts'; +import { ProgramMetadataRepository } from './program/ProgramMetadataRepository.ts'; +import { ProgramGroupingUpsertRepository } from './program/ProgramGroupingUpsertRepository.ts'; +import { ProgramSearchRepository } from './program/ProgramSearchRepository.ts'; +import { ProgramStateRepository } from './program/ProgramStateRepository.ts'; +import { BasicChannelRepository } from './channel/BasicChannelRepository.ts'; +import { ChannelProgramRepository } from './channel/ChannelProgramRepository.ts'; +import { LineupRepository } from './channel/LineupRepository.ts'; +import { ChannelConfigRepository } from './channel/ChannelConfigRepository.ts'; const DBModule = new ContainerModule((bind) => { + // ProgramDB sub-repositories (must be registered before ProgramDB itself) + bind(KEYS.BasicProgramRepository).to(BasicProgramRepository).inSingletonScope(); + bind(KEYS.ProgramGroupingRepository).to(ProgramGroupingRepository).inSingletonScope(); + bind(KEYS.ProgramExternalIdRepository).to(ProgramExternalIdRepository).inSingletonScope(); + bind(KEYS.ProgramMetadataRepository).to(ProgramMetadataRepository).inSingletonScope(); + bind(KEYS.ProgramGroupingUpsertRepository).to(ProgramGroupingUpsertRepository).inSingletonScope(); + bind(KEYS.ProgramSearchRepository).to(ProgramSearchRepository).inSingletonScope(); + bind(KEYS.ProgramStateRepository).to(ProgramStateRepository).inSingletonScope(); + bind(KEYS.ProgramUpsertRepository).to(ProgramUpsertRepository).inSingletonScope(); + + // ChannelDB sub-repositories (must be registered before ChannelDB itself) + // LineupRepository must come before BasicChannelRepository (it's injected into it) + bind(KEYS.LineupRepository).to(LineupRepository).inSingletonScope(); + bind(KEYS.ChannelConfigRepository).to(ChannelConfigRepository).inSingletonScope(); + bind(KEYS.ChannelProgramRepository).to(ChannelProgramRepository).inSingletonScope(); + bind(KEYS.BasicChannelRepository).to(BasicChannelRepository).inSingletonScope(); + + // Main DB facades bind(KEYS.ProgramDB).to(ProgramDB).inSingletonScope(); bind(KEYS.ChannelDB).to(ChannelDB).inSingletonScope(); bind(DBAccess).toSelf().inSingletonScope(); diff --git a/server/src/db/ProgramDB.test.ts b/server/src/db/ProgramDB.test.ts index facfaab1..3da1ba66 100644 --- a/server/src/db/ProgramDB.test.ts +++ b/server/src/db/ProgramDB.test.ts @@ -14,6 +14,14 @@ import { LoggerFactory } from '../util/logging/LoggerFactory.ts'; import { DBAccess } from './DBAccess.ts'; import { IProgramDB } from './interfaces/IProgramDB.ts'; import { ProgramDB } from './ProgramDB.ts'; +import { BasicProgramRepository } from './program/BasicProgramRepository.ts'; +import { ProgramGroupingRepository } from './program/ProgramGroupingRepository.ts'; +import { ProgramExternalIdRepository } from './program/ProgramExternalIdRepository.ts'; +import { ProgramUpsertRepository } from './program/ProgramUpsertRepository.ts'; +import { ProgramMetadataRepository } from './program/ProgramMetadataRepository.ts'; +import { ProgramGroupingUpsertRepository } from './program/ProgramGroupingUpsertRepository.ts'; +import { ProgramSearchRepository } from './program/ProgramSearchRepository.ts'; +import { ProgramStateRepository } from './program/ProgramStateRepository.ts'; import { NewArtwork } from './schema/Artwork.ts'; import { MediaSourceId, @@ -94,16 +102,36 @@ const test = baseTest.extend({ const dbAccess = DBAccess.instance; const logger = LoggerFactory.child({ className: 'ProgramDB' }); - // Mock the task factories required by ProgramDB + // Mock the task factories required by ProgramUpsertRepository const mockTaskFactory = () => ({ enqueue: async () => {} }) as any; - const programDb = new ProgramDB( - logger, - mockTaskFactory, - mockTaskFactory, + const metadataRepo = new ProgramMetadataRepository(dbAccess.drizzle!); + const externalIdRepo = new ProgramExternalIdRepository(logger, dbAccess.db!, dbAccess.drizzle!); + const groupingUpsertRepo = new ProgramGroupingUpsertRepository( dbAccess.db!, - () => ({}) as any, // ProgramDaoMinterFactory dbAccess.drizzle!, + metadataRepo, + ); + const upsertRepo = new ProgramUpsertRepository( + logger, + dbAccess.db!, + dbAccess.drizzle!, + mockTaskFactory, + mockTaskFactory, + () => ({}) as any, // ProgramDaoMinterFactory + externalIdRepo, + metadataRepo, + groupingUpsertRepo, + ); + const programDb = new ProgramDB( + new BasicProgramRepository(dbAccess.db!, dbAccess.drizzle!), + new ProgramGroupingRepository(logger, dbAccess.db!, dbAccess.drizzle!), + externalIdRepo, + upsertRepo, + metadataRepo, + groupingUpsertRepo, + new ProgramSearchRepository(dbAccess.db!, dbAccess.drizzle!), + new ProgramStateRepository(dbAccess.drizzle!), ); await use(programDb); diff --git a/server/src/db/ProgramDB.ts b/server/src/db/ProgramDB.ts index 2b17999b..62a8b108 100644 --- a/server/src/db/ProgramDB.ts +++ b/server/src/db/ProgramDB.ts @@ -7,493 +7,136 @@ import type { UpsertResult, WithChannelIdFilter, } from '@/db/interfaces/IProgramDB.js'; -import { GlobalScheduler } from '@/services/Scheduler.js'; -import { ReconcileProgramDurationsTask } from '@/tasks/ReconcileProgramDurationsTask.js'; -import { AnonymousTask } from '@/tasks/Task.js'; -import { JellyfinTaskQueue, PlexTaskQueue } from '@/tasks/TaskQueue.js'; -import { SaveJellyfinProgramExternalIdsTask } from '@/tasks/jellyfin/SaveJellyfinProgramExternalIdsTask.js'; -import { SavePlexProgramExternalIdsTask } from '@/tasks/plex/SavePlexProgramExternalIdsTask.js'; -import { autoFactoryKey, KEYS } from '@/types/inject.js'; -import { MarkNonNullable, Maybe, PagedResult } from '@/types/util.js'; -import { Timer } from '@/util/Timer.js'; -import { devAssert } from '@/util/debug.js'; -import { type Logger } from '@/util/logging/LoggerFactory.js'; -import { createExternalId } from '@tunarr/shared'; -import { seq } from '@tunarr/shared/util'; -import { - ChannelProgram, - ContentProgram, - isContentProgram, - untag, -} from '@tunarr/types'; -import { isValidSingleExternalIdType } from '@tunarr/types/schemas'; -import { RunResult } from 'better-sqlite3'; -import dayjs from 'dayjs'; -import { - and, - asc, - count, - countDistinct, - isNull as dbIsNull, - eq, - inArray, - or, - sql, -} from 'drizzle-orm'; -import { - BaseSQLiteDatabase, - SelectedFields, - SQLiteSelectBuilder, -} from 'drizzle-orm/sqlite-core'; -import { inject, injectable, interfaces } from 'inversify'; -import { - CaseWhenBuilder, - InsertResult, - Kysely, - NotNull, - UpdateResult, -} from 'kysely'; -import { jsonArrayFrom } from 'kysely/helpers/sqlite'; -import { - chunk, - compact, - concat, - difference, - filter, - first, - flatMap, - flatten, - forEach, - groupBy, - head, - isArray, - isEmpty, - isNil, - isNull, - isUndefined, - keys, - last, - map, - mapValues, - omit, - orderBy, - partition, - reduce, - reject, - round, - some, - sum, - uniq, - uniqBy, -} from 'lodash-es'; -import { +import { KEYS } from '@/types/inject.js'; +import type { MarkNonNullable, Maybe, PagedResult } from '@/types/util.js'; +import type { ChannelProgram } from '@tunarr/types'; +import { inject, injectable } from 'inversify'; +import type { Dictionary, MarkOptional, MarkRequired, StrictExclude, } from 'ts-essentials'; -import { match, P } from 'ts-pattern'; -import { v4 } from 'uuid'; -import { typedProperty } from '../types/path.ts'; -import { - createManyRelationAgg, - mapRawJsonRelationResult, -} from '../util/drizzleUtil.ts'; -import { getNumericEnvVar, TUNARR_ENV_VARS } from '../util/env.ts'; -import { - flatMapAsyncSeq, - groupByUniq, - groupByUniqProp, - isDefined, - isNonEmptyString, - mapAsyncSeq, - mapToObj, - programExternalIdString, - run, - unzip, -} from '../util/index.ts'; -import { ProgramGroupingMinter } from './converters/ProgramGroupingMinter.ts'; -import { ProgramDaoMinter } from './converters/ProgramMinter.ts'; -import { ProgramExternalIdType } from './custom_types/ProgramExternalIdType.ts'; -import { - ProgramSourceType, - programSourceTypeFromString, -} from './custom_types/ProgramSourceType.ts'; -import { PageParams } from './interfaces/IChannelDB.ts'; -import { - AllProgramFields, - ProgramUpsertFields, - selectProgramsBuilder, - withProgramByExternalId, - withProgramExternalIds, -} from './programQueryHelpers.ts'; -import { Artwork, NewArtwork } from './schema/Artwork.ts'; -import { ChannelPrograms } from './schema/ChannelPrograms.ts'; -import { Credit, NewCredit } from './schema/Credit.ts'; -import { - EntityGenre, - Genre, - NewGenre, - NewGenreEntity, -} from './schema/Genre.ts'; -import { RemoteMediaSourceType } from './schema/MediaSource.ts'; -import { - NewProgramDao, - Program, - ProgramDao, - ProgramType, - ProgramDao as RawProgram, -} from './schema/Program.ts'; -import { NewProgramChapter, ProgramChapter } from './schema/ProgramChapter.ts'; -import { +import type { ProgramExternalIdType } from './custom_types/ProgramExternalIdType.js'; +import type { PageParams } from './interfaces/IChannelDB.js'; +import { BasicProgramRepository } from './program/BasicProgramRepository.ts'; +import { ProgramExternalIdRepository } from './program/ProgramExternalIdRepository.ts'; +import { ProgramGroupingRepository } from './program/ProgramGroupingRepository.ts'; +import { ProgramGroupingUpsertRepository } from './program/ProgramGroupingUpsertRepository.ts'; +import { ProgramMetadataRepository } from './program/ProgramMetadataRepository.ts'; +import { ProgramSearchRepository } from './program/ProgramSearchRepository.ts'; +import { ProgramStateRepository } from './program/ProgramStateRepository.ts'; +import { ProgramUpsertRepository } from './program/ProgramUpsertRepository.ts'; +import type { NewArtwork } from './schema/Artwork.js'; +import type { NewGenre } from './schema/Genre.js'; +import type { RemoteMediaSourceType } from './schema/MediaSource.js'; +import type { ProgramDao, ProgramType } from './schema/Program.js'; +import type { MinimalProgramExternalId, NewProgramExternalId, NewSingleOrMultiExternalId, ProgramExternalId, - toInsertableProgramExternalId, -} from './schema/ProgramExternalId.ts'; -import { - NewProgramGrouping, - NewProgramGroupingOrm, - ProgramGrouping, - ProgramGroupingOrm, - ProgramGroupingType, - type ProgramGroupingTypes, -} from './schema/ProgramGrouping.ts'; -import { - NewProgramGroupingExternalId, - NewSingleOrMultiProgramGroupingExternalId, - ProgramGroupingExternalId, - ProgramGroupingExternalIdOrm, - toInsertableProgramGroupingExternalId, -} from './schema/ProgramGroupingExternalId.ts'; -import { - NewProgramMediaFile, - ProgramMediaFile, -} from './schema/ProgramMediaFile.ts'; -import { - NewProgramMediaStream, - ProgramMediaStream, -} from './schema/ProgramMediaStream.ts'; -import { - NewProgramSubtitles, - ProgramSubtitles, -} from './schema/ProgramSubtitles.ts'; -import { ProgramVersion } from './schema/ProgramVersion.ts'; -import { - NewStudio, - NewStudioEntity, - Studio, - StudioEntity, -} from './schema/Studio.ts'; -import { NewTag, NewTagRelation, Tag, TagRelations } from './schema/Tag.ts'; -import { +} from './schema/ProgramExternalId.js'; +import type { ProgramGroupingType } from './schema/ProgramGrouping.js'; +import type { MediaSourceId, - MediaSourceName, MediaSourceType, + ProgramExternalIdSourceType, ProgramState, RemoteSourceType, } from './schema/base.js'; -import { DB } from './schema/db.ts'; import type { MusicAlbumOrm, NewProgramGroupingWithRelations, - NewProgramVersion, NewProgramWithRelations, ProgramGroupingOrmWithRelations, ProgramGroupingWithExternalIds, ProgramWithExternalIds, + ProgramWithRelations, ProgramWithRelationsOrm, TvSeasonOrm, -} from './schema/derivedTypes.ts'; -import { DrizzleDBAccess, schema } from './schema/index.ts'; - -type MintedNewProgramInfo = { - program: NewProgramDao; - externalIds: NewSingleOrMultiExternalId[]; - apiProgram: ContentProgram; -}; - -type ContentProgramWithHierarchy = Omit< - MarkRequired, - 'subtype' -> & { - subtype: 'episode' | 'track'; -}; - -type ProgramRelationCaseBuilder = CaseWhenBuilder< - DB, - 'program', - unknown, - string | null ->; - -type RelevantProgramWithHierarchy = { - program: RawProgram; - programWithHierarchy: ContentProgramWithHierarchy & { - grandparentKey: string; - parentKey: string; - }; -}; - -// Keep this low to make bun sqlite happy. -const DEFAULT_PROGRAM_GROUPING_UPDATE_CHUNK_SIZE = 100; +} from './schema/derivedTypes.js'; @injectable() export class ProgramDB implements IProgramDB { - private timer: Timer; // = new Timer(this.logger); - constructor( - @inject(KEYS.Logger) private logger: Logger, - @inject(autoFactoryKey(SavePlexProgramExternalIdsTask)) - private savePlexProgramExternalIdsTaskFactory: interfaces.AutoFactory, - @inject(autoFactoryKey(SaveJellyfinProgramExternalIdsTask)) - private saveJellyfinProgramExternalIdsTask: interfaces.AutoFactory, - @inject(KEYS.Database) private db: Kysely, - @inject(KEYS.ProgramDaoMinterFactory) - private programMinterFactory: interfaces.AutoFactory, - @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, - ) { - this.timer = new Timer(this.logger); - } + @inject(KEYS.BasicProgramRepository) + private readonly basicProg: BasicProgramRepository, + @inject(KEYS.ProgramGroupingRepository) + private readonly progGrouping: ProgramGroupingRepository, + @inject(KEYS.ProgramExternalIdRepository) + private readonly externalIdRepo: ProgramExternalIdRepository, + @inject(KEYS.ProgramUpsertRepository) + private readonly upsertRepo: ProgramUpsertRepository, + @inject(KEYS.ProgramMetadataRepository) + private readonly metadataRepo: ProgramMetadataRepository, + @inject(KEYS.ProgramGroupingUpsertRepository) + private readonly groupingUpsertRepo: ProgramGroupingUpsertRepository, + @inject(KEYS.ProgramSearchRepository) + private readonly searchRepo: ProgramSearchRepository, + @inject(KEYS.ProgramStateRepository) + private readonly stateRepo: ProgramStateRepository, + ) {} - async getProgramById( + getProgramById( id: string, ): Promise>> { - return this.drizzleDB.query.program.findFirst({ - where: (fields, { eq }) => eq(fields.uuid, id), - with: { - externalIds: true, - artwork: true, - subtitles: true, - credits: true, - versions: { - with: { - mediaStreams: true, - mediaFiles: true, - chapters: true, - }, - }, - }, - }); + return this.basicProg.getProgramById(id); } - async getProgramExternalIds( + getProgramExternalIds( id: string, externalIdTypes?: ProgramExternalIdType[], - ) { - return await this.db - .selectFrom('programExternalId') - .selectAll() - .where('programExternalId.programUuid', '=', id) - .$if(!isEmpty(externalIdTypes), (qb) => - qb.where('programExternalId.sourceType', 'in', externalIdTypes!), - ) - .execute(); + ): Promise { + return this.basicProg.getProgramExternalIds(id, externalIdTypes); } - async getShowIdFromTitle(title: string) { - const matchedGrouping = await this.db - .selectFrom('programGrouping') - .select('uuid') - .where('title', '=', title) - .where('type', '=', ProgramGroupingType.Show) - .executeTakeFirst(); - - return matchedGrouping?.uuid; + getShowIdFromTitle(title: string): Promise> { + return this.basicProg.getShowIdFromTitle(title); } - async updateProgramDuration(programId: string, duration: number) { - await this.db - .updateTable('program') - .where('uuid', '=', programId) - .set({ - duration, - }) - .executeTakeFirst(); + updateProgramDuration(programId: string, duration: number): Promise { + return this.basicProg.updateProgramDuration(programId, duration); } - async getProgramsByIds( + getProgramsByIds( ids: string[] | readonly string[], - // joins: DBQueryConfig<'many', true, (typeof schema)['programRelations']>['with'], - batchSize: number = 500, + batchSize?: number, ): Promise { - const results: ProgramWithRelationsOrm[] = []; - for (const idChunk of chunk(ids, batchSize)) { - const res = await this.drizzleDB.query.program.findMany({ - where: (fields, { inArray }) => inArray(fields.uuid, idChunk), - with: { - album: { - with: { - artwork: true, - }, - }, - artist: true, - season: true, - show: { - with: { - artwork: true, - }, - }, - externalIds: true, - artwork: true, - tags: { - with: { - tag: true, - }, - }, - }, - }); - results.push(...res); - } - return results; + return this.basicProg.getProgramsByIds(ids, batchSize); } - async getProgramGrouping(id: string) { - return this.drizzleDB.query.programGrouping.findFirst({ - where: (fields, { eq }) => eq(fields.uuid, id), - with: { - externalIds: true, - artwork: true, - }, - }); + getProgramGrouping( + id: string, + ): Promise> { + return this.progGrouping.getProgramGrouping(id); } - async getProgramGroupings( + getProgramGroupings( ids: string[], ): Promise> { - if (ids.length === 0) { - return {}; - } - - const uniqueIds = uniq(ids); - - const results = await Promise.allSettled( - chunk(uniqueIds, 1000).map((idChunk) => { - return this.drizzleDB.query.programGrouping.findMany({ - where: (fields, { inArray }) => inArray(fields.uuid, idChunk), - with: { - externalIds: true, - artwork: true, - artist: true, - show: true, - credits: true, - tags: { - with: { - tag: true, - }, - }, - }, - }); - }), - ); - - const map: Record = {}; - for (const result of results) { - if (result.status === 'rejected') { - this.logger.error( - result.reason, - 'Error while querying for program groupings. Returning partial data.', - ); - continue; - } - for (const grouping of result.value) { - map[grouping.uuid] = grouping; - } - } - return map; + return this.progGrouping.getProgramGroupings(ids); } - async getProgramGroupingByExternalId( + getProgramGroupingByExternalId( eid: ProgramGroupingExternalIdLookup, ): Promise> { - return await this.drizzleDB.query.programGroupingExternalId - .findFirst({ - where: (row, { and, or, eq }) => - and( - eq(row.externalKey, eid.externalKey), - eq(row.sourceType, eid.sourceType), - or( - eq(row.externalSourceId, untag(eid.externalSourceId)), - eq(row.mediaSourceId, eid.externalSourceId), - ), - ), - with: { - grouping: { - with: { - externalIds: true, - }, - }, - }, - }) - .then((result) => result?.grouping ?? undefined); + return this.progGrouping.getProgramGroupingByExternalId(eid); } - async getProgramGroupingsByExternalIds( + getProgramGroupingsByExternalIds( eids: | Set<[RemoteSourceType, MediaSourceId, string]> | Set, - chunkSize: number = 100, - ) { - const allIds = [...eids]; - const programs: MarkRequired< - ProgramGroupingOrmWithRelations, - 'externalIds' - >[] = []; - for (const idChunk of chunk(allIds, chunkSize)) { - const results = - await this.drizzleDB.query.programGroupingExternalId.findMany({ - where: (fields, { or, and, eq }) => { - const ands = idChunk.map(([ps, es, ek]) => - and( - eq(fields.externalKey, ek), - eq(fields.sourceType, ps), - eq(fields.mediaSourceId, es), - ), - ); - return or(...ands); - }, - columns: {}, - with: { - grouping: { - with: { - artist: true, - show: true, - externalIds: true, - }, - }, - }, - }); - programs.push(...seq.collect(results, (r) => r.grouping)); - } - - return programs; + chunkSize?: number, + ): Promise { + return this.progGrouping.getProgramGroupingsByExternalIds(eids, chunkSize); } - async getProgramParent( + getProgramParent( programId: string, ): Promise> { - const p = await selectProgramsBuilder(this.db, { - joins: { tvSeason: true, trackAlbum: true }, - }) - .where('program.uuid', '=', programId) - .executeTakeFirst() - .then((program) => program?.tvSeason ?? program?.trackAlbum); - - // It would be better if we didn'thave to do this in two queries... - if (p) { - const eids = await this.db - .selectFrom('programGroupingExternalId') - .where('groupUuid', '=', p.uuid) - .selectAll() - .execute(); - return { - ...p, - externalIds: eids, - }; - } - - return; + return this.progGrouping.getProgramParent(programId); } getChildren( @@ -516,7 +159,7 @@ export class ProgramDB implements IProgramDB { parentType: 'artist' | 'show', params?: WithChannelIdFilter, ): Promise>; - async getChildren( + getChildren( parentId: string, parentType: ProgramGroupingType, params?: WithChannelIdFilter, @@ -524,394 +167,63 @@ export class ProgramDB implements IProgramDB { | PagedResult | PagedResult > { - if (parentType === 'album' || parentType === 'season') { - return this.getTerminalChildren(parentId, parentType, params); - } else { - return this.getGroupingChildren(parentId, parentType, params); - } - } - - private async getGroupingChildren( - parentId: string, - parentType: ProgramGroupingTypes['Show'] | ProgramGroupingTypes['Artist'], - params?: WithChannelIdFilter, - ) { - const childType = parentType === 'artist' ? 'album' : 'season'; - function builder< - TSelection extends SelectedFields | undefined, - TResultType extends 'sync' | 'async', - TRunResult, - >(f: SQLiteSelectBuilder) { - return f - .from(Program) - .where( - and( - eq( - Program.type, - parentType === ProgramGroupingType.Show - ? ProgramType.Episode - : ProgramType.Track, - ), - eq( - parentType === ProgramGroupingType.Show - ? Program.tvShowUuid - : Program.artistUuid, - parentId, - ), - params?.channelId - ? eq(ChannelPrograms.channelUuid, params.channelId) - : undefined, - ), - ); - } - - const sq = this.drizzleDB - .select() - .from(ProgramGroupingExternalId) - .where(eq(ProgramGroupingExternalId.groupUuid, ProgramGrouping.uuid)) - .as('sq'); - - const baseQuery = builder( - this.drizzleDB.select({ - grouping: ProgramGrouping, - externalIds: createManyRelationAgg(sq, 'external_ids'), - artwork: createManyRelationAgg( - this.drizzleDB - .select() - .from(Artwork) - .where(eq(ProgramGrouping.uuid, Artwork.groupingId)) - .as('artwork'), - 'artwork', - ), - }), - ) - .innerJoin( - ProgramGrouping, - eq( - childType === 'season' ? Program.seasonUuid : Program.albumUuid, - ProgramGrouping.uuid, - ), - ) - .orderBy(asc(ProgramGrouping.index)) - .offset(params?.offset ?? 0) - .limit(params?.limit ?? 1_000_000) - .groupBy(ProgramGrouping.uuid); - - const baseCountQuery = builder( - this.drizzleDB.select({ - count: countDistinct(ProgramGrouping.uuid), - }), - ) - .innerJoin( - ProgramGrouping, - eq( - childType === 'season' ? Program.seasonUuid : Program.albumUuid, - ProgramGrouping.uuid, - ), - ) - .groupBy(ProgramGrouping.uuid); - - if (params?.channelId) { - const res = await baseQuery.innerJoin( - ChannelPrograms, - eq(ChannelPrograms.programUuid, Program.uuid), - ); - - const cq = baseCountQuery.innerJoin( - ChannelPrograms, - eq(ChannelPrograms.programUuid, Program.uuid), - ); - - const programs = res.map(({ grouping, externalIds, artwork }) => { - const withRelations = grouping as ProgramGroupingOrmWithRelations; - withRelations.externalIds = mapRawJsonRelationResult( - externalIds, - ProgramGroupingExternalId, - ); - withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); - return withRelations; - }); - - return { - total: sum((await cq).map(({ count }) => count)), - results: programs, - }; - } else { - const res = await baseQuery; - - const programs = res.map(({ grouping, externalIds, artwork }) => { - const withRelations = grouping as ProgramGroupingOrmWithRelations; - withRelations.externalIds = mapRawJsonRelationResult( - externalIds, - ProgramGroupingExternalId, - ); - withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); - return withRelations; - }); - - return { - total: sum((await baseCountQuery).map(({ count }) => count)), - results: programs, - }; - } - } - - private async getTerminalChildren( - parentId: string, - parentType: ProgramGroupingTypes['Season'] | ProgramGroupingTypes['Album'], - params?: WithChannelIdFilter, - ) { - function builder< - TSelection extends SelectedFields | undefined, - TResultType extends 'sync' | 'async', - TRunResult, - >(f: SQLiteSelectBuilder) { - return f - .from(Program) - .where( - and( - eq( - Program.type, - parentType === ProgramGroupingType.Album - ? ProgramType.Track - : ProgramType.Episode, - ), - eq( - parentType === ProgramGroupingType.Album - ? Program.albumUuid - : Program.seasonUuid, - parentId, - ), - params?.channelId - ? eq(ChannelPrograms.channelUuid, params.channelId) - : undefined, - ), - ); - } - - const sq = this.drizzleDB - .select() - .from(ProgramExternalId) - .where(eq(ProgramExternalId.programUuid, Program.uuid)) - .as('sq'); - - const baseQuery = builder( - this.drizzleDB.select({ - program: Program, - externalIds: createManyRelationAgg(sq, 'external_ids'), - artwork: createManyRelationAgg( - this.drizzleDB - .select() - .from(Artwork) - .where(eq(Artwork.programId, Program.uuid)) - .as('artwork'), - 'artwork', - ), - }), - ).orderBy(asc(Program.seasonNumber), asc(Program.episode)); - - const baseCountQuery = builder( - this.drizzleDB.select({ - count: count(), - }), + return this.progGrouping.getChildren( + parentId, + parentType as 'season' | 'album', + params, ); - - if (params?.channelId) { - const res = await baseQuery - .offset(params?.offset ?? 0) - .limit(params?.limit ?? 1_000_000) - .innerJoin( - ChannelPrograms, - eq(ChannelPrograms.programUuid, Program.uuid), - ); - - const cq = baseCountQuery.innerJoin( - ChannelPrograms, - eq(ChannelPrograms.programUuid, Program.uuid), - ); - - const programs = res.map(({ program, externalIds, artwork }) => { - const withRelations: ProgramWithRelationsOrm = program; - withRelations.externalIds = mapRawJsonRelationResult( - externalIds, - ProgramExternalId, - ); - withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); - return withRelations; - }); - - console.log(programs); - - return { - total: sum((await cq).map(({ count }) => count)), - results: programs, - }; - } else { - const res = await baseQuery; - - const programs = res.map(({ program, externalIds, artwork }) => { - const withRelations: ProgramWithRelationsOrm = program; - withRelations.externalIds = mapRawJsonRelationResult( - externalIds, - ProgramExternalId, - ); - withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); - return withRelations; - }); - - return { - total: sum((await baseCountQuery).map(({ count }) => count)), - results: programs, - }; - } } - async lookupByExternalId(eid: { + lookupByExternalId(eid: { sourceType: RemoteSourceType; - externalSourceId: MediaSourceId; + externalSourceId: string; externalKey: string; - }) { - return first( - await this.lookupByExternalIds( - new Set([[eid.sourceType, eid.externalSourceId, eid.externalKey]]), - ), + }): Promise>> { + return this.externalIdRepo.lookupByExternalId( + eid as Parameters[0], ); } - async lookupByExternalIds( + lookupByExternalIds( ids: | Set<[RemoteSourceType, MediaSourceId, string]> | Set, - chunkSize: number = 200, - ) { - const allIds = [...ids]; - const programs: MarkRequired[] = []; - for (const idChunk of chunk(allIds, chunkSize)) { - const results = await this.drizzleDB.query.programExternalId.findMany({ - where: (fields, { or, and, eq }) => { - const ands = idChunk.map(([ps, es, ek]) => - and( - eq(fields.externalKey, ek), - eq(fields.sourceType, ps), - eq(fields.mediaSourceId, es), - ), - ); - return or(...ands); - }, - with: { - program: { - with: { - album: true, - artist: true, - season: true, - show: true, - externalIds: true, - tags: { - with: { - tag: true, - }, - }, - artwork: true, - credits: true, - genres: { - with: { - genre: true, - }, - }, - studios: { - with: { - studio: true, - }, - }, - versions: true, - }, - }, - }, - }); - programs.push(...seq.collect(results, (r) => r.program)); - } - - return programs; + chunkSize?: number, + ): Promise[]> { + return this.externalIdRepo.lookupByExternalIds(ids, chunkSize); } - async lookupByMediaSource( + lookupByMediaSource( sourceType: RemoteMediaSourceType, sourceId: MediaSourceId, - programType: Maybe, - chunkSize: number = 200, + mediaType?: ProgramType, + chunkSize?: number, ): Promise { - const programs: ProgramDao[] = []; - let chunk: ProgramDao[] = []; - let lastId: Maybe; - do { - const result = await this.db - .selectFrom('programExternalId') - .select('programExternalId.uuid') - .select((eb) => - withProgramByExternalId(eb, { joins: {} }, (qb) => - qb.$if(!!programType, (eb) => - eb.where('program.type', '=', programType!), - ), - ), - ) - .where('programExternalId.sourceType', '=', sourceType) - .where('programExternalId.mediaSourceId', '=', sourceId) - .$if(!!lastId, (x) => x.where('programExternalId.uuid', '>', lastId!)) - .orderBy('programExternalId.uuid asc') - .limit(chunkSize) - .execute(); - chunk = seq.collect(result, (eid) => eid.program); - programs.push(...chunk); - lastId = last(result)?.uuid; - } while (chunk.length > 0); - - return programs; - } - - async programIdsByExternalIds( - ids: Set<[string, MediaSourceId, string]>, - chunkSize: number = 50, - ) { - if (ids.size === 0) { - return {}; - } - - const externalIds = await flatMapAsyncSeq( - chunk([...ids], chunkSize), - (idChunk) => { - return this.db - .selectFrom('programExternalId') - .selectAll() - .where((eb) => - eb.or( - map(idChunk, ([ps, es, ek]) => { - return eb.and([ - eb('programExternalId.externalKey', '=', ek), - eb('programExternalId.mediaSourceId', '=', es), - eb( - 'programExternalId.sourceType', - '=', - programSourceTypeFromString(ps)!, - ), - ]); - }), - ), - ) - .execute(); - }, - ); - - return mapValues( - groupByUniq(externalIds, (eid) => - createExternalId(eid.sourceType, eid.mediaSourceId!, eid.externalKey), - ), - (eid) => eid.programUuid, + return this.externalIdRepo.lookupByMediaSource( + sourceType, + sourceId, + mediaType, + chunkSize, ); } - async updateProgramPlexRatingKey( + programIdsByExternalIds( + ids: Set<[string, string, string]>, + chunkSize?: number, + ): Promise< + Record<`${ProgramExternalIdSourceType}.${string}.${string}`, string> + > { + return this.externalIdRepo.programIdsByExternalIds( + ids as Set<[string, MediaSourceId, string]>, + chunkSize ?? 50, + ) as Promise< + Record<`${ProgramExternalIdSourceType}.${string}.${string}`, string> + >; + } + + updateProgramPlexRatingKey( programId: string, - serverId: MediaSourceId, + plexServerName: string, details: MarkOptional< Pick< ProgramExternalId, @@ -919,241 +231,41 @@ export class ProgramDB implements IProgramDB { >, 'directFilePath' | 'externalFilePath' >, - ) { - const existingRatingKey = await this.db - .selectFrom('programExternalId') - .selectAll() - .where((eb) => - eb.and({ - programUuid: programId, - mediaSourceId: serverId, - sourceType: ProgramExternalIdType.PLEX, - }), - ) - .executeTakeFirst(); - - if (isNil(existingRatingKey)) { - const now = +dayjs(); - return await this.db - .insertInto('programExternalId') - .values({ - uuid: v4(), - createdAt: now, - updatedAt: now, - programUuid: programId, - sourceType: ProgramExternalIdType.PLEX, - mediaSourceId: serverId, - ...details, - }) - .returningAll() - .executeTakeFirstOrThrow(); - } else { - await this.db - .updateTable('programExternalId') - .set({ - externalKey: details.externalKey, - }) - .$if(isNonEmptyString(details.externalFilePath), (_) => - _.set({ - externalFilePath: details.externalFilePath!, - }), - ) - .$if(isNonEmptyString(details.directFilePath), (_) => - _.set({ - directFilePath: details.directFilePath!, - }), - ) - .where('uuid', '=', existingRatingKey.uuid) - .executeTakeFirst(); - return await this.db - .selectFrom('programExternalId') - .selectAll() - .where('uuid', '=', existingRatingKey.uuid) - .executeTakeFirstOrThrow(); - } + ): Promise { + return this.externalIdRepo.updateProgramPlexRatingKey( + programId, + plexServerName as import('./schema/base.js').MediaSourceId, + details, + ); } - async replaceProgramExternalId( + replaceProgramExternalId( programId: string, newExternalId: NewProgramExternalId, oldExternalId?: MinimalProgramExternalId, - ) { - await this.db.transaction().execute(async (tx) => { - if (oldExternalId) { - await tx - .deleteFrom('programExternalId') - .where('programExternalId.programUuid', '=', programId) - .where( - 'programExternalId.externalKey', - '=', - oldExternalId.externalKey, - ) - .where( - 'programExternalId.externalSourceId', - '=', - oldExternalId.externalSourceId, - ) - .where('programExternalId.sourceType', '=', oldExternalId.sourceType) - // TODO: Blocked on https://github.com/oven-sh/bun/issues/16909 - // .limit(1) - .execute(); - } - await tx.insertInto('programExternalId').values(newExternalId).execute(); - }); + ): Promise { + return this.externalIdRepo.replaceProgramExternalId( + programId, + newExternalId, + oldExternalId, + ); } - async upsertContentPrograms( + upsertProgramExternalIds( + externalIds: NewSingleOrMultiExternalId[], + chunkSize?: number, + ): Promise> { + return this.externalIdRepo.upsertProgramExternalIds(externalIds, chunkSize); + } + + upsertContentPrograms( programs: ChannelProgram[], - programUpsertBatchSize: number = 100, + programUpsertBatchSize?: number, ): Promise[]> { - if (isEmpty(programs)) { - return []; - } - - const start = performance.now(); - // TODO: Wrap all of this stuff in a class and use its own logger - const [, nonPersisted] = partition(programs, (p) => p.persisted); - const minter = this.programMinterFactory(); - - const [contentPrograms, invalidPrograms] = partition( - uniqBy(filter(nonPersisted, isContentProgram), (p) => p.uniqueId), - (p) => - isNonEmptyString(p.externalSourceType) && - isNonEmptyString(p.externalSourceId) && - isNonEmptyString(p.externalKey) && - p.duration > 0, + return this.upsertRepo.upsertContentPrograms( + programs, + programUpsertBatchSize, ); - - if (!isEmpty(invalidPrograms)) { - this.logger.warn( - 'Found %d invalid programs when saving:\n%O', - invalidPrograms.length, - invalidPrograms, - ); - } - - const programsToPersist: MintedNewProgramInfo[] = seq.collect( - contentPrograms, - (p) => { - const program = minter.contentProgramDtoToDao(p); - if (!program) { - return; - } - const externalIds = minter.mintExternalIds( - program.externalSourceId, - program.mediaSourceId, - program.uuid, - p, - ); - return { program, externalIds, apiProgram: p }; - }, - ); - - const programInfoByUniqueId = groupByUniq( - programsToPersist, - ({ program }) => programExternalIdString(program), - ); - - this.logger.debug('Upserting %d programs', programsToPersist.length); - - const upsertedPrograms: MarkNonNullable[] = []; - await this.timer.timeAsync('programUpsert', async () => { - for (const c of chunk(programsToPersist, programUpsertBatchSize)) { - upsertedPrograms.push( - ...(await this.db.transaction().execute((tx) => - tx - .insertInto('program') - .values(map(c, 'program')) - // .onConflict((oc) => - // oc - // .columns(['sourceType', 'externalSourceId', 'externalKey']) - // .doUpdateSet((eb) => - // mapToObj(ProgramUpsertFields, (f) => ({ - // [f.replace('excluded.', '')]: eb.ref(f), - // })), - // ), - // ) - .onConflict((oc) => - oc - .columns(['sourceType', 'mediaSourceId', 'externalKey']) - .doUpdateSet((eb) => - mapToObj(ProgramUpsertFields, (f) => ({ - [f.replace('excluded.', '')]: eb.ref(f), - })), - ), - ) - .returningAll() - .$narrowType<{ mediaSourceId: NotNull }>() - .execute(), - )), - ); - } - }); - - const programExternalIds = flatMap(upsertedPrograms, (program) => { - const eids = - programInfoByUniqueId[programExternalIdString(program)]?.externalIds ?? - []; - forEach(eids, (eid) => { - eid.programUuid = program.uuid; - }); - return eids; - }); - - await this.timer.timeAsync('programGroupings', () => - this.handleProgramGroupings(upsertedPrograms, programInfoByUniqueId), - ); - - const [requiredExternalIds, backgroundExternalIds] = partition( - programExternalIds, - (p) => p.sourceType === 'plex' || p.sourceType === 'jellyfin', - ); - - // Fail hard on not saving Plex / Jellyfin program external IDs. We need them for streaming - // TODO: We could optimize further here by only saving IDs necessary for streaming - await this.timer.timeAsync( - `upsert ${requiredExternalIds.length} external ids`, - () => this.upsertProgramExternalIds(requiredExternalIds, 200), - ); - - this.schedulePlexExternalIdsTask(upsertedPrograms); - this.scheduleJellyfinExternalIdsTask(upsertedPrograms); - - setImmediate(() => { - this.logger.debug('Scheduling follow-up program tasks...'); - - GlobalScheduler.scheduleOneOffTask( - autoFactoryKey(ReconcileProgramDurationsTask), - dayjs().add(500, 'ms'), - [], - ); - - PlexTaskQueue.resume(); - JellyfinTaskQueue.resume(); - - this.logger.debug('Upserting external IDs in background'); - - GlobalScheduler.scheduleOneOffTask( - 'UpsertExternalIds', - dayjs().add(100), - undefined, - AnonymousTask('UpsertExternalIds', () => - this.timer.timeAsync( - `background external ID upsert (${backgroundExternalIds.length} ids)`, - () => this.upsertProgramExternalIds(backgroundExternalIds), - ), - ), - ); - }); - - const end = performance.now(); - this.logger.debug( - 'upsertContentPrograms took %d millis. %d upsertedPrograms', - round(end - start, 3), - upsertedPrograms.length, - ); - - return upsertedPrograms; } upsertPrograms( @@ -1163,2162 +275,137 @@ export class ProgramDB implements IProgramDB { programs: NewProgramWithRelations[], programUpsertBatchSize?: number, ): Promise; - async upsertPrograms( - requests: NewProgramWithRelations | NewProgramWithRelations[], - programUpsertBatchSize: number = 100, + upsertPrograms( + programs: NewProgramWithRelations | NewProgramWithRelations[], + programUpsertBatchSize?: number, ): Promise { - const wasSingleRequest = !isArray(requests); - requests = isArray(requests) ? requests : [requests]; - if (isEmpty(requests)) { - return []; - } - - const db = this.db; - - // Group related items by canonicalId because the UUID we get back - // from the upsert may not be the one we generated (if an existing entry) - // already exists - const requestsByCanonicalId = groupByUniq( - requests, - ({ program }) => program.canonicalId, - ); - - const result = await Promise.all( - chunk(requests, programUpsertBatchSize).map(async (c) => { - const chunkResult = await db.transaction().execute((tx) => - tx - .insertInto('program') - .values(c.map(({ program }) => program)) - .onConflict((oc) => - oc - .columns(['sourceType', 'mediaSourceId', 'externalKey']) - .doUpdateSet((eb) => - mapToObj(ProgramUpsertFields, (f) => ({ - [f.replace('excluded.', '')]: eb.ref(f), - })), - ), - ) - .returningAll() - // All new programs must have mediaSourceId and canonicalId. This is enforced - // by the NewProgramDao type - .$narrowType<{ mediaSourceId: NotNull; canonicalId: NotNull }>() - .execute(), - ); - - const allExternalIds = flatten(c.map((program) => program.externalIds)); - const versionsToInsert: NewProgramVersion[] = []; - const artworkToInsert: NewArtwork[] = []; - const subtitlesToInsert: NewProgramSubtitles[] = []; - const creditsToInsert: NewCredit[] = []; - const genresToInsert: Dictionary = {}; - const studiosToInsert: Dictionary = {}; - const tagsToInsert: Dictionary = {}; - for (const program of chunkResult) { - const key = program.canonicalId; - const request: Maybe = - requestsByCanonicalId[key]; - const eids = request?.externalIds ?? []; - for (const eid of eids) { - eid.programUuid = program.uuid; - } - - for (const version of request?.versions ?? []) { - version.programId = program.uuid; - versionsToInsert.push(version); - } - - for (const art of request?.artwork ?? []) { - art.programId = program.uuid; - artworkToInsert.push(art); - } - - for (const subtitle of request?.subtitles ?? []) { - subtitle.programId = program.uuid; - subtitlesToInsert.push(subtitle); - } - - for (const { credit, artwork } of request?.credits ?? []) { - credit.programId = program.uuid; - creditsToInsert.push(credit); - artworkToInsert.push(...artwork); - } - - for (const genre of request?.genres ?? []) { - genresToInsert[program.uuid] ??= []; - genresToInsert[program.uuid]?.push(genre); - } - - for (const studio of request?.studios ?? []) { - studiosToInsert[program.uuid] ??= []; - studiosToInsert[program.uuid]?.push(studio); - } - - for (const tag of request?.tags ?? []) { - tagsToInsert[program.uuid] ??= []; - tagsToInsert[program.uuid]?.push(tag); - } - } - - const externalIdsByProgramId = - await this.upsertProgramExternalIds(allExternalIds); - - await this.upsertProgramVersions(versionsToInsert); - - // Credits must come before artwork because some art may - // rely on credit IDs - await this.upsertCredits(creditsToInsert); - - await this.upsertArtwork(artworkToInsert); - - await this.upsertSubtitles(subtitlesToInsert); - - for (const [programId, genres] of Object.entries(genresToInsert)) { - await this.upsertProgramGenres(programId, genres); - } - - for (const [programId, studios] of Object.entries(studiosToInsert)) { - await this.upsertProgramStudios(programId, studios); - } - - for (const [programId, tags] of Object.entries(tagsToInsert)) { - await this.upsertProgramTags(programId, tags); - } - - return chunkResult.map( - (upsertedProgram) => - ({ - ...upsertedProgram, - externalIds: externalIdsByProgramId[upsertedProgram.uuid] ?? [], - }) satisfies ProgramWithExternalIds, - ); - }), - ).then(flatten); - - if (wasSingleRequest) { - return head(result)!; + if (Array.isArray(programs)) { + return this.upsertRepo.upsertPrograms(programs, programUpsertBatchSize); } else { - return result; + return this.upsertRepo.upsertPrograms(programs); } } - private async upsertProgramVersions(versions: NewProgramVersion[]) { - if (versions.length === 0) { - this.logger.warn('No program versions passed for item'); - return []; - } - - const insertedVersions: ProgramVersion[] = []; - await this.db.transaction().execute(async (tx) => { - const byProgramId = groupByUniq(versions, (version) => version.programId); - for (const batch of chunk(Object.entries(byProgramId), 50)) { - const [programIds, versionBatch] = unzip(batch); - // We probably need to delete here, because we never really delete - // programs on the upsert path. - await tx - .deleteFrom('programVersion') - .where('programId', 'in', programIds) - .executeTakeFirstOrThrow(); - - const insertResult = await tx - .insertInto('programVersion') - .values( - versionBatch.map((version) => - omit(version, ['chapters', 'mediaStreams', 'mediaFiles']), - ), - ) - .returningAll() - .execute(); - - await this.upsertProgramMediaStreams( - versionBatch.flatMap(({ mediaStreams }) => mediaStreams), - tx, - ); - await this.upsertProgramChapters( - versionBatch.flatMap(({ chapters }) => chapters ?? []), - tx, - ); - await this.upsertProgramMediaFiles( - versionBatch.flatMap(({ mediaFiles }) => mediaFiles), - tx, - ); - - insertedVersions.push(...insertResult); - } - }); - return insertedVersions; + upsertArtwork(artwork: NewArtwork[]): Promise { + return this.metadataRepo.upsertArtwork(artwork).then(() => {}); } - private async upsertProgramMediaStreams( - streams: NewProgramMediaStream[], - tx: Kysely = this.db, - ) { - if (streams.length === 0) { - this.logger.warn('No media streams passed for version'); - return []; - } - - const byVersionId = groupBy(streams, (stream) => stream.programVersionId); - const inserted: ProgramMediaStream[] = []; - for (const batch of chunk(Object.entries(byVersionId), 50)) { - const [_, streams] = unzip(batch); - // TODO: Do we need to delete first? - // await tx.deleteFrom('programMediaStream').where('programVersionId', 'in', versionIds).executeTakeFirstOrThrow(); - inserted.push( - ...(await tx - .insertInto('programMediaStream') - .values(flatten(streams)) - .returningAll() - .execute()), - ); - } - return inserted; - } - - private async upsertProgramChapters( - chapters: NewProgramChapter[], - tx: Kysely = this.db, - ) { - if (chapters.length === 0) { - return []; - } - - const byVersionId = groupBy(chapters, (stream) => stream.programVersionId); - const inserted: ProgramChapter[] = []; - for (const batch of chunk(Object.entries(byVersionId), 50)) { - const [_, streams] = unzip(batch); - // TODO: Do we need to delete first? - // await tx.deleteFrom('programMediaStream').where('programVersionId', 'in', versionIds).executeTakeFirstOrThrow(); - inserted.push( - ...(await tx - .insertInto('programChapter') - .values(flatten(streams)) - .returningAll() - .execute()), - ); - } - return inserted; - } - - private async upsertProgramMediaFiles( - files: NewProgramMediaFile[], - tx: Kysely = this.db, - ) { - if (files.length === 0) { - this.logger.warn('No media files passed for version'); - return []; - } - - const byVersionId = groupBy(files, (stream) => stream.programVersionId); - const inserted: ProgramMediaFile[] = []; - for (const batch of chunk(Object.entries(byVersionId), 50)) { - const [_, files] = unzip(batch); - // TODO: Do we need to delete first? - // await tx.deleteFrom('programMediaStream').where('programVersionId', 'in', versionIds).executeTakeFirstOrThrow(); - inserted.push( - ...(await tx - .insertInto('programMediaFile') - .values(flatten(files)) - .returningAll() - .execute()), - ); - } - return inserted; - } - - async upsertArtwork(artwork: NewArtwork[]) { - if (artwork.length === 0) { - return; - } - - const programArt = groupBy( - artwork.filter((art) => isNonEmptyString(art.programId)), - (art) => art.programId, - ); - const groupArt = groupBy( - artwork.filter((art) => isNonEmptyString(art.groupingId)), - (art) => art.groupingId, - ); - const creditArt = groupBy( - artwork.filter((art) => isNonEmptyString(art.creditId)), - (art) => art.creditId, - ); - - return await this.drizzleDB.transaction(async (tx) => { - for (const batch of chunk(keys(programArt), 50)) { - await tx.delete(Artwork).where(inArray(Artwork.programId, batch)); - } - for (const batch of chunk(keys(groupArt), 50)) { - await tx.delete(Artwork).where(inArray(Artwork.groupingId, batch)); - } - for (const batch of chunk(keys(creditArt), 50)) { - await tx.delete(Artwork).where(inArray(Artwork.creditId, batch)); - } - const inserted: Artwork[] = []; - for (const batch of chunk(artwork, 50)) { - const batchResult = await this.drizzleDB - .insert(Artwork) - .values(batch) - .onConflictDoUpdate({ - target: Artwork.uuid, - set: { - cachePath: sql`excluded.cache_path`, - groupingId: sql`excluded.grouping_id`, - programId: sql`excluded.program_id`, - updatedAt: sql`excluded.updated_at`, - sourcePath: sql`excluded.source_path`, - }, - }) - .returning(); - inserted.push(...batchResult); - } - return inserted; - }); - } - - async upsertProgramGenres(programId: string, genres: NewGenre[]) { - return this.upsertProgramGenresInternal('program', programId, genres); - } - - async upsertProgramGroupingGenres(groupingId: string, genres: NewGenre[]) { - return this.upsertProgramGenresInternal('grouping', groupingId, genres); - } - - private async upsertProgramGenresInternal( - entityType: 'program' | 'grouping', - joinId: string, + upsertProgramGroupingGenres( + groupingId: string, genres: NewGenre[], - ) { - if (genres.length === 0) { - return; - } - - const incomingByName = groupByUniq(genres, (g) => g.name); - const existingGenresByName: Dictionary = {}; - for (const genreChunk of chunk(genres, 100)) { - const names = genreChunk.map((g) => g.name); - const results = await this.drizzleDB - .select() - .from(Genre) - .where(inArray(Genre.name, names)); - for (const result of results) { - existingGenresByName[result.name] = result; - } - } - - const newGenreNames = new Set( - difference(keys(incomingByName), keys(existingGenresByName)), - ); - - const relations: NewGenreEntity[] = []; - for (const name of Object.keys(incomingByName)) { - const genreId = newGenreNames.has(name) - ? incomingByName[name]!.uuid - : existingGenresByName[name]!.uuid; - relations.push({ - genreId, - programId: entityType === 'program' ? joinId : null, - groupId: entityType === 'grouping' ? joinId : null, - }); - } - - return this.drizzleDB.transaction(async (tx) => { - const col = - entityType === 'grouping' ? EntityGenre.groupId : EntityGenre.programId; - await tx.delete(EntityGenre).where(eq(col, joinId)); - if (newGenreNames.size > 0) { - await tx - .insert(Genre) - .values( - [...newGenreNames.values()].map((name) => incomingByName[name]!), - ) - .onConflictDoNothing(); - } - if (relations.length > 0) { - await tx.insert(EntityGenre).values(relations).onConflictDoNothing(); - } - }); + ): Promise { + return this.metadataRepo + .upsertProgramGroupingGenres(groupingId, genres) + .then(() => {}); } - async upsertProgramStudios(programId: string, studios: NewStudio[]) { - return this.upsertProgramStudiosInternal('program', programId, studios); - } - - async upsertProgramGroupingStudios(groupingId: string, studios: NewStudio[]) { - return this.upsertProgramStudiosInternal('grouping', groupingId, studios); - } - - private async upsertProgramStudiosInternal( - entityType: 'program' | 'grouping', - joinId: string, - studios: NewStudio[], - ) { - if (studios.length === 0) { - return; - } - - const incomingByName = groupByUniq(studios, (g) => g.name); - const existingStudiosByName: Dictionary = {}; - for (const studioChunk of chunk(studios, 100)) { - const names = studioChunk.map((g) => g.name); - const results = await this.drizzleDB - .select() - .from(Studio) - .where(inArray(Studio.name, names)); - for (const result of results) { - existingStudiosByName[result.name] = result; - } - } - - const newStudioNames = new Set( - difference(keys(incomingByName), keys(existingStudiosByName)), - ); - - const relations: NewStudioEntity[] = []; - for (const name of Object.keys(incomingByName)) { - const studioId = newStudioNames.has(name) - ? incomingByName[name]!.uuid - : existingStudiosByName[name]!.uuid; - relations.push({ - studioId, - programId: entityType === 'program' ? joinId : null, - groupId: entityType === 'grouping' ? joinId : null, - }); - } - - return this.drizzleDB.transaction(async (tx) => { - const col = - entityType === 'grouping' - ? StudioEntity.groupId - : StudioEntity.programId; - await tx.delete(StudioEntity).where(eq(col, joinId)); - if (newStudioNames.size > 0) { - await tx - .insert(Studio) - .values( - [...newStudioNames.values()].map((name) => incomingByName[name]!), - ) - .onConflictDoNothing(); - } - if (relations.length > 0) { - await tx.insert(StudioEntity).values(relations).onConflictDoNothing(); - } - }); - } - - async upsertProgramTags(programId: string, tags: NewTag[]) { - return this.upsertProgramTagsInternal('program', programId, tags); - } - - async upsertProgramGroupingTags(groupingId: string, tags: NewTag[]) { - return this.upsertProgramTagsInternal('grouping', groupingId, tags); - } - - private async upsertProgramTagsInternal( - entityType: 'program' | 'grouping', - joinId: string, - tags: NewTag[], - ) { - if (tags.length === 0) { - return; - } - - const incomingByName = groupByUniq(tags, (g) => g.tag); - const existingTagsByName: Dictionary = {}; - for (const tagChunk of chunk(tags, 100)) { - const names = tagChunk.map((g) => g.tag); - const results = await this.drizzleDB - .select() - .from(Tag) - .where(inArray(Tag.tag, names)); - for (const result of results) { - existingTagsByName[result.tag] = result; - } - } - - const newTagNames = new Set( - difference(keys(incomingByName), keys(existingTagsByName)), - ); - - const relations: NewTagRelation[] = []; - for (const name of Object.keys(incomingByName)) { - const tagId = newTagNames.has(name) - ? incomingByName[name]!.uuid - : existingTagsByName[name]!.uuid; - relations.push({ - tagId, - programId: entityType === 'program' ? joinId : null, - groupingId: entityType === 'grouping' ? joinId : null, - source: 'media', - }); - } - - return this.drizzleDB.transaction(async (tx) => { - const col = - entityType === 'grouping' - ? TagRelations.groupingId - : TagRelations.programId; - await tx - .delete(TagRelations) - .where(and(eq(col, joinId), eq(TagRelations.source, 'media'))); - if (newTagNames.size > 0) { - await tx - .insert(Tag) - .values( - [...newTagNames.values()].map((name) => incomingByName[name]!), - ) - .onConflictDoNothing(); - } - if (relations.length > 0) { - await tx.insert(TagRelations).values(relations).onConflictDoNothing(); - } - }); - } - - private async upsertSubtitles(subtitles: NewProgramSubtitles[]) { - if (subtitles.length === 0) { - return; - } - - const grouped = groupBy(subtitles, (sub) => sub.programId); - for (const [programId, programSubtitles] of Object.entries(grouped)) { - const existingSubsForProgram = - await this.drizzleDB.query.programSubtitles.findMany({ - where: (fields, { eq }) => eq(fields.programId, programId), - }); - - // Embedded subtitles are unique by stream index - // Sidecar are unique by path. - const [existingEmbedded, _] = partition( - existingSubsForProgram, - (sub) => !isNil(sub.streamIndex), - ); - const [incomingEmbedded, incomingExternal] = partition( - programSubtitles, - (sub) => !isNil(sub.streamIndex), - ); - - const existingIndexes = new Set( - seq.collect(existingEmbedded, (sub) => sub.streamIndex), - ); - const incomingIndexes = new Set( - seq.collect(incomingEmbedded, (sub) => sub.streamIndex), - ); - - const newIndexes = incomingIndexes.difference(existingIndexes); - const removedIndexes = existingIndexes.difference(newIndexes); - const updatedIndexes = incomingIndexes.difference( - newIndexes.union(removedIndexes), - ); - - const inserts = incomingEmbedded.filter((s) => - newIndexes.has(s.streamIndex!), - ); - const removes = existingEmbedded.filter((s) => - removedIndexes.has(s.streamIndex!), - ); - - const updates: ProgramSubtitles[] = []; - for (const updatedIndex of updatedIndexes.values()) { - const incoming = incomingEmbedded.find( - (s) => s.streamIndex === updatedIndex, - ); - const existing = existingEmbedded.find( - (s) => s.streamIndex === updatedIndex, - ); - if (!existing || !incoming) { - continue; // Shouldn't happen - } - - if (existing.isExtracted) { - const needsExtraction = - existing.subtitleType !== incoming.subtitleType || - existing.codec !== incoming.subtitleType || - existing.language !== incoming.language || - existing.forced !== incoming.forced || - existing.sdh !== incoming.sdh || - existing.default !== incoming.default; - if (needsExtraction) { - existing.isExtracted = false; - existing.path = incoming.path ?? null; - } else if ( - isNonEmptyString(incoming.path) && - existing.path !== incoming.path - ) { - existing.isExtracted = false; - existing.path = incoming.path; - } - } - - existing.codec = incoming.codec; - existing.language = incoming.language; - existing.subtitleType = incoming.subtitleType; - existing.updatedAt = incoming.updatedAt; - if (isDefined(incoming.default)) { - existing.default = incoming.default; - } - - if (isDefined(incoming.sdh)) { - existing.sdh = incoming.sdh; - } - - if (isDefined(incoming.forced)) { - existing.forced = incoming.forced; - } - - updates.push(existing); - } - - await this.drizzleDB.transaction(async (tx) => { - if (inserts.length > 0) { - await tx.insert(ProgramSubtitles).values(inserts); - } - if (removes.length > 0) { - await tx.delete(ProgramSubtitles).where( - inArray( - ProgramSubtitles.uuid, - removes.map((s) => s.uuid), - ), - ); - } - - if (updates.length > 0) { - for (const update of updates) { - await tx - .update(ProgramSubtitles) - .set(update) - .where(eq(ProgramSubtitles.uuid, update.uuid)); - } - } - - await tx - .delete(ProgramSubtitles) - .where( - and( - eq(ProgramSubtitles.subtitleType, 'sidecar'), - eq(ProgramSubtitles.programId, programId), - ), - ); - - if (incomingExternal.length > 0) { - await tx.insert(ProgramSubtitles).values(incomingExternal); - } - }); - } - } - - async upsertCredits(credits: NewCredit[]) { - if (credits.length === 0) { - return; - } - - const programCredits = groupBy( - credits.filter((credit) => isNonEmptyString(credit.programId)), - (credit) => credit.programId, - ); - const groupCredits = groupBy( - credits.filter((credit) => isNonEmptyString(credit.groupingId)), - (credit) => credit.groupingId, - ); - - return await this.drizzleDB.transaction(async (tx) => { - for (const batch of chunk(keys(programCredits), 50)) { - await tx.delete(Credit).where(inArray(Credit.programId, batch)); - } - for (const batch of chunk(keys(groupCredits), 50)) { - await tx.delete(Credit).where(inArray(Credit.groupingId, batch)); - } - const inserted: Credit[] = []; - for (const batch of chunk(credits, 50)) { - const batchResult = await this.drizzleDB - .insert(Credit) - .values(batch) - .returning(); - inserted.push(...batchResult); - } - return inserted; - }); - } - - async upsertProgramExternalIds( - externalIds: NewSingleOrMultiExternalId[], - chunkSize: number = 100, - ): Promise> { - if (isEmpty(externalIds)) { - return {}; - } - - const logger = this.logger; - - const [singles, multiples] = partition( - externalIds, - (id) => id.type === 'single', - ); - - let singleIdPromise: Promise; - if (!isEmpty(singles)) { - singleIdPromise = mapAsyncSeq( - chunk(singles, chunkSize), - (singleChunk) => { - return this.db.transaction().execute((tx) => - tx - .insertInto('programExternalId') - .values(singleChunk.map(toInsertableProgramExternalId)) - // .onConflict((oc) => - // oc - // .columns(['programUuid', 'sourceType', 'externalSourceId']) - // .where('externalSourceId', 'is', null) - // .doUpdateSet((eb) => ({ - // updatedAt: eb.ref('excluded.updatedAt'), - // externalFilePath: eb.ref('excluded.externalFilePath'), - // directFilePath: eb.ref('excluded.directFilePath'), - // programUuid: eb.ref('excluded.programUuid'), - // })), - // ) - .onConflict((oc) => - oc - .columns(['programUuid', 'sourceType']) - .where('mediaSourceId', 'is', null) - .doUpdateSet((eb) => ({ - updatedAt: eb.ref('excluded.updatedAt'), - externalFilePath: eb.ref('excluded.externalFilePath'), - directFilePath: eb.ref('excluded.directFilePath'), - programUuid: eb.ref('excluded.programUuid'), - })), - ) - .returningAll() - .execute(), - ); - }, - ).then(flatten); - } else { - singleIdPromise = Promise.resolve([]); - } - - let multiIdPromise: Promise; - if (!isEmpty(multiples)) { - multiIdPromise = mapAsyncSeq( - chunk(multiples, chunkSize), - (multiChunk) => { - return this.db.transaction().execute((tx) => - tx - .insertInto('programExternalId') - .values(multiChunk.map(toInsertableProgramExternalId)) - // .onConflict((oc) => - // oc - // .columns(['programUuid', 'sourceType', 'externalSourceId']) - // .where('externalSourceId', 'is not', null) - // .doUpdateSet((eb) => ({ - // updatedAt: eb.ref('excluded.updatedAt'), - // externalFilePath: eb.ref('excluded.externalFilePath'), - // directFilePath: eb.ref('excluded.directFilePath'), - // programUuid: eb.ref('excluded.programUuid'), - // })), - // ) - .onConflict((oc) => - oc - .columns(['programUuid', 'sourceType', 'mediaSourceId']) - .where('mediaSourceId', 'is not', null) - .doUpdateSet((eb) => ({ - updatedAt: eb.ref('excluded.updatedAt'), - externalFilePath: eb.ref('excluded.externalFilePath'), - directFilePath: eb.ref('excluded.directFilePath'), - programUuid: eb.ref('excluded.programUuid'), - })), - ) - .returningAll() - .execute(), - ); - }, - ).then(flatten); - } else { - multiIdPromise = Promise.resolve([]); - } - - const [singleResult, multiResult] = await Promise.allSettled([ - singleIdPromise, - multiIdPromise, - ]); - - const allExternalIds: ProgramExternalId[] = []; - if (singleResult.status === 'rejected') { - logger.error(singleResult.reason, 'Error saving external IDs'); - } else { - logger.trace('Upserted %d external IDs', singleResult.value.length); - allExternalIds.push(...singleResult.value); - } - - if (multiResult.status === 'rejected') { - logger.error(multiResult.reason, 'Error saving external IDs'); - } else { - logger.trace('Upserted %d external IDs', multiResult.value.length); - allExternalIds.push(...multiResult.value); - } - - return groupBy(allExternalIds, (eid) => eid.programUuid); - } - - async getProgramsForMediaSource( - mediaSourceId: MediaSourceId, + getProgramsForMediaSource( + mediaSourceId: string, type?: ProgramType, - ) { - return this.db - .selectFrom('mediaSource') - .where('mediaSource.uuid', '=', mediaSourceId) - .select((eb) => - jsonArrayFrom( - eb - .selectFrom('program') - .select(AllProgramFields) - .$if(isDefined(type), (eb) => eb.where('program.type', '=', type!)) - .whereRef('mediaSource.uuid', '=', 'program.mediaSourceId'), - ).as('programs'), - ) - .executeTakeFirst() - .then((dbResult) => dbResult?.programs ?? []); + ): Promise { + return this.searchRepo.getProgramsForMediaSource( + mediaSourceId as unknown as MediaSourceId, + type, + ); } - async getMediaSourceLibraryPrograms(libraryId: string) { - return selectProgramsBuilder(this.db, { includeGroupingExternalIds: true }) - .where('libraryId', '=', libraryId) - .selectAll() - .select(withProgramExternalIds) - .execute(); + getMediaSourceLibraryPrograms( + libraryId: string, + ): Promise { + return this.searchRepo.getMediaSourceLibraryPrograms(libraryId); } - async getProgramInfoForMediaSource( + getProgramInfoForMediaSource( mediaSourceId: MediaSourceId, type: ProgramType, parentFilter?: [ProgramGroupingType, string], - ) { - const results = await this.drizzleDB.query.program.findMany({ - where: (fields, { eq, and, isNotNull }) => { - const parentField = match([type, parentFilter?.[0]]) - .with(['episode', 'show'], () => fields.tvShowUuid) - .with(['episode', 'season'], () => fields.seasonUuid) - .with(['track', 'album'], () => fields.albumUuid) - .with(['track', 'artist'], () => fields.artistUuid) - .otherwise(() => null); - - return and( - eq(fields.mediaSourceId, mediaSourceId), - eq(fields.type, type), - isNotNull(fields.canonicalId), - parentField && parentFilter - ? eq(parentField, parentFilter[1]) - : undefined, - ); - }, - }); - - const grouped: Dictionary = {}; - for (const result of results) { - if (!result.canonicalId || !result.libraryId) { - continue; - } - grouped[result.externalKey] = { - canonicalId: result.canonicalId, - externalKey: result.externalKey, - libraryId: result.libraryId, - uuid: result.uuid, - }; - } - - return grouped; + ): Promise> { + return this.searchRepo.getProgramInfoForMediaSource( + mediaSourceId, + type, + parentFilter, + ); } - async getProgramInfoForMediaSourceLibrary( + getProgramInfoForMediaSourceLibrary( mediaSourceLibraryId: string, type: ProgramType, parentFilter?: [ProgramGroupingType, string], - ) { - const grouped: Dictionary = {}; - for await (const result of this.getProgramInfoForMediaSourceLibraryAsync( + ): Promise> { + return this.searchRepo.getProgramInfoForMediaSourceLibrary( mediaSourceLibraryId, type, parentFilter, - )) { - grouped[result.externalKey] = { - canonicalId: result.canonicalId, - externalKey: result.externalKey, - libraryId: result.libraryId, - uuid: result.uuid, - }; - } - - return grouped; + ); } - async *getProgramInfoForMediaSourceLibraryAsync( + getProgramInfoForMediaSourceLibraryAsync( mediaSourceLibraryId: string, type: ProgramType, parentFilter?: [ProgramGroupingType, string], ): AsyncGenerator { - let lastId: Maybe; - for (;;) { - const page = await this.drizzleDB.query.program.findMany({ - where: (fields, { eq, and, isNotNull, gt }) => { - const parentField = match([type, parentFilter?.[0]]) - .with(['episode', 'show'], () => fields.tvShowUuid) - .with(['episode', 'season'], () => fields.seasonUuid) - .with(['track', 'album'], () => fields.albumUuid) - .with(['track', 'artist'], () => fields.artistUuid) - .otherwise(() => null); - - return and( - eq(fields.libraryId, mediaSourceLibraryId), - eq(fields.type, type), - isNotNull(fields.canonicalId), - parentField && parentFilter - ? eq(parentField, parentFilter[1]) - : undefined, - lastId ? gt(fields.uuid, lastId) : undefined, - ); - }, - orderBy: (fields, ops) => ops.asc(fields.uuid), - columns: { - uuid: true, - canonicalId: true, - libraryId: true, - externalKey: true, - }, - limit: 500, - }); - - if (page.length === 0) { - return; - } - - lastId = last(page)?.uuid; - for (const item of page) { - yield { - externalKey: item.externalKey, - canonicalId: item.canonicalId, - uuid: item.uuid, - libraryId: item.libraryId, - }; - } - } + return this.searchRepo.getProgramInfoForMediaSourceLibraryAsync( + mediaSourceLibraryId, + type, + parentFilter, + ); } - async getExistingProgramGroupingDetails( + getExistingProgramGroupingDetails( mediaSourceLibraryId: string, type: ProgramGroupingType, sourceType: StrictExclude, parentFilter?: string, - ) { - const results = await this.drizzleDB.query.programGrouping.findMany({ - where: (fields, { and, eq, isNotNull }) => { - const parentField = match(type) - // .returnType() - .with('album', () => fields.artistUuid) - .with('season', () => fields.showUuid) - .otherwise(() => null); - return and( - eq(fields.libraryId, mediaSourceLibraryId), - eq(fields.type, type), - isNotNull(fields.canonicalId), - parentField && parentFilter - ? eq(parentField, parentFilter) - : undefined, - ); - }, - with: { - externalIds: { - where: (fields, { eq }) => eq(fields.sourceType, sourceType), - }, - }, - columns: { - uuid: true, - canonicalId: true, - libraryId: true, - externalKey: true, - }, - }); - - const grouped: Dictionary = {}; - for (const result of results) { - const key = result.externalKey ?? head(result.externalIds)?.externalKey; - if (!key) { - continue; - } - - grouped[key] = { - canonicalId: result.canonicalId, - externalKey: key, - libraryId: result.libraryId!, - uuid: result.uuid, - }; - } - - return grouped; + ): Promise> { + return this.searchRepo.getExistingProgramGroupingDetails( + mediaSourceLibraryId, + type, + sourceType, + parentFilter, + ); } - async upsertProgramGrouping( + upsertProgramGrouping( newGroupingAndRelations: NewProgramGroupingWithRelations, - forceUpdate: boolean = false, + forceUpdate?: boolean, ): Promise> { - let entity: Maybe = - await this.getProgramGrouping( - newGroupingAndRelations.programGrouping.uuid, - ); - let shouldUpdate = forceUpdate; - let wasInserted = false, - wasUpdated = false; - const { programGrouping: dao, externalIds } = newGroupingAndRelations; - - if (!entity && dao.sourceType === 'local') { - const incomingYear = newGroupingAndRelations.programGrouping.year; - entity = await this.drizzleDB.query.programGrouping.findFirst({ - where: (fields, { eq, and, isNull }) => { - const parentClause = match(newGroupingAndRelations.programGrouping) - .with({ type: 'season', showUuid: P.nonNullable }, (season) => - compact([ - eq(fields.showUuid, season.showUuid), - season.index ? eq(fields.index, season.index) : null, - ]), - ) - .with({ type: 'album', artistUuid: P.nonNullable }, (album) => [ - eq(fields.artistUuid, album.artistUuid), - ]) - .otherwise(() => []); - return and( - eq( - fields.libraryId, - newGroupingAndRelations.programGrouping.libraryId, - ), - eq(fields.title, newGroupingAndRelations.programGrouping.title), - eq(fields.type, newGroupingAndRelations.programGrouping.type), - eq(fields.sourceType, 'local'), - isNil(incomingYear) - ? isNull(fields.year) - : eq(fields.year, incomingYear), - ...parentClause, - ); - }, - with: { - externalIds: true, - }, - }); - } else if (!entity && dao.sourceType !== 'local') { - entity = await this.getProgramGroupingByExternalId({ - sourceType: dao.sourceType, - externalKey: dao.externalKey, - externalSourceId: dao.mediaSourceId, - }); - if (entity) { - // let wasUpdated = false; - const missingAssociation = - (entity.type === 'season' && - isDefined(dao.showUuid) && - dao.showUuid !== entity.showUuid) || - (entity.type === 'album' && - isDefined(dao.artistUuid) && - dao.artistUuid !== entity.artistUuid); - const differentVersion = entity.canonicalId !== dao.canonicalId; - shouldUpdate ||= differentVersion || missingAssociation; - } - } - - if (entity && shouldUpdate) { - newGroupingAndRelations.programGrouping.uuid = entity.uuid; - for (const externalId of newGroupingAndRelations.externalIds) { - externalId.groupUuid = entity.uuid; - } - entity = await this.drizzleDB.transaction(async (tx) => { - const updated = await this.updateProgramGrouping( - newGroupingAndRelations, - entity!, - tx, - ); - const upsertedExternalIds = await this.updateProgramGroupingExternalIds( - entity!.externalIds, - externalIds, - tx, - ); - return { - ...updated, - externalIds: upsertedExternalIds, - } satisfies ProgramGroupingOrmWithRelations; - }); - - wasUpdated = true; - } else if (!entity) { - entity = await this.drizzleDB.transaction(async (tx) => { - const grouping = head( - await tx - .insert(ProgramGrouping) - .values(omit(dao, 'externalIds')) - .returning(), - )!; - const insertedExternalIds: ProgramGroupingExternalIdOrm[] = []; - if (externalIds.length > 0) { - insertedExternalIds.push( - ...(await this.upsertProgramGroupingExternalIdsChunkOrm( - externalIds, - tx, - )), - ); - } - - return { - ...grouping, - externalIds: insertedExternalIds, - } satisfies ProgramGroupingOrmWithRelations; - }); - - wasInserted = true; - shouldUpdate = true; - } - - if (shouldUpdate) { - newGroupingAndRelations.credits.forEach((credit) => { - credit.credit.groupingId = entity.uuid; - }); - - newGroupingAndRelations.artwork.forEach((artwork) => { - artwork.groupingId = entity.uuid; - }); - - await this.upsertCredits( - newGroupingAndRelations.credits.map(({ credit }) => credit), - ); - - await this.upsertArtwork( - newGroupingAndRelations.artwork.concat( - newGroupingAndRelations.credits.flatMap(({ artwork }) => artwork), - ), - ); - - await this.upsertProgramGroupingGenres( - entity.uuid, - newGroupingAndRelations.genres, - ); - - await this.upsertProgramGroupingStudios( - entity.uuid, - newGroupingAndRelations.studios, - ); - - await this.upsertProgramGroupingTags( - entity.uuid, - newGroupingAndRelations.tags, - ); - } - - return { - entity, - wasInserted, - wasUpdated, - }; + return this.groupingUpsertRepo.upsertProgramGrouping( + newGroupingAndRelations, + forceUpdate, + ); } - private async updateProgramGrouping( - { programGrouping: incoming }: NewProgramGroupingWithRelations, - existing: ProgramGroupingOrmWithRelations, - tx: BaseSQLiteDatabase<'sync', RunResult, typeof schema> = this.drizzleDB, - ): Promise { - const update: NewProgramGroupingOrm = { - ...omit(existing, 'externalIds'), - index: incoming.index, - title: incoming.title, - summary: incoming.summary, - icon: incoming.icon, - year: incoming.year, - // relations - artistUuid: incoming.artistUuid, - showUuid: incoming.showUuid, - canonicalId: incoming.canonicalId, - mediaSourceId: incoming.mediaSourceId, - libraryId: incoming.libraryId, - sourceType: incoming.sourceType, - externalKey: incoming.externalKey, - plot: incoming.plot, - rating: incoming.rating, - releaseDate: incoming.releaseDate, - tagline: incoming.tagline, - updatedAt: incoming.updatedAt, - state: incoming.state, - }; - - return head( - await tx - .update(ProgramGrouping) - .set(update) - .where(eq(ProgramGrouping.uuid, existing.uuid)) - .limit(1) - .returning(), - )!; + getProgramGroupingChildCounts( + groupIds: string[], + ): Promise> { + return this.progGrouping.getProgramGroupingChildCounts(groupIds); } - private async updateProgramGroupingExternalIds( - existingIds: ProgramGroupingExternalId[], - newIds: NewSingleOrMultiProgramGroupingExternalId[], - tx: BaseSQLiteDatabase<'sync', RunResult, typeof schema> = this.drizzleDB, - ): Promise { - devAssert( - uniq(seq.collect(existingIds, (id) => id.mediaSourceId)).length <= 1, - ); - devAssert(uniq(existingIds.map((id) => id.libraryId)).length <= 1); - devAssert(uniq(newIds.map((id) => id.libraryId)).length <= 1); - - const newByUniqueId: Record< - string, - NewSingleOrMultiProgramGroupingExternalId - > = groupByUniq(newIds, (id) => { - switch (id.type) { - case 'single': - return id.sourceType; - case 'multi': - return `${id.sourceType}|${id.mediaSourceId}`; - } - }); - const newUniqueIds = new Set(keys(newByUniqueId)); - - const existingByUniqueId: Record = - groupByUniq(existingIds, (id) => { - if (isValidSingleExternalIdType(id.sourceType)) { - return id.sourceType; - } else { - return `${id.sourceType}|${id.mediaSourceId}`; - } - }); - const existingUniqueIds = new Set(keys(existingByUniqueId)); - - const deletedUniqueKeys = existingUniqueIds.difference(newUniqueIds); - const addedUniqueKeys = newUniqueIds.difference(existingUniqueIds); - const updatedKeys = existingUniqueIds.intersection(newUniqueIds); - - // TODO: This stinks, consider adding a unique ID - const deletedIds = [...deletedUniqueKeys.values()].map( - (key) => existingByUniqueId[key]!, - ); - await Promise.all( - chunk(deletedIds, 100).map((idChunk) => { - const clauses = idChunk.map((id) => - and( - id.mediaSourceId - ? eq(ProgramGroupingExternalId.mediaSourceId, id.mediaSourceId) - : dbIsNull(ProgramGroupingExternalId.mediaSourceId), - id.libraryId - ? eq(ProgramGroupingExternalId.libraryId, id.libraryId) - : dbIsNull(ProgramGroupingExternalId.libraryId), - eq(ProgramGroupingExternalId.externalKey, id.externalKey), - id.externalSourceId - ? eq( - ProgramGroupingExternalId.externalSourceId, - id.externalSourceId, - ) - : dbIsNull(ProgramGroupingExternalId.externalSourceId), - eq(ProgramGroupingExternalId.sourceType, id.sourceType), - ), - ); - - return tx - .delete(ProgramGroupingExternalId) - .where(or(...clauses)) - .execute(); - }), - ); - - const addedIds = [...addedUniqueKeys.union(updatedKeys).values()].map( - (key) => newByUniqueId[key]!, - ); - - return await Promise.all( - chunk(addedIds, 100).map((idChunk) => - this.upsertProgramGroupingExternalIdsChunkOrm(idChunk, tx), - ), - ).then((_) => _.flat()); - } - - async getProgramGroupingChildCounts(groupingIds: string[]) { - if (isEmpty(groupingIds)) { - return {}; - } - - const uniqueIds = uniq(groupingIds); - - const allResults = await Promise.allSettled( - chunk(uniqueIds, 1000).map((idChunk) => - this.db - .selectFrom('programGrouping as pg') - .where('pg.uuid', 'in', idChunk) - .leftJoin('program as p', (j) => - j.on((eb) => - eb.or([ - eb('pg.uuid', '=', eb.ref('p.tvShowUuid')), - eb('pg.uuid', '=', eb.ref('p.artistUuid')), - eb('pg.uuid', '=', eb.ref('p.seasonUuid')), - eb('pg.uuid', '=', eb.ref('p.albumUuid')), - ]), - ), - ) - .leftJoin('programGrouping as pg2', (j) => - j.on((eb) => - eb.or([ - eb('pg.uuid', '=', eb.ref('pg2.artistUuid')), - eb('pg.uuid', '=', eb.ref('pg2.showUuid')), - ]), - ), - ) - .select(['pg.uuid as uuid', 'pg.type as type']) - .select((eb) => - eb.fn.count('p.uuid').distinct().as('programCount'), - ) - .select((eb) => - eb.fn.count('pg2.uuid').distinct().as('childGroupCount'), - ) - .groupBy('pg.uuid') - .execute(), - ), - ); - - const map: Record = {}; - - for (const result of allResults) { - if (result.status === 'rejected') { - this.logger.error( - result.reason, - 'Failed querying program grouping children. Continuing with partial results', - ); - continue; - } - - for (const counts of result.value) { - map[counts.uuid] = { - type: counts.type, - childCount: - counts.type === 'season' || counts.type === 'album' - ? counts.programCount - : counts.childGroupCount, - grandchildCount: - counts.type === 'artist' || counts.type === 'show' - ? counts.programCount - : undefined, - }; - } - } - - return map; - } - - async getProgramGroupingDescendants( + getProgramGroupingDescendants( groupId: string, groupTypeHint?: ProgramGroupingType, ): Promise { - const programs = await this.drizzleDB.query.program.findMany({ - where: (fields, { or, eq }) => { - if (groupTypeHint) { - switch (groupTypeHint) { - case 'show': - return eq(fields.tvShowUuid, groupId); - case 'season': - return eq(fields.seasonUuid, groupId); - case 'artist': - return eq(fields.artistUuid, groupId); - case 'album': - return eq(fields.albumUuid, groupId); - } - } else { - return or( - eq(fields.albumUuid, groupId), - eq(fields.artistUuid, groupId), - eq(fields.tvShowUuid, groupId), - eq(fields.seasonUuid, groupId), - ); - } - }, - with: { - album: - isUndefined(groupTypeHint) || - groupTypeHint === 'album' || - groupTypeHint === 'artist' - ? true - : undefined, - artist: - isUndefined(groupTypeHint) || - groupTypeHint === 'album' || - groupTypeHint === 'artist' - ? true - : undefined, - season: - isUndefined(groupTypeHint) || - groupTypeHint === 'show' || - groupTypeHint === 'season' - ? true - : undefined, - show: - isUndefined(groupTypeHint) || - groupTypeHint === 'show' || - groupTypeHint === 'season' - ? true - : undefined, - externalIds: true, - }, - }); - - return orderBy( - programs, - [(p) => p.season?.index ?? p.seasonNumber ?? 1, (p) => p.episode ?? 1], - ['asc', 'asc'], + return this.progGrouping.getProgramGroupingDescendants( + groupId, + groupTypeHint, ); } - async updateProgramsState( + updateProgramsState( programIds: string[], newState: ProgramState, ): Promise { - if (programIds.length === 0) { - return; - } - - for (const idChunk of chunk(programIds, 100)) { - await this.drizzleDB - .update(Program) - .set({ - state: newState, - }) - .where(inArray(Program.uuid, idChunk)) - .execute(); - } + return this.stateRepo.updateProgramsState(programIds, newState); } - async updateGroupingsState( + updateGroupingsState( groupingIds: string[], newState: ProgramState, ): Promise { - if (groupingIds.length === 0) { - return; - } - - for (const idChunk of chunk(groupingIds, 100)) { - await this.drizzleDB - .update(ProgramGrouping) - .set({ - state: newState, - }) - .where(inArray(ProgramGrouping.uuid, idChunk)) - .execute(); - } + return this.stateRepo.updateGroupingsState(groupingIds, newState); } - async emptyTrashPrograms() { - await this.drizzleDB.delete(Program).where(eq(Program.state, 'missing')); - } - - private async handleProgramGroupings( - upsertedPrograms: MarkNonNullable[], - programInfos: Record, - ) { - const programsBySourceAndServer = mapValues( - groupBy(upsertedPrograms, 'sourceType'), - (ps) => groupBy(ps, typedProperty('mediaSourceId')), - ); - - for (const [sourceType, byServerId] of Object.entries( - programsBySourceAndServer, - )) { - for (const [serverId, programs] of Object.entries(byServerId)) { - // Making an assumption that these are all the same... this field will - // go away soon anyway - const serverName = head(programs)!.externalSourceId; - // This is just extra safety because lodash erases the type in groupBy - const typ = programSourceTypeFromString(sourceType); - if (!typ) { - return; - } - - await this.handleSingleSourceProgramGroupings( - programs, - programInfos, - typ, - serverName, - serverId as MediaSourceId, // We know this is true above, types get lost from Object.entries - ); - } - } - } - - private async handleSingleSourceProgramGroupings( - upsertedPrograms: MarkNonNullable[], - programInfos: Record, - mediaSourceType: ProgramSourceType, - mediaSourceName: MediaSourceName, - mediaSourceId: MediaSourceId, - ) { - const grandparentRatingKeyToParentRatingKey: Record< - string, - Set - > = {}; - const grandparentRatingKeyToProgramId: Record> = {}; - const parentRatingKeyToProgramId: Record> = {}; - - const relevantPrograms: RelevantProgramWithHierarchy[] = seq.collect( - upsertedPrograms, - (program) => { - if ( - program.type === ProgramType.Movie || - program.type === ProgramType.MusicVideo || - program.type === ProgramType.OtherVideo - ) { - return; - } - - const info = programInfos[programExternalIdString(program)]; - if (!info) { - return; - } - - if ( - info.apiProgram.subtype === ProgramType.Movie || - info.apiProgram.subtype === ProgramType.MusicVideo || - info.apiProgram.subtype === ProgramType.OtherVideo - ) { - return; - } - - const [grandparentKey, parentKey] = [ - info.apiProgram.grandparent?.externalKey, - info.apiProgram.parent?.externalKey, - ]; - - if (!grandparentKey || !parentKey) { - this.logger.warn( - 'Unexpected null/empty parent keys: %O', - info.apiProgram, - ); - return; - } - - return { - program, - programWithHierarchy: { - ...(info.apiProgram as ContentProgramWithHierarchy), - grandparentKey, - parentKey, - }, - }; - }, - ); - - const upsertedProgramById = groupByUniqProp( - map(relevantPrograms, ({ program }) => program), - 'uuid', - ); - - for (const { - program, - programWithHierarchy: { grandparentKey, parentKey }, - } of relevantPrograms) { - if (isNonEmptyString(grandparentKey)) { - (grandparentRatingKeyToProgramId[grandparentKey] ??= new Set()).add( - program.uuid, - ); - - const set = (grandparentRatingKeyToParentRatingKey[grandparentKey] ??= - new Set()); - if (isNonEmptyString(parentKey)) { - set.add(parentKey); - } - } - - if (isNonEmptyString(parentKey)) { - (parentRatingKeyToProgramId[parentKey] ??= new Set()).add(program.uuid); - } - } - - const allGroupingKeys = concat( - keys(grandparentRatingKeyToParentRatingKey), - keys(parentRatingKeyToProgramId), - ); - - const existingGroupings = await this.timer.timeAsync( - `selecting grouping external ids (${allGroupingKeys.length})`, - () => - this.drizzleDB.query.programGroupingExternalId.findMany({ - where: (fields, { eq, and, inArray }) => - and( - eq(fields.sourceType, mediaSourceType), - eq(fields.mediaSourceId, mediaSourceId), - inArray(fields.externalKey, allGroupingKeys), - ), - with: { - grouping: true, - }, - }), - ); - - const foundGroupingRatingKeys = map(existingGroupings, 'externalKey'); - const missingGroupingRatingKeys = difference( - allGroupingKeys, - foundGroupingRatingKeys, - ); - const grandparentKeys = new Set(keys(grandparentRatingKeyToProgramId)); - const missingGrandparents = filter(missingGroupingRatingKeys, (s) => - grandparentKeys.has(s), - ); - - const updatesByType: Record> = { - album: new Set(), - artist: new Set(), - season: new Set(), - show: new Set(), - } as const; - - for (const group of existingGroupings) { - for (const { - program: upsertedProgram, - programWithHierarchy: { grandparentKey, parentKey }, - } of relevantPrograms) { - if (group.externalKey === grandparentKey) { - switch (upsertedProgram.type) { - case ProgramType.Episode: - upsertedProgram.tvShowUuid = group.groupUuid; - updatesByType[ProgramGroupingType.Show].add(upsertedProgram.uuid); - break; - case ProgramType.Track: - upsertedProgram.artistUuid = group.groupUuid; - updatesByType[ProgramGroupingType.Artist].add( - upsertedProgram.uuid, - ); - break; - case 'movie': - case 'music_video': - case 'other_video': - default: - this.logger.warn( - 'Unexpected program type %s when calculating hierarchy. id = %s', - upsertedProgram.type, - upsertedProgram.uuid, - ); - break; - } - } else if (group.externalKey === parentKey) { - switch (upsertedProgram.type) { - case ProgramType.Episode: - upsertedProgram.seasonUuid = group.groupUuid; - updatesByType[ProgramGroupingType.Season].add( - upsertedProgram.uuid, - ); - break; - case ProgramType.Track: - upsertedProgram.albumUuid = group.groupUuid; - updatesByType[ProgramGroupingType.Album].add( - upsertedProgram.uuid, - ); - break; - case 'movie': - case 'music_video': - case 'other_video': - default: - this.logger.warn( - 'Unexpected program type %s when calculating hierarchy. id = %s', - upsertedProgram.type, - upsertedProgram.uuid, - ); - break; - } - } - } - } - - // New ones - const groupings: NewProgramGrouping[] = []; - const externalIds: NewProgramGroupingExternalId[] = []; - for (const missingGrandparent of missingGrandparents) { - const matchingPrograms = filter( - relevantPrograms, - ({ programWithHierarchy: { grandparentKey } }) => - grandparentKey === missingGrandparent, - ); - - if (isEmpty(matchingPrograms)) { - continue; - } - - const grandparentGrouping = ProgramGroupingMinter.mintGrandparentGrouping( - matchingPrograms[0]!.programWithHierarchy, - ); - - if (isNull(grandparentGrouping)) { - devAssert(false); - continue; - } - - matchingPrograms.forEach(({ program }) => { - if (grandparentGrouping.type === ProgramGroupingType.Artist) { - program.artistUuid = grandparentGrouping.uuid; - updatesByType[ProgramGroupingType.Artist].add(program.uuid); - } else if (grandparentGrouping.type === ProgramGroupingType.Show) { - program.tvShowUuid = grandparentGrouping.uuid; - updatesByType[ProgramGroupingType.Show].add(program.uuid); - } - }); - - const parentKeys = [ - ...(grandparentRatingKeyToParentRatingKey[missingGrandparent] ?? - new Set()), - ]; - const parents = reject(parentKeys, (parent) => - foundGroupingRatingKeys.includes(parent), - ); - - // const existingParents = seq.collect( - // existingParentKeys, - // (key) => existingGroupingsByKey[key], - // ); - // Fix mappings if we have to... - // const existingParentsNeedingUpdate = existingParents.filter(parent => { - // if (parent.type === ProgramGroupingType.Album && parent.artistUuid !== grandparentGrouping.uuid) { - // parent.artistUuid = grandparentGrouping.uuid; - // return true; - // } else if (parent.type === ProgramGroupingType.Season && parent.showUuid !== grandparentGrouping.uuid) { - // return true; - // } - // return false; - // }); - - for (const parentKey of parents) { - const programIds = parentRatingKeyToProgramId[parentKey]; - if (!programIds || programIds.size === 0) { - devAssert(false); - continue; - } - - const programs = filter(relevantPrograms, ({ program }) => - programIds.has(program.uuid), - ); - - // Also should never happen... - if (isEmpty(programs)) { - devAssert(false); - continue; - } - - devAssert( - () => uniq(map(programs, ({ program: p }) => p.type)).length === 1, - ); - - const parentGrouping = ProgramGroupingMinter.mintParentGrouping( - programs[0]!.programWithHierarchy, - ); - - if (!parentGrouping) { - continue; - } - - programs.forEach(({ program }) => { - if (program.type === ProgramType.Episode) { - program.seasonUuid = parentGrouping.uuid; - updatesByType[ProgramGroupingType.Season].add(program.uuid); - } else if (program.type === ProgramType.Track) { - program.albumUuid = parentGrouping.uuid; - updatesByType[ProgramGroupingType.Album].add(program.uuid); - } - }); - - if (parentGrouping.type === ProgramGroupingType.Season) { - parentGrouping.showUuid = grandparentGrouping.uuid; - } else if (parentGrouping.type === ProgramGroupingType.Album) { - parentGrouping.artistUuid = grandparentGrouping.uuid; - } - - groupings.push(parentGrouping); - externalIds.push( - ...ProgramGroupingMinter.mintGroupingExternalIds( - programs[0]!.programWithHierarchy, - parentGrouping.uuid, - mediaSourceName, - mediaSourceId, - 'parent', - ), - ); - } - - groupings.push(grandparentGrouping); - externalIds.push( - ...ProgramGroupingMinter.mintGroupingExternalIds( - matchingPrograms[0]!.programWithHierarchy, - grandparentGrouping.uuid, - mediaSourceName, - mediaSourceId, - 'grandparent', - ), - ); - } - - if (!isEmpty(groupings)) { - await this.timer.timeAsync('upsert program_groupings', () => - this.db - .transaction() - .execute((tx) => - tx - .insertInto('programGrouping') - .values(groupings) - .executeTakeFirstOrThrow(), - ), - ); - } - - if (!isEmpty(externalIds)) { - await this.timer.timeAsync('upsert program_grouping external ids', () => - Promise.all( - chunk( - externalIds, //.map(toInsertableProgramGroupingExternalId), - 100, - ).map((externalIds) => - this.db - .transaction() - .execute((tx) => - this.upsertProgramGroupingExternalIdsChunk(externalIds, tx), - ), - ), - ), - ); - } - - const hasUpdates = some(updatesByType, (updates) => updates.size > 0); - - if (hasUpdates) { - // Surprisingly it's faster to do these all at once... - await this.timer.timeAsync('update program relations', () => - this.db.transaction().execute(async (tx) => { - // For each program, we produce 3 SQL variables: when = ?, then = ?, and uuid in [?]. - // We have to chunk by type in order to ensure we don't go over the variable limit - const tvShowIdUpdates = [...updatesByType[ProgramGroupingType.Show]]; - - const chunkSize = run(() => { - const envVal = getNumericEnvVar( - TUNARR_ENV_VARS.DEBUG__PROGRAM_GROUPING_UPDATE_CHUNK_SIZE, - ); - - if (isNonEmptyString(envVal) && !isNaN(parseInt(envVal))) { - return Math.min(10_000, parseInt(envVal)); - } - return DEFAULT_PROGRAM_GROUPING_UPDATE_CHUNK_SIZE; - }); - - const updates: Promise[] = []; - - if (!isEmpty(tvShowIdUpdates)) { - // Should produce up to 30_000 variables each iteration... - for (const idChunk of chunk(tvShowIdUpdates, chunkSize)) { - updates.push( - tx - .updateTable('program') - .set((eb) => ({ - tvShowUuid: reduce( - idChunk, - (acc, curr) => - acc - .when('program.uuid', '=', curr) - .then(upsertedProgramById[curr]!.tvShowUuid), - eb.case() as unknown as ProgramRelationCaseBuilder, - ) - .else(eb.ref('program.tvShowUuid')) - .end(), - })) - .where('program.uuid', 'in', idChunk) - .execute(), - ); - } - } - - const seasonIdUpdates = [ - ...updatesByType[ProgramGroupingType.Season], - ]; - - if (!isEmpty(seasonIdUpdates)) { - // Should produce up to 30_000 variables each iteration... - for (const idChunk of chunk(seasonIdUpdates, chunkSize)) { - updates.push( - tx - .updateTable('program') - .set((eb) => ({ - seasonUuid: reduce( - idChunk, - (acc, curr) => - acc - .when('program.uuid', '=', curr) - .then(upsertedProgramById[curr]!.seasonUuid), - eb.case() as unknown as ProgramRelationCaseBuilder, - ) - .else(eb.ref('program.seasonUuid')) - .end(), - })) - .where('program.uuid', 'in', idChunk) - .execute(), - ); - } - } - - const musicArtistUpdates = [ - ...updatesByType[ProgramGroupingType.Artist], - ]; - - if (!isEmpty(musicArtistUpdates)) { - // Should produce up to 30_000 variables each iteration... - for (const idChunk of chunk(musicArtistUpdates, chunkSize)) { - updates.push( - tx - .updateTable('program') - .set((eb) => ({ - artistUuid: reduce( - idChunk, - (acc, curr) => - acc - .when('program.uuid', '=', curr) - .then(upsertedProgramById[curr]!.artistUuid), - eb.case() as unknown as ProgramRelationCaseBuilder, - ) - .else(eb.ref('program.artistUuid')) - .end(), - })) - .where('program.uuid', 'in', idChunk) - .execute(), - ); - } - } - - const musicAlbumUpdates = [ - ...updatesByType[ProgramGroupingType.Album], - ]; - - if (!isEmpty(musicAlbumUpdates)) { - // Should produce up to 30_000 variables each iteration... - for (const idChunk of chunk(musicAlbumUpdates, chunkSize)) { - updates.push( - tx - .updateTable('program') - .set((eb) => ({ - albumUuid: reduce( - idChunk, - (acc, curr) => - acc - .when('program.uuid', '=', curr) - .then(upsertedProgramById[curr]!.albumUuid), - eb.case() as unknown as ProgramRelationCaseBuilder, - ) - .else(eb.ref('program.albumUuid')) - .end(), - })) - .where('program.uuid', 'in', idChunk) - .execute(), - ); - } - } - - await Promise.all(updates); - }), - ); - } - } - - private async upsertProgramGroupingExternalIdsChunkOrm( - ids: ( - | NewSingleOrMultiProgramGroupingExternalId - | NewProgramGroupingExternalId - )[], - tx: BaseSQLiteDatabase<'sync', RunResult, typeof schema> = this.drizzleDB, - ): Promise { - if (ids.length === 0) { - return []; - } - - const [singles, multiples] = partition(ids, (id) => - isValidSingleExternalIdType(id.sourceType), - ); - - const promises: Promise[] = []; - - if (singles.length > 0) { - promises.push( - tx - .insert(ProgramGroupingExternalId) - .values(singles.map(toInsertableProgramGroupingExternalId)) - .onConflictDoUpdate({ - target: [ - ProgramGroupingExternalId.groupUuid, - ProgramGroupingExternalId.sourceType, - ], - targetWhere: sql`media_source_id is null`, - set: { - updatedAt: sql`excluded.updated_at`, - externalFilePath: sql`excluded.external_file_path`, - groupUuid: sql`excluded.group_uuid`, - externalKey: sql`excluded.external_key`, - }, - }) - .returning() - .execute(), - // .onConflict((oc) => - // oc - // .columns(['groupUuid', 'sourceType']) - // .where('mediaSourceId', 'is', null) - // .doUpdateSet((eb) => ({ - // updatedAt: eb.ref('excluded.updatedAt'), - // externalFilePath: eb.ref('excluded.externalFilePath'), - // groupUuid: eb.ref('excluded.groupUuid'), - // externalKey: eb.ref('excluded.externalKey'), - // })), - // ) - // .executeTakeFirstOrThrow(), - ); - } - - if (multiples.length > 0) { - promises.push( - tx - .insert(ProgramGroupingExternalId) - .values(multiples.map(toInsertableProgramGroupingExternalId)) - .onConflictDoUpdate({ - target: [ - ProgramGroupingExternalId.groupUuid, - ProgramGroupingExternalId.sourceType, - ProgramGroupingExternalId.mediaSourceId, - ], - targetWhere: sql`media_source_id is not null`, - set: { - updatedAt: sql`excluded.updated_at`, - externalFilePath: sql`excluded.external_file_path`, - groupUuid: sql`excluded.group_uuid`, - externalKey: sql`excluded.external_key`, - }, - }) - .returning() - .execute(), - // .onConflict((oc) => - // oc - // .columns(['groupUuid', 'sourceType', 'mediaSourceId']) - // .where('mediaSourceId', 'is not', null) - // .doUpdateSet((eb) => ({ - // updatedAt: eb.ref('excluded.updatedAt'), - // externalFilePath: eb.ref('excluded.externalFilePath'), - // groupUuid: eb.ref('excluded.groupUuid'), - // externalKey: eb.ref('excluded.externalKey'), - // })), - // ) - // .executeTakeFirstOrThrow(), - ); - } - - return (await Promise.all(promises)).flat(); - } - - private async upsertProgramGroupingExternalIdsChunk( - ids: ( - | NewSingleOrMultiProgramGroupingExternalId - | NewProgramGroupingExternalId - )[], - tx: Kysely = this.db, - ): Promise { - if (ids.length === 0) { - return; - } - - const [singles, multiples] = partition(ids, (id) => - isValidSingleExternalIdType(id.sourceType), - ); - - const promises: Promise[] = []; - - if (singles.length > 0) { - promises.push( - tx - .insertInto('programGroupingExternalId') - .values(singles.map(toInsertableProgramGroupingExternalId)) - .onConflict((oc) => - oc - .columns(['groupUuid', 'sourceType']) - .where('mediaSourceId', 'is', null) - .doUpdateSet((eb) => ({ - updatedAt: eb.ref('excluded.updatedAt'), - externalFilePath: eb.ref('excluded.externalFilePath'), - groupUuid: eb.ref('excluded.groupUuid'), - externalKey: eb.ref('excluded.externalKey'), - })), - ) - .executeTakeFirstOrThrow(), - ); - } - - if (multiples.length > 0) { - promises.push( - tx - .insertInto('programGroupingExternalId') - .values(multiples.map(toInsertableProgramGroupingExternalId)) - .onConflict((oc) => - oc - .columns(['groupUuid', 'sourceType', 'mediaSourceId']) - .where('mediaSourceId', 'is not', null) - .doUpdateSet((eb) => ({ - updatedAt: eb.ref('excluded.updatedAt'), - externalFilePath: eb.ref('excluded.externalFilePath'), - groupUuid: eb.ref('excluded.groupUuid'), - externalKey: eb.ref('excluded.externalKey'), - })), - ) - .executeTakeFirstOrThrow(), - ); - } - - await Promise.all(promises); - } - - private schedulePlexExternalIdsTask(upsertedPrograms: ProgramDao[]) { - PlexTaskQueue.pause(); - this.timer.timeSync('schedule Plex external IDs tasks', () => { - forEach( - filter(upsertedPrograms, { sourceType: ProgramSourceType.PLEX }), - (program) => { - try { - const task = this.savePlexProgramExternalIdsTaskFactory(); - task.logLevel = 'trace'; - PlexTaskQueue.add(task, { programId: program.uuid }).catch((e) => { - this.logger.error( - e, - 'Error saving external IDs for program %O', - program, - ); - }); - } catch (e) { - this.logger.error( - e, - 'Failed to schedule external IDs task for persisted program: %O', - program, - ); - } - }, - ); - }); - } - - private scheduleJellyfinExternalIdsTask(upsertedPrograms: ProgramDao[]) { - JellyfinTaskQueue.pause(); - this.timer.timeSync('Schedule Jellyfin external IDs tasks', () => { - forEach( - filter(upsertedPrograms, (p) => p.sourceType === 'jellyfin'), - (program) => { - try { - const task = this.saveJellyfinProgramExternalIdsTask(); - JellyfinTaskQueue.add(task, { programId: program.uuid }).catch( - (e) => { - this.logger.error( - e, - 'Error saving external IDs for program %O', - program, - ); - }, - ); - } catch (e) { - this.logger.error( - e, - 'Failed to schedule external IDs task for persisted program: %O', - program, - ); - } - }, - ); - }); + emptyTrashPrograms(): Promise { + return this.stateRepo.emptyTrashPrograms(); } } diff --git a/server/src/db/channel/BasicChannelRepository.ts b/server/src/db/channel/BasicChannelRepository.ts new file mode 100644 index 00000000..e4804b0a --- /dev/null +++ b/server/src/db/channel/BasicChannelRepository.ts @@ -0,0 +1,532 @@ +import { ChannelQueryBuilder } from '@/db/ChannelQueryBuilder.js'; +import { CacheImageService } from '@/services/cacheImageService.js'; +import { ChannelNotFoundError } from '@/types/errors.js'; +import { KEYS } from '@/types/inject.js'; +import { Result } from '@/types/result.js'; +import { Maybe } from '@/types/util.js'; +import dayjs from '@/util/dayjs.js'; +import { booleanToNumber } from '@/util/sqliteUtil.js'; +import type { SaveableChannel, Watermark } from '@tunarr/types'; +import { eq } from 'drizzle-orm'; +import { inject, injectable } from 'inversify'; +import { Kysely } from 'kysely'; +import { jsonArrayFrom } from 'kysely/helpers/sqlite'; +import { + isEmpty, + isNil, + isNumber, + isString, + isUndefined, + map, + sum, +} from 'lodash-es'; +import { MarkRequired } from 'ts-essentials'; +import { v4 } from 'uuid'; +import { isDefined, isNonEmptyString } from '../../util/index.ts'; +import { ChannelAndLineup } from '../interfaces/IChannelDB.ts'; +import { + Channel, + ChannelOrm, + ChannelUpdate, + NewChannel, +} from '../schema/Channel.ts'; +import { NewChannelFillerShow } from '../schema/ChannelFillerShow.ts'; +import { + ChannelWithRelations, + ChannelOrmWithTranscodeConfig, +} from '../schema/derivedTypes.ts'; +import { + NewChannelSubtitlePreference, +} from '../schema/SubtitlePreferences.ts'; +import type { DB } from '../schema/db.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; +import { LineupRepository } from './LineupRepository.ts'; + +function sanitizeChannelWatermark( + watermark: Maybe, +): Maybe { + if (isUndefined(watermark)) { + return; + } + + const validFadePoints = (watermark.fadeConfig ?? []).filter( + (conf) => conf.periodMins > 0, + ); + + return { + ...watermark, + fadeConfig: isEmpty(validFadePoints) ? undefined : validFadePoints, + }; +} + +function updateRequestToChannel(updateReq: SaveableChannel): ChannelUpdate { + const sanitizedWatermark = sanitizeChannelWatermark(updateReq.watermark); + + return { + number: updateReq.number, + watermark: sanitizedWatermark + ? JSON.stringify(sanitizedWatermark) + : undefined, + icon: JSON.stringify(updateReq.icon), + guideMinimumDuration: updateReq.guideMinimumDuration, + groupTitle: updateReq.groupTitle, + disableFillerOverlay: booleanToNumber(updateReq.disableFillerOverlay), + startTime: +dayjs(updateReq.startTime).second(0).millisecond(0), + offline: JSON.stringify(updateReq.offline), + name: updateReq.name, + duration: updateReq.duration, + stealth: booleanToNumber(updateReq.stealth), + fillerRepeatCooldown: updateReq.fillerRepeatCooldown, + guideFlexTitle: updateReq.guideFlexTitle, + transcodeConfigId: updateReq.transcodeConfigId, + streamMode: updateReq.streamMode, + subtitlesEnabled: booleanToNumber(updateReq.subtitlesEnabled), + } satisfies ChannelUpdate; +} + +function createRequestToChannel(saveReq: SaveableChannel): NewChannel { + const now = +dayjs(); + + return { + uuid: v4(), + createdAt: now, + updatedAt: now, + number: saveReq.number, + watermark: saveReq.watermark ? JSON.stringify(saveReq.watermark) : null, + icon: JSON.stringify(saveReq.icon), + guideMinimumDuration: saveReq.guideMinimumDuration, + groupTitle: saveReq.groupTitle, + disableFillerOverlay: saveReq.disableFillerOverlay ? 1 : 0, + startTime: saveReq.startTime, + offline: JSON.stringify(saveReq.offline), + name: saveReq.name, + duration: saveReq.duration, + stealth: saveReq.stealth ? 1 : 0, + fillerRepeatCooldown: saveReq.fillerRepeatCooldown, + guideFlexTitle: saveReq.guideFlexTitle, + streamMode: saveReq.streamMode, + transcodeConfigId: saveReq.transcodeConfigId, + subtitlesEnabled: booleanToNumber(saveReq.subtitlesEnabled), + } satisfies NewChannel; +} + +@injectable() +export class BasicChannelRepository { + constructor( + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + @inject(CacheImageService) private cacheImageService: CacheImageService, + @inject(KEYS.LineupRepository) private lineupRepository: LineupRepository, + ) {} + + async channelExists(channelId: string): Promise { + const channel = await this.db + .selectFrom('channel') + .where('channel.uuid', '=', channelId) + .select('uuid') + .executeTakeFirst(); + return !isNil(channel); + } + + getChannelOrm( + id: string | number, + ): Promise> { + return this.drizzleDB.query.channels.findFirst({ + where: (channel, { eq }) => { + return isString(id) ? eq(channel.uuid, id) : eq(channel.number, id); + }, + with: { + transcodeConfig: true, + }, + }); + } + + getChannel(id: string | number): Promise>; + getChannel( + id: string | number, + includeFiller: true, + ): Promise>>; + async getChannel( + id: string | number, + includeFiller: boolean = false, + ): Promise> { + return this.db + .selectFrom('channel') + .$if(isString(id), (eb) => eb.where('channel.uuid', '=', id as string)) + .$if(isNumber(id), (eb) => eb.where('channel.number', '=', id as number)) + .$if(includeFiller, (eb) => + eb.select((qb) => + jsonArrayFrom( + qb + .selectFrom('channelFillerShow') + .whereRef('channel.uuid', '=', 'channelFillerShow.channelUuid') + .select([ + 'channelFillerShow.channelUuid', + 'channelFillerShow.fillerShowUuid', + 'channelFillerShow.cooldown', + 'channelFillerShow.weight', + ]), + ).as('fillerShows'), + ), + ) + .selectAll() + .executeTakeFirst(); + } + + getChannelBuilder(id: string | number) { + return ChannelQueryBuilder.createForIdOrNumber(this.db, id); + } + + getAllChannels(): Promise { + return this.drizzleDB.query.channels + .findMany({ + orderBy: (fields, { asc }) => asc(fields.number), + }) + .execute(); + } + + async saveChannel( + createReq: SaveableChannel, + ): Promise> { + const existing = await this.getChannel(createReq.number); + if (!isNil(existing)) { + throw new Error( + `Channel with number ${createReq.number} already exists: ${existing.name}`, + ); + } + + const channel = await this.db.transaction().execute(async (tx) => { + const channel = await tx + .insertInto('channel') + .values(createRequestToChannel(createReq)) + .returningAll() + .executeTakeFirst(); + + if (!channel) { + throw new Error('Error while saving new channel.'); + } + + if (!isEmpty(createReq.fillerCollections)) { + await tx + .insertInto('channelFillerShow') + .values( + map( + createReq.fillerCollections, + (fc) => + ({ + channelUuid: channel.uuid, + cooldown: fc.cooldownSeconds, + fillerShowUuid: fc.id, + weight: fc.weight, + }) satisfies NewChannelFillerShow, + ), + ) + .execute(); + } + + const subtitlePreferences = createReq.subtitlePreferences?.map( + (pref) => + ({ + channelId: channel.uuid, + uuid: v4(), + languageCode: pref.langugeCode, + allowExternal: booleanToNumber(pref.allowExternal), + allowImageBased: booleanToNumber(pref.allowImageBased), + filterType: pref.filter, + priority: pref.priority, + }) satisfies NewChannelSubtitlePreference, + ); + if (subtitlePreferences) { + await tx + .insertInto('channelSubtitlePreferences') + .values(subtitlePreferences) + .executeTakeFirstOrThrow(); + } + + return channel; + }); + + await this.lineupRepository.createLineup(channel.uuid); + + if (isDefined(createReq.onDemand) && createReq.onDemand.enabled) { + const db = await this.lineupRepository.getFileDb(channel.uuid); + await db.update((lineup) => { + lineup.onDemandConfig = { + state: 'paused', + cursor: 0, + }; + }); + } + + return { + channel, + lineup: (await this.lineupRepository.getFileDb(channel.uuid)).data, + }; + } + + async updateChannel( + id: string, + updateReq: SaveableChannel, + ): Promise> { + const channel = await this.getChannel(id); + + if (isNil(channel)) { + throw new ChannelNotFoundError(id); + } + + const update = updateRequestToChannel(updateReq); + + if ( + isNonEmptyString(updateReq.watermark?.url) && + URL.canParse(updateReq.watermark.url) + ) { + const url = updateReq.watermark?.url; + const parsed = new URL(url); + if (!parsed.hostname.includes('localhost')) { + await Result.attemptAsync(() => + this.cacheImageService.getOrDownloadImageUrl(url), + ); + } + } + + await this.db.transaction().execute(async (tx) => { + await tx + .updateTable('channel') + .where('channel.uuid', '=', id) + .set(update) + .executeTakeFirstOrThrow(); + + if (!isEmpty(updateReq.fillerCollections)) { + const channelFillerShows = map( + updateReq.fillerCollections, + (filler) => + ({ + cooldown: filler.cooldownSeconds, + channelUuid: channel.uuid, + fillerShowUuid: filler.id, + weight: filler.weight, + }) satisfies NewChannelFillerShow, + ); + + await tx + .deleteFrom('channelFillerShow') + .where('channelFillerShow.channelUuid', '=', channel.uuid) + .executeTakeFirstOrThrow(); + await tx + .insertInto('channelFillerShow') + .values(channelFillerShows) + .executeTakeFirstOrThrow(); + } + const subtitlePreferences = updateReq.subtitlePreferences?.map( + (pref) => + ({ + channelId: channel.uuid, + uuid: v4(), + languageCode: pref.langugeCode, + allowExternal: booleanToNumber(pref.allowExternal), + allowImageBased: booleanToNumber(pref.allowImageBased), + filterType: pref.filter, + priority: pref.priority, + }) satisfies NewChannelSubtitlePreference, + ); + await tx + .deleteFrom('channelSubtitlePreferences') + .where('channelSubtitlePreferences.channelId', '=', channel.uuid) + .executeTakeFirstOrThrow(); + if (subtitlePreferences) { + await tx + .insertInto('channelSubtitlePreferences') + .values(subtitlePreferences) + .executeTakeFirstOrThrow(); + } + }); + + if (isDefined(updateReq.onDemand)) { + const db = await this.lineupRepository.getFileDb(id); + await db.update((lineup) => { + if (updateReq.onDemand?.enabled ?? false) { + lineup.onDemandConfig = { + state: 'paused', + cursor: 0, + }; + } else { + delete lineup['onDemandConfig']; + } + }); + } + + return { + channel: (await this.getChannel(id, true))!, + lineup: await this.lineupRepository.loadLineup(id), + }; + } + + updateChannelDuration(id: string, newDur: number): Promise { + return this.drizzleDB + .update(Channel) + .set({ + duration: newDur, + }) + .where(eq(Channel.uuid, id)) + .limit(1) + .execute() + .then((_) => _.changes); + } + + async updateChannelStartTime(id: string, newTime: number): Promise { + return this.db + .updateTable('channel') + .where('channel.uuid', '=', id) + .set('startTime', newTime) + .executeTakeFirst() + .then(() => {}); + } + + async syncChannelDuration(id: string): Promise { + const channelAndLineup = await this.lineupRepository.loadChannelAndLineup(id); + if (!channelAndLineup) { + return false; + } + const { channel, lineup } = channelAndLineup; + const lineupDuration = sum(map(lineup.items, (item) => item.durationMs)); + if (lineupDuration !== channel.duration) { + await this.db + .updateTable('channel') + .where('channel.uuid', '=', id) + .set('duration', lineupDuration) + .executeTakeFirst(); + return true; + } + return false; + } + + async copyChannel(id: string): Promise> { + const channel = await this.getChannel(id); + if (!channel) { + throw new Error(`Cannot copy channel: channel ID: ${id} not found`); + } + + const lineup = await this.lineupRepository.loadLineup(id); + + const newChannelId = v4(); + const now = +dayjs(); + const newChannel = await this.db.transaction().execute(async (tx) => { + const { number: maxId } = await tx + .selectFrom('channel') + .select('number') + .orderBy('number desc') + .limit(1) + .executeTakeFirstOrThrow(); + const newChannel = await tx + .insertInto('channel') + .values({ + ...channel, + uuid: newChannelId, + name: `${channel.name} - Copy`, + number: maxId + 1, + icon: JSON.stringify(channel.icon), + offline: JSON.stringify(channel.offline), + watermark: JSON.stringify(channel.watermark), + createdAt: now, + updatedAt: now, + transcoding: null, + }) + .returningAll() + .executeTakeFirstOrThrow(); + + await tx + .insertInto('channelFillerShow') + .columns(['channelUuid', 'cooldown', 'fillerShowUuid', 'weight']) + .expression((eb) => + eb + .selectFrom('channelFillerShow') + .select([ + eb.val(newChannelId).as('channelUuid'), + 'channelFillerShow.cooldown', + 'channelFillerShow.fillerShowUuid', + 'channelFillerShow.weight', + ]) + .where('channelFillerShow.channelUuid', '=', channel.uuid), + ) + .executeTakeFirstOrThrow(); + + await tx + .insertInto('channelPrograms') + .columns(['channelUuid', 'programUuid']) + .expression((eb) => + eb + .selectFrom('channelPrograms') + .select([ + eb.val(newChannelId).as('channelUuid'), + 'channelPrograms.programUuid', + ]) + .where('channelPrograms.channelUuid', '=', channel.uuid), + ) + .executeTakeFirstOrThrow(); + + await tx + .insertInto('channelCustomShows') + .columns(['channelUuid', 'customShowUuid']) + .expression((eb) => + eb + .selectFrom('channelCustomShows') + .select([ + eb.val(newChannelId).as('channelUuid'), + 'channelCustomShows.customShowUuid', + ]) + .where('channelCustomShows.channelUuid', '=', channel.uuid), + ) + .executeTakeFirstOrThrow(); + + return newChannel; + }); + + const newLineup = await this.lineupRepository.saveLineup(newChannel.uuid, lineup); + + return { + channel: newChannel, + lineup: newLineup, + }; + } + + async deleteChannel( + channelId: string, + blockOnLineupUpdates: boolean = false, + ): Promise { + let marked = false; + try { + await this.lineupRepository.markLineupFileForDeletion(channelId); + marked = true; + + await this.db.transaction().execute(async (tx) => { + await tx + .deleteFrom('channelSubtitlePreferences') + .where('channelId', '=', channelId) + .executeTakeFirstOrThrow(); + await tx + .deleteFrom('channel') + .where('uuid', '=', channelId) + .limit(1) + .executeTakeFirstOrThrow(); + }); + + const removeRefs = () => + this.lineupRepository.removeRedirectReferences(channelId).catch(() => { + // Errors are logged inside removeRedirectReferences + }); + + if (blockOnLineupUpdates) { + await removeRefs(); + } else { + setTimeout(() => { + removeRefs().catch(() => {}); + }); + } + } catch (e) { + if (marked) { + await this.lineupRepository.restoreLineupFile(channelId); + } + throw e; + } + } +} diff --git a/server/src/db/channel/ChannelConfigRepository.ts b/server/src/db/channel/ChannelConfigRepository.ts new file mode 100644 index 00000000..9bc2afa7 --- /dev/null +++ b/server/src/db/channel/ChannelConfigRepository.ts @@ -0,0 +1,23 @@ +import { KEYS } from '@/types/inject.js'; +import { inject, injectable } from 'inversify'; +import type { Kysely } from 'kysely'; +import type { DB } from '../schema/db.ts'; +import type { ChannelSubtitlePreferences } from '../schema/SubtitlePreferences.ts'; + +@injectable() +export class ChannelConfigRepository { + constructor( + @inject(KEYS.Database) private db: Kysely, + ) {} + + async getChannelSubtitlePreferences( + id: string, + ): Promise { + return this.db + .selectFrom('channelSubtitlePreferences') + .selectAll() + .where('channelId', '=', id) + .orderBy('priority asc') + .execute(); + } +} diff --git a/server/src/db/channel/ChannelProgramRepository.ts b/server/src/db/channel/ChannelProgramRepository.ts new file mode 100644 index 00000000..d1a0159c --- /dev/null +++ b/server/src/db/channel/ChannelProgramRepository.ts @@ -0,0 +1,478 @@ +import { KEYS } from '@/types/inject.js'; +import type { Maybe, PagedResult } from '@/types/util.js'; +import type { ContentProgramType } from '@tunarr/types/schemas'; +import { and, asc, count, countDistinct, eq, isNotNull } from 'drizzle-orm'; +import { inject, injectable } from 'inversify'; +import type { Kysely } from 'kysely'; +import { chunk, flatten, groupBy, sum, uniq } from 'lodash-es'; +import type { MarkRequired } from 'ts-essentials'; +import { + createManyRelationAgg, + mapRawJsonRelationResult, +} from '../../util/drizzleUtil.ts'; +import type { PageParams } from '../interfaces/IChannelDB.ts'; +import { withFallbackPrograms, withPrograms } from '../programQueryHelpers.ts'; +import { Artwork } from '../schema/Artwork.ts'; +import { ChannelPrograms } from '../schema/ChannelPrograms.ts'; +import type { ProgramDao } from '../schema/Program.ts'; +import { Program, ProgramType } from '../schema/Program.ts'; +import type { ProgramExternalId } from '../schema/ProgramExternalId.ts'; +import { + ProgramGrouping, + ProgramGroupingType, +} from '../schema/ProgramGrouping.ts'; +import { ProgramGroupingExternalIdOrm } from '../schema/ProgramGroupingExternalId.ts'; +import type { DB } from '../schema/db.ts'; +import type { + ChannelOrmWithPrograms, + ChannelOrmWithRelations, + ChannelWithPrograms, + MusicArtistOrm, + MusicArtistWithExternalIds, + ProgramGroupingOrmWithRelations, + ProgramWithRelationsOrm, + TvSeasonOrm, + TvShowOrm, +} from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; + +@injectable() +export class ChannelProgramRepository { + constructor( + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + ) {} + + async getChannelAndPrograms( + uuid: string, + typeFilter?: ContentProgramType, + ): Promise>> { + const channelsAndPrograms = await this.drizzleDB.query.channels.findFirst({ + where: (fields, { eq }) => eq(fields.uuid, uuid), + with: { + channelPrograms: { + with: { + program: { + with: { + show: true, + season: true, + artist: true, + album: true, + externalIds: true, + }, + }, + }, + }, + }, + orderBy: (fields, { asc }) => asc(fields.number), + }); + + if (channelsAndPrograms) { + const programs = typeFilter + ? channelsAndPrograms.channelPrograms + .map(({ program }) => program) + .filter((p) => p.type === typeFilter) + : channelsAndPrograms.channelPrograms.map(({ program }) => program); + return { + ...channelsAndPrograms, + programs, + } satisfies MarkRequired; + } + + return; + } + + async getChannelAndProgramsOld( + uuid: string, + ): Promise { + return this.db + .selectFrom('channel') + .selectAll(['channel']) + .where('channel.uuid', '=', uuid) + .leftJoin( + 'channelPrograms', + 'channel.uuid', + 'channelPrograms.channelUuid', + ) + .select((eb) => + withPrograms(eb, { + joins: { + customShows: true, + tvShow: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + tvSeason: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + trackArtist: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + trackAlbum: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + }, + }), + ) + .groupBy('channel.uuid') + .orderBy('channel.number asc') + .executeTakeFirst(); + } + + async getChannelTvShows( + id: string, + pageParams?: PageParams, + ): Promise> { + const groups = await this.drizzleDB + .select({ + programGrouping: ProgramGrouping, + artwork: createManyRelationAgg( + this.drizzleDB + .select() + .from(Artwork) + .where(eq(ProgramGrouping.uuid, Artwork.groupingId)) + .as('artwork'), + 'artwork', + ), + }) + .from(ChannelPrograms) + .where( + and( + eq(ChannelPrograms.channelUuid, id), + eq(Program.type, ProgramType.Episode), + isNotNull(Program.tvShowUuid), + eq(ProgramGrouping.type, ProgramGroupingType.Show), + ), + ) + .groupBy(Program.tvShowUuid) + .orderBy(asc(ProgramGrouping.uuid)) + .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) + .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.tvShowUuid)) + .offset(pageParams?.offset ?? 0) + .limit(pageParams?.limit ?? 1_000_000); + + const countPromise = this.drizzleDB + .select({ + count: countDistinct(ProgramGrouping.uuid), + }) + .from(ChannelPrograms) + .where( + and( + eq(ChannelPrograms.channelUuid, id), + eq(Program.type, ProgramType.Episode), + isNotNull(Program.tvShowUuid), + eq(ProgramGrouping.type, ProgramGroupingType.Show), + ), + ) + .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) + .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.tvShowUuid)); + + const externalIdQueries: Promise[] = []; + const seasonQueries: Promise[] = []; + for (const groupChunk of chunk(groups, 100)) { + const ids = groupChunk.map(({ programGrouping }) => programGrouping.uuid); + externalIdQueries.push( + this.drizzleDB.query.programGroupingExternalId.findMany({ + where: (fields, { inArray }) => inArray(fields.groupUuid, ids), + }), + ); + seasonQueries.push( + this.drizzleDB.query.programGrouping.findMany({ + where: (fields, { eq, and, inArray }) => + and( + eq(fields.type, ProgramGroupingType.Season), + inArray(fields.showUuid, ids), + ), + with: { + externalIds: true, + }, + }), + ); + } + + const [externalIdResults, seasonResults] = await Promise.all([ + Promise.all(externalIdQueries).then(flatten), + Promise.all(seasonQueries).then(flatten), + ]); + + const externalIdsByGroupId = groupBy( + externalIdResults, + (id) => id.groupUuid, + ); + const seasonByGroupId = groupBy(seasonResults, (season) => season.showUuid); + + const shows: TvShowOrm[] = []; + for (const { programGrouping, artwork } of groups) { + if (programGrouping.type === 'show') { + const seasons = + seasonByGroupId[programGrouping.uuid]?.filter( + (group): group is TvSeasonOrm => group.type === 'season', + ) ?? []; + shows.push({ + ...programGrouping, + type: 'show', + externalIds: externalIdsByGroupId[programGrouping.uuid] ?? [], + seasons, + artwork: mapRawJsonRelationResult(artwork, Artwork), + }); + } + } + + return { + total: sum((await countPromise).map(({ count }) => count)), + results: shows, + }; + } + + async getChannelMusicArtists( + id: string, + pageParams?: PageParams, + ): Promise> { + const groups = await this.drizzleDB + .select({ + programGrouping: ProgramGrouping, + }) + .from(ChannelPrograms) + .where( + and( + eq(ChannelPrograms.channelUuid, id), + eq(Program.type, ProgramType.Track), + isNotNull(Program.artistUuid), + eq(ProgramGrouping.type, ProgramGroupingType.Artist), + ), + ) + .groupBy(Program.artistUuid) + .orderBy(asc(ProgramGrouping.uuid)) + .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) + .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.artistUuid)) + .offset(pageParams?.offset ?? 0) + .limit(pageParams?.limit ?? 1_000_000); + + const countPromise = this.drizzleDB + .select({ + count: count(), + }) + .from(ChannelPrograms) + .where( + and( + eq(ChannelPrograms.channelUuid, id), + eq(Program.type, ProgramType.Episode), + isNotNull(Program.tvShowUuid), + eq(ProgramGrouping.type, ProgramGroupingType.Show), + ), + ) + .innerJoin(Program, eq(Program.uuid, ChannelPrograms.programUuid)) + .innerJoin(ProgramGrouping, eq(ProgramGrouping.uuid, Program.tvShowUuid)); + + const externalIdQueries: Promise[] = []; + const albumQueries: Promise[] = []; + for (const groupChunk of chunk(groups, 100)) { + const ids = groupChunk.map(({ programGrouping }) => programGrouping.uuid); + externalIdQueries.push( + this.drizzleDB.query.programGroupingExternalId.findMany({ + where: (fields, { inArray }) => inArray(fields.groupUuid, ids), + }), + ); + albumQueries.push( + this.drizzleDB.query.programGrouping.findMany({ + where: (fields, { eq, and, inArray }) => + and( + eq(fields.type, ProgramGroupingType.Season), + inArray(fields.showUuid, ids), + ), + with: { + externalIds: true, + }, + }), + ); + } + + const [externalIdResults, albumResults] = await Promise.all([ + Promise.all(externalIdQueries).then(flatten), + Promise.all(albumQueries).then(flatten), + ]); + + const externalIdsByGroupId = groupBy( + externalIdResults, + (id) => id.groupUuid, + ); + const seasonByGroupId = groupBy(albumResults, (season) => season.showUuid); + + const artists: MusicArtistOrm[] = []; + for (const { programGrouping } of groups) { + if (programGrouping.type === 'artist') { + const albums = + seasonByGroupId[programGrouping.uuid]?.filter( + (group): group is MusicAlbumOrm => group.type === 'album', + ) ?? []; + artists.push({ + ...programGrouping, + type: 'artist', + externalIds: externalIdsByGroupId[programGrouping.uuid] ?? [], + albums, + }); + } + } + + return { + total: sum((await countPromise).map(({ count }) => count)), + results: artists, + }; + } + + async getChannelPrograms( + id: string, + pageParams?: PageParams, + typeFilter?: ContentProgramType, + ): Promise> { + let query = this.drizzleDB + .select({ programId: ChannelPrograms.programUuid, count: count() }) + .from(ChannelPrograms) + .where( + and( + eq(ChannelPrograms.channelUuid, id), + typeFilter ? eq(Program.type, typeFilter) : undefined, + ), + ) + .innerJoin(Program, eq(ChannelPrograms.programUuid, Program.uuid)) + .$dynamic(); + + const countResult = (await query.execute())[0]?.count ?? 0; + + if (pageParams) { + query = query + .groupBy(Program.uuid) + .orderBy(asc(Program.title)) + .offset(pageParams.offset) + .limit(pageParams.limit); + } + + const results = await query.execute(); + + const materialized: ProgramWithRelationsOrm[] = []; + for (const idChunk of chunk( + results.map(({ programId }) => programId), + 100, + )) { + materialized.push( + ...(await this.drizzleDB.query.program.findMany({ + where: (fields, { inArray }) => inArray(fields.uuid, idChunk), + with: { + externalIds: true, + album: true, + artist: true, + season: true, + show: true, + artwork: true, + subtitles: true, + credits: true, + versions: { + with: { + mediaStreams: true, + mediaFiles: true, + chapters: true, + }, + }, + }, + orderBy: (fields, { asc }) => asc(fields.uuid), + })), + ); + } + + return { results: materialized, total: countResult }; + } + + getChannelProgramExternalIds(uuid: string): Promise { + return this.db + .selectFrom('channelPrograms') + .where('channelUuid', '=', uuid) + .innerJoin( + 'programExternalId', + 'channelPrograms.programUuid', + 'programExternalId.programUuid', + ) + .selectAll('programExternalId') + .execute(); + } + + async getChannelFallbackPrograms(uuid: string): Promise { + const result = await this.db + .selectFrom('channelFallback') + .where('channelFallback.channelUuid', '=', uuid) + .select(withFallbackPrograms) + .groupBy('channelFallback.channelUuid') + .executeTakeFirst(); + return result?.programs ?? []; + } + + async replaceChannelPrograms( + channelId: string, + programIds: string[], + ): Promise { + const uniqueIds = uniq(programIds); + await this.drizzleDB.transaction(async (tx) => { + await tx + .delete(ChannelPrograms) + .where(eq(ChannelPrograms.channelUuid, channelId)); + for (const c of chunk(uniqueIds, 250)) { + await tx + .insert(ChannelPrograms) + .values(c.map((id) => ({ channelUuid: channelId, programUuid: id }))); + } + }); + } + + findChannelsForProgramId(programId: string): Promise { + return this.drizzleDB.query.channelPrograms + .findMany({ + where: (cp, { eq }) => eq(cp.programUuid, programId), + with: { + channel: true, + }, + }) + .then((result) => result.map((row) => row.channel)); + } + + async getAllChannelsAndPrograms(): Promise { + return await this.drizzleDB.query.channels + .findMany({ + with: { + channelPrograms: { + with: { + program: { + with: { + album: true, + artist: true, + show: true, + season: true, + externalIds: true, + }, + }, + }, + }, + }, + orderBy: (fields, { asc }) => asc(fields.number), + }) + .then((result) => { + return result.map((channel) => { + const { omit } = require('lodash-es'); + const withoutJoinTable = omit(channel, 'channelPrograms'); + return { + ...withoutJoinTable, + programs: channel.channelPrograms.map((cp) => cp.program), + } satisfies ChannelOrmWithPrograms; + }); + }); + } +} diff --git a/server/src/db/channel/LineupRepository.ts b/server/src/db/channel/LineupRepository.ts new file mode 100644 index 00000000..a4fbcb71 --- /dev/null +++ b/server/src/db/channel/LineupRepository.ts @@ -0,0 +1,1225 @@ +import type { IProgramDB } from '@/db/interfaces/IProgramDB.js'; +import { FileSystemService } from '@/services/FileSystemService.js'; +import { KEYS } from '@/types/inject.js'; +import { jsonSchema } from '@/types/schemas.js'; +import { Nullable } from '@/types/util.js'; +import { Timer } from '@/util/Timer.js'; +import { asyncPool } from '@/util/asyncPool.js'; +import dayjs from '@/util/dayjs.js'; +import { fileExists } from '@/util/fsUtil.js'; +import { LoggerFactory } from '@/util/logging/LoggerFactory.js'; +import { MutexMap } from '@/util/mutexMap.js'; +import { seq } from '@tunarr/shared/util'; +import { + ChannelProgram, + ChannelProgramming, + CondensedChannelProgram, + CondensedChannelProgramming, + ContentProgram, +} from '@tunarr/types'; +import { UpdateChannelProgrammingRequest } from '@tunarr/types/api'; +import { inject, injectable, interfaces } from 'inversify'; +import { Kysely } from 'kysely'; +import { + drop, + entries, + filter, + forEach, + groupBy, + isEmpty, + isNil, + isString, + isUndefined, + map, + mapValues, + nth, + omitBy, + partition, + omit, + reject, + sum, + sumBy, + take, + uniq, + uniqBy, +} from 'lodash-es'; +import { Low } from 'lowdb'; +import fs from 'node:fs/promises'; +import { join } from 'node:path'; +import { MarkRequired } from 'ts-essentials'; +import { match } from 'ts-pattern'; +import { MaterializeLineupCommand } from '../../commands/MaterializeLineupCommand.ts'; +import { ProgramConverter } from '../converters/ProgramConverter.ts'; +import { + ContentItem, + CurrentLineupSchemaVersion, + isContentItem, + isOfflineItem, + isRedirectItem, + Lineup, + LineupItem, + LineupSchema, + PendingProgram, +} from '../derived_types/Lineup.ts'; +import { IWorkerPool } from '../../interfaces/IWorkerPool.ts'; +import { + ChannelAndLineup, + ChannelAndRawLineup, + UpdateChannelLineupRequest, +} from '../interfaces/IChannelDB.ts'; +import { SchemaBackedDbAdapter } from '../json/SchemaBackedJsonDBAdapter.ts'; +import { calculateStartTimeOffsets } from '../lineupUtil.ts'; +import { + AllProgramGroupingFields, + withPrograms, + withTrackAlbum, + withTrackArtist, + withTvSeason, + withTvShow, +} from '../programQueryHelpers.ts'; +import { + Channel, + ChannelOrm, +} from '../schema/Channel.ts'; +import { NewChannelProgram } from '../schema/ChannelPrograms.ts'; +import { DB } from '../schema/db.ts'; +import { DrizzleDBAccess } from '../schema/index.ts'; +import { + ChannelOrmWithPrograms, + ChannelWithPrograms, +} from '../schema/derivedTypes.ts'; +import { + asyncMapToRecord, + groupByFunc, + groupByUniqProp, + isDefined, + isNonEmptyString, + mapReduceAsyncSeq, + programExternalIdString, + run, +} from '../../util/index.ts'; +import { typedProperty } from '@/types/path.js'; +import { globalOptions } from '@/globals.js'; +import { eq } from 'drizzle-orm'; +import { chunk } from 'lodash-es'; + +// Module-level cache shared within this module +const fileDbCache: Record> = {}; +const fileDbLocks = new MutexMap(); + +const SqliteMaxDepthLimit = 1000; + +type ProgramRelationOperation = { operation: 'add' | 'remove'; id: string }; + +function channelProgramToLineupItemFunc( + dbIdByUniqueId: Record, +): (p: ChannelProgram) => LineupItem { + return (p) => + match(p) + .returnType() + .with({ type: 'content' }, (program) => ({ + type: 'content', + id: program.persisted ? program.id! : dbIdByUniqueId[program.uniqueId]!, + durationMs: program.duration, + })) + .with({ type: 'custom' }, (program) => ({ + type: 'content', + durationMs: program.duration, + id: program.id, + customShowId: program.customShowId, + })) + .with({ type: 'filler' }, (program) => ({ + type: 'content', + durationMs: program.duration, + id: program.id, + fillerListId: program.fillerListId, + fillerType: program.fillerType, + })) + .with({ type: 'redirect' }, (program) => ({ + type: 'redirect', + channel: program.channel, + durationMs: program.duration, + })) + .with({ type: 'flex' }, (program) => ({ + type: 'offline', + durationMs: program.duration, + })) + .exhaustive(); +} + +@injectable() +export class LineupRepository { + private logger = LoggerFactory.child({ + caller: import.meta, + className: this.constructor.name, + }); + + private timer = new Timer(this.logger, 'trace'); + + constructor( + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + @inject(FileSystemService) private fileSystemService: FileSystemService, + @inject(KEYS.WorkerPoolFactory) + private workerPoolProvider: interfaces.AutoFactory, + @inject(MaterializeLineupCommand) + private materializeLineupCommand: MaterializeLineupCommand, + @inject(KEYS.ProgramDB) private programDB: IProgramDB, + @inject(ProgramConverter) private programConverter: ProgramConverter, + ) {} + + async createLineup(channelId: string): Promise { + const db = await this.getFileDb(channelId); + await db.write(); + } + + async getFileDb(channelId: string, forceRead: boolean = false): Promise> { + return await fileDbLocks.getOrCreateLock(channelId).then((lock) => + lock.runExclusive(async () => { + const existing = fileDbCache[channelId]; + if (isDefined(existing)) { + if (forceRead) { + await existing.read(); + } + return existing; + } + + const defaultValue = { + items: [], + startTimeOffsets: [], + lastUpdated: dayjs().valueOf(), + version: CurrentLineupSchemaVersion, + }; + const db = new Low( + new SchemaBackedDbAdapter( + LineupSchema, + this.fileSystemService.getChannelLineupPath(channelId), + defaultValue, + ), + defaultValue, + ); + await db.read(); + fileDbCache[channelId] = db; + return db; + }), + ); + } + + async markLineupFileForDeletion( + channelId: string, + isDelete: boolean = true, + ): Promise { + const path = join( + globalOptions().databaseDirectory, + `channel-lineups/${channelId}.json${isDelete ? '' : '.bak'}`, + ); + try { + if (await fileExists(path)) { + const newPath = isDelete ? `${path}.bak` : path.replace('.bak', ''); + await fs.rename(path, newPath); + } + if (isDelete) { + delete fileDbCache[channelId]; + } else { + await this.getFileDb(channelId); + } + } catch (e) { + this.logger.error( + e, + `Error while trying to ${ + isDelete ? 'mark' : 'unmark' + } Channel %s lineup json for deletion`, + channelId, + ); + } + } + + async restoreLineupFile(channelId: string): Promise { + return this.markLineupFileForDeletion(channelId, false); + } + + async removeRedirectReferences(toChannel: string): Promise { + const allChannels = await this.drizzleDB.query.channels + .findMany({ + orderBy: (fields, { asc }) => asc(fields.number), + }) + .execute(); + + const ops = asyncPool( + reject(allChannels, { uuid: toChannel }), + async (channel) => { + const lineup = await this.loadLineup(channel.uuid); + let changed = false; + const newLineup: LineupItem[] = map(lineup.items, (item) => { + if (item.type === 'redirect' && item.channel === toChannel) { + changed = true; + return { + type: 'offline' as const, + durationMs: item.durationMs, + }; + } else { + return item; + } + }); + if (changed) { + return this.saveLineup(channel.uuid, { ...lineup, items: newLineup }); + } + return; + }, + { concurrency: 2 }, + ); + + for await (const updateResult of ops) { + if (updateResult.isFailure()) { + this.logger.error( + 'Error removing redirect references for channel %s from channel %s', + toChannel, + updateResult.error.input.uuid, + ); + } + } + } + + async saveLineup( + channelId: string, + newLineup: UpdateChannelLineupRequest, + ): Promise { + const db = await this.getFileDb(channelId); + await db.update((data) => { + if (isDefined(newLineup.items)) { + data.items = newLineup.items; + data.startTimeOffsets = + newLineup.startTimeOffsets ?? + calculateStartTimeOffsets(newLineup.items); + } + + if (isDefined(newLineup.schedule)) { + if (newLineup.schedule === null) { + data.schedule = undefined; + } else { + data.schedule = newLineup.schedule; + } + } + + if (isDefined(newLineup.pendingPrograms)) { + data.pendingPrograms = + newLineup.pendingPrograms === null + ? undefined + : newLineup.pendingPrograms; + } + + if (isDefined(newLineup.onDemandConfig)) { + data.onDemandConfig = + newLineup.onDemandConfig === null + ? undefined + : newLineup.onDemandConfig; + } + + data.version = newLineup?.version ?? data.version; + + data.lastUpdated = dayjs().valueOf(); + }); + + if (isDefined(newLineup.items)) { + const newDur = sum(newLineup.items.map((item) => item.durationMs)); + await this.updateChannelDuration(channelId, newDur); + } + return db.data; + } + + private updateChannelDuration(id: string, newDur: number): Promise { + return this.drizzleDB + .update(Channel) + .set({ duration: newDur }) + .where(eq(Channel.uuid, id)) + .limit(1) + .execute() + .then((_) => _.changes); + } + + async updateLineupConfig< + Key extends keyof Omit< + Lineup, + 'items' | 'startTimeOffsets' | 'pendingPrograms' + >, + >(id: string, key: Key, conf: Lineup[Key]): Promise { + const lineupDb = await this.getFileDb(id); + return await lineupDb.update((existing) => { + existing[key] = conf; + }); + } + + async setChannelPrograms( + channel: Channel, + lineup: readonly LineupItem[], + ): Promise; + async setChannelPrograms( + channel: string | Channel, + lineup: readonly LineupItem[], + startTime?: number, + ): Promise; + async setChannelPrograms( + channel: string | Channel, + lineup: readonly LineupItem[], + startTime?: number, + ): Promise { + const loadedChannel = await run(async () => { + if (isString(channel)) { + return this.db + .selectFrom('channel') + .where('channel.uuid', '=', channel) + .selectAll() + .executeTakeFirst(); + } else { + return channel; + } + }); + + if (isNil(loadedChannel)) { + return null; + } + + const allIds = uniq(map(filter(lineup, isContentItem), 'id')); + + return await this.db.transaction().execute(async (tx) => { + if (!isUndefined(startTime)) { + loadedChannel.startTime = startTime; + } + loadedChannel.duration = sumBy(lineup, typedProperty('durationMs')); + const updatedChannel = await tx + .updateTable('channel') + .where('channel.uuid', '=', loadedChannel.uuid) + .set('duration', sumBy(lineup, typedProperty('durationMs'))) + .$if(isDefined(startTime), (_) => _.set('startTime', startTime!)) + .returningAll() + .executeTakeFirst(); + + for (const idChunk of chunk(allIds, 500)) { + await tx + .deleteFrom('channelPrograms') + .where('channelUuid', '=', loadedChannel.uuid) + .where('programUuid', 'not in', idChunk) + .execute(); + } + + for (const idChunk of chunk(allIds, 500)) { + await tx + .insertInto('channelPrograms') + .values( + map(idChunk, (id) => ({ + programUuid: id, + channelUuid: loadedChannel.uuid, + })), + ) + .onConflict((oc) => oc.doNothing()) + .executeTakeFirst(); + } + + return updatedChannel ?? null; + }); + } + + async addPendingPrograms( + channelId: string, + pendingPrograms: PendingProgram[], + ): Promise { + if (pendingPrograms.length === 0) { + return; + } + + const db = await this.getFileDb(channelId); + return await db.update((data) => { + if (isUndefined(data.pendingPrograms)) { + data.pendingPrograms = [...pendingPrograms]; + } else { + data.pendingPrograms.push(...pendingPrograms); + } + }); + } + + async removeProgramsFromLineup( + channelId: string, + programIds: string[], + ): Promise { + if (programIds.length === 0) { + return; + } + + const idSet = new Set(programIds); + const lineup = await this.loadLineup(channelId); + lineup.items = map(lineup.items, (item) => { + if (isContentItem(item) && idSet.has(item.id)) { + return { + type: 'offline', + durationMs: item.durationMs, + }; + } else { + return item; + } + }); + await this.saveLineup(channelId, lineup); + } + + async removeProgramsFromAllLineups(programIds: string[]): Promise { + if (isEmpty(programIds)) { + return; + } + + const lineups = await this.loadAllLineups(); + + const programsToRemove = new Set(programIds); + for (const [channelId, { lineup }] of Object.entries(lineups)) { + const newLineupItems: LineupItem[] = lineup.items.map((item) => { + switch (item.type) { + case 'content': { + if (programsToRemove.has(item.id)) { + return { + type: 'offline', + durationMs: item.durationMs, + }; + } + return item; + } + case 'offline': + case 'redirect': + return item; + } + }); + + await this.saveLineup(channelId, { + ...lineup, + items: newLineupItems, + }); + + const duration = sum(newLineupItems.map((item) => item.durationMs)); + + await this.db + .updateTable('channel') + .set({ duration }) + .where('uuid', '=', channelId) + .limit(1) + .executeTakeFirst(); + } + } + + async loadAllLineups(): Promise> { + const allChannels = await this.drizzleDB.query.channels + .findMany({ orderBy: (fields, { asc }) => asc(fields.number) }) + .execute(); + return mapReduceAsyncSeq( + allChannels, + async (channel) => ({ + channel, + lineup: await this.loadLineup(channel.uuid), + }), + (prev, { channel, lineup }) => { + prev[channel.uuid] = { channel, lineup }; + return prev; + }, + {} as Record, + ); + } + + async loadAllLineupConfigs( + forceRead: boolean = false, + ): Promise> { + const allChannels = await this.drizzleDB.query.channels + .findMany({ orderBy: (fields, { asc }) => asc(fields.number) }) + .execute(); + return asyncMapToRecord( + allChannels, + async (channel) => ({ + channel, + lineup: await this.loadLineup(channel.uuid, forceRead), + }), + ({ channel }) => channel.uuid, + ); + } + + async loadAllRawLineups(): Promise> { + const allChannels = await this.drizzleDB.query.channels + .findMany({ orderBy: (fields, { asc }) => asc(fields.number) }) + .execute(); + return asyncMapToRecord( + allChannels, + async (channel) => { + if ( + !(await fileExists( + this.fileSystemService.getChannelLineupPath(channel.uuid), + )) + ) { + await this.createLineup(channel.uuid); + } + + return { + channel, + lineup: jsonSchema.parse( + JSON.parse( + ( + await fs.readFile( + this.fileSystemService.getChannelLineupPath(channel.uuid), + ) + ).toString('utf-8'), + ), + ), + }; + }, + ({ channel }) => channel.uuid, + ); + } + + async loadLineup(channelId: string, forceRead: boolean = false): Promise { + const db = await this.getFileDb(channelId, forceRead); + return db.data; + } + + async loadChannelAndLineup( + channelId: string, + ): Promise | null> { + const channel = await this.db + .selectFrom('channel') + .where('channel.uuid', '=', channelId) + .selectAll() + .executeTakeFirst(); + if (isNil(channel)) { + return null; + } + + return { + channel, + lineup: await this.loadLineup(channelId), + }; + } + + async loadChannelAndLineupOrm( + channelId: string, + ): Promise | null> { + const channel = await this.drizzleDB.query.channels.findFirst({ + where: (ch, { eq }) => eq(ch.uuid, channelId), + with: { transcodeConfig: true }, + }); + if (isNil(channel)) { + return null; + } + + return { + channel, + lineup: await this.loadLineup(channelId), + }; + } + + async loadChannelWithProgamsAndLineup( + channelId: string, + ): Promise<{ channel: ChannelOrmWithPrograms; lineup: Lineup } | null> { + const channelsAndPrograms = await this.drizzleDB.query.channels.findFirst({ + where: (fields, { eq }) => eq(fields.uuid, channelId), + with: { + channelPrograms: { + with: { + program: { + with: { + show: true, + season: true, + artist: true, + album: true, + externalIds: true, + }, + }, + }, + }, + }, + }); + + if (isNil(channelsAndPrograms)) { + return null; + } + + const withoutJoinTable = omit(channelsAndPrograms, 'channelPrograms'); + const channel: ChannelOrmWithPrograms = { + ...withoutJoinTable, + programs: channelsAndPrograms.channelPrograms.map((cp) => cp.program), + }; + + return { + channel, + lineup: await this.loadLineup(channelId), + }; + } + + async loadAndMaterializeLineup( + channelId: string, + offset: number = 0, + limit: number = -1, + ): Promise { + const channel = await this.db + .selectFrom('channel') + .selectAll(['channel']) + .where('channel.uuid', '=', channelId) + .leftJoin( + 'channelPrograms', + 'channel.uuid', + 'channelPrograms.channelUuid', + ) + .select((eb) => + withPrograms(eb, { + joins: { + customShows: true, + tvShow: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + tvSeason: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + trackArtist: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + trackAlbum: [ + 'programGrouping.uuid', + 'programGrouping.title', + 'programGrouping.summary', + 'programGrouping.type', + ], + }, + }), + ) + .groupBy('channel.uuid') + .orderBy('channel.number asc') + .executeTakeFirst(); + + if (isNil(channel)) { + return null; + } + + const lineup = await this.loadLineup(channelId); + const len = lineup.items.length; + const cleanOffset = offset < 0 ? 0 : offset; + const cleanLimit = limit < 0 ? len : limit; + + const { lineup: apiLineup, offsets } = await this.buildApiLineup( + channel, + take(drop(lineup.items, cleanOffset), cleanLimit), + ); + + return { + icon: channel.icon, + name: channel.name, + number: channel.number, + totalPrograms: len, + programs: apiLineup, + startTimeOffsets: offsets, + }; + } + + async loadCondensedLineup( + channelId: string, + offset: number = 0, + limit: number = -1, + ): Promise { + const lineup = await this.timer.timeAsync('loadLineup', () => + this.loadLineup(channelId), + ); + + const len = lineup.items.length; + const cleanOffset = offset < 0 ? 0 : offset; + const cleanLimit = limit < 0 ? len : limit; + const pagedLineup = take(drop(lineup.items, cleanOffset), cleanLimit); + + const channel = await this.timer.timeAsync('select channel', () => + this.db + .selectFrom('channel') + .where('channel.uuid', '=', channelId) + .selectAll() + .executeTakeFirst(), + ); + + if (isNil(channel)) { + return null; + } + + const contentItems = filter(pagedLineup, isContentItem); + + const directPrograms = await this.timer.timeAsync('direct', () => + this.db + .selectFrom('channelPrograms') + .where('channelUuid', '=', channelId) + .innerJoin('program', 'channelPrograms.programUuid', 'program.uuid') + .selectAll('program') + .select((eb) => [ + withTvShow(eb, AllProgramGroupingFields, true), + withTvSeason(eb, AllProgramGroupingFields, true), + withTrackAlbum(eb, AllProgramGroupingFields, true), + withTrackArtist(eb, AllProgramGroupingFields, true), + ]) + .execute(), + ); + + const externalIds = await this.timer.timeAsync('eids', () => + this.db + .selectFrom('channelPrograms') + .where('channelUuid', '=', channelId) + .innerJoin( + 'programExternalId', + 'channelPrograms.programUuid', + 'programExternalId.programUuid', + ) + .selectAll('programExternalId') + .execute(), + ); + + const externalIdsByProgramId = groupBy( + externalIds, + (eid) => eid.programUuid, + ); + + const programsById = groupByUniqProp(directPrograms, 'uuid'); + + const materializedPrograms = this.timer.timeSync('program convert', () => { + const ret: Record = {}; + forEach(uniqBy(contentItems, 'id'), (item) => { + const program = programsById[item.id]; + if (!program) { + return; + } + + const converted = this.programConverter.programDaoToContentProgram( + program, + externalIdsByProgramId[program.uuid] ?? [], + ); + + if (converted) { + ret[converted.id] = converted; + } + }); + + return ret; + }); + + const { lineup: condensedLineup, offsets } = await this.timer.timeAsync( + 'build condensed lineup', + () => + this.buildCondensedLineup( + channel, + new Set([...seq.collect(directPrograms, (p) => p.uuid)]), + pagedLineup, + ), + ); + + let apiOffsets: number[]; + if (lineup.startTimeOffsets) { + apiOffsets = take(drop(lineup.startTimeOffsets, cleanOffset), cleanLimit); + } else { + const scale = sumBy( + take(lineup.items, cleanOffset - 1), + (i) => i.durationMs, + ); + apiOffsets = map(offsets, (o) => o + scale); + } + + return { + icon: channel.icon, + name: channel.name, + number: channel.number, + totalPrograms: len, + programs: omitBy(materializedPrograms, isNil), + lineup: condensedLineup, + startTimeOffsets: apiOffsets, + schedule: lineup.schedule, + }; + } + + async updateLineup( + id: string, + req: UpdateChannelProgrammingRequest, + ): Promise> { + const channel = await this.drizzleDB.query.channels.findFirst({ + where: (fields, { eq }) => eq(fields.uuid, id), + with: { + channelPrograms: { + columns: { + programUuid: true, + }, + }, + }, + }); + + const lineup = await this.loadLineup(id); + + if (isNil(channel)) { + return null; + } + + const updateChannel = async (lineup: readonly LineupItem[]) => { + return await this.db.transaction().execute(async (tx) => { + await tx + .updateTable('channel') + .where('channel.uuid', '=', id) + .set({ + duration: sumBy(lineup, typedProperty('durationMs')), + }) + .limit(1) + .executeTakeFirstOrThrow(); + + const allNewIds = new Set([ + ...uniq(map(filter(lineup, isContentItem), (p) => p.id)), + ]); + + const existingIds = new Set([ + ...channel.channelPrograms.map((cp) => cp.programUuid), + ]); + + const removeOperations: ProgramRelationOperation[] = map( + reject([...existingIds], (existingId) => allNewIds.has(existingId)), + (removalId) => ({ + operation: 'remove' as const, + id: removalId, + }), + ); + + const addOperations: ProgramRelationOperation[] = map( + reject([...allNewIds], (newId) => existingIds.has(newId)), + (addId) => ({ + operation: 'add' as const, + id: addId, + }), + ); + + for (const ops of chunk( + [...addOperations, ...removeOperations], + SqliteMaxDepthLimit / 2, + )) { + const [adds, removes] = partition( + ops, + ({ operation }) => operation === 'add', + ); + + if (!isEmpty(removes)) { + await tx + .deleteFrom('channelPrograms') + .where('channelPrograms.programUuid', 'in', map(removes, 'id')) + .where('channelPrograms.channelUuid', '=', id) + .execute(); + } + + if (!isEmpty(adds)) { + await tx + .insertInto('channelPrograms') + .values( + map( + adds, + ({ id }) => + ({ + channelUuid: channel.uuid, + programUuid: id, + }) satisfies NewChannelProgram, + ), + ) + .execute(); + } + } + return channel; + }); + }; + + const createNewLineup = async ( + programs: ChannelProgram[], + lineupPrograms: ChannelProgram[] = programs, + ) => { + const upsertedPrograms = + await this.programDB.upsertContentPrograms(programs); + const dbIdByUniqueId = groupByFunc( + upsertedPrograms, + programExternalIdString, + (p) => p.uuid, + ); + return map(lineupPrograms, channelProgramToLineupItemFunc(dbIdByUniqueId)); + }; + + const upsertPrograms = async (programs: ChannelProgram[]) => { + const upsertedPrograms = + await this.programDB.upsertContentPrograms(programs); + return groupByFunc( + upsertedPrograms, + programExternalIdString, + (p) => p.uuid, + ); + }; + + if (req.type === 'manual') { + const newLineupItems = await run(async () => { + const newItems = await this.timer.timeAsync( + 'createNewLineup', + async () => { + const programs = req.programs; + const dbIdByUniqueId = await upsertPrograms(programs); + const convertFunc = channelProgramToLineupItemFunc(dbIdByUniqueId); + return seq.collect(req.lineup, (lineupItem) => { + switch (lineupItem.type) { + case 'index': { + const program = nth(programs, lineupItem.index); + if (program) { + return convertFunc({ + ...program, + duration: lineupItem.duration ?? program.duration, + }); + } + return null; + } + case 'persisted': { + return { + type: 'content', + id: lineupItem.programId, + customShowId: lineupItem.customShowId, + durationMs: lineupItem.duration, + } satisfies ContentItem; + } + } + }); + }, + ); + if (req.append) { + const existingLineup = await this.loadLineup(channel.uuid); + return [...existingLineup.items, ...newItems]; + } else { + return newItems; + } + }); + + const updatedChannel = await this.timer.timeAsync('updateChannel', () => + updateChannel(newLineupItems), + ); + + await this.timer.timeAsync('saveLineup', () => + this.saveLineup(id, { + items: newLineupItems, + onDemandConfig: isDefined(lineup.onDemandConfig) + ? { + ...lineup.onDemandConfig, + cursor: 0, + } + : undefined, + }), + ); + + return { + channel: updatedChannel, + newLineup: newLineupItems, + }; + } else if (req.type === 'time' || req.type === 'random') { + let programs: ChannelProgram[]; + if (req.type === 'time') { + const { result } = await this.workerPoolProvider().queueTask({ + type: 'time-slots', + request: { + type: 'programs', + programIds: req.programs, + schedule: req.schedule, + seed: req.seed, + startTime: channel.startTime, + }, + }); + + programs = MaterializeLineupCommand.expandLineup( + result.lineup, + await this.materializeLineupCommand.execute({ + lineup: result.lineup, + }), + ); + } else { + const { result } = await this.workerPoolProvider().queueTask({ + type: 'schedule-slots', + request: { + type: 'programs', + programIds: req.programs, + startTime: channel.startTime, + schedule: req.schedule, + seed: req.seed, + }, + }); + programs = MaterializeLineupCommand.expandLineup( + result.lineup, + await this.materializeLineupCommand.execute({ + lineup: result.lineup, + }), + ); + } + + const newLineup = await createNewLineup(programs); + + const updatedChannel = await updateChannel(newLineup); + await this.saveLineup(id, { + items: newLineup, + schedule: req.schedule, + }); + + return { + channel: updatedChannel, + newLineup, + }; + } + + return null; + } + + private async buildApiLineup( + channel: ChannelWithPrograms, + lineup: LineupItem[], + ): Promise<{ lineup: ChannelProgram[]; offsets: number[] }> { + const allChannels = await this.db + .selectFrom('channel') + .select(['channel.uuid', 'channel.number', 'channel.name']) + .execute(); + let lastOffset = 0; + const offsets: number[] = []; + + const programsById = groupByUniqProp(channel.programs, 'uuid'); + + const programs: ChannelProgram[] = []; + + for (const item of lineup) { + const apiItem = match(item) + .with({ type: 'content' }, (contentItem) => { + const fullProgram = programsById[contentItem.id]; + if (!fullProgram) { + return null; + } + return this.programConverter.programDaoToContentProgram( + fullProgram, + fullProgram.externalIds ?? [], + ); + }) + .otherwise((item) => + this.programConverter.lineupItemToChannelProgram( + channel, + item, + allChannels, + ), + ); + + if (apiItem) { + offsets.push(lastOffset); + lastOffset += item.durationMs; + programs.push(apiItem); + } + } + + return { lineup: programs, offsets }; + } + + private async buildCondensedLineup( + channel: Channel, + dbProgramIds: Set, + lineup: LineupItem[], + ): Promise<{ lineup: CondensedChannelProgram[]; offsets: number[] }> { + let lastOffset = 0; + const offsets: number[] = []; + + const customShowLineupItemsByShowId = mapValues( + groupBy( + filter( + lineup, + (l): l is MarkRequired => + l.type === 'content' && isNonEmptyString(l.customShowId), + ), + (i) => i.customShowId, + ), + (items) => uniqBy(items, 'id'), + ); + + const customShowIndexes: Record> = {}; + for (const [customShowId, items] of entries( + customShowLineupItemsByShowId, + )) { + customShowIndexes[customShowId] = {}; + + const results = await this.db + .selectFrom('customShowContent') + .select(['customShowContent.contentUuid', 'customShowContent.index']) + .where('customShowContent.contentUuid', 'in', map(items, 'id')) + .where('customShowContent.customShowUuid', '=', customShowId) + .groupBy('customShowContent.contentUuid') + .execute(); + + const byItemId: Record = {}; + for (const { contentUuid, index } of results) { + byItemId[contentUuid] = index; + } + + customShowIndexes[customShowId] = byItemId; + } + + const allChannels = await this.db + .selectFrom('channel') + .select(['uuid', 'name', 'number']) + .execute(); + + const channelsById = groupByUniqProp(allChannels, 'uuid'); + + const programs = seq.collect(lineup, (item) => { + let p: CondensedChannelProgram | null = null; + if (isOfflineItem(item)) { + p = this.programConverter.offlineLineupItemToProgram(channel, item); + } else if (isRedirectItem(item)) { + if (channelsById[item.channel]) { + p = this.programConverter.redirectLineupItemToProgram( + item, + channelsById[item.channel]!, + ); + } else { + this.logger.warn( + 'Found dangling redirect program. Bad ID = %s', + item.channel, + ); + p = { + persisted: true, + type: 'flex', + duration: item.durationMs, + }; + } + } else if (item.customShowId) { + p = { + persisted: true, + type: 'custom', + customShowId: item.customShowId, + duration: item.durationMs, + index: customShowIndexes[item.customShowId]![item.id] ?? -1, + id: item.id, + }; + } else if (item.fillerListId) { + p = { + persisted: true, + type: 'filler', + fillerListId: item.fillerListId, + fillerType: item.fillerType, + id: item.id, + duration: item.durationMs, + }; + } else { + if (dbProgramIds.has(item.id)) { + p = { + persisted: true, + type: 'content', + id: item.id, + duration: item.durationMs, + }; + } + } + + if (p) { + offsets.push(lastOffset); + lastOffset += item.durationMs; + } + + return p; + }); + return { lineup: programs, offsets }; + } +} diff --git a/server/src/db/program/BasicProgramRepository.ts b/server/src/db/program/BasicProgramRepository.ts new file mode 100644 index 00000000..46e76809 --- /dev/null +++ b/server/src/db/program/BasicProgramRepository.ts @@ -0,0 +1,114 @@ +import type { ProgramExternalIdType } from '@/db/custom_types/ProgramExternalIdType.js'; +import { KEYS } from '@/types/inject.js'; +import type { Maybe } from '@/types/util.js'; +import type { ProgramExternalId } from '../schema/ProgramExternalId.ts'; +import { ProgramGroupingType } from '../schema/ProgramGrouping.ts'; +import type { ProgramWithRelationsOrm } from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; +import type { DB } from '../schema/db.ts'; +import { inject, injectable } from 'inversify'; +import type { Kysely } from 'kysely'; +import { chunk, isEmpty } from 'lodash-es'; +import type { MarkRequired } from 'ts-essentials'; + +@injectable() +export class BasicProgramRepository { + constructor( + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + ) {} + + async getProgramById( + id: string, + ): Promise>> { + return this.drizzleDB.query.program.findFirst({ + where: (fields, { eq }) => eq(fields.uuid, id), + with: { + externalIds: true, + artwork: true, + subtitles: true, + credits: true, + versions: { + with: { + mediaStreams: true, + mediaFiles: true, + chapters: true, + }, + }, + }, + }); + } + + async getProgramExternalIds( + id: string, + externalIdTypes?: ProgramExternalIdType[], + ): Promise { + return await this.db + .selectFrom('programExternalId') + .selectAll() + .where('programExternalId.programUuid', '=', id) + .$if(!isEmpty(externalIdTypes), (qb) => + qb.where('programExternalId.sourceType', 'in', externalIdTypes!), + ) + .execute(); + } + + async getShowIdFromTitle(title: string): Promise> { + const matchedGrouping = await this.db + .selectFrom('programGrouping') + .select('uuid') + .where('title', '=', title) + .where('type', '=', ProgramGroupingType.Show) + .executeTakeFirst(); + + return matchedGrouping?.uuid; + } + + async updateProgramDuration( + programId: string, + duration: number, + ): Promise { + await this.db + .updateTable('program') + .where('uuid', '=', programId) + .set({ + duration, + }) + .executeTakeFirst(); + } + + async getProgramsByIds( + ids: string[] | readonly string[], + batchSize: number = 500, + ): Promise { + const results: ProgramWithRelationsOrm[] = []; + for (const idChunk of chunk(ids, batchSize)) { + const res = await this.drizzleDB.query.program.findMany({ + where: (fields, { inArray }) => inArray(fields.uuid, idChunk), + with: { + album: { + with: { + artwork: true, + }, + }, + artist: true, + season: true, + show: { + with: { + artwork: true, + }, + }, + externalIds: true, + artwork: true, + tags: { + with: { + tag: true, + }, + }, + }, + }); + results.push(...res); + } + return results; + } +} diff --git a/server/src/db/program/ProgramExternalIdRepository.ts b/server/src/db/program/ProgramExternalIdRepository.ts new file mode 100644 index 00000000..7b81668d --- /dev/null +++ b/server/src/db/program/ProgramExternalIdRepository.ts @@ -0,0 +1,375 @@ +import { KEYS } from '@/types/inject.js'; +import type { Logger } from '@/util/logging/LoggerFactory.js'; +import { seq } from '@tunarr/shared/util'; +import { flatMapAsyncSeq, isNonEmptyString } from '../../util/index.ts'; +import { createExternalId } from '@tunarr/shared'; +import { inject, injectable } from 'inversify'; +import type { Kysely } from 'kysely'; +import { chunk, first, isEmpty, isNil, last, map, mapValues } from 'lodash-es'; +import type { MarkOptional, MarkRequired } from 'ts-essentials'; +import dayjs from 'dayjs'; +import { v4 } from 'uuid'; +import { ProgramExternalIdType } from '../custom_types/ProgramExternalIdType.ts'; +import { programSourceTypeFromString } from '../custom_types/ProgramSourceType.ts'; +import { + withProgramByExternalId, +} from '../programQueryHelpers.ts'; +import type { + MinimalProgramExternalId, + NewProgramExternalId, + NewSingleOrMultiExternalId, + ProgramExternalId, +} from '../schema/ProgramExternalId.ts'; +import { toInsertableProgramExternalId } from '../schema/ProgramExternalId.ts'; +import type { MediaSourceId, RemoteSourceType } from '../schema/base.ts'; +import type { DB } from '../schema/db.ts'; +import type { + ProgramWithRelationsOrm, +} from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; +import type { ProgramType } from '../schema/Program.ts'; +import { groupByUniq } from '../../util/index.ts'; +import type { RemoteMediaSourceType } from '../schema/MediaSource.ts'; +import type { ProgramDao } from '../schema/Program.ts'; +import type { Dictionary } from 'ts-essentials'; +import { groupBy, flatten, partition } from 'lodash-es'; +import { mapAsyncSeq } from '../../util/index.ts'; + +@injectable() +export class ProgramExternalIdRepository { + constructor( + @inject(KEYS.Logger) private logger: Logger, + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + ) {} + + async lookupByExternalId(eid: { + sourceType: RemoteSourceType; + externalSourceId: MediaSourceId; + externalKey: string; + }) { + return first( + await this.lookupByExternalIds( + new Set([[eid.sourceType, eid.externalSourceId, eid.externalKey]]), + ), + ); + } + + async lookupByExternalIds( + ids: + | Set<[RemoteSourceType, MediaSourceId, string]> + | Set, + chunkSize: number = 200, + ) { + const allIds = [...ids]; + const programs: MarkRequired[] = []; + for (const idChunk of chunk(allIds, chunkSize)) { + const results = await this.drizzleDB.query.programExternalId.findMany({ + where: (fields, { or, and, eq }) => { + const ands = idChunk.map(([ps, es, ek]) => + and( + eq(fields.externalKey, ek), + eq(fields.sourceType, ps), + eq(fields.mediaSourceId, es), + ), + ); + return or(...ands); + }, + with: { + program: { + with: { + album: true, + artist: true, + season: true, + show: true, + externalIds: true, + tags: { + with: { + tag: true, + }, + }, + artwork: true, + credits: true, + genres: { + with: { + genre: true, + }, + }, + studios: { + with: { + studio: true, + }, + }, + versions: true, + }, + }, + }, + }); + programs.push(...seq.collect(results, (r) => r.program)); + } + + return programs; + } + + async lookupByMediaSource( + sourceType: RemoteMediaSourceType, + sourceId: MediaSourceId, + programType: ProgramType | undefined, + chunkSize: number = 200, + ): Promise { + const programs: ProgramDao[] = []; + let chunk: ProgramDao[] = []; + let lastId: string | undefined; + do { + const result = await this.db + .selectFrom('programExternalId') + .select('programExternalId.uuid') + .select((eb) => + withProgramByExternalId(eb, { joins: {} }, (qb) => + qb.$if(!!programType, (eb) => + eb.where('program.type', '=', programType!), + ), + ), + ) + .where('programExternalId.sourceType', '=', sourceType) + .where('programExternalId.mediaSourceId', '=', sourceId) + .$if(!!lastId, (x) => x.where('programExternalId.uuid', '>', lastId!)) + .orderBy('programExternalId.uuid asc') + .limit(chunkSize) + .execute(); + chunk = seq.collect(result, (eid) => eid.program); + programs.push(...chunk); + lastId = last(result)?.uuid; + } while (chunk.length > 0); + + return programs; + } + + async programIdsByExternalIds( + ids: Set<[string, MediaSourceId, string]>, + chunkSize: number = 50, + ) { + if (ids.size === 0) { + return {}; + } + + const externalIds = await flatMapAsyncSeq( + chunk([...ids], chunkSize), + (idChunk) => { + return this.db + .selectFrom('programExternalId') + .selectAll() + .where((eb) => + eb.or( + map(idChunk, ([ps, es, ek]) => { + return eb.and([ + eb('programExternalId.externalKey', '=', ek), + eb('programExternalId.mediaSourceId', '=', es), + eb( + 'programExternalId.sourceType', + '=', + programSourceTypeFromString(ps)!, + ), + ]); + }), + ), + ) + .execute(); + }, + ); + + return mapValues( + groupByUniq(externalIds, (eid) => + createExternalId(eid.sourceType, eid.mediaSourceId!, eid.externalKey), + ), + (eid) => eid.programUuid, + ); + } + + async updateProgramPlexRatingKey( + programId: string, + serverId: MediaSourceId, + details: MarkOptional< + Pick< + ProgramExternalId, + 'externalKey' | 'directFilePath' | 'externalFilePath' + >, + 'directFilePath' | 'externalFilePath' + >, + ) { + const existingRatingKey = await this.db + .selectFrom('programExternalId') + .selectAll() + .where((eb) => + eb.and({ + programUuid: programId, + mediaSourceId: serverId, + sourceType: ProgramExternalIdType.PLEX, + }), + ) + .executeTakeFirst(); + + if (isNil(existingRatingKey)) { + const now = +dayjs(); + return await this.db + .insertInto('programExternalId') + .values({ + uuid: v4(), + createdAt: now, + updatedAt: now, + programUuid: programId, + sourceType: ProgramExternalIdType.PLEX, + mediaSourceId: serverId, + ...details, + }) + .returningAll() + .executeTakeFirstOrThrow(); + } else { + await this.db + .updateTable('programExternalId') + .set({ + externalKey: details.externalKey, + }) + .$if(isNonEmptyString(details.externalFilePath), (_) => + _.set({ + externalFilePath: details.externalFilePath!, + }), + ) + .$if(isNonEmptyString(details.directFilePath), (_) => + _.set({ + directFilePath: details.directFilePath!, + }), + ) + .where('uuid', '=', existingRatingKey.uuid) + .executeTakeFirst(); + return await this.db + .selectFrom('programExternalId') + .selectAll() + .where('uuid', '=', existingRatingKey.uuid) + .executeTakeFirstOrThrow(); + } + } + + async replaceProgramExternalId( + programId: string, + newExternalId: NewProgramExternalId, + oldExternalId?: MinimalProgramExternalId, + ) { + await this.db.transaction().execute(async (tx) => { + if (oldExternalId) { + await tx + .deleteFrom('programExternalId') + .where('programExternalId.programUuid', '=', programId) + .where( + 'programExternalId.externalKey', + '=', + oldExternalId.externalKey, + ) + .where( + 'programExternalId.externalSourceId', + '=', + oldExternalId.externalSourceId, + ) + .where('programExternalId.sourceType', '=', oldExternalId.sourceType) + .execute(); + } + await tx.insertInto('programExternalId').values(newExternalId).execute(); + }); + } + + async upsertProgramExternalIds( + externalIds: NewSingleOrMultiExternalId[], + chunkSize: number = 100, + ): Promise> { + if (isEmpty(externalIds)) { + return {}; + } + + const logger = this.logger; + + const [singles, multiples] = partition( + externalIds, + (id) => id.type === 'single', + ); + + let singleIdPromise: Promise; + if (!isEmpty(singles)) { + singleIdPromise = mapAsyncSeq( + chunk(singles, chunkSize), + (singleChunk) => { + return this.db.transaction().execute((tx) => + tx + .insertInto('programExternalId') + .values(singleChunk.map(toInsertableProgramExternalId)) + .onConflict((oc) => + oc + .columns(['programUuid', 'sourceType']) + .where('mediaSourceId', 'is', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + directFilePath: eb.ref('excluded.directFilePath'), + programUuid: eb.ref('excluded.programUuid'), + })), + ) + .returningAll() + .execute(), + ); + }, + ).then(flatten); + } else { + singleIdPromise = Promise.resolve([]); + } + + let multiIdPromise: Promise; + if (!isEmpty(multiples)) { + multiIdPromise = mapAsyncSeq( + chunk(multiples, chunkSize), + (multiChunk) => { + return this.db.transaction().execute((tx) => + tx + .insertInto('programExternalId') + .values(multiChunk.map(toInsertableProgramExternalId)) + .onConflict((oc) => + oc + .columns(['programUuid', 'sourceType', 'mediaSourceId']) + .where('mediaSourceId', 'is not', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + directFilePath: eb.ref('excluded.directFilePath'), + programUuid: eb.ref('excluded.programUuid'), + })), + ) + .returningAll() + .execute(), + ); + }, + ).then(flatten); + } else { + multiIdPromise = Promise.resolve([]); + } + + const [singleResult, multiResult] = await Promise.allSettled([ + singleIdPromise, + multiIdPromise, + ]); + + const allExternalIds: ProgramExternalId[] = []; + if (singleResult.status === 'rejected') { + logger.error(singleResult.reason, 'Error saving external IDs'); + } else { + logger.trace('Upserted %d external IDs', singleResult.value.length); + allExternalIds.push(...singleResult.value); + } + + if (multiResult.status === 'rejected') { + logger.error(multiResult.reason, 'Error saving external IDs'); + } else { + logger.trace('Upserted %d external IDs', multiResult.value.length); + allExternalIds.push(...multiResult.value); + } + + return groupBy(allExternalIds, (eid) => eid.programUuid); + } +} diff --git a/server/src/db/program/ProgramGroupingRepository.ts b/server/src/db/program/ProgramGroupingRepository.ts new file mode 100644 index 00000000..75318b76 --- /dev/null +++ b/server/src/db/program/ProgramGroupingRepository.ts @@ -0,0 +1,610 @@ +import type { + ProgramGroupingExternalIdLookup, + WithChannelIdFilter, +} from '@/db/interfaces/IProgramDB.js'; +import { KEYS } from '@/types/inject.js'; +import type { Maybe, PagedResult } from '@/types/util.js'; +import { type Logger } from '@/util/logging/LoggerFactory.js'; +import { seq } from '@tunarr/shared/util'; +import { untag } from '@tunarr/types'; +import { + and, + asc, + count, + countDistinct, + eq, +} from 'drizzle-orm'; +import type { SelectedFields, SQLiteSelectBuilder } from 'drizzle-orm/sqlite-core'; +import { inject, injectable } from 'inversify'; +import type { Kysely } from 'kysely'; +import { chunk, isEmpty, isUndefined, orderBy, sum, uniq } from 'lodash-es'; +import { Artwork } from '../schema/Artwork.ts'; +import { ChannelPrograms } from '../schema/ChannelPrograms.ts'; +import { Program, ProgramType } from '../schema/Program.ts'; +import { + ProgramGrouping, + ProgramGroupingType, + type ProgramGroupingTypes, +} from '../schema/ProgramGrouping.ts'; +import { ProgramExternalId } from '../schema/ProgramExternalId.ts'; +import { ProgramGroupingExternalId } from '../schema/ProgramGroupingExternalId.ts'; +import type { MediaSourceId, RemoteSourceType } from '../schema/base.ts'; +import type { DB } from '../schema/db.ts'; +import type { + MusicAlbumOrm, + ProgramGroupingOrmWithRelations, + ProgramGroupingWithExternalIds, + ProgramWithRelationsOrm, + TvSeasonOrm, +} from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; +import type { MarkRequired } from 'ts-essentials'; +import { + createManyRelationAgg, + mapRawJsonRelationResult, +} from '../../util/drizzleUtil.ts'; +import { selectProgramsBuilder } from '../programQueryHelpers.ts'; +import type { PageParams } from '../interfaces/IChannelDB.ts'; +import type { + ProgramGroupingChildCounts, +} from '../interfaces/IProgramDB.ts'; + +@injectable() +export class ProgramGroupingRepository { + constructor( + @inject(KEYS.Logger) private logger: Logger, + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + ) {} + + async getProgramGrouping(id: string) { + return this.drizzleDB.query.programGrouping.findFirst({ + where: (fields, { eq }) => eq(fields.uuid, id), + with: { + externalIds: true, + artwork: true, + }, + }); + } + + async getProgramGroupings( + ids: string[], + ): Promise> { + if (ids.length === 0) { + return {}; + } + + const uniqueIds = uniq(ids); + + const results = await Promise.allSettled( + chunk(uniqueIds, 1000).map((idChunk) => { + return this.drizzleDB.query.programGrouping.findMany({ + where: (fields, { inArray }) => inArray(fields.uuid, idChunk), + with: { + externalIds: true, + artwork: true, + artist: true, + show: true, + credits: true, + tags: { + with: { + tag: true, + }, + }, + }, + }); + }), + ); + + const map: Record = {}; + for (const result of results) { + if (result.status === 'rejected') { + this.logger.error( + result.reason, + 'Error while querying for program groupings. Returning partial data.', + ); + continue; + } + for (const grouping of result.value) { + map[grouping.uuid] = grouping; + } + } + return map; + } + + async getProgramGroupingByExternalId( + eid: ProgramGroupingExternalIdLookup, + ): Promise> { + return await this.drizzleDB.query.programGroupingExternalId + .findFirst({ + where: (row, { and, or, eq }) => + and( + eq(row.externalKey, eid.externalKey), + eq(row.sourceType, eid.sourceType), + or( + eq(row.externalSourceId, untag(eid.externalSourceId)), + eq(row.mediaSourceId, eid.externalSourceId), + ), + ), + with: { + grouping: { + with: { + externalIds: true, + }, + }, + }, + }) + .then((result) => result?.grouping ?? undefined); + } + + async getProgramGroupingsByExternalIds( + eids: + | Set<[RemoteSourceType, MediaSourceId, string]> + | Set, + chunkSize: number = 100, + ) { + const allIds = [...eids]; + const programs: MarkRequired< + ProgramGroupingOrmWithRelations, + 'externalIds' + >[] = []; + for (const idChunk of chunk(allIds, chunkSize)) { + const results = + await this.drizzleDB.query.programGroupingExternalId.findMany({ + where: (fields, { or, and, eq }) => { + const ands = idChunk.map(([ps, es, ek]) => + and( + eq(fields.externalKey, ek), + eq(fields.sourceType, ps), + eq(fields.mediaSourceId, es), + ), + ); + return or(...ands); + }, + columns: {}, + with: { + grouping: { + with: { + artist: true, + show: true, + externalIds: true, + }, + }, + }, + }); + programs.push(...seq.collect(results, (r) => r.grouping)); + } + + return programs; + } + + async getProgramParent( + programId: string, + ): Promise> { + const p = await selectProgramsBuilder(this.db, { + joins: { tvSeason: true, trackAlbum: true }, + }) + .where('program.uuid', '=', programId) + .executeTakeFirst() + .then((program) => program?.tvSeason ?? program?.trackAlbum); + + if (p) { + const eids = await this.db + .selectFrom('programGroupingExternalId') + .where('groupUuid', '=', p.uuid) + .selectAll() + .execute(); + return { + ...p, + externalIds: eids, + }; + } + + return; + } + + getChildren( + parentId: string, + parentType: 'season' | 'album', + params?: WithChannelIdFilter, + ): Promise>; + getChildren( + parentId: string, + parentType: 'artist', + params?: WithChannelIdFilter, + ): Promise>; + getChildren( + parentId: string, + parentType: 'show', + params?: WithChannelIdFilter, + ): Promise>; + getChildren( + parentId: string, + parentType: 'artist' | 'show', + params?: WithChannelIdFilter, + ): Promise>; + async getChildren( + parentId: string, + parentType: ProgramGroupingType, + params?: WithChannelIdFilter, + ): Promise< + | PagedResult + | PagedResult + > { + if (parentType === 'album' || parentType === 'season') { + return this.getTerminalChildren(parentId, parentType, params); + } else { + return this.getGroupingChildren(parentId, parentType, params); + } + } + + private async getGroupingChildren( + parentId: string, + parentType: ProgramGroupingTypes['Show'] | ProgramGroupingTypes['Artist'], + params?: WithChannelIdFilter, + ) { + const childType = parentType === 'artist' ? 'album' : 'season'; + function builder< + TSelection extends SelectedFields | undefined, + TResultType extends 'sync' | 'async', + TRunResult, + >(f: SQLiteSelectBuilder) { + return f + .from(Program) + .where( + and( + eq( + Program.type, + parentType === ProgramGroupingType.Show + ? ProgramType.Episode + : ProgramType.Track, + ), + eq( + parentType === ProgramGroupingType.Show + ? Program.tvShowUuid + : Program.artistUuid, + parentId, + ), + params?.channelId + ? eq(ChannelPrograms.channelUuid, params.channelId) + : undefined, + ), + ); + } + + const sq = this.drizzleDB + .select() + .from(ProgramGroupingExternalId) + .where(eq(ProgramGroupingExternalId.groupUuid, ProgramGrouping.uuid)) + .as('sq'); + + const baseQuery = builder( + this.drizzleDB.select({ + grouping: ProgramGrouping, + externalIds: createManyRelationAgg(sq, 'external_ids'), + artwork: createManyRelationAgg( + this.drizzleDB + .select() + .from(Artwork) + .where(eq(ProgramGrouping.uuid, Artwork.groupingId)) + .as('artwork'), + 'artwork', + ), + }), + ) + .innerJoin( + ProgramGrouping, + eq( + childType === 'season' ? Program.seasonUuid : Program.albumUuid, + ProgramGrouping.uuid, + ), + ) + .orderBy(asc(ProgramGrouping.index)) + .offset(params?.offset ?? 0) + .limit(params?.limit ?? 1_000_000) + .groupBy(ProgramGrouping.uuid); + + const baseCountQuery = builder( + this.drizzleDB.select({ + count: countDistinct(ProgramGrouping.uuid), + }), + ) + .innerJoin( + ProgramGrouping, + eq( + childType === 'season' ? Program.seasonUuid : Program.albumUuid, + ProgramGrouping.uuid, + ), + ) + .groupBy(ProgramGrouping.uuid); + + if (params?.channelId) { + const res = await baseQuery.innerJoin( + ChannelPrograms, + eq(ChannelPrograms.programUuid, Program.uuid), + ); + + const cq = baseCountQuery.innerJoin( + ChannelPrograms, + eq(ChannelPrograms.programUuid, Program.uuid), + ); + + const programs = res.map(({ grouping, externalIds, artwork }) => { + const withRelations = grouping as ProgramGroupingOrmWithRelations; + withRelations.externalIds = mapRawJsonRelationResult( + externalIds, + ProgramGroupingExternalId, + ); + withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); + return withRelations; + }); + + return { + total: sum((await cq).map(({ count }) => count)), + results: programs, + }; + } else { + const res = await baseQuery; + + const programs = res.map(({ grouping, externalIds, artwork }) => { + const withRelations = grouping as ProgramGroupingOrmWithRelations; + withRelations.externalIds = mapRawJsonRelationResult( + externalIds, + ProgramGroupingExternalId, + ); + withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); + return withRelations; + }); + + return { + total: sum((await baseCountQuery).map(({ count }) => count)), + results: programs, + }; + } + } + + private async getTerminalChildren( + parentId: string, + parentType: ProgramGroupingTypes['Season'] | ProgramGroupingTypes['Album'], + params?: WithChannelIdFilter, + ) { + function builder< + TSelection extends SelectedFields | undefined, + TResultType extends 'sync' | 'async', + TRunResult, + >(f: SQLiteSelectBuilder) { + return f + .from(Program) + .where( + and( + eq( + Program.type, + parentType === ProgramGroupingType.Album + ? ProgramType.Track + : ProgramType.Episode, + ), + eq( + parentType === ProgramGroupingType.Album + ? Program.albumUuid + : Program.seasonUuid, + parentId, + ), + params?.channelId + ? eq(ChannelPrograms.channelUuid, params.channelId) + : undefined, + ), + ); + } + + const sq = this.drizzleDB + .select() + .from(ProgramExternalId) + .where(eq(ProgramExternalId.programUuid, Program.uuid)) + .as('sq'); + + const baseQuery = builder( + this.drizzleDB.select({ + program: Program, + externalIds: createManyRelationAgg(sq, 'external_ids'), + artwork: createManyRelationAgg( + this.drizzleDB + .select() + .from(Artwork) + .where(eq(Artwork.programId, Program.uuid)) + .as('artwork'), + 'artwork', + ), + }), + ).orderBy(asc(Program.seasonNumber), asc(Program.episode)); + + const baseCountQuery = builder( + this.drizzleDB.select({ + count: count(), + }), + ); + + if (params?.channelId) { + const res = await baseQuery + .offset(params?.offset ?? 0) + .limit(params?.limit ?? 1_000_000) + .innerJoin( + ChannelPrograms, + eq(ChannelPrograms.programUuid, Program.uuid), + ); + + const cq = baseCountQuery.innerJoin( + ChannelPrograms, + eq(ChannelPrograms.programUuid, Program.uuid), + ); + + const programs = res.map(({ program, externalIds, artwork }) => { + const withRelations: ProgramWithRelationsOrm = program; + withRelations.externalIds = mapRawJsonRelationResult( + externalIds, + ProgramExternalId, + ); + withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); + return withRelations; + }); + + console.log(programs); + + return { + total: sum((await cq).map(({ count }) => count)), + results: programs, + }; + } else { + const res = await baseQuery; + + const programs = res.map(({ program, externalIds, artwork }) => { + const withRelations: ProgramWithRelationsOrm = program; + withRelations.externalIds = mapRawJsonRelationResult( + externalIds, + ProgramExternalId, + ); + withRelations.artwork = mapRawJsonRelationResult(artwork, Artwork); + return withRelations; + }); + + return { + total: sum((await baseCountQuery).map(({ count }) => count)), + results: programs, + }; + } + } + + async getProgramGroupingChildCounts( + groupingIds: string[], + ): Promise> { + if (isEmpty(groupingIds)) { + return {}; + } + + const uniqueIds = uniq(groupingIds); + + const allResults = await Promise.allSettled( + chunk(uniqueIds, 1000).map((idChunk) => + this.db + .selectFrom('programGrouping as pg') + .where('pg.uuid', 'in', idChunk) + .leftJoin('program as p', (j) => + j.on((eb) => + eb.or([ + eb('pg.uuid', '=', eb.ref('p.tvShowUuid')), + eb('pg.uuid', '=', eb.ref('p.artistUuid')), + eb('pg.uuid', '=', eb.ref('p.seasonUuid')), + eb('pg.uuid', '=', eb.ref('p.albumUuid')), + ]), + ), + ) + .leftJoin('programGrouping as pg2', (j) => + j.on((eb) => + eb.or([ + eb('pg.uuid', '=', eb.ref('pg2.artistUuid')), + eb('pg.uuid', '=', eb.ref('pg2.showUuid')), + ]), + ), + ) + .select(['pg.uuid as uuid', 'pg.type as type']) + .select((eb) => + eb.fn.count('p.uuid').distinct().as('programCount'), + ) + .select((eb) => + eb.fn.count('pg2.uuid').distinct().as('childGroupCount'), + ) + .groupBy('pg.uuid') + .execute(), + ), + ); + + const map: Record = {}; + + for (const result of allResults) { + if (result.status === 'rejected') { + this.logger.error( + result.reason, + 'Failed querying program grouping children. Continuing with partial results', + ); + continue; + } + + for (const counts of result.value) { + map[counts.uuid] = { + type: counts.type, + childCount: + counts.type === 'season' || counts.type === 'album' + ? counts.programCount + : counts.childGroupCount, + grandchildCount: + counts.type === 'artist' || counts.type === 'show' + ? counts.programCount + : undefined, + }; + } + } + + return map; + } + + async getProgramGroupingDescendants( + groupId: string, + groupTypeHint?: ProgramGroupingType, + ): Promise { + const programs = await this.drizzleDB.query.program.findMany({ + where: (fields, { or, eq }) => { + if (groupTypeHint) { + switch (groupTypeHint) { + case 'show': + return eq(fields.tvShowUuid, groupId); + case 'season': + return eq(fields.seasonUuid, groupId); + case 'artist': + return eq(fields.artistUuid, groupId); + case 'album': + return eq(fields.albumUuid, groupId); + } + } else { + return or( + eq(fields.albumUuid, groupId), + eq(fields.artistUuid, groupId), + eq(fields.tvShowUuid, groupId), + eq(fields.seasonUuid, groupId), + ); + } + }, + with: { + album: + isUndefined(groupTypeHint) || + groupTypeHint === 'album' || + groupTypeHint === 'artist' + ? true + : undefined, + artist: + isUndefined(groupTypeHint) || + groupTypeHint === 'album' || + groupTypeHint === 'artist' + ? true + : undefined, + season: + isUndefined(groupTypeHint) || + groupTypeHint === 'show' || + groupTypeHint === 'season' + ? true + : undefined, + show: + isUndefined(groupTypeHint) || + groupTypeHint === 'show' || + groupTypeHint === 'season' + ? true + : undefined, + externalIds: true, + }, + }); + + return orderBy( + programs, + [(p) => p.season?.index ?? p.seasonNumber ?? 1, (p) => p.episode ?? 1], + ['asc', 'asc'], + ); + } +} diff --git a/server/src/db/program/ProgramGroupingUpsertRepository.ts b/server/src/db/program/ProgramGroupingUpsertRepository.ts new file mode 100644 index 00000000..00d50927 --- /dev/null +++ b/server/src/db/program/ProgramGroupingUpsertRepository.ts @@ -0,0 +1,494 @@ +import type { + ProgramGroupingExternalIdLookup, + UpsertResult, +} from '@/db/interfaces/IProgramDB.js'; +import { KEYS } from '@/types/inject.js'; +import { devAssert } from '@/util/debug.js'; +import { seq } from '@tunarr/shared/util'; +import { untag } from '@tunarr/types'; +import { isValidSingleExternalIdType } from '@tunarr/types/schemas'; +import type { RunResult } from 'better-sqlite3'; +import { and, isNull as dbIsNull, eq, or, sql } from 'drizzle-orm'; +import type { BaseSQLiteDatabase } from 'drizzle-orm/sqlite-core'; +import { inject, injectable } from 'inversify'; +import type { InsertResult, Kysely } from 'kysely'; +import { + chunk, + compact, + head, + isNil, + keys, + omit, + partition, + uniq, +} from 'lodash-es'; +import { P, match } from 'ts-pattern'; +import { groupByUniq, isDefined } from '../../util/index.ts'; +import { + ProgramGrouping, + type NewProgramGroupingOrm, + type ProgramGroupingOrm, +} from '../schema/ProgramGrouping.ts'; +import { + ProgramGroupingExternalId, + toInsertableProgramGroupingExternalId, + type NewProgramGroupingExternalId, + type NewSingleOrMultiProgramGroupingExternalId, + type ProgramGroupingExternalIdOrm, +} from '../schema/ProgramGroupingExternalId.ts'; +import type { DB } from '../schema/db.ts'; +import type { + NewProgramGroupingWithRelations, + ProgramGroupingOrmWithRelations, +} from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess, schema } from '../schema/index.ts'; +import { ProgramMetadataRepository } from './ProgramMetadataRepository.ts'; + +@injectable() +export class ProgramGroupingUpsertRepository { + constructor( + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + @inject(KEYS.ProgramMetadataRepository) + private metadataRepo: ProgramMetadataRepository, + ) {} + + async upsertProgramGrouping( + newGroupingAndRelations: NewProgramGroupingWithRelations, + forceUpdate: boolean = false, + ): Promise> { + let entity: ProgramGroupingOrmWithRelations | undefined = + await this.getProgramGrouping( + newGroupingAndRelations.programGrouping.uuid, + ); + let shouldUpdate = forceUpdate; + let wasInserted = false, + wasUpdated = false; + const { programGrouping: dao, externalIds } = newGroupingAndRelations; + + if (!entity && dao.sourceType === 'local') { + const incomingYear = newGroupingAndRelations.programGrouping.year; + entity = await this.drizzleDB.query.programGrouping.findFirst({ + where: (fields, { eq, and, isNull }) => { + const parentClause = match(newGroupingAndRelations.programGrouping) + .with({ type: 'season', showUuid: P.nonNullable }, (season) => + compact([ + eq(fields.showUuid, season.showUuid), + season.index ? eq(fields.index, season.index) : null, + ]), + ) + .with({ type: 'album', artistUuid: P.nonNullable }, (album) => [ + eq(fields.artistUuid, album.artistUuid), + ]) + .otherwise(() => []); + return and( + eq( + fields.libraryId, + newGroupingAndRelations.programGrouping.libraryId, + ), + eq(fields.title, newGroupingAndRelations.programGrouping.title), + eq(fields.type, newGroupingAndRelations.programGrouping.type), + eq(fields.sourceType, 'local'), + isNil(incomingYear) + ? isNull(fields.year) + : eq(fields.year, incomingYear), + ...parentClause, + ); + }, + with: { + externalIds: true, + }, + }); + } else if (!entity && dao.sourceType !== 'local') { + entity = await this.getProgramGroupingByExternalId({ + sourceType: dao.sourceType, + externalKey: dao.externalKey, + externalSourceId: dao.mediaSourceId, + }); + if (entity) { + const missingAssociation = + (entity.type === 'season' && + isDefined(dao.showUuid) && + dao.showUuid !== entity.showUuid) || + (entity.type === 'album' && + isDefined(dao.artistUuid) && + dao.artistUuid !== entity.artistUuid); + const differentVersion = entity.canonicalId !== dao.canonicalId; + shouldUpdate ||= differentVersion || missingAssociation; + } + } + + if (entity && shouldUpdate) { + newGroupingAndRelations.programGrouping.uuid = entity.uuid; + for (const externalId of newGroupingAndRelations.externalIds) { + externalId.groupUuid = entity.uuid; + } + entity = await this.drizzleDB.transaction(async (tx) => { + const updated = await this.updateProgramGrouping( + newGroupingAndRelations, + entity!, + tx, + ); + const upsertedExternalIds = await this.updateProgramGroupingExternalIds( + entity!.externalIds, + externalIds, + tx, + ); + return { + ...updated, + externalIds: upsertedExternalIds, + } satisfies ProgramGroupingOrmWithRelations; + }); + + wasUpdated = true; + } else if (!entity) { + entity = await this.drizzleDB.transaction(async (tx) => { + const grouping = head( + await tx + .insert(ProgramGrouping) + .values(omit(dao, 'externalIds')) + .returning(), + )!; + const insertedExternalIds: ProgramGroupingExternalIdOrm[] = []; + if (externalIds.length > 0) { + insertedExternalIds.push( + ...(await this.upsertProgramGroupingExternalIdsChunkOrm( + externalIds, + tx, + )), + ); + } + + return { + ...grouping, + externalIds: insertedExternalIds, + } satisfies ProgramGroupingOrmWithRelations; + }); + + wasInserted = true; + shouldUpdate = true; + } + + if (shouldUpdate) { + newGroupingAndRelations.credits.forEach((credit) => { + credit.credit.groupingId = entity.uuid; + }); + + newGroupingAndRelations.artwork.forEach((artwork) => { + artwork.groupingId = entity.uuid; + }); + + await this.metadataRepo.upsertCredits( + newGroupingAndRelations.credits.map(({ credit }) => credit), + ); + + await this.metadataRepo.upsertArtwork( + newGroupingAndRelations.artwork.concat( + newGroupingAndRelations.credits.flatMap(({ artwork }) => artwork), + ), + ); + + await this.metadataRepo.upsertProgramGroupingGenres( + entity.uuid, + newGroupingAndRelations.genres, + ); + + await this.metadataRepo.upsertProgramGroupingStudios( + entity.uuid, + newGroupingAndRelations.studios, + ); + + await this.metadataRepo.upsertProgramGroupingTags( + entity.uuid, + newGroupingAndRelations.tags, + ); + } + + return { + entity: entity, + wasInserted, + wasUpdated, + }; + } + + private async getProgramGrouping( + id: string, + ): Promise { + return this.drizzleDB.query.programGrouping.findFirst({ + where: (fields, { eq }) => eq(fields.uuid, id), + with: { + externalIds: true, + artwork: true, + }, + }); + } + + private async getProgramGroupingByExternalId( + eid: ProgramGroupingExternalIdLookup, + ): Promise { + return await this.drizzleDB.query.programGroupingExternalId + .findFirst({ + where: (row, { and, or, eq }) => + and( + eq(row.externalKey, eid.externalKey), + eq(row.sourceType, eid.sourceType), + or( + eq(row.externalSourceId, untag(eid.externalSourceId)), + eq(row.mediaSourceId, eid.externalSourceId), + ), + ), + with: { + grouping: { + with: { + externalIds: true, + }, + }, + }, + }) + .then((result) => result?.grouping ?? undefined); + } + + private async updateProgramGrouping( + { programGrouping: incoming }: NewProgramGroupingWithRelations, + existing: ProgramGroupingOrmWithRelations, + tx: BaseSQLiteDatabase<'sync', RunResult, typeof schema> = this.drizzleDB, + ): Promise { + const update: NewProgramGroupingOrm = { + ...omit(existing, 'externalIds'), + index: incoming.index, + title: incoming.title, + summary: incoming.summary, + icon: incoming.icon, + year: incoming.year, + artistUuid: incoming.artistUuid, + showUuid: incoming.showUuid, + canonicalId: incoming.canonicalId, + mediaSourceId: incoming.mediaSourceId, + libraryId: incoming.libraryId, + sourceType: incoming.sourceType, + externalKey: incoming.externalKey, + plot: incoming.plot, + rating: incoming.rating, + releaseDate: incoming.releaseDate, + tagline: incoming.tagline, + updatedAt: incoming.updatedAt, + state: incoming.state, + }; + + return head( + await tx + .update(ProgramGrouping) + .set(update) + .where(eq(ProgramGrouping.uuid, existing.uuid)) + .limit(1) + .returning(), + )!; + } + + private async updateProgramGroupingExternalIds( + existingIds: ProgramGroupingExternalId[], + newIds: NewSingleOrMultiProgramGroupingExternalId[], + tx: BaseSQLiteDatabase<'sync', RunResult, typeof schema> = this.drizzleDB, + ): Promise { + devAssert( + uniq(seq.collect(existingIds, (id) => id.mediaSourceId)).length <= 1, + ); + devAssert(uniq(existingIds.map((id) => id.libraryId)).length <= 1); + devAssert(uniq(newIds.map((id) => id.libraryId)).length <= 1); + + const newByUniqueId: Record< + string, + NewSingleOrMultiProgramGroupingExternalId + > = groupByUniq(newIds, (id) => { + switch (id.type) { + case 'single': + return id.sourceType; + case 'multi': + return `${id.sourceType}|${id.mediaSourceId}`; + } + }); + const newUniqueIds = new Set(keys(newByUniqueId)); + + const existingByUniqueId: Record = + groupByUniq(existingIds, (id) => { + if (isValidSingleExternalIdType(id.sourceType)) { + return id.sourceType; + } else { + return `${id.sourceType}|${id.mediaSourceId}`; + } + }); + const existingUniqueIds = new Set(keys(existingByUniqueId)); + + const deletedUniqueKeys = existingUniqueIds.difference(newUniqueIds); + const addedUniqueKeys = newUniqueIds.difference(existingUniqueIds); + const updatedKeys = existingUniqueIds.intersection(newUniqueIds); + + const deletedIds = [...deletedUniqueKeys.values()].map( + (key) => existingByUniqueId[key]!, + ); + await Promise.all( + chunk(deletedIds, 100).map((idChunk) => { + const clauses = idChunk.map((id) => + and( + id.mediaSourceId + ? eq(ProgramGroupingExternalId.mediaSourceId, id.mediaSourceId) + : dbIsNull(ProgramGroupingExternalId.mediaSourceId), + id.libraryId + ? eq(ProgramGroupingExternalId.libraryId, id.libraryId) + : dbIsNull(ProgramGroupingExternalId.libraryId), + eq(ProgramGroupingExternalId.externalKey, id.externalKey), + id.externalSourceId + ? eq( + ProgramGroupingExternalId.externalSourceId, + id.externalSourceId, + ) + : dbIsNull(ProgramGroupingExternalId.externalSourceId), + eq(ProgramGroupingExternalId.sourceType, id.sourceType), + ), + ); + + return tx + .delete(ProgramGroupingExternalId) + .where(or(...clauses)) + .execute(); + }), + ); + + const addedIds = [...addedUniqueKeys.union(updatedKeys).values()].map( + (key) => newByUniqueId[key]!, + ); + + return await Promise.all( + chunk(addedIds, 100).map((idChunk) => + this.upsertProgramGroupingExternalIdsChunkOrm(idChunk, tx), + ), + ).then((_) => _.flat()); + } + + async upsertProgramGroupingExternalIdsChunkOrm( + ids: ( + | NewSingleOrMultiProgramGroupingExternalId + | NewProgramGroupingExternalId + )[], + tx: BaseSQLiteDatabase<'sync', RunResult, typeof schema> = this.drizzleDB, + ): Promise { + if (ids.length === 0) { + return []; + } + + const [singles, multiples] = partition(ids, (id) => + isValidSingleExternalIdType(id.sourceType), + ); + + const promises: Promise[] = []; + + if (singles.length > 0) { + promises.push( + tx + .insert(ProgramGroupingExternalId) + .values(singles.map(toInsertableProgramGroupingExternalId)) + .onConflictDoUpdate({ + target: [ + ProgramGroupingExternalId.groupUuid, + ProgramGroupingExternalId.sourceType, + ], + targetWhere: sql`media_source_id is null`, + set: { + updatedAt: sql`excluded.updated_at`, + externalFilePath: sql`excluded.external_file_path`, + groupUuid: sql`excluded.group_uuid`, + externalKey: sql`excluded.external_key`, + }, + }) + .returning() + .execute(), + ); + } + + if (multiples.length > 0) { + promises.push( + tx + .insert(ProgramGroupingExternalId) + .values(multiples.map(toInsertableProgramGroupingExternalId)) + .onConflictDoUpdate({ + target: [ + ProgramGroupingExternalId.groupUuid, + ProgramGroupingExternalId.sourceType, + ProgramGroupingExternalId.mediaSourceId, + ], + targetWhere: sql`media_source_id is not null`, + set: { + updatedAt: sql`excluded.updated_at`, + externalFilePath: sql`excluded.external_file_path`, + groupUuid: sql`excluded.group_uuid`, + externalKey: sql`excluded.external_key`, + }, + }) + .returning() + .execute(), + ); + } + + return (await Promise.all(promises)).flat(); + } + + async upsertProgramGroupingExternalIdsChunk( + ids: ( + | NewSingleOrMultiProgramGroupingExternalId + | NewProgramGroupingExternalId + )[], + tx: Kysely = this.db, + ): Promise { + if (ids.length === 0) { + return; + } + + const [singles, multiples] = partition(ids, (id) => + isValidSingleExternalIdType(id.sourceType), + ); + + const promises: Promise[] = []; + + if (singles.length > 0) { + promises.push( + tx + .insertInto('programGroupingExternalId') + .values(singles.map(toInsertableProgramGroupingExternalId)) + .onConflict((oc) => + oc + .columns(['groupUuid', 'sourceType']) + .where('mediaSourceId', 'is', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + groupUuid: eb.ref('excluded.groupUuid'), + externalKey: eb.ref('excluded.externalKey'), + })), + ) + .executeTakeFirstOrThrow(), + ); + } + + if (multiples.length > 0) { + promises.push( + tx + .insertInto('programGroupingExternalId') + .values(multiples.map(toInsertableProgramGroupingExternalId)) + .onConflict((oc) => + oc + .columns(['groupUuid', 'sourceType', 'mediaSourceId']) + .where('mediaSourceId', 'is not', null) + .doUpdateSet((eb) => ({ + updatedAt: eb.ref('excluded.updatedAt'), + externalFilePath: eb.ref('excluded.externalFilePath'), + groupUuid: eb.ref('excluded.groupUuid'), + externalKey: eb.ref('excluded.externalKey'), + })), + ) + .executeTakeFirstOrThrow(), + ); + } + + await Promise.all(promises); + } +} diff --git a/server/src/db/program/ProgramMetadataRepository.ts b/server/src/db/program/ProgramMetadataRepository.ts new file mode 100644 index 00000000..d021c622 --- /dev/null +++ b/server/src/db/program/ProgramMetadataRepository.ts @@ -0,0 +1,444 @@ +import { KEYS } from '@/types/inject.js'; +import { seq } from '@tunarr/shared/util'; +import { and, eq, inArray, sql } from 'drizzle-orm'; +import { inject, injectable } from 'inversify'; +import { chunk, difference, groupBy, isNil, keys, partition } from 'lodash-es'; +import type { Dictionary } from 'ts-essentials'; +import { groupByUniq, isDefined, isNonEmptyString } from '../../util/index.ts'; +import { Artwork, type NewArtwork } from '../schema/Artwork.ts'; +import { Credit, type NewCredit } from '../schema/Credit.ts'; +import { + EntityGenre, + Genre, + type NewGenre, + type NewGenreEntity, +} from '../schema/Genre.ts'; +import { + NewProgramSubtitles, + ProgramSubtitles, +} from '../schema/ProgramSubtitles.ts'; +import { + NewStudio, + NewStudioEntity, + Studio, + StudioEntity, +} from '../schema/Studio.ts'; +import { NewTag, NewTagRelation, Tag, TagRelations } from '../schema/Tag.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; + +@injectable() +export class ProgramMetadataRepository { + constructor(@inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess) {} + + async upsertArtwork(artwork: NewArtwork[]) { + if (artwork.length === 0) { + return; + } + + const programArt = groupBy( + artwork.filter((art) => isNonEmptyString(art.programId)), + (art) => art.programId, + ); + const groupArt = groupBy( + artwork.filter((art) => isNonEmptyString(art.groupingId)), + (art) => art.groupingId, + ); + const creditArt = groupBy( + artwork.filter((art) => isNonEmptyString(art.creditId)), + (art) => art.creditId, + ); + + return await this.drizzleDB.transaction(async (tx) => { + for (const batch of chunk(keys(programArt), 50)) { + await tx.delete(Artwork).where(inArray(Artwork.programId, batch)); + } + for (const batch of chunk(keys(groupArt), 50)) { + await tx.delete(Artwork).where(inArray(Artwork.groupingId, batch)); + } + for (const batch of chunk(keys(creditArt), 50)) { + await tx.delete(Artwork).where(inArray(Artwork.creditId, batch)); + } + const inserted: Artwork[] = []; + for (const batch of chunk(artwork, 50)) { + const batchResult = await this.drizzleDB + .insert(Artwork) + .values(batch) + .onConflictDoUpdate({ + target: Artwork.uuid, + set: { + cachePath: sql`excluded.cache_path`, + groupingId: sql`excluded.grouping_id`, + programId: sql`excluded.program_id`, + updatedAt: sql`excluded.updated_at`, + sourcePath: sql`excluded.source_path`, + }, + }) + .returning(); + inserted.push(...batchResult); + } + return inserted; + }); + } + + async upsertProgramGenres(programId: string, genres: NewGenre[]) { + return this.upsertProgramGenresInternal('program', programId, genres); + } + + async upsertProgramGroupingGenres(groupingId: string, genres: NewGenre[]) { + return this.upsertProgramGenresInternal('grouping', groupingId, genres); + } + + private async upsertProgramGenresInternal( + entityType: 'program' | 'grouping', + joinId: string, + genres: NewGenre[], + ) { + if (genres.length === 0) { + return; + } + + const incomingByName = groupByUniq(genres, (g) => g.name); + const existingGenresByName: Dictionary = {}; + for (const genreChunk of chunk(genres, 100)) { + const names = genreChunk.map((g) => g.name); + const results = await this.drizzleDB + .select() + .from(Genre) + .where(inArray(Genre.name, names)); + for (const result of results) { + existingGenresByName[result.name] = result; + } + } + + const newGenreNames = new Set( + difference(keys(incomingByName), keys(existingGenresByName)), + ); + + const relations: NewGenreEntity[] = []; + for (const name of Object.keys(incomingByName)) { + const genreId = newGenreNames.has(name) + ? incomingByName[name]!.uuid + : existingGenresByName[name]!.uuid; + relations.push({ + genreId, + programId: entityType === 'program' ? joinId : null, + groupId: entityType === 'grouping' ? joinId : null, + }); + } + + return this.drizzleDB.transaction(async (tx) => { + const col = + entityType === 'grouping' ? EntityGenre.groupId : EntityGenre.programId; + await tx.delete(EntityGenre).where(eq(col, joinId)); + if (newGenreNames.size > 0) { + await tx + .insert(Genre) + .values( + [...newGenreNames.values()].map((name) => incomingByName[name]!), + ) + .onConflictDoNothing(); + } + if (relations.length > 0) { + await tx.insert(EntityGenre).values(relations).onConflictDoNothing(); + } + }); + } + + async upsertProgramStudios(programId: string, studios: NewStudio[]) { + return this.upsertProgramStudiosInternal('program', programId, studios); + } + + async upsertProgramGroupingStudios(groupingId: string, studios: NewStudio[]) { + return this.upsertProgramStudiosInternal('grouping', groupingId, studios); + } + + private async upsertProgramStudiosInternal( + entityType: 'program' | 'grouping', + joinId: string, + studios: NewStudio[], + ) { + if (studios.length === 0) { + return; + } + + const incomingByName = groupByUniq(studios, (g) => g.name); + const existingStudiosByName: Dictionary = {}; + for (const studioChunk of chunk(studios, 100)) { + const names = studioChunk.map((g) => g.name); + const results = await this.drizzleDB + .select() + .from(Studio) + .where(inArray(Studio.name, names)); + for (const result of results) { + existingStudiosByName[result.name] = result; + } + } + + const newStudioNames = new Set( + difference(keys(incomingByName), keys(existingStudiosByName)), + ); + + const relations: NewStudioEntity[] = []; + for (const name of Object.keys(incomingByName)) { + const studioId = newStudioNames.has(name) + ? incomingByName[name]!.uuid + : existingStudiosByName[name]!.uuid; + relations.push({ + studioId, + programId: entityType === 'program' ? joinId : null, + groupId: entityType === 'grouping' ? joinId : null, + }); + } + + return this.drizzleDB.transaction(async (tx) => { + const col = + entityType === 'grouping' + ? StudioEntity.groupId + : StudioEntity.programId; + await tx.delete(StudioEntity).where(eq(col, joinId)); + if (newStudioNames.size > 0) { + await tx + .insert(Studio) + .values( + [...newStudioNames.values()].map((name) => incomingByName[name]!), + ) + .onConflictDoNothing(); + } + if (relations.length > 0) { + await tx.insert(StudioEntity).values(relations).onConflictDoNothing(); + } + }); + } + + async upsertProgramTags(programId: string, tags: NewTag[]) { + return this.upsertProgramTagsInternal('program', programId, tags); + } + + async upsertProgramGroupingTags(groupingId: string, tags: NewTag[]) { + return this.upsertProgramTagsInternal('grouping', groupingId, tags); + } + + private async upsertProgramTagsInternal( + entityType: 'program' | 'grouping', + joinId: string, + tags: NewTag[], + ) { + if (tags.length === 0) { + return; + } + + const incomingByName = groupByUniq(tags, (g) => g.tag); + const existingTagsByName: Dictionary = {}; + for (const tagChunk of chunk(tags, 100)) { + const names = tagChunk.map((g) => g.tag); + const results = await this.drizzleDB + .select() + .from(Tag) + .where(inArray(Tag.tag, names)); + for (const result of results) { + existingTagsByName[result.tag] = result; + } + } + + const newTagNames = new Set( + difference(keys(incomingByName), keys(existingTagsByName)), + ); + + const relations: NewTagRelation[] = []; + for (const name of Object.keys(incomingByName)) { + const tagId = newTagNames.has(name) + ? incomingByName[name]!.uuid + : existingTagsByName[name]!.uuid; + relations.push({ + tagId, + programId: entityType === 'program' ? joinId : null, + groupingId: entityType === 'grouping' ? joinId : null, + source: 'media', + }); + } + + return this.drizzleDB.transaction(async (tx) => { + const col = + entityType === 'grouping' + ? TagRelations.groupingId + : TagRelations.programId; + await tx + .delete(TagRelations) + .where(and(eq(col, joinId), eq(TagRelations.source, 'media'))); + if (newTagNames.size > 0) { + await tx + .insert(Tag) + .values( + [...newTagNames.values()].map((name) => incomingByName[name]!), + ) + .onConflictDoNothing(); + } + if (relations.length > 0) { + await tx.insert(TagRelations).values(relations).onConflictDoNothing(); + } + }); + } + + async upsertSubtitles(subtitles: NewProgramSubtitles[]) { + if (subtitles.length === 0) { + return; + } + + const grouped = groupBy(subtitles, (sub) => sub.programId); + for (const [programId, programSubtitles] of Object.entries(grouped)) { + const existingSubsForProgram = + await this.drizzleDB.query.programSubtitles.findMany({ + where: (fields, { eq }) => eq(fields.programId, programId), + }); + + const [existingEmbedded, _] = partition( + existingSubsForProgram, + (sub) => !isNil(sub.streamIndex), + ); + const [incomingEmbedded, incomingExternal] = partition( + programSubtitles, + (sub) => !isNil(sub.streamIndex), + ); + + const existingIndexes = new Set( + seq.collect(existingEmbedded, (sub) => sub.streamIndex), + ); + const incomingIndexes = new Set( + seq.collect(incomingEmbedded, (sub) => sub.streamIndex), + ); + + const newIndexes = incomingIndexes.difference(existingIndexes); + const removedIndexes = existingIndexes.difference(newIndexes); + const updatedIndexes = incomingIndexes.difference( + newIndexes.union(removedIndexes), + ); + + const inserts = incomingEmbedded.filter((s) => + newIndexes.has(s.streamIndex!), + ); + const removes = existingEmbedded.filter((s) => + removedIndexes.has(s.streamIndex!), + ); + + const updates: ProgramSubtitles[] = []; + for (const updatedIndex of updatedIndexes.values()) { + const incoming = incomingEmbedded.find( + (s) => s.streamIndex === updatedIndex, + ); + const existing = existingEmbedded.find( + (s) => s.streamIndex === updatedIndex, + ); + if (!existing || !incoming) { + continue; + } + + if (existing.isExtracted) { + const needsExtraction = + existing.subtitleType !== incoming.subtitleType || + existing.codec !== incoming.subtitleType || + existing.language !== incoming.language || + existing.forced !== incoming.forced || + existing.sdh !== incoming.sdh || + existing.default !== incoming.default; + if (needsExtraction) { + existing.isExtracted = false; + existing.path = incoming.path ?? null; + } else if ( + isNonEmptyString(incoming.path) && + existing.path !== incoming.path + ) { + existing.isExtracted = false; + existing.path = incoming.path; + } + } + + existing.codec = incoming.codec; + existing.language = incoming.language; + existing.subtitleType = incoming.subtitleType; + existing.updatedAt = incoming.updatedAt; + if (isDefined(incoming.default)) { + existing.default = incoming.default; + } + + if (isDefined(incoming.sdh)) { + existing.sdh = incoming.sdh; + } + + if (isDefined(incoming.forced)) { + existing.forced = incoming.forced; + } + + updates.push(existing); + } + + await this.drizzleDB.transaction(async (tx) => { + if (inserts.length > 0) { + await tx.insert(ProgramSubtitles).values(inserts); + } + if (removes.length > 0) { + await tx.delete(ProgramSubtitles).where( + inArray( + ProgramSubtitles.uuid, + removes.map((s) => s.uuid), + ), + ); + } + + if (updates.length > 0) { + for (const update of updates) { + await tx + .update(ProgramSubtitles) + .set(update) + .where(eq(ProgramSubtitles.uuid, update.uuid)); + } + } + + await tx + .delete(ProgramSubtitles) + .where( + and( + eq(ProgramSubtitles.subtitleType, 'sidecar'), + eq(ProgramSubtitles.programId, programId), + ), + ); + + if (incomingExternal.length > 0) { + await tx.insert(ProgramSubtitles).values(incomingExternal); + } + }); + } + } + + async upsertCredits(credits: NewCredit[]) { + if (credits.length === 0) { + return; + } + + const programCredits = groupBy( + credits.filter((credit) => isNonEmptyString(credit.programId)), + (credit) => credit.programId, + ); + const groupCredits = groupBy( + credits.filter((credit) => isNonEmptyString(credit.groupingId)), + (credit) => credit.groupingId, + ); + + return await this.drizzleDB.transaction(async (tx) => { + for (const batch of chunk(keys(programCredits), 50)) { + await tx.delete(Credit).where(inArray(Credit.programId, batch)); + } + for (const batch of chunk(keys(groupCredits), 50)) { + await tx.delete(Credit).where(inArray(Credit.groupingId, batch)); + } + const inserted: Credit[] = []; + for (const batch of chunk(credits, 50)) { + const batchResult = await this.drizzleDB + .insert(Credit) + .values(batch) + .returning(); + inserted.push(...batchResult); + } + return inserted; + }); + } +} diff --git a/server/src/db/program/ProgramSearchRepository.ts b/server/src/db/program/ProgramSearchRepository.ts new file mode 100644 index 00000000..6cc49f22 --- /dev/null +++ b/server/src/db/program/ProgramSearchRepository.ts @@ -0,0 +1,229 @@ +import type { + ProgramCanonicalIdLookupResult, + ProgramGroupingCanonicalIdLookupResult, +} from '@/db/interfaces/IProgramDB.js'; +import { KEYS } from '@/types/inject.js'; +import { inject, injectable } from 'inversify'; +import type { Kysely } from 'kysely'; +import { jsonArrayFrom } from 'kysely/helpers/sqlite'; +import { last } from 'lodash-es'; +import type { Dictionary, StrictExclude } from 'ts-essentials'; +import { match } from 'ts-pattern'; +import { + AllProgramFields, + selectProgramsBuilder, + withProgramExternalIds, +} from '../programQueryHelpers.ts'; +import type { ProgramType } from '../schema/Program.ts'; +import type { ProgramGroupingType } from '../schema/ProgramGrouping.ts'; +import type { MediaSourceId, MediaSourceType } from '../schema/base.ts'; +import type { DB } from '../schema/db.ts'; +import type { ProgramWithRelations } from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; +import { isDefined } from '../../util/index.ts'; +import type { ProgramDao } from '../schema/Program.ts'; + +@injectable() +export class ProgramSearchRepository { + constructor( + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + ) {} + + async getProgramsForMediaSource( + mediaSourceId: MediaSourceId, + type?: ProgramType, + ): Promise { + return this.db + .selectFrom('mediaSource') + .where('mediaSource.uuid', '=', mediaSourceId) + .select((eb) => + jsonArrayFrom( + eb + .selectFrom('program') + .select(AllProgramFields) + .$if(isDefined(type), (eb) => eb.where('program.type', '=', type!)) + .whereRef('mediaSource.uuid', '=', 'program.mediaSourceId'), + ).as('programs'), + ) + .executeTakeFirst() + .then((dbResult) => dbResult?.programs ?? []); + } + + async getMediaSourceLibraryPrograms( + libraryId: string, + ): Promise { + return selectProgramsBuilder(this.db, { includeGroupingExternalIds: true }) + .where('libraryId', '=', libraryId) + .selectAll() + .select(withProgramExternalIds) + .execute(); + } + + async getProgramInfoForMediaSource( + mediaSourceId: MediaSourceId, + type: ProgramType, + parentFilter?: [ProgramGroupingType, string], + ): Promise> { + const results = await this.drizzleDB.query.program.findMany({ + where: (fields, { eq, and, isNotNull }) => { + const parentField = match([type, parentFilter?.[0]]) + .with(['episode', 'show'], () => fields.tvShowUuid) + .with(['episode', 'season'], () => fields.seasonUuid) + .with(['track', 'album'], () => fields.albumUuid) + .with(['track', 'artist'], () => fields.artistUuid) + .otherwise(() => null); + + return and( + eq(fields.mediaSourceId, mediaSourceId), + eq(fields.type, type), + isNotNull(fields.canonicalId), + parentField && parentFilter + ? eq(parentField, parentFilter[1]) + : undefined, + ); + }, + }); + + const grouped: Dictionary = {}; + for (const result of results) { + if (!result.canonicalId || !result.libraryId) { + continue; + } + grouped[result.externalKey] = { + canonicalId: result.canonicalId, + externalKey: result.externalKey, + libraryId: result.libraryId, + uuid: result.uuid, + }; + } + + return grouped; + } + + async getProgramInfoForMediaSourceLibrary( + mediaSourceLibraryId: string, + type: ProgramType, + parentFilter?: [ProgramGroupingType, string], + ): Promise> { + const grouped: Dictionary = {}; + for await (const result of this.getProgramInfoForMediaSourceLibraryAsync( + mediaSourceLibraryId, + type, + parentFilter, + )) { + grouped[result.externalKey] = { + canonicalId: result.canonicalId, + externalKey: result.externalKey, + libraryId: result.libraryId, + uuid: result.uuid, + }; + } + + return grouped; + } + + async *getProgramInfoForMediaSourceLibraryAsync( + mediaSourceLibraryId: string, + type: ProgramType, + parentFilter?: [ProgramGroupingType, string], + ): AsyncGenerator { + let lastId: string | undefined; + for (;;) { + const page = await this.drizzleDB.query.program.findMany({ + where: (fields, { eq, and, isNotNull, gt }) => { + const parentField = match([type, parentFilter?.[0]]) + .with(['episode', 'show'], () => fields.tvShowUuid) + .with(['episode', 'season'], () => fields.seasonUuid) + .with(['track', 'album'], () => fields.albumUuid) + .with(['track', 'artist'], () => fields.artistUuid) + .otherwise(() => null); + + return and( + eq(fields.libraryId, mediaSourceLibraryId), + eq(fields.type, type), + isNotNull(fields.canonicalId), + parentField && parentFilter + ? eq(parentField, parentFilter[1]) + : undefined, + lastId ? gt(fields.uuid, lastId) : undefined, + ); + }, + orderBy: (fields, ops) => ops.asc(fields.uuid), + columns: { + uuid: true, + canonicalId: true, + libraryId: true, + externalKey: true, + }, + limit: 500, + }); + + if (page.length === 0) { + return; + } + + lastId = last(page)?.uuid; + for (const item of page) { + yield { + externalKey: item.externalKey, + canonicalId: item.canonicalId, + uuid: item.uuid, + libraryId: item.libraryId, + }; + } + } + } + + async getExistingProgramGroupingDetails( + mediaSourceLibraryId: string, + type: ProgramGroupingType, + sourceType: StrictExclude, + parentFilter?: string, + ): Promise> { + const results = await this.drizzleDB.query.programGrouping.findMany({ + where: (fields, { and, eq, isNotNull }) => { + const parentField = match(type) + .with('album', () => fields.artistUuid) + .with('season', () => fields.showUuid) + .otherwise(() => null); + return and( + eq(fields.libraryId, mediaSourceLibraryId), + eq(fields.type, type), + isNotNull(fields.canonicalId), + parentField && parentFilter + ? eq(parentField, parentFilter) + : undefined, + ); + }, + with: { + externalIds: { + where: (fields, { eq }) => eq(fields.sourceType, sourceType), + }, + }, + columns: { + uuid: true, + canonicalId: true, + libraryId: true, + externalKey: true, + }, + }); + + const grouped: Dictionary = {}; + for (const result of results) { + const key = result.externalKey ?? result.externalIds[0]?.externalKey; + if (!key) { + continue; + } + + grouped[key] = { + canonicalId: result.canonicalId, + externalKey: key, + libraryId: result.libraryId!, + uuid: result.uuid, + }; + } + + return grouped; + } +} diff --git a/server/src/db/program/ProgramStateRepository.ts b/server/src/db/program/ProgramStateRepository.ts new file mode 100644 index 00000000..ca8ec486 --- /dev/null +++ b/server/src/db/program/ProgramStateRepository.ts @@ -0,0 +1,57 @@ +import { KEYS } from '@/types/inject.js'; +import { eq, inArray } from 'drizzle-orm'; +import { inject, injectable } from 'inversify'; +import { chunk } from 'lodash-es'; +import { Program } from '../schema/Program.ts'; +import { ProgramGrouping } from '../schema/ProgramGrouping.ts'; +import type { ProgramState } from '../schema/base.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; + +@injectable() +export class ProgramStateRepository { + constructor( + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + ) {} + + async updateProgramsState( + programIds: string[], + newState: ProgramState, + ): Promise { + if (programIds.length === 0) { + return; + } + + for (const idChunk of chunk(programIds, 100)) { + await this.drizzleDB + .update(Program) + .set({ + state: newState, + }) + .where(inArray(Program.uuid, idChunk)) + .execute(); + } + } + + async updateGroupingsState( + groupingIds: string[], + newState: ProgramState, + ): Promise { + if (groupingIds.length === 0) { + return; + } + + for (const idChunk of chunk(groupingIds, 100)) { + await this.drizzleDB + .update(ProgramGrouping) + .set({ + state: newState, + }) + .where(inArray(ProgramGrouping.uuid, idChunk)) + .execute(); + } + } + + async emptyTrashPrograms(): Promise { + await this.drizzleDB.delete(Program).where(eq(Program.state, 'missing')); + } +} diff --git a/server/src/db/program/ProgramUpsertRepository.ts b/server/src/db/program/ProgramUpsertRepository.ts new file mode 100644 index 00000000..ba6060e4 --- /dev/null +++ b/server/src/db/program/ProgramUpsertRepository.ts @@ -0,0 +1,1080 @@ +import { GlobalScheduler } from '@/services/Scheduler.js'; +import { ReconcileProgramDurationsTask } from '@/tasks/ReconcileProgramDurationsTask.js'; +import { AnonymousTask } from '@/tasks/Task.js'; +import { JellyfinTaskQueue, PlexTaskQueue } from '@/tasks/TaskQueue.js'; +import { SaveJellyfinProgramExternalIdsTask } from '@/tasks/jellyfin/SaveJellyfinProgramExternalIdsTask.js'; +import { SavePlexProgramExternalIdsTask } from '@/tasks/plex/SavePlexProgramExternalIdsTask.js'; +import { autoFactoryKey, KEYS } from '@/types/inject.js'; +import type { MarkNonNullable, Maybe } from '@/types/util.js'; +import { Timer } from '@/util/Timer.js'; +import { + devAssert, +} from '@/util/debug.js'; +import { type Logger } from '@/util/logging/LoggerFactory.js'; +import { seq } from '@tunarr/shared/util'; +import type { ChannelProgram, ContentProgram } from '@tunarr/types'; +import { isContentProgram } from '@tunarr/types'; +import dayjs from 'dayjs'; +import { inject, injectable, interfaces } from 'inversify'; +import type { Kysely, NotNull } from 'kysely'; +import { UpdateResult } from 'kysely'; +import { + chunk, + concat, + difference, + filter, + flatMap, + flatten, + forEach, + groupBy, + head, + isArray, + isEmpty, + keys, + map, + mapValues, + omit, + partition, + reduce, + reject, + round, + some, + uniq, + uniqBy, +} from 'lodash-es'; +import type { Dictionary, MarkRequired } from 'ts-essentials'; +import { typedProperty } from '../../types/path.ts'; +import { getNumericEnvVar, TUNARR_ENV_VARS } from '../../util/env.ts'; +import { + groupByUniq, + groupByUniqProp, + isNonEmptyString, + mapToObj, + unzip as myUnzip, + programExternalIdString, + run, +} from '../../util/index.ts'; +import { ProgramGroupingMinter } from '../converters/ProgramGroupingMinter.ts'; +import { ProgramDaoMinter } from '../converters/ProgramMinter.ts'; +import { + ProgramSourceType, + programSourceTypeFromString, +} from '../custom_types/ProgramSourceType.ts'; +import { + ProgramUpsertFields, +} from '../programQueryHelpers.ts'; +import type { NewArtwork } from '../schema/Artwork.ts'; +import type { NewCredit } from '../schema/Credit.ts'; +import type { NewGenre } from '../schema/Genre.ts'; +import type { + NewProgramDao, + ProgramDao, +} from '../schema/Program.ts'; +import type { NewProgramChapter, ProgramChapter } from '../schema/ProgramChapter.ts'; +import type { NewSingleOrMultiExternalId } from '../schema/ProgramExternalId.ts'; +import type { NewProgramGrouping } from '../schema/ProgramGrouping.ts'; +import { ProgramGroupingType } from '../schema/ProgramGrouping.ts'; +import type { NewSingleOrMultiProgramGroupingExternalId } from '../schema/ProgramGroupingExternalId.ts'; +import type { NewProgramMediaFile } from '../schema/ProgramMediaFile.ts'; +import type { NewProgramMediaStream, ProgramMediaStream } from '../schema/ProgramMediaStream.ts'; +import type { NewProgramSubtitles } from '../schema/ProgramSubtitles.ts'; +import type { ProgramVersion } from '../schema/ProgramVersion.ts'; +import type { NewStudio } from '../schema/Studio.ts'; +import type { NewTag } from '../schema/Tag.ts'; +import type { MediaSourceId, MediaSourceName } from '../schema/base.ts'; +import type { DB } from '../schema/db.ts'; +import type { + NewProgramVersion, + NewProgramWithRelations, + ProgramWithExternalIds, +} from '../schema/derivedTypes.ts'; +import type { DrizzleDBAccess } from '../schema/index.ts'; +import { ProgramExternalIdRepository } from './ProgramExternalIdRepository.ts'; +import { ProgramGroupingUpsertRepository } from './ProgramGroupingUpsertRepository.ts'; +import { ProgramMetadataRepository } from './ProgramMetadataRepository.ts'; + +// Keep this low to make bun sqlite happy. +const DEFAULT_PROGRAM_GROUPING_UPDATE_CHUNK_SIZE = 100; + +type MintedNewProgramInfo = { + program: NewProgramDao; + externalIds: NewSingleOrMultiExternalId[]; + apiProgram: ContentProgram; +}; + +type ContentProgramWithHierarchy = Omit< + MarkRequired, + 'subtype' +> & { + subtype: 'episode' | 'track'; +}; + +type RelevantProgramWithHierarchy = { + program: ProgramDao; + programWithHierarchy: ContentProgramWithHierarchy & { + grandparentKey: string; + parentKey: string; + }; +}; + +type ProgramRelationCaseBuilder = CaseWhenBuilder< + DB, + 'program', + unknown, + string | null +>; + +@injectable() +export class ProgramUpsertRepository { + private timer: Timer; + + constructor( + @inject(KEYS.Logger) private logger: Logger, + @inject(KEYS.Database) private db: Kysely, + @inject(KEYS.DrizzleDB) private drizzleDB: DrizzleDBAccess, + @inject(autoFactoryKey(SavePlexProgramExternalIdsTask)) + private savePlexProgramExternalIdsTaskFactory: interfaces.AutoFactory, + @inject(autoFactoryKey(SaveJellyfinProgramExternalIdsTask)) + private saveJellyfinProgramExternalIdsTask: interfaces.AutoFactory, + @inject(KEYS.ProgramDaoMinterFactory) + private programMinterFactory: interfaces.AutoFactory, + @inject(KEYS.ProgramExternalIdRepository) + private externalIdRepo: ProgramExternalIdRepository, + @inject(KEYS.ProgramMetadataRepository) + private metadataRepo: ProgramMetadataRepository, + @inject(KEYS.ProgramGroupingUpsertRepository) + private groupingUpsertRepo: ProgramGroupingUpsertRepository, + ) { + this.timer = new Timer(this.logger); + } + + async upsertContentPrograms( + programs: ChannelProgram[], + programUpsertBatchSize: number = 100, + ): Promise[]> { + if (isEmpty(programs)) { + return []; + } + + const start = performance.now(); + const [, nonPersisted] = partition(programs, (p) => p.persisted); + const minter = this.programMinterFactory(); + + const [contentPrograms, invalidPrograms] = partition( + uniqBy(filter(nonPersisted, isContentProgram), (p) => p.uniqueId), + (p) => + isNonEmptyString(p.externalSourceType) && + isNonEmptyString(p.externalSourceId) && + isNonEmptyString(p.externalKey) && + p.duration > 0, + ); + + if (!isEmpty(invalidPrograms)) { + this.logger.warn( + 'Found %d invalid programs when saving:\n%O', + invalidPrograms.length, + invalidPrograms, + ); + } + + const programsToPersist: MintedNewProgramInfo[] = seq.collect( + contentPrograms, + (p) => { + const program = minter.contentProgramDtoToDao(p); + if (!program) { + return; + } + const externalIds = minter.mintExternalIds( + program.externalSourceId, + program.mediaSourceId, + program.uuid, + p, + ); + return { program, externalIds, apiProgram: p }; + }, + ); + + const programInfoByUniqueId = groupByUniq( + programsToPersist, + ({ program }) => programExternalIdString(program), + ); + + this.logger.debug('Upserting %d programs', programsToPersist.length); + + const upsertedPrograms: MarkNonNullable[] = []; + await this.timer.timeAsync('programUpsert', async () => { + for (const c of chunk(programsToPersist, programUpsertBatchSize)) { + upsertedPrograms.push( + ...(await this.db.transaction().execute((tx) => + tx + .insertInto('program') + .values(map(c, 'program')) + .onConflict((oc) => + oc + .columns(['sourceType', 'mediaSourceId', 'externalKey']) + .doUpdateSet((eb) => + mapToObj(ProgramUpsertFields, (f) => ({ + [f.replace('excluded.', '')]: eb.ref(f), + })), + ), + ) + .returningAll() + .$narrowType<{ mediaSourceId: NotNull }>() + .execute(), + )), + ); + } + }); + + const programExternalIds = flatMap(upsertedPrograms, (program) => { + const eids = + programInfoByUniqueId[programExternalIdString(program)]?.externalIds ?? + []; + forEach(eids, (eid) => { + eid.programUuid = program.uuid; + }); + return eids; + }); + + await this.timer.timeAsync('programGroupings', () => + this.handleProgramGroupings(upsertedPrograms, programInfoByUniqueId), + ); + + const [requiredExternalIds, backgroundExternalIds] = partition( + programExternalIds, + (p) => p.sourceType === 'plex' || p.sourceType === 'jellyfin', + ); + + await this.timer.timeAsync( + `upsert ${requiredExternalIds.length} external ids`, + () => this.externalIdRepo.upsertProgramExternalIds(requiredExternalIds, 200), + ); + + this.schedulePlexExternalIdsTask(upsertedPrograms); + this.scheduleJellyfinExternalIdsTask(upsertedPrograms); + + setImmediate(() => { + this.logger.debug('Scheduling follow-up program tasks...'); + + GlobalScheduler.scheduleOneOffTask( + autoFactoryKey(ReconcileProgramDurationsTask), + dayjs().add(500, 'ms'), + [], + ); + + PlexTaskQueue.resume(); + JellyfinTaskQueue.resume(); + + this.logger.debug('Upserting external IDs in background'); + + GlobalScheduler.scheduleOneOffTask( + 'UpsertExternalIds', + dayjs().add(100), + undefined, + AnonymousTask('UpsertExternalIds', () => + this.timer.timeAsync( + `background external ID upsert (${backgroundExternalIds.length} ids)`, + () => this.externalIdRepo.upsertProgramExternalIds(backgroundExternalIds), + ), + ), + ); + }); + + const end = performance.now(); + this.logger.debug( + 'upsertContentPrograms took %d millis. %d upsertedPrograms', + round(end - start, 3), + upsertedPrograms.length, + ); + + return upsertedPrograms; + } + + upsertPrograms( + request: NewProgramWithRelations, + ): Promise; + upsertPrograms( + programs: NewProgramWithRelations[], + programUpsertBatchSize?: number, + ): Promise; + async upsertPrograms( + requests: NewProgramWithRelations | NewProgramWithRelations[], + programUpsertBatchSize: number = 100, + ): Promise { + const wasSingleRequest = !isArray(requests); + requests = isArray(requests) ? requests : [requests]; + if (isEmpty(requests)) { + return []; + } + + const db = this.db; + + const requestsByCanonicalId = groupByUniq( + requests, + ({ program }) => program.canonicalId, + ); + + const result = await Promise.all( + chunk(requests, programUpsertBatchSize).map(async (c) => { + const chunkResult = await db.transaction().execute((tx) => + tx + .insertInto('program') + .values(c.map(({ program }) => program)) + .onConflict((oc) => + oc + .columns(['sourceType', 'mediaSourceId', 'externalKey']) + .doUpdateSet((eb) => + mapToObj(ProgramUpsertFields, (f) => ({ + [f.replace('excluded.', '')]: eb.ref(f), + })), + ), + ) + .returningAll() + .$narrowType<{ mediaSourceId: NotNull; canonicalId: NotNull }>() + .execute(), + ); + + const allExternalIds = flatten(c.map((program) => program.externalIds)); + const versionsToInsert: NewProgramVersion[] = []; + const artworkToInsert: NewArtwork[] = []; + const subtitlesToInsert: NewProgramSubtitles[] = []; + const creditsToInsert: NewCredit[] = []; + const genresToInsert: Dictionary = {}; + const studiosToInsert: Dictionary = {}; + const tagsToInsert: Dictionary = {}; + for (const program of chunkResult) { + const key = program.canonicalId; + const request: Maybe = + requestsByCanonicalId[key]; + const eids = request?.externalIds ?? []; + for (const eid of eids) { + eid.programUuid = program.uuid; + } + + for (const version of request?.versions ?? []) { + version.programId = program.uuid; + versionsToInsert.push(version); + } + + for (const art of request?.artwork ?? []) { + art.programId = program.uuid; + artworkToInsert.push(art); + } + + for (const subtitle of request?.subtitles ?? []) { + subtitle.programId = program.uuid; + subtitlesToInsert.push(subtitle); + } + + for (const { credit, artwork } of request?.credits ?? []) { + credit.programId = program.uuid; + creditsToInsert.push(credit); + artworkToInsert.push(...artwork); + } + + for (const genre of request?.genres ?? []) { + genresToInsert[program.uuid] ??= []; + genresToInsert[program.uuid]?.push(genre); + } + + for (const studio of request?.studios ?? []) { + studiosToInsert[program.uuid] ??= []; + studiosToInsert[program.uuid]?.push(studio); + } + + for (const tag of request?.tags ?? []) { + tagsToInsert[program.uuid] ??= []; + tagsToInsert[program.uuid]?.push(tag); + } + } + + const externalIdsByProgramId = + await this.externalIdRepo.upsertProgramExternalIds(allExternalIds); + + await this.upsertProgramVersions(versionsToInsert); + + await this.metadataRepo.upsertCredits(creditsToInsert); + + await this.metadataRepo.upsertArtwork(artworkToInsert); + + await this.metadataRepo.upsertSubtitles(subtitlesToInsert); + + for (const [programId, genres] of Object.entries(genresToInsert)) { + await this.metadataRepo.upsertProgramGenres(programId, genres); + } + + for (const [programId, studios] of Object.entries(studiosToInsert)) { + await this.metadataRepo.upsertProgramStudios(programId, studios); + } + + for (const [programId, tags] of Object.entries(tagsToInsert)) { + await this.metadataRepo.upsertProgramTags(programId, tags); + } + + return chunkResult.map( + (upsertedProgram) => + ({ + ...upsertedProgram, + externalIds: externalIdsByProgramId[upsertedProgram.uuid] ?? [], + }) satisfies ProgramWithExternalIds, + ); + }), + ).then(flatten); + + if (wasSingleRequest) { + return head(result)!; + } else { + return result; + } + } + + private async upsertProgramVersions(versions: NewProgramVersion[]) { + if (versions.length === 0) { + this.logger.warn('No program versions passed for item'); + return []; + } + + const insertedVersions: ProgramVersion[] = []; + await this.db.transaction().execute(async (tx) => { + const byProgramId = groupByUniq(versions, (version) => version.programId); + for (const batch of chunk(Object.entries(byProgramId), 50)) { + const [programIds, versionBatch] = myUnzip(batch); + await tx + .deleteFrom('programVersion') + .where('programId', 'in', programIds) + .executeTakeFirstOrThrow(); + + const insertResult = await tx + .insertInto('programVersion') + .values( + versionBatch.map((version) => + omit(version, ['chapters', 'mediaStreams', 'mediaFiles']), + ), + ) + .returningAll() + .execute(); + + await this.upsertProgramMediaStreams( + versionBatch.flatMap(({ mediaStreams }) => mediaStreams), + tx, + ); + await this.upsertProgramChapters( + versionBatch.flatMap(({ chapters }) => chapters ?? []), + tx, + ); + await this.upsertProgramMediaFiles( + versionBatch.flatMap(({ mediaFiles }) => mediaFiles), + tx, + ); + + insertedVersions.push(...insertResult); + } + }); + return insertedVersions; + } + + private async upsertProgramMediaStreams( + streams: NewProgramMediaStream[], + tx: Kysely = this.db, + ) { + if (streams.length === 0) { + this.logger.warn('No media streams passed for version'); + return []; + } + + const byVersionId = groupBy(streams, (stream) => stream.programVersionId); + const inserted: ProgramMediaStream[] = []; + for (const batch of chunk(Object.entries(byVersionId), 50)) { + const [_, streams] = myUnzip(batch); + inserted.push( + ...(await tx + .insertInto('programMediaStream') + .values(flatten(streams)) + .returningAll() + .execute()), + ); + } + return inserted; + } + + private async upsertProgramChapters( + chapters: NewProgramChapter[], + tx: Kysely = this.db, + ) { + if (chapters.length === 0) { + return []; + } + + const byVersionId = groupBy(chapters, (stream) => stream.programVersionId); + const inserted: ProgramChapter[] = []; + for (const batch of chunk(Object.entries(byVersionId), 50)) { + const [_, streams] = myUnzip(batch); + inserted.push( + ...(await tx + .insertInto('programChapter') + .values(flatten(streams)) + .returningAll() + .execute()), + ); + } + return inserted; + } + + private async upsertProgramMediaFiles( + files: NewProgramMediaFile[], + tx: Kysely = this.db, + ) { + if (files.length === 0) { + this.logger.warn('No media files passed for version'); + return []; + } + + const byVersionId = groupBy(files, (stream) => stream.programVersionId); + const inserted: ProgramMediaFile[] = []; + for (const batch of chunk(Object.entries(byVersionId), 50)) { + const [_, files] = myUnzip(batch); + inserted.push( + ...(await tx + .insertInto('programMediaFile') + .values(flatten(files)) + .returningAll() + .execute()), + ); + } + return inserted; + } + + private async handleProgramGroupings( + upsertedPrograms: MarkNonNullable[], + programInfos: Record, + ) { + const bySourceAndServer = mapValues( + groupBy(upsertedPrograms, 'sourceType'), + (ps) => groupBy(ps, typedProperty('mediaSourceId')), + ); + + for (const [sourceType, byServerId] of Object.entries(bySourceAndServer)) { + for (const [serverId, programs] of Object.entries(byServerId)) { + const serverName = head(programs)!.externalSourceId; + const typ = programSourceTypeFromString(sourceType); + if (!typ) { + return; + } + + await this.handleSingleSourceProgramGroupings( + programs, + programInfos, + typ, + serverName, + serverId as MediaSourceId, + ); + } + } + } + + private async handleSingleSourceProgramGroupings( + upsertedPrograms: MarkNonNullable[], + programInfos: Record, + mediaSourceType: ProgramSourceType, + mediaSourceName: MediaSourceName, + mediaSourceId: MediaSourceId, + ) { + const grandparentRatingKeyToParentRatingKey: Record< + string, + Set + > = {}; + const grandparentRatingKeyToProgramId: Record> = {}; + const parentRatingKeyToProgramId: Record> = {}; + + const relevantPrograms: RelevantProgramWithHierarchy[] = seq.collect( + upsertedPrograms, + (program) => { + if ( + program.type === 'movie' || + program.type === 'music_video' || + program.type === 'other_video' + ) { + return; + } + + const info = programInfos[programExternalIdString(program)]; + if (!info) { + return; + } + + if ( + info.apiProgram.subtype === 'movie' || + info.apiProgram.subtype === 'music_video' || + info.apiProgram.subtype === 'other_video' + ) { + return; + } + + const [grandparentKey, parentKey] = [ + info.apiProgram.grandparent?.externalKey, + info.apiProgram.parent?.externalKey, + ]; + + if (!grandparentKey || !parentKey) { + this.logger.warn( + 'Unexpected null/empty parent keys: %O', + info.apiProgram, + ); + return; + } + + return { + program, + programWithHierarchy: { + ...(info.apiProgram as ContentProgramWithHierarchy), + grandparentKey, + parentKey, + }, + }; + }, + ); + + const upsertedProgramById = groupByUniqProp( + map(relevantPrograms, ({ program }) => program), + 'uuid', + ); + + for (const { + program, + programWithHierarchy: { grandparentKey, parentKey }, + } of relevantPrograms) { + if (isNonEmptyString(grandparentKey)) { + (grandparentRatingKeyToProgramId[grandparentKey] ??= new Set()).add( + program.uuid, + ); + + const set = (grandparentRatingKeyToParentRatingKey[grandparentKey] ??= + new Set()); + if (isNonEmptyString(parentKey)) { + set.add(parentKey); + } + } + + if (isNonEmptyString(parentKey)) { + (parentRatingKeyToProgramId[parentKey] ??= new Set()).add(program.uuid); + } + } + + const allGroupingKeys = concat( + keys(grandparentRatingKeyToParentRatingKey), + keys(parentRatingKeyToProgramId), + ); + + const existingGroupings = await this.timer.timeAsync( + `selecting grouping external ids (${allGroupingKeys.length})`, + () => + this.drizzleDB.query.programGroupingExternalId.findMany({ + where: (fields, { eq, and, inArray }) => + and( + eq(fields.sourceType, mediaSourceType), + eq(fields.mediaSourceId, mediaSourceId), + inArray(fields.externalKey, allGroupingKeys), + ), + with: { + grouping: true, + }, + }), + ); + + const foundGroupingRatingKeys = map(existingGroupings, 'externalKey'); + const missingGroupingRatingKeys = difference( + allGroupingKeys, + foundGroupingRatingKeys, + ); + const grandparentKeys = new Set(keys(grandparentRatingKeyToProgramId)); + const missingGrandparents = filter(missingGroupingRatingKeys, (s) => + grandparentKeys.has(s), + ); + + const updatesByType: Record> = { + album: new Set(), + artist: new Set(), + season: new Set(), + show: new Set(), + } as const; + + for (const group of existingGroupings) { + for (const { + program: upsertedProgram, + programWithHierarchy: { grandparentKey, parentKey }, + } of relevantPrograms) { + if (group.externalKey === grandparentKey) { + switch (upsertedProgram.type) { + case 'episode': + upsertedProgram.tvShowUuid = group.groupUuid; + updatesByType[ProgramGroupingType.Show].add(upsertedProgram.uuid); + break; + case 'track': + upsertedProgram.artistUuid = group.groupUuid; + updatesByType[ProgramGroupingType.Artist].add( + upsertedProgram.uuid, + ); + break; + case 'movie': + case 'music_video': + case 'other_video': + default: + this.logger.warn( + 'Unexpected program type %s when calculating hierarchy. id = %s', + upsertedProgram.type, + upsertedProgram.uuid, + ); + break; + } + } else if (group.externalKey === parentKey) { + switch (upsertedProgram.type) { + case 'episode': + upsertedProgram.seasonUuid = group.groupUuid; + updatesByType[ProgramGroupingType.Season].add( + upsertedProgram.uuid, + ); + break; + case 'track': + upsertedProgram.albumUuid = group.groupUuid; + updatesByType[ProgramGroupingType.Album].add( + upsertedProgram.uuid, + ); + break; + case 'movie': + case 'music_video': + case 'other_video': + default: + this.logger.warn( + 'Unexpected program type %s when calculating hierarchy. id = %s', + upsertedProgram.type, + upsertedProgram.uuid, + ); + break; + } + } + } + } + + // New ones + const groupings: NewProgramGrouping[] = []; + const externalIds: NewSingleOrMultiProgramGroupingExternalId[] = []; + for (const missingGrandparent of missingGrandparents) { + const matchingPrograms = filter( + relevantPrograms, + ({ programWithHierarchy: { grandparentKey } }) => + grandparentKey === missingGrandparent, + ); + + if (isEmpty(matchingPrograms)) { + continue; + } + + const grandparentGrouping = ProgramGroupingMinter.mintGrandparentGrouping( + matchingPrograms[0]!.programWithHierarchy, + ); + + if (grandparentGrouping === null) { + devAssert(false); + continue; + } + + matchingPrograms.forEach(({ program }) => { + if (grandparentGrouping.type === ProgramGroupingType.Artist) { + program.artistUuid = grandparentGrouping.uuid; + updatesByType[ProgramGroupingType.Artist].add(program.uuid); + } else if (grandparentGrouping.type === ProgramGroupingType.Show) { + program.tvShowUuid = grandparentGrouping.uuid; + updatesByType[ProgramGroupingType.Show].add(program.uuid); + } + }); + + const parentKeys = [ + ...(grandparentRatingKeyToParentRatingKey[missingGrandparent] ?? + new Set()), + ]; + const parents = reject(parentKeys, (parent) => + foundGroupingRatingKeys.includes(parent), + ); + + for (const parentKey of parents) { + const programIds = parentRatingKeyToProgramId[parentKey]; + if (!programIds || programIds.size === 0) { + devAssert(false); + continue; + } + + const programs = filter(relevantPrograms, ({ program }) => + programIds.has(program.uuid), + ); + + if (isEmpty(programs)) { + devAssert(false); + continue; + } + + devAssert( + () => uniq(map(programs, ({ program: p }) => p.type)).length === 1, + ); + + const parentGrouping = ProgramGroupingMinter.mintParentGrouping( + programs[0]!.programWithHierarchy, + ); + + if (!parentGrouping) { + continue; + } + + programs.forEach(({ program }) => { + if (program.type === 'episode') { + program.seasonUuid = parentGrouping.uuid; + updatesByType[ProgramGroupingType.Season].add(program.uuid); + } else if (program.type === 'track') { + program.albumUuid = parentGrouping.uuid; + updatesByType[ProgramGroupingType.Album].add(program.uuid); + } + }); + + if (parentGrouping.type === ProgramGroupingType.Season) { + parentGrouping.showUuid = grandparentGrouping.uuid; + } else if (parentGrouping.type === ProgramGroupingType.Album) { + parentGrouping.artistUuid = grandparentGrouping.uuid; + } + + groupings.push(parentGrouping); + externalIds.push( + ...ProgramGroupingMinter.mintGroupingExternalIds( + programs[0]!.programWithHierarchy, + parentGrouping.uuid, + mediaSourceName, + mediaSourceId, + 'parent', + ), + ); + } + + groupings.push(grandparentGrouping); + externalIds.push( + ...ProgramGroupingMinter.mintGroupingExternalIds( + matchingPrograms[0]!.programWithHierarchy, + grandparentGrouping.uuid, + mediaSourceName, + mediaSourceId, + 'grandparent', + ), + ); + } + + if (!isEmpty(groupings)) { + await this.timer.timeAsync('upsert program_groupings', () => + this.db + .transaction() + .execute((tx) => + tx + .insertInto('programGrouping') + .values(groupings) + .executeTakeFirstOrThrow(), + ), + ); + } + + if (!isEmpty(externalIds)) { + await this.timer.timeAsync('upsert program_grouping external ids', () => + Promise.all( + chunk(externalIds, 100).map((externalIdsChunk) => + this.db + .transaction() + .execute((tx) => + this.groupingUpsertRepo.upsertProgramGroupingExternalIdsChunk(externalIdsChunk, tx), + ), + ), + ), + ); + } + + const hasUpdates = some(updatesByType, (updates) => updates.size > 0); + + if (hasUpdates) { + await this.timer.timeAsync('update program relations', () => + this.db.transaction().execute(async (tx) => { + const tvShowIdUpdates = [...updatesByType[ProgramGroupingType.Show]]; + + const chunkSize = run(() => { + const envVal = getNumericEnvVar( + TUNARR_ENV_VARS.DEBUG__PROGRAM_GROUPING_UPDATE_CHUNK_SIZE, + ); + + if (isNonEmptyString(envVal) && !isNaN(parseInt(envVal))) { + return Math.min(10_000, parseInt(envVal)); + } + return DEFAULT_PROGRAM_GROUPING_UPDATE_CHUNK_SIZE; + }); + + const updates: Promise[] = []; + + if (!isEmpty(tvShowIdUpdates)) { + for (const idChunk of chunk(tvShowIdUpdates, chunkSize)) { + updates.push( + tx + .updateTable('program') + .set((eb) => ({ + tvShowUuid: reduce( + idChunk, + (acc, curr) => + acc + .when('program.uuid', '=', curr) + .then(upsertedProgramById[curr]!.tvShowUuid), + eb.case() as unknown as ProgramRelationCaseBuilder, + ) + .else(eb.ref('program.tvShowUuid')) + .end(), + })) + .where('program.uuid', 'in', idChunk) + .execute(), + ); + } + } + + const seasonIdUpdates = [ + ...updatesByType[ProgramGroupingType.Season], + ]; + + if (!isEmpty(seasonIdUpdates)) { + for (const idChunk of chunk(seasonIdUpdates, chunkSize)) { + updates.push( + tx + .updateTable('program') + .set((eb) => ({ + seasonUuid: reduce( + idChunk, + (acc, curr) => + acc + .when('program.uuid', '=', curr) + .then(upsertedProgramById[curr]!.seasonUuid), + eb.case() as unknown as ProgramRelationCaseBuilder, + ) + .else(eb.ref('program.seasonUuid')) + .end(), + })) + .where('program.uuid', 'in', idChunk) + .execute(), + ); + } + } + + const musicArtistUpdates = [ + ...updatesByType[ProgramGroupingType.Artist], + ]; + + if (!isEmpty(musicArtistUpdates)) { + for (const idChunk of chunk(musicArtistUpdates, chunkSize)) { + updates.push( + tx + .updateTable('program') + .set((eb) => ({ + artistUuid: reduce( + idChunk, + (acc, curr) => + acc + .when('program.uuid', '=', curr) + .then(upsertedProgramById[curr]!.artistUuid), + eb.case() as unknown as ProgramRelationCaseBuilder, + ) + .else(eb.ref('program.artistUuid')) + .end(), + })) + .where('program.uuid', 'in', idChunk) + .execute(), + ); + } + } + + const musicAlbumUpdates = [ + ...updatesByType[ProgramGroupingType.Album], + ]; + + if (!isEmpty(musicAlbumUpdates)) { + for (const idChunk of chunk(musicAlbumUpdates, chunkSize)) { + updates.push( + tx + .updateTable('program') + .set((eb) => ({ + albumUuid: reduce( + idChunk, + (acc, curr) => + acc + .when('program.uuid', '=', curr) + .then(upsertedProgramById[curr]!.albumUuid), + eb.case() as unknown as ProgramRelationCaseBuilder, + ) + .else(eb.ref('program.albumUuid')) + .end(), + })) + .where('program.uuid', 'in', idChunk) + .execute(), + ); + } + } + + await Promise.all(updates); + }), + ); + } + } + + private schedulePlexExternalIdsTask(upsertedPrograms: ProgramDao[]) { + PlexTaskQueue.pause(); + this.timer.timeSync('schedule Plex external IDs tasks', () => { + forEach( + filter(upsertedPrograms, { sourceType: ProgramSourceType.PLEX }), + (program) => { + try { + const task = this.savePlexProgramExternalIdsTaskFactory(); + task.logLevel = 'trace'; + PlexTaskQueue.add(task, { programId: program.uuid }).catch((e) => { + this.logger.error( + e, + 'Error saving external IDs for program %O', + program, + ); + }); + } catch (e) { + this.logger.error( + e, + 'Failed to schedule external IDs task for persisted program: %O', + program, + ); + } + }, + ); + }); + } + + private scheduleJellyfinExternalIdsTask(upsertedPrograms: ProgramDao[]) { + JellyfinTaskQueue.pause(); + this.timer.timeSync('Schedule Jellyfin external IDs tasks', () => { + forEach( + filter(upsertedPrograms, (p) => p.sourceType === 'jellyfin'), + (program) => { + try { + const task = this.saveJellyfinProgramExternalIdsTask(); + JellyfinTaskQueue.add(task, { programId: program.uuid }).catch( + (e) => { + this.logger.error( + e, + 'Error saving external IDs for program %O', + program, + ); + }, + ); + } catch (e) { + this.logger.error( + e, + 'Failed to schedule external IDs task for persisted program: %O', + program, + ); + } + }, + ); + }); + } +} diff --git a/server/src/types/inject.ts b/server/src/types/inject.ts index 0adf1d2d..60d24404 100644 --- a/server/src/types/inject.ts +++ b/server/src/types/inject.ts @@ -86,6 +86,22 @@ const KEYS = { // Tasks Task: Symbol.for('Task'), StartupTasks: Symbol.for('StartupTasks'), + + // ProgramDB repositories + BasicProgramRepository: Symbol.for('BasicProgramRepository'), + ProgramGroupingRepository: Symbol.for('ProgramGroupingRepository'), + ProgramExternalIdRepository: Symbol.for('ProgramExternalIdRepository'), + ProgramUpsertRepository: Symbol.for('ProgramUpsertRepository'), + ProgramMetadataRepository: Symbol.for('ProgramMetadataRepository'), + ProgramGroupingUpsertRepository: Symbol.for('ProgramGroupingUpsertRepository'), + ProgramSearchRepository: Symbol.for('ProgramSearchRepository'), + ProgramStateRepository: Symbol.for('ProgramStateRepository'), + + // ChannelDB repositories + BasicChannelRepository: Symbol.for('BasicChannelRepository'), + ChannelProgramRepository: Symbol.for('ChannelProgramRepository'), + LineupRepository: Symbol.for('LineupRepository'), + ChannelConfigRepository: Symbol.for('ChannelConfigRepository'), }; // eslint-disable-next-line @typescript-eslint/no-explicit-any