feat: add health check for Meilisearch service health

server/src/services/ISearchService.ts

@@ -1,5 +1,8 @@
+export type SearchHealthStatus = 'starting' | 'healthy' | 'degraded' | 'error';
+
 export interface ISearchService {
   start(): Promise<void>;
   restart(): Promise<void>;
   stop(): void;
+  getHealthStatus(): SearchHealthStatus;
 }

server/src/services/MeilisearchService.ts

@@ -84,7 +84,7 @@ import { fileExists } from '../util/fsUtil.ts';
 import { isNonEmptyString, isWindows, wait } from '../util/index.ts';
 import { Logger } from '../util/logging/LoggerFactory.ts';
 import { FileSystemService } from './FileSystemService.ts';
-import { ISearchService } from './ISearchService.ts';
+import { ISearchService, SearchHealthStatus } from './ISearchService.ts';
 import { SearchParser } from './search/SearchParser.ts';
 
 type FlattenArrayTypes<T> = {
@@ -389,6 +389,11 @@ export class MeilisearchService implements ISearchService {
   private proc?: ChildProcessWrapper;
   private port: number;
   #client: MeiliSearch;
+  private executablePath: string = '';
+  private spawnArgs: readonly string[] = [];
+  private logFilePath: string = '';
+  private healthStatus: SearchHealthStatus = 'starting';
+  private healthPollInterval?: NodeJS.Timeout;
 
   constructor(
     @inject(KEYS.Logger) private logger: Logger,
@@ -520,13 +525,13 @@ export class MeilisearchService implements ISearchService {
           args.push('--experimental-reduce-indexing-memory-usage');
         }
 
-        const searchServerLogFile = path.join(
+        this.logFilePath = path.join(
           this.settingsDB.systemSettings().logging.logsDirectory,
           'meilisearch.log',
         );
 
-        if (await fileExists(searchServerLogFile)) {
-          await fs.truncate(searchServerLogFile);
+        if (await fileExists(this.logFilePath)) {
+          await fs.truncate(this.logFilePath);
         }
 
         let executablePath: Maybe<string>;
@@ -568,19 +573,26 @@ export class MeilisearchService implements ISearchService {
           );
         }
 
+        this.executablePath = executablePath;
+        this.spawnArgs = args;
+
         this.logger.trace(
           'Starting meilisearch with args: %s %s',
-          executablePath,
+          this.executablePath,
           args.join(' '),
         );
-        this.proc = await this.childProcessHelper.spawn(executablePath, args, {
-          maxAttempts: 3,
-          additionalOpts: {
-            cwd: this.serverOptions.databaseDirectory,
+        this.proc = await this.childProcessHelper.spawn(
+          this.executablePath,
+          this.spawnArgs,
+          {
+            maxAttempts: 3,
+            additionalOpts: {
+              cwd: this.serverOptions.databaseDirectory,
+            },
           },
-        });
+        );
         this.logger.info('Meilisearch service started on port %d', this.port);
-        const outStream = createWriteStream(searchServerLogFile);
+        const outStream = createWriteStream(this.logFilePath);
         this.proc.process?.stdout.pipe(outStream);
         this.proc.process?.stderr.pipe(outStream);
       }
@@ -593,18 +605,74 @@ export class MeilisearchService implements ISearchService {
         this.logger.debug('Got health result from Meilisearch: %O', result);
       });
 
+      this.healthStatus = 'healthy';
+      if (isMainThread) {
+        this.#startHealthPoll();
+      }
+
       return;
     });
   }
 
+  #startHealthPoll() {
+    this.healthPollInterval = setInterval(() => {
+      this.healthCheckFunc().catch(console.error);
+    }, 30_000);
+  }
+
+  private healthCheckFunc = async () => {
+    try {
+      await this.client().health();
+      this.healthStatus = 'healthy';
+    } catch (err) {
+      this.logger.warn(
+        err,
+        'Meilisearch health poll failed. Attempting restart.',
+      );
+      this.healthStatus = 'degraded';
+      // try {
+      //   await this.restart();
+      // } catch (restartErr) {
+      //   this.logger.fatal(restartErr, 'Meilisearch restart failed.');
+      //   this.healthStatus = 'error';
+      // }
+    }
+  };
+
   async restart() {
-    // TODO: implement
+    if (!isMainThread) return;
+    this.logger.info('Restarting Meilisearch process...');
+    this.proc?.kill();
+    this.proc = await this.childProcessHelper.spawn(
+      this.executablePath,
+      this.spawnArgs,
+      {
+        maxAttempts: 3,
+        additionalOpts: { cwd: this.serverOptions.databaseDirectory },
+      },
+    );
+    const outStream = createWriteStream(this.logFilePath, { flags: 'a' });
+    this.proc.process?.stdout.pipe(outStream);
+    this.proc.process?.stderr.pipe(outStream);
+    await retry(async () => {
+      await this.client().health();
+    });
+    this.healthStatus = 'healthy';
+    this.logger.info('Meilisearch restarted successfully.');
   }
 
   stop() {
+    if (this.healthPollInterval) {
+      clearInterval(this.healthPollInterval);
+      this.healthPollInterval = undefined;
+    }
     this.proc?.kill();
   }
 
+  getHealthStatus(): SearchHealthStatus {
+    return this.healthStatus;
+  }
+
   async getMeilisearchVersion(): Promise<Maybe<string>> {
     const versionPath = path.join(this.dbPath, 'VERSION');
     return fs

server/src/services/health_checks/HealthCheckModule.ts

@@ -8,6 +8,7 @@ import { KEYS } from '@/types/inject.js';
 import { ContainerModule } from 'inversify';
 import { BaseImageHealthCheck } from './BaseImageHealthCheck.ts';
 import { FfmpegTranscodeDirectoryHealthCheck } from './FfmpegTranscodeDirectoryHealthCheck.ts';
+import { SearchHealthCheck } from './SearchHealthCheck.ts';
 
 const HealthCheckModule = new ContainerModule((bind) => {
   bind<HealthCheck>(KEYS.HealthCheck).to(FfmpegDebugLoggingHealthCheck);
@@ -17,6 +18,7 @@ const HealthCheckModule = new ContainerModule((bind) => {
   bind<HealthCheck>(KEYS.HealthCheck).to(HardwareAccelerationHealthCheck);
   bind<HealthCheck>(KEYS.HealthCheck).to(MissingSeasonNumbersHealthCheck);
   bind<HealthCheck>(KEYS.HealthCheck).to(FfmpegTranscodeDirectoryHealthCheck);
+  bind<HealthCheck>(KEYS.HealthCheck).to(SearchHealthCheck);
 });
 
 export { HealthCheckModule };

server/src/services/health_checks/SearchHealthCheck.ts

@@ -0,0 +1,47 @@
+import { MeilisearchService } from '@/services/MeilisearchService.js';
+import { inject, injectable } from 'inversify';
+import {
+  HealthCheck,
+  HealthCheckResult,
+  HealthyHealthCheckResult,
+  healthCheckResult,
+} from './HealthCheck.ts';
+
+@injectable()
+export class SearchHealthCheck implements HealthCheck {
+  readonly id = 'SearchServer';
+
+  constructor(
+    @inject(MeilisearchService) private searchService: MeilisearchService,
+  ) {}
+
+  getStatus(): Promise<HealthCheckResult> {
+    switch (this.searchService.getHealthStatus()) {
+      case 'starting':
+        return Promise.resolve(
+          healthCheckResult({
+            type: 'info',
+            context: 'Search server is starting up.',
+          }),
+        );
+      case 'healthy':
+        return Promise.resolve(HealthyHealthCheckResult);
+      case 'degraded':
+        return Promise.resolve(
+          healthCheckResult({
+            type: 'warning',
+            context:
+              'Search server failed a health check and a restart was attempted. Search may be temporarily unavailable.',
+          }),
+        );
+      case 'error':
+        return Promise.resolve(
+          healthCheckResult({
+            type: 'error',
+            context:
+              'Search server is not responding and could not be restarted. Search functionality is unavailable. Check server logs.',
+          }),
+        );
+    }
+  }
+}

server/src/util/ChildProcessHelper.ts

@@ -26,6 +26,7 @@ type SpawnOpts = {
 
 type ChildProcessEvents = {
   restart: () => void;
+  fail: () => void;
 };
 
 abstract class ITypedEventEmitter extends (events.EventEmitter as new () => TypedEventEmitter<ChildProcessEvents>) {}
@@ -109,6 +110,7 @@ export class ChildProcessWrapper extends ITypedEventEmitter {
         const bufferedBytes = bufferedOut.getLastN().toString('utf-8');
         this.logger.error(bufferedBytes);
         console.error(bufferedBytes);
+        this.emit('fail');
       }
 
       if (!this.wasAborted && this.opts.restartOnFailure) {

web/src/pages/system/StatusPage.tsx

@@ -44,6 +44,7 @@ const HardwareAccelerationCheck = 'HardwareAcceleration';
 const FfmpegDebugLoggingCheck = 'FfmpegDebugLogging';
 const FfmpegTranscodeDirectory = 'FfmpegTranscodeDirectory';
 const BaseImageHealthCheck = 'BaseImageHealthCheck';
+const SearchServerCheck = 'SearchServer';
 
 const AllKnownChecks = [
   FfmpegVersionCheck,
@@ -53,6 +54,7 @@ const AllKnownChecks = [
   // MissingSeasonNumbersCheck,
   // MissingProgramAssociationsHealthCheck,
   BaseImageHealthCheck,
+  SearchServerCheck,
 ] as const;
 
 const CopyToClipboardButton = (
@@ -127,6 +129,7 @@ export const StatusPage = () => {
           // )
           .with(FfmpegTranscodeDirectory, () => 'FFmpeg Transcode Directory')
           .with(BaseImageHealthCheck, () => 'Base Docker Image Tag')
+          .with(SearchServerCheck, () => 'Search Server')
           .exhaustive();
 
         const fixer = match(check)