diff --git a/apps/server/src/modules/backends/directory/directory-backend.ts b/apps/server/src/modules/backends/directory/directory-backend.ts index ce57108..902faa4 100644 --- a/apps/server/src/modules/backends/directory/directory-backend.ts +++ b/apps/server/src/modules/backends/directory/directory-backend.ts @@ -3,6 +3,7 @@ import * as npath from "node:path"; import { BACKEND_STATUS, type BackendConfig } from "@ironmount/schemas"; import type { VolumeBackend } from "../backend"; import { logger } from "../../../utils/logger"; +import { toMessage } from "../../../utils/errors"; const mount = async (_config: BackendConfig, path: string) => { logger.info("Mounting directory volume..."); @@ -27,7 +28,7 @@ const checkHealth = async (path: string) => { return { status: BACKEND_STATUS.mounted }; } catch (error) { logger.error("Directory health check failed:", error); - return { status: BACKEND_STATUS.error, error: error instanceof Error ? error.message : String(error) }; + return { status: BACKEND_STATUS.error, error: toMessage(error) }; } }; diff --git a/apps/server/src/modules/backends/nfs/nfs-backend.ts b/apps/server/src/modules/backends/nfs/nfs-backend.ts index 69de508..f2748a2 100644 --- a/apps/server/src/modules/backends/nfs/nfs-backend.ts +++ b/apps/server/src/modules/backends/nfs/nfs-backend.ts @@ -8,6 +8,7 @@ import { logger } from "../../../utils/logger"; import { promisify } from "node:util"; import { withTimeout } from "../../../utils/timeout"; import { OPERATION_TIMEOUT } from "../../../core/constants"; +import { toMessage } from "../../../utils/errors"; const execFile = promisify(execFileCb); @@ -121,8 +122,8 @@ const checkHealth = async (path: string) => { try { return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check"); } catch (error) { - logger.error("NFS volume health check failed:", error); - return { status: BACKEND_STATUS.error, error: error instanceof Error ? error.message : String(error) }; + logger.error("NFS volume health check failed:", toMessage(error)); + return { status: BACKEND_STATUS.error, error: toMessage(error) }; } }; diff --git a/apps/server/src/modules/lifecycle/startup.ts b/apps/server/src/modules/lifecycle/startup.ts index 69e7378..e3af703 100644 --- a/apps/server/src/modules/lifecycle/startup.ts +++ b/apps/server/src/modules/lifecycle/startup.ts @@ -2,8 +2,8 @@ import { eq, or } from "drizzle-orm"; import { db } from "../../db/db"; import { logger } from "../../utils/logger"; import { volumesTable } from "../../db/schema"; -import { createVolumeBackend } from "../backends/backend"; import { schedule, getTasks } from "node-cron"; +import { volumeService } from "../volumes/volume.service"; export const startup = async () => { const volumes = await db.query.volumesTable.findMany({ @@ -11,55 +11,26 @@ export const startup = async () => { }); for (const volume of volumes) { - try { - const backend = createVolumeBackend(volume); - await backend.mount(); - await db - .update(volumesTable) - .set({ status: "mounted", lastHealthCheck: new Date(), lastError: null }) - .where(eq(volumesTable.name, volume.name)); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - logger.error(`Failed to mount volume ${volume.name}:`, errorMessage); - - await db - .update(volumesTable) - .set({ status: "error", lastError: errorMessage }) - .where(eq(volumesTable.name, volume.name)); - } + await volumeService.mountVolume(volume.name); } - // const tasks = getTasks(); - // logger.info("Existing scheduled tasks:", tasks); - // tasks.forEach((task) => task.destroy()); - // - // schedule("* * * * *", async () => { - // logger.info("Running health check for all volumes..."); - // - // const volumes = await db.query.volumesTable.findMany({ - // where: or(eq(volumesTable.status, "mounted")), - // }); - // - // for (const volume of volumes) { - // try { - // const backend = createVolumeBackend(volume); - // const health = await backend.checkHealth(); - // - // if (health.status !== volume.status || health.error) { - // await db - // .update(volumesTable) - // .set({ status: health.status, lastError: health.error, lastHealthCheck: new Date() }) - // .where(eq(volumesTable.name, volume.name)); - // - // logger.info(`Volume ${volume.name} status updated to ${health.status}`); - // } - // } catch (error) { - // logger.error(`Health check failed for volume ${volume.name}:`, error); - // await db - // .update(volumesTable) - // .set({ status: "unmounted", lastError: (error as Error).message, lastHealthCheck: new Date() }) - // .where(eq(volumesTable.name, volume.name)); - // } - // } - // }); + const existingTasks = getTasks(); + existingTasks.forEach((task) => task.destroy()); + + schedule("* * * * *", async () => { + logger.info("Running health check for all volumes..."); + + const volumes = await db.query.volumesTable.findMany({ + where: or(eq(volumesTable.status, "mounted")), + }); + + for (const volume of volumes) { + const { error } = await volumeService.checkHealth(volume.name); + if (error && volume.autoRemount) { + // TODO: retry with backoff based on last health check time + // Until we reach the max backoff and it'll try every 10 minutes + await volumeService.mountVolume(volume.name); + } + } + }); }; diff --git a/apps/server/src/modules/volumes/volume.service.ts b/apps/server/src/modules/volumes/volume.service.ts index 639ee14..1f36be9 100644 --- a/apps/server/src/modules/volumes/volume.service.ts +++ b/apps/server/src/modules/volumes/volume.service.ts @@ -10,6 +10,7 @@ import { db } from "../../db/db"; import { volumesTable } from "../../db/schema"; import { createVolumeBackend } from "../backends/backend"; import { logger } from "../../utils/logger"; +import { toMessage } from "../../utils/errors"; const listVolumes = async () => { const volumes = await db.query.volumesTable.findMany({}); @@ -77,15 +78,11 @@ const mountVolume = async (name: string) => { } const backend = createVolumeBackend(volume); - await backend.unmount().catch((_) => { - // Ignore unmount errors - }); - await backend.mount(); await db .update(volumesTable) - .set({ status: "mounted", lastHealthCheck: new Date() }) + .set({ status: "mounted", lastHealthCheck: new Date(), lastError: null }) .where(eq(volumesTable.name, name)); return { status: 200 }; @@ -173,39 +170,6 @@ const updateVolume = async (name: string, backendConfig: BackendConfig) => { } }; -const updateVolumeStatus = async (name: string, status: "mounted" | "unmounted" | "error", error?: string) => { - await db - .update(volumesTable) - .set({ - status, - lastHealthCheck: new Date(), - lastError: error ?? null, - updatedAt: new Date(), - }) - .where(eq(volumesTable.name, name)); -}; - -const getVolumeStatus = async (name: string) => { - const volume = await db.query.volumesTable.findFirst({ - where: eq(volumesTable.name, name), - }); - - if (!volume) { - return { error: new NotFoundError("Volume not found") }; - } - - const backend = createVolumeBackend(volume); - const healthResult = await backend.checkHealth(); - await updateVolumeStatus(name, healthResult.status, healthResult.error); - - return { - name: volume.name, - status: healthResult.status, - lastHealthCheck: new Date(), - error: healthResult.error, - }; -}; - const testConnection = async (backendConfig: BackendConfig) => { let tempDir: string | null = null; @@ -238,7 +202,7 @@ const testConnection = async (backendConfig: BackendConfig) => { } catch (error) { return { success: false, - message: error instanceof Error ? error.message : "Connection failed", + message: toMessage(error), }; } finally { if (tempDir) { @@ -246,13 +210,42 @@ const testConnection = async (backendConfig: BackendConfig) => { await fs.access(tempDir); await fs.rm(tempDir, { recursive: true, force: true }); } catch (cleanupError) { - // Ignore cleanup errors if directory doesn't exist or can't be removed logger.warn("Failed to cleanup temp directory:", cleanupError); } } } }; +const checkHealth = async (name: string) => { + try { + const volume = await db.query.volumesTable.findFirst({ + where: eq(volumesTable.name, name), + }); + + if (!volume) { + return { error: new NotFoundError("Volume not found") }; + } + + const backend = createVolumeBackend(volume); + const { error } = await backend.checkHealth(); + + if (error) { + await db + .update(volumesTable) + .set({ status: "error", lastError: error, lastHealthCheck: new Date() }) + .where(eq(volumesTable.name, volume.name)); + + return { error }; + } + + await db.update(volumesTable).set({ lastHealthCheck: new Date() }).where(eq(volumesTable.name, volume.name)); + + return { status: 200 }; + } catch (err) { + return { error: new InternalServerError("Health check failed", { cause: err }) }; + } +}; + export const volumeService = { listVolumes, createVolume, @@ -261,7 +254,6 @@ export const volumeService = { getVolume, updateVolume, testConnection, - updateVolumeStatus, - getVolumeStatus, unmountVolume, + checkHealth, }; diff --git a/apps/server/src/utils/errors.ts b/apps/server/src/utils/errors.ts index 3aea02c..18e51cd 100644 --- a/apps/server/src/utils/errors.ts +++ b/apps/server/src/utils/errors.ts @@ -13,3 +13,7 @@ export const handleServiceError = (error: unknown) => { logger.error("Unhandled service error:", error); return { message: "Internal Server Error", status: 500 as const }; }; + +export const toMessage = (err: unknown): string => { + return err instanceof Error ? err.message : String(err); +};