feat: proper timeout and order of operations

This commit is contained in:
Nicolas Meienberger
2025-09-23 22:39:42 +02:00
parent 3734ba2925
commit 5695a8c700
22 changed files with 343 additions and 85 deletions

View File

@@ -1,3 +1,4 @@
import { formatDistanceToNow } from "date-fns";
import { ScanHeartIcon } from "lucide-react";
import type { GetVolumeResponse } from "~/api-client";
import { Button } from "~/components/ui/button";
@@ -9,6 +10,10 @@ type Props = {
};
export const HealthchecksCard = ({ volume }: Props) => {
const timeAgo = formatDistanceToNow(volume.lastHealthCheck, {
addSuffix: true,
});
return (
<Card className="p-6 flex-1 flex flex-col">
<div className="flex flex-col flex-1 justify-start">
@@ -17,9 +22,7 @@ export const HealthchecksCard = ({ volume }: Props) => {
<h2 className="text-lg font-medium">Health Checks</h2>
</span>
<span className="">Status: {volume.status ?? "Unknown"}</span>
<span className="text-sm text-muted-foreground mb-4">
Last checked: {new Date(volume.lastHealthCheck).toLocaleString()}
</span>
<span className="text-sm text-muted-foreground mb-4">Checked {timeAgo || "never"}</span>
<span className="flex items-center">
Enable auto remount
<Switch className="ml-auto cursor-pointer" checked={volume.autoRemount} />

View File

@@ -26,6 +26,7 @@
"arktype": "^2.1.20",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"date-fns": "^4.1.0",
"isbot": "^5.1.27",
"lucide-react": "^0.539.0",
"next-themes": "^0.4.6",

View File

@@ -0,0 +1 @@
ALTER TABLE `volumes_table` ADD `auto_remount` integer DEFAULT 1 NOT NULL;

View File

@@ -0,0 +1,118 @@
{
"version": "6",
"dialect": "sqlite",
"id": "b7f1ccb8-7bb3-486f-a103-b95b331a121f",
"prevId": "00a82d1d-4745-4487-83e4-42bb7aaa3e95",
"tables": {
"volumes_table": {
"name": "volumes_table",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"name": {
"name": "name",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"path": {
"name": "path",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"type": {
"name": "type",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"status": {
"name": "status",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "'unmounted'"
},
"last_error": {
"name": "last_error",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"last_health_check": {
"name": "last_health_check",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
},
"created_at": {
"name": "created_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
},
"updated_at": {
"name": "updated_at",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": "(unixepoch())"
},
"config": {
"name": "config",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"auto_remount": {
"name": "auto_remount",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false,
"default": 1
}
},
"indexes": {
"volumes_table_name_unique": {
"name": "volumes_table_name_unique",
"columns": [
"name"
],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

View File

@@ -22,6 +22,13 @@
"when": 1756930554198,
"tag": "0002_cheerful_randall",
"breakpoints": true
},
{
"idx": 3,
"version": "6",
"when": 1758653407064,
"tag": "0003_mature_hellcat",
"breakpoints": true
}
]
}

View File

@@ -2,8 +2,8 @@
"name": "@ironmount/server",
"type": "module",
"scripts": {
"dev": "bun run --hot src/index.ts",
"typecheck": "tsc --noEmit",
"dev": "bun run --watch src/index.ts",
"tsc": "tsc --noEmit",
"gen:migrations": "drizzle-kit generate"
},
"dependencies": {
@@ -16,6 +16,7 @@
"hono": "^4.9.2",
"hono-openapi": "^0.4.8",
"http-errors-enhanced": "^3.0.2",
"node-cron": "^4.2.1",
"slugify": "^1.6.6",
"winston": "^3.17.0"
},

View File

@@ -7,9 +7,7 @@ const envSchema = type({
}).pipe((s) => ({
__prod__: s.NODE_ENV === "production",
environment: s.NODE_ENV,
dbFileName: "/data/ironmount.db",
volumeRootHost: s.VOLUME_ROOT,
volumeRootContainer: "/mounts",
}));
const parseConfig = (env: unknown) => {

View File

@@ -0,0 +1,3 @@
export const OPERATION_TIMEOUT = 5000;
export const VOLUME_MOUNT_BASE = "/mounts";
export const DATABASE_URL = "/data/ironmount.db";

View File

@@ -2,10 +2,10 @@ import "dotenv/config";
import { Database } from "bun:sqlite";
import { drizzle } from "drizzle-orm/bun-sqlite";
import { migrate } from "drizzle-orm/bun-sqlite/migrator";
import { config } from "../core/config";
import * as schema from "./schema";
import { DATABASE_URL } from "../core/constants";
const sqlite = new Database(config.dbFileName);
const sqlite = new Database(DATABASE_URL);
export const db = drizzle({ client: sqlite, schema });

View File

@@ -13,6 +13,7 @@ export const volumesTable = sqliteTable("volumes_table", {
createdAt: int("created_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`),
updatedAt: int("updated_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`),
config: text("config", { mode: "json" }).$type<typeof volumeConfigSchema.inferOut>().notNull(),
autoRemount: int("auto_remount").notNull().default(1),
});
export type Volume = typeof volumesTable.$inferSelect;

View File

@@ -2,16 +2,21 @@ import type { BackendStatus } from "@ironmount/schemas";
import type { Volume } from "../../db/schema";
import { makeDirectoryBackend } from "./directory/directory-backend";
import { makeNfsBackend } from "./nfs/nfs-backend";
import { config } from "../../core/config";
import { VOLUME_MOUNT_BASE } from "../../core/constants";
type OperationResult = {
error?: string;
status: BackendStatus;
};
export type VolumeBackend = {
mount: () => Promise<void>;
unmount: () => Promise<void>;
checkHealth: () => Promise<{ error?: string; status: BackendStatus }>;
mount: () => Promise<OperationResult>;
unmount: () => Promise<OperationResult>;
checkHealth: () => Promise<OperationResult>;
};
export const createVolumeBackend = (volume: Volume): VolumeBackend => {
const path = `${config.volumeRootContainer}/${volume.name}/_data`;
const path = `${VOLUME_MOUNT_BASE}/${volume.name}/_data`;
switch (volume.config.backend) {
case "nfs": {

View File

@@ -7,10 +7,12 @@ import { logger } from "../../../utils/logger";
const mount = async (_config: BackendConfig, path: string) => {
logger.info("Mounting directory volume...");
await fs.mkdir(path, { recursive: true });
return { status: BACKEND_STATUS.mounted };
};
const unmount = async () => {
logger.info("Cannot unmount directory volume.");
return { status: BACKEND_STATUS.unmounted };
};
const checkHealth = async (path: string) => {

View File

@@ -1,95 +1,125 @@
import { exec } from "node:child_process";
import { exec, execFile as execFileCb } from "node:child_process";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as npath from "node:path";
import { BACKEND_STATUS, type BackendConfig } from "@ironmount/schemas";
import type { VolumeBackend } from "../backend";
import { logger } from "../../../utils/logger";
import { promisify } from "node:util";
import { withTimeout } from "../../../utils/timeout";
import { OPERATION_TIMEOUT } from "../../../core/constants";
const execFile = promisify(execFileCb);
const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting volume ${path}...`);
if (config.backend !== "nfs") {
throw new Error("Invalid backend config for NFS");
logger.error("Provided config is not for NFS backend");
return { status: BACKEND_STATUS.error, error: "Provided config is not for NFS backend" };
}
if (os.platform() !== "linux") {
logger.error("NFS mounting is only supported on Linux hosts.");
return;
return { status: BACKEND_STATUS.error, error: "NFS mounting is only supported on Linux hosts." };
}
const { status } = await checkHealth(path);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
await unmount(path);
const run = async () => {
await fs.mkdir(path, { recursive: true });
const source = `${config.server}:${config.exportPath}`;
const options = [`vers=${config.version}`, `port=${config.port}`];
const cmd = `mount -t nfs -o ${options.join(",")} ${source} ${path}`;
const args = ["-t", "nfs", "-o", options.join(","), source, path];
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Mount command timed out"));
}, 5000);
logger.debug(`Mounting volume ${path}...`);
logger.info(`Executing mount: mount ${args.join(" ")}`);
exec(cmd, (error, stdout, stderr) => {
logger.info("Mount command executed:", { cmd, error, stdout, stderr });
clearTimeout(timeout);
const { stderr } = await execFile("mount", args, {
timeout: OPERATION_TIMEOUT,
maxBuffer: 1024 * 1024,
});
if (error) {
logger.error(`Error mounting NFS volume: ${stderr}`);
return reject(new Error(`Failed to mount NFS volume: ${stderr}`));
if (stderr?.trim()) {
logger.warn(stderr.trim());
}
logger.info(`NFS volume at ${path} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS mount");
} catch (err: any) {
const msg = err.stderr?.toString().trim() || err.message;
logger.error("Error mounting NFS volume", { error: msg });
return { status: BACKEND_STATUS.error, error: msg };
}
logger.info(`NFS volume mounted successfully: ${stdout}`);
resolve();
});
});
};
const unmount = async (path: string) => {
if (os.platform() !== "linux") {
logger.error("NFS unmounting is only supported on Linux hosts.");
return;
return { status: BACKEND_STATUS.error, error: "NFS unmounting is only supported on Linux hosts." };
}
const run = async () => {
try {
await fs.access(path);
} catch {
logger.warn(`Path ${path} does not exist. Skipping unmount.`);
return;
return { status: BACKEND_STATUS.unmounted };
}
const cmd = `umount -f ${path}`;
const { stderr } = await execFile("umount", ["-l", "-f", path], {
timeout: OPERATION_TIMEOUT,
maxBuffer: 1024 * 1024,
});
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Mount command timed out"));
}, 5000);
exec(cmd, (error, stdout, stderr) => {
logger.info("Unmount command executed:", { cmd, error, stdout, stderr });
clearTimeout(timeout);
if (error) {
logger.error(`Error unmounting NFS volume: ${stderr}`);
return reject(new Error(`Failed to unmount NFS volume: ${stderr}`));
if (stderr?.trim()) {
logger.warn(stderr.trim());
}
fs.rmdir(path).catch((rmdirError) => {
logger.error(`Failed to remove directory ${path}:`, rmdirError);
});
await fs.rmdir(path);
logger.info(`NFS volume unmounted successfully: ${stdout}`);
resolve();
});
});
logger.info(`NFS volume at ${path} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS unmount");
} catch (err: any) {
const msg = err.stderr?.toString().trim() || err.message;
logger.error("Error unmounting NFS volume", { path, error: msg });
return { status: BACKEND_STATUS.error, error: msg };
}
};
const checkHealth = async (path: string) => {
try {
const run = async () => {
logger.debug(`Checking health of NFS volume at ${path}...`);
await fs.access(path);
// Try to create a temporary file to ensure the mount is writable
const testFilePath = npath.join(path, `.healthcheck-${Date.now()}`);
const testFilePath = npath.join(path, `.healthcheck-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
await fs.writeFile(testFilePath, "healthcheck");
await fs.unlink(testFilePath);
logger.debug(`NFS volume at ${path} is healthy and mounted.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check");
} catch (error) {
logger.error("NFS volume health check failed:", error);
return { status: BACKEND_STATUS.error, error: error instanceof Error ? error.message : String(error) };

View File

@@ -1,22 +1,65 @@
import { eq } from "drizzle-orm";
import { eq, or } from "drizzle-orm";
import { db } from "../../db/db";
import { logger } from "../../utils/logger";
import { volumesTable } from "../../db/schema";
import { createVolumeBackend } from "../backends/backend";
import { schedule, getTasks } from "node-cron";
export const startup = async () => {
logger.info("Mounting all volumes...");
const volumes = await db.query.volumesTable.findMany({ where: eq(volumesTable.status, "mounted") });
const volumes = await db.query.volumesTable.findMany({
where: or(eq(volumesTable.status, "mounted"), eq(volumesTable.autoRemount, 1)),
});
for (const volume of volumes) {
try {
const backend = createVolumeBackend(volume);
await backend.mount();
logger.info(`Mounted volume ${volume.name} successfully`);
await db
.update(volumesTable)
.set({ status: "mounted", lastHealthCheck: new Date(), lastError: null })
.where(eq(volumesTable.name, volume.name));
} catch (error) {
logger.error(`Failed to mount volume ${volume.name}:`, error);
await db.update(volumesTable).set({ status: "unmounted" }).where(eq(volumesTable.name, volume.name));
const errorMessage = error instanceof Error ? error.message : String(error);
logger.error(`Failed to mount volume ${volume.name}:`, errorMessage);
await db
.update(volumesTable)
.set({ status: "error", lastError: errorMessage })
.where(eq(volumesTable.name, volume.name));
}
}
// const tasks = getTasks();
// logger.info("Existing scheduled tasks:", tasks);
// tasks.forEach((task) => task.destroy());
//
// schedule("* * * * *", async () => {
// logger.info("Running health check for all volumes...");
//
// const volumes = await db.query.volumesTable.findMany({
// where: or(eq(volumesTable.status, "mounted")),
// });
//
// for (const volume of volumes) {
// try {
// const backend = createVolumeBackend(volume);
// const health = await backend.checkHealth();
//
// if (health.status !== volume.status || health.error) {
// await db
// .update(volumesTable)
// .set({ status: health.status, lastError: health.error, lastHealthCheck: new Date() })
// .where(eq(volumesTable.name, volume.name));
//
// logger.info(`Volume ${volume.name} status updated to ${health.status}`);
// }
// } catch (error) {
// logger.error(`Health check failed for volume ${volume.name}:`, error);
// await db
// .update(volumesTable)
// .set({ status: "unmounted", lastError: (error as Error).message, lastHealthCheck: new Date() })
// .where(eq(volumesTable.name, volume.name));
// }
// }
// });
};

View File

@@ -148,7 +148,7 @@ const updateVolume = async (name: string, backendConfig: BackendConfig) => {
return { error: new NotFoundError("Volume not found") };
}
const updated = await db
const [updated] = await db
.update(volumesTable)
.set({
config: backendConfig,
@@ -159,7 +159,11 @@ const updateVolume = async (name: string, backendConfig: BackendConfig) => {
.where(eq(volumesTable.name, name))
.returning();
return { volume: updated[0] };
if (!updated) {
return { error: new InternalServerError("Failed to update volume") };
}
return { volume: updated };
} catch (error) {
return {
error: new InternalServerError("Failed to update volume", {
@@ -219,6 +223,7 @@ const testConnection = async (backendConfig: BackendConfig) => {
type: backendConfig.backend,
status: "unmounted" as const,
lastError: null,
autoRemount: 0,
};
const backend = createVolumeBackend(mockVolume);

View File

@@ -6,12 +6,12 @@ const printConsole = printf((info) => `${info.level} > ${info.message}`);
const consoleFormat = combine(colorize(), printConsole);
const winstonLogger = createLogger({
level: "info",
level: "debug",
format: format.json(),
transports: [new transports.Console({ level: "info", format: consoleFormat })],
transports: [new transports.Console({ level: "debug", format: consoleFormat })],
});
const log = (level: "info" | "warn" | "error", messages: unknown[]) => {
const log = (level: "info" | "warn" | "error" | "debug", messages: unknown[]) => {
const stringMessages = messages.flatMap((m) => {
if (m instanceof Error) {
return [m.message, m.stack];
@@ -24,10 +24,11 @@ const log = (level: "info" | "warn" | "error", messages: unknown[]) => {
return m;
});
winstonLogger.log(level, stringMessages.join(" | "));
winstonLogger.log(level, stringMessages.join(" "));
};
export const logger = {
debug: (...messages: unknown[]) => log("debug", messages),
info: (...messages: unknown[]) => log("info", messages),
warn: (...messages: unknown[]) => log("warn", messages),
error: (...messages: unknown[]) => log("error", messages),

View File

@@ -0,0 +1,17 @@
class TimeoutError extends Error {
code = "ETIMEOUT";
constructor(message: string) {
super(message);
this.name = "TimeoutError";
}
}
export async function withTimeout<T>(promise: Promise<T>, ms: number, label = "operation"): Promise<T> {
let timer: NodeJS.Timeout | undefined;
const timeout = new Promise<T>((_, reject) => {
timer = setTimeout(() => reject(new TimeoutError(`${label} timed out after ${ms}ms`)), ms);
});
return Promise.race([promise, timeout]).finally(() => {
if (timer) clearTimeout(timer);
});
}

View File

@@ -1,7 +1,18 @@
{
"compilerOptions": {
"esModuleInterop": true,
"skipLibCheck": true,
"target": "es2022",
"allowJs": true,
"resolveJsonModule": true,
"moduleDetection": "force",
"isolatedModules": true,
"verbatimModuleSyntax": true,
"strict": true,
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx"
"noUncheckedIndexedAccess": true,
"noImplicitOverride": true,
"module": "preserve",
"noEmit": true,
"lib": ["es2022"]
}
}

View File

@@ -28,6 +28,7 @@
"arktype": "^2.1.20",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"date-fns": "^4.1.0",
"isbot": "^5.1.27",
"lucide-react": "^0.539.0",
"next-themes": "^0.4.6",
@@ -63,6 +64,7 @@
"hono": "^4.9.2",
"hono-openapi": "^0.4.8",
"http-errors-enhanced": "^3.0.2",
"node-cron": "^4.2.1",
"slugify": "^1.6.6",
"winston": "^3.17.0",
},
@@ -534,6 +536,8 @@
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
"date-fns": ["date-fns@4.1.0", "", {}, "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg=="],
"debug": ["debug@4.4.1", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ=="],
"dedent": ["dedent@1.6.0", "", { "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, "optionalPeers": ["babel-plugin-macros"] }, "sha512-F1Z+5UCFpmQUzJa11agbyPVMbpgT/qA3/SKyJ1jyBgm7dUcUEa8v9JwDkerSQXfakBwFljIxhOJqGkjUwZ9FSA=="],
@@ -776,6 +780,8 @@
"next-themes": ["next-themes@0.4.6", "", { "peerDependencies": { "react": "^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc", "react-dom": "^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc" } }, "sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA=="],
"node-cron": ["node-cron@4.2.1", "", {}, "sha512-lgimEHPE/QDgFlywTd8yTR61ptugX3Qer29efeyWw2rv259HtGBNn1vZVmp8lB9uo9wC0t/AT4iGqXxia+CJFg=="],
"node-fetch-native": ["node-fetch-native@1.6.7", "", {}, "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q=="],
"node-releases": ["node-releases@2.0.19", "", {}, "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw=="],

View File

@@ -24,4 +24,4 @@ services:
- ./data:/data
- /home/nicolas/ironmount/tmp:/mounts:rshared
- /home/nicolas/mounts:/mounts:rshared

View File

@@ -4,6 +4,7 @@
"packageManager": "bun@1.2.15",
"scripts": {
"dev": "turbo dev",
"tsc": "turbo run tsc",
"gen:api-client": "openapi-ts"
},
"workspaces": [

View File

@@ -6,6 +6,10 @@
"cache": false,
"persistent": true,
"env": ["*"]
},
"tsc": {
"cache": false,
"persistent": true
}
}
}