feat: proper timeout and order of operations

This commit is contained in:
Nicolas Meienberger
2025-09-23 22:39:42 +02:00
parent 3734ba2925
commit 5695a8c700
22 changed files with 343 additions and 85 deletions

View File

@@ -2,16 +2,21 @@ import type { BackendStatus } from "@ironmount/schemas";
import type { Volume } from "../../db/schema";
import { makeDirectoryBackend } from "./directory/directory-backend";
import { makeNfsBackend } from "./nfs/nfs-backend";
import { config } from "../../core/config";
import { VOLUME_MOUNT_BASE } from "../../core/constants";
type OperationResult = {
error?: string;
status: BackendStatus;
};
export type VolumeBackend = {
mount: () => Promise<void>;
unmount: () => Promise<void>;
checkHealth: () => Promise<{ error?: string; status: BackendStatus }>;
mount: () => Promise<OperationResult>;
unmount: () => Promise<OperationResult>;
checkHealth: () => Promise<OperationResult>;
};
export const createVolumeBackend = (volume: Volume): VolumeBackend => {
const path = `${config.volumeRootContainer}/${volume.name}/_data`;
const path = `${VOLUME_MOUNT_BASE}/${volume.name}/_data`;
switch (volume.config.backend) {
case "nfs": {

View File

@@ -7,10 +7,12 @@ import { logger } from "../../../utils/logger";
const mount = async (_config: BackendConfig, path: string) => {
logger.info("Mounting directory volume...");
await fs.mkdir(path, { recursive: true });
return { status: BACKEND_STATUS.mounted };
};
const unmount = async () => {
logger.info("Cannot unmount directory volume.");
return { status: BACKEND_STATUS.unmounted };
};
const checkHealth = async (path: string) => {

View File

@@ -1,95 +1,125 @@
import { exec } from "node:child_process";
import { exec, execFile as execFileCb } from "node:child_process";
import * as fs from "node:fs/promises";
import * as os from "node:os";
import * as npath from "node:path";
import { BACKEND_STATUS, type BackendConfig } from "@ironmount/schemas";
import type { VolumeBackend } from "../backend";
import { logger } from "../../../utils/logger";
import { promisify } from "node:util";
import { withTimeout } from "../../../utils/timeout";
import { OPERATION_TIMEOUT } from "../../../core/constants";
const execFile = promisify(execFileCb);
const mount = async (config: BackendConfig, path: string) => {
logger.debug(`Mounting volume ${path}...`);
if (config.backend !== "nfs") {
throw new Error("Invalid backend config for NFS");
logger.error("Provided config is not for NFS backend");
return { status: BACKEND_STATUS.error, error: "Provided config is not for NFS backend" };
}
if (os.platform() !== "linux") {
logger.error("NFS mounting is only supported on Linux hosts.");
return;
return { status: BACKEND_STATUS.error, error: "NFS mounting is only supported on Linux hosts." };
}
await fs.mkdir(path, { recursive: true });
const { status } = await checkHealth(path);
if (status === "mounted") {
return { status: BACKEND_STATUS.mounted };
}
const source = `${config.server}:${config.exportPath}`;
const options = [`vers=${config.version}`, `port=${config.port}`];
const cmd = `mount -t nfs -o ${options.join(",")} ${source} ${path}`;
logger.debug(`Trying to unmount any existing mounts at ${path} before mounting...`);
await unmount(path);
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Mount command timed out"));
}, 5000);
const run = async () => {
await fs.mkdir(path, { recursive: true });
exec(cmd, (error, stdout, stderr) => {
logger.info("Mount command executed:", { cmd, error, stdout, stderr });
clearTimeout(timeout);
const source = `${config.server}:${config.exportPath}`;
const options = [`vers=${config.version}`, `port=${config.port}`];
const args = ["-t", "nfs", "-o", options.join(","), source, path];
if (error) {
logger.error(`Error mounting NFS volume: ${stderr}`);
return reject(new Error(`Failed to mount NFS volume: ${stderr}`));
}
logger.info(`NFS volume mounted successfully: ${stdout}`);
resolve();
logger.debug(`Mounting volume ${path}...`);
logger.info(`Executing mount: mount ${args.join(" ")}`);
const { stderr } = await execFile("mount", args, {
timeout: OPERATION_TIMEOUT,
maxBuffer: 1024 * 1024,
});
});
if (stderr?.trim()) {
logger.warn(stderr.trim());
}
logger.info(`NFS volume at ${path} mounted successfully.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS mount");
} catch (err: any) {
const msg = err.stderr?.toString().trim() || err.message;
logger.error("Error mounting NFS volume", { error: msg });
return { status: BACKEND_STATUS.error, error: msg };
}
};
const unmount = async (path: string) => {
if (os.platform() !== "linux") {
logger.error("NFS unmounting is only supported on Linux hosts.");
return;
return { status: BACKEND_STATUS.error, error: "NFS unmounting is only supported on Linux hosts." };
}
const run = async () => {
try {
await fs.access(path);
} catch {
logger.warn(`Path ${path} does not exist. Skipping unmount.`);
return { status: BACKEND_STATUS.unmounted };
}
const { stderr } = await execFile("umount", ["-l", "-f", path], {
timeout: OPERATION_TIMEOUT,
maxBuffer: 1024 * 1024,
});
if (stderr?.trim()) {
logger.warn(stderr.trim());
}
await fs.rmdir(path);
logger.info(`NFS volume at ${path} unmounted successfully.`);
return { status: BACKEND_STATUS.unmounted };
};
try {
await fs.access(path);
} catch {
logger.warn(`Path ${path} does not exist. Skipping unmount.`);
return;
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS unmount");
} catch (err: any) {
const msg = err.stderr?.toString().trim() || err.message;
logger.error("Error unmounting NFS volume", { path, error: msg });
return { status: BACKEND_STATUS.error, error: msg };
}
const cmd = `umount -f ${path}`;
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Mount command timed out"));
}, 5000);
exec(cmd, (error, stdout, stderr) => {
logger.info("Unmount command executed:", { cmd, error, stdout, stderr });
clearTimeout(timeout);
if (error) {
logger.error(`Error unmounting NFS volume: ${stderr}`);
return reject(new Error(`Failed to unmount NFS volume: ${stderr}`));
}
fs.rmdir(path).catch((rmdirError) => {
logger.error(`Failed to remove directory ${path}:`, rmdirError);
});
logger.info(`NFS volume unmounted successfully: ${stdout}`);
resolve();
});
});
};
const checkHealth = async (path: string) => {
try {
const run = async () => {
logger.debug(`Checking health of NFS volume at ${path}...`);
await fs.access(path);
// Try to create a temporary file to ensure the mount is writable
const testFilePath = npath.join(path, `.healthcheck-${Date.now()}`);
const testFilePath = npath.join(path, `.healthcheck-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
await fs.writeFile(testFilePath, "healthcheck");
await fs.unlink(testFilePath);
logger.debug(`NFS volume at ${path} is healthy and mounted.`);
return { status: BACKEND_STATUS.mounted };
};
try {
return await withTimeout(run(), OPERATION_TIMEOUT, "NFS health check");
} catch (error) {
logger.error("NFS volume health check failed:", error);
return { status: BACKEND_STATUS.error, error: error instanceof Error ? error.message : String(error) };