ondra/ironmount
1
1
Fork 0
mirror of https://github.com/nicotsx/ironmount.git synced 2025-12-10 12:10:51 +01:00
Code Issues Packages Projects Releases 1 Wiki Activity

feat: throttle logs during backup

Browse Source
This commit is contained in:
Nicolas Meienberger
2025-11-09 11:33:14 +01:00
parent 1152939373
commit 5f35cfd4c2
4 changed files with 97 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
apps/server/package.json
View File

@@ -15,6 +15,7 @@
"dockerode": "^4.0.8",
"dotenv": "^17.2.1",
"drizzle-orm": "^0.44.6",
"es-toolkit": "^1.41.0",
"hono": "^4.9.2",
"hono-openapi": "^1.1.0",
"http-errors-enhanced": "^3.0.2",

98
apps/server/src/utils/restic.ts
View File

@@ -1,7 +1,7 @@
import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { spawn } from "node:child_process";
import { throttle } from "es-toolkit";
import type { RepositoryConfig } from "@ironmount/schemas/restic";
import { type } from "arktype";
import { $ } from "bun";
@@ -9,6 +9,7 @@ import { REPOSITORY_BASE, RESTIC_PASS_FILE } from "../core/constants";
import { logger } from "./logger";
import { cryptoUtils } from "./crypto";
import type { RetentionPolicy } from "../modules/backups/backups.dto";
import { safeSpawn } from "./spawn";
const backupOutputSchema = type({
message_type: "'summary'",
@@ -149,67 +150,41 @@ const backup = async (
args.push("--json");
return new Promise((resolve, reject) => {
const child = spawn("restic", args, {
env: { ...process.env, ...env },
signal: options?.signal,
});
const logData = throttle((data: string) => {
logger.info(data.trim());
}, 5000);
let stdout = "";
let stdout = "";
child.stdout.on("data", (data) => {
stdout = data.toString();
logger.info(data.toString());
});
child.stderr.on("data", (data) => {
logger.error(data.toString());
});
child.on("error", async (error) => {
if (includeFile) {
await fs.unlink(includeFile).catch(() => {});
}
if (error.name === "AbortError") {
logger.info("Restic backup process was aborted");
reject(error);
} else {
logger.error(`Restic backup process error: ${error.message}`);
reject(new Error(`Restic backup process error: ${error.message}`));
}
});
child.on("close", async (code) => {
if (includeFile) {
await fs.unlink(includeFile).catch(() => {});
}
if (code !== 0) {
logger.error(`Restic backup failed with exit code ${code}`);
reject(new Error(`Restic backup failed`));
return;
}
try {
const lastLine = stdout.trim();
const resSummary = JSON.parse(lastLine ?? "{}");
const result = backupOutputSchema(resSummary);
if (result instanceof type.errors) {
logger.error(`Restic backup output validation failed: ${result}`);
reject(new Error(`Restic backup output validation failed: ${result}`));
return;
}
resolve(result);
} catch (error) {
logger.error(`Failed to parse restic backup output: ${error}`);
reject(new Error(`Failed to parse restic backup output: ${error}`));
}
});
await safeSpawn({
command: "restic",
args,
env,
signal: options?.signal,
onStdout: (data) => {
stdout = data;
logData(data);
},
onStderr: (error) => {
logger.error(error.trim());
},
finally: async () => {
includeFile && (await fs.unlink(includeFile).catch(() => {}));
},
});
const lastLine = stdout.trim();
const resSummary = JSON.parse(lastLine ?? "{}");
const result = backupOutputSchema(resSummary);
if (result instanceof type.errors) {
logger.error(`Restic backup output validation failed: ${result}`);
throw new Error(`Restic backup output validation failed: ${result}`);
}
return result;
};
const restoreOutputSchema = type({
@@ -370,7 +345,6 @@ const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra:
args.push("--prune");
args.push("--json");
// await $`restic unlock --repo ${repoUrl}`.env(env).nothrow();
const res = await $`restic ${args}`.env(env).nothrow();
if (res.exitCode !== 0) {
@@ -465,7 +439,7 @@ const unlock = async (config: RepositoryConfig) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const res = await $`restic unlock --repo ${repoUrl} --json`.env(env).nothrow();
const res = await $`restic unlock --repo ${repoUrl} --remove-all --json`.env(env).nothrow();
if (res.exitCode !== 0) {
logger.error(`Restic unlock failed: ${res.stderr}`);
@@ -501,7 +475,7 @@ const check = async (config: RepositoryConfig, options?: { readData?: boolean })
};
}
const hasErrors = stdout.includes("error") || stdout.includes("Fatal");
const hasErrors = stdout.includes("Fatal");
logger.info(`Restic check completed for repository: ${repoUrl}`);
return {

56
apps/server/src/utils/spawn.ts Normal file
View File

@@ -0,0 +1,56 @@
import { spawn } from "node:child_process";
interface Params {
command: string;
args: string[];
env?: NodeJS.ProcessEnv;
signal?: AbortSignal;
onStdout?: (data: string) => void;
onStderr?: (error: string) => void;
onError?: (error: Error) => Promise<void> | void;
onClose?: (code: number | null) => Promise<void> | void;
finally?: () => Promise<void> | void;
}
export const safeSpawn = (params: Params) => {
const { command, args, env = {}, signal, ...callbacks } = params;
return new Promise((resolve, reject) => {
const child = spawn(command, args, {
env: { ...process.env, ...env },
signal: signal,
});
child.stdout.on("data", (data) => {
if (callbacks.onStdout) {
callbacks.onStdout(data.toString());
}
});
child.stderr.on("data", (data) => {
if (callbacks.onStderr) {
callbacks.onStderr(data.toString());
}
});
child.on("error", async (error) => {
if (callbacks.onError) {
await callbacks.onError(error);
}
if (callbacks.finally) {
await callbacks.finally();
}
reject(error);
});
child.on("close", async (code) => {
if (callbacks.onClose) {
await callbacks.onClose(code);
}
if (callbacks.finally) {
await callbacks.finally();
}
resolve(code);
});
});
};