mirror of
https://github.com/nicotsx/ironmount.git
synced 2025-12-10 12:10:51 +01:00
refactor: unify backend and frontend servers (#3)
* refactor: unify backend and frontend servers * refactor: correct paths for openapi & drizzle * refactor: move api-client to client * fix: drizzle paths * chore: fix linting issues * fix: form reset issue
This commit is contained in:
61
app/server/utils/crypto.ts
Normal file
61
app/server/utils/crypto.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import crypto from "node:crypto";
|
||||
import { RESTIC_PASS_FILE } from "../core/constants";
|
||||
|
||||
const algorithm = "aes-256-gcm" as const;
|
||||
const keyLength = 32;
|
||||
const encryptionPrefix = "encv1";
|
||||
|
||||
/**
|
||||
* Given a string, encrypts it using a randomly generated salt
|
||||
*/
|
||||
const encrypt = async (data: string) => {
|
||||
if (!data) {
|
||||
return data;
|
||||
}
|
||||
|
||||
if (data.startsWith(encryptionPrefix)) {
|
||||
return data;
|
||||
}
|
||||
|
||||
const secret = (await Bun.file(RESTIC_PASS_FILE).text()).trim();
|
||||
|
||||
const salt = crypto.randomBytes(16);
|
||||
const key = crypto.pbkdf2Sync(secret, salt, 100000, keyLength, "sha256");
|
||||
const iv = crypto.randomBytes(12);
|
||||
|
||||
const cipher = crypto.createCipheriv(algorithm, key, iv);
|
||||
const encrypted = Buffer.concat([cipher.update(data), cipher.final()]);
|
||||
|
||||
const tag = cipher.getAuthTag();
|
||||
return `${encryptionPrefix}:${salt.toString("hex")}:${iv.toString("hex")}:${encrypted.toString("hex")}:${tag.toString("hex")}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Given an encrypted string, decrypts it using the salt stored in the string
|
||||
*/
|
||||
const decrypt = async (encryptedData: string) => {
|
||||
const secret = await Bun.file(RESTIC_PASS_FILE).text();
|
||||
|
||||
const parts = encryptedData.split(":").slice(1); // Remove prefix
|
||||
const saltHex = parts.shift() as string;
|
||||
const salt = Buffer.from(saltHex, "hex");
|
||||
|
||||
const key = crypto.pbkdf2Sync(secret, salt, 100000, keyLength, "sha256");
|
||||
|
||||
const iv = Buffer.from(parts.shift() as string, "hex");
|
||||
const encrypted = Buffer.from(parts.shift() as string, "hex");
|
||||
const tag = Buffer.from(parts.shift() as string, "hex");
|
||||
const decipher = crypto.createDecipheriv(algorithm, key, iv);
|
||||
|
||||
decipher.setAuthTag(tag);
|
||||
|
||||
let decrypted = decipher.update(encrypted);
|
||||
decrypted = Buffer.concat([decrypted, decipher.final()]);
|
||||
|
||||
return decrypted.toString();
|
||||
};
|
||||
|
||||
export const cryptoUtils = {
|
||||
encrypt,
|
||||
decrypt,
|
||||
};
|
||||
19
app/server/utils/errors.ts
Normal file
19
app/server/utils/errors.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { ConflictError, NotFoundError } from "http-errors-enhanced";
|
||||
import { sanitizeSensitiveData } from "./sanitize";
|
||||
|
||||
export const handleServiceError = (error: unknown) => {
|
||||
if (error instanceof ConflictError) {
|
||||
return { message: sanitizeSensitiveData(error.message), status: 409 as const };
|
||||
}
|
||||
|
||||
if (error instanceof NotFoundError) {
|
||||
return { message: sanitizeSensitiveData(error.message), status: 404 as const };
|
||||
}
|
||||
|
||||
return { message: sanitizeSensitiveData(toMessage(error)), status: 500 as const };
|
||||
};
|
||||
|
||||
export const toMessage = (err: unknown): string => {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return sanitizeSensitiveData(message);
|
||||
};
|
||||
36
app/server/utils/logger.ts
Normal file
36
app/server/utils/logger.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { createLogger, format, transports } from "winston";
|
||||
import { sanitizeSensitiveData } from "./sanitize";
|
||||
|
||||
const { printf, combine, colorize } = format;
|
||||
|
||||
const printConsole = printf((info) => `${info.level} > ${info.message}`);
|
||||
const consoleFormat = combine(colorize(), printConsole);
|
||||
|
||||
const winstonLogger = createLogger({
|
||||
level: "debug",
|
||||
format: format.json(),
|
||||
transports: [new transports.Console({ level: "debug", format: consoleFormat })],
|
||||
});
|
||||
|
||||
const log = (level: "info" | "warn" | "error" | "debug", messages: unknown[]) => {
|
||||
const stringMessages = messages.flatMap((m) => {
|
||||
if (m instanceof Error) {
|
||||
return [sanitizeSensitiveData(m.message), m.stack ? sanitizeSensitiveData(m.stack) : undefined].filter(Boolean);
|
||||
}
|
||||
|
||||
if (typeof m === "object") {
|
||||
return sanitizeSensitiveData(JSON.stringify(m, null, 2));
|
||||
}
|
||||
|
||||
return sanitizeSensitiveData(String(m));
|
||||
});
|
||||
|
||||
winstonLogger.log(level, stringMessages.join(" "));
|
||||
};
|
||||
|
||||
export const logger = {
|
||||
debug: (...messages: unknown[]) => log("debug", messages),
|
||||
info: (...messages: unknown[]) => log("info", messages),
|
||||
warn: (...messages: unknown[]) => log("warn", messages),
|
||||
error: (...messages: unknown[]) => log("error", messages),
|
||||
};
|
||||
85
app/server/utils/mountinfo.ts
Normal file
85
app/server/utils/mountinfo.ts
Normal file
@@ -0,0 +1,85 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
type MountInfo = {
|
||||
mountPoint: string;
|
||||
fstype: string;
|
||||
};
|
||||
|
||||
export type StatFs = {
|
||||
total: number;
|
||||
used: number;
|
||||
free: number;
|
||||
};
|
||||
|
||||
function isPathWithin(base: string, target: string): boolean {
|
||||
const rel = path.posix.relative(base, target);
|
||||
return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel));
|
||||
}
|
||||
|
||||
function unescapeMount(s: string): string {
|
||||
return s.replace(/\\([0-7]{3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8)));
|
||||
}
|
||||
|
||||
export async function readMountInfo(): Promise<MountInfo[]> {
|
||||
const text = await fs.readFile("/proc/self/mountinfo", "utf-8");
|
||||
const result: MountInfo[] = [];
|
||||
|
||||
for (const line of text.split("\n")) {
|
||||
if (!line) continue;
|
||||
const sep = line.indexOf(" - ");
|
||||
|
||||
if (sep === -1) continue;
|
||||
|
||||
const left = line.slice(0, sep).split(" ");
|
||||
const right = line.slice(sep + 3).split(" ");
|
||||
|
||||
// [0]=mount ID, [1]=parent ID, [2]=major:minor, [3]=root, [4]=mount point, [5]=mount options, ...
|
||||
const mpRaw = left[4];
|
||||
const fstype = right[0];
|
||||
|
||||
if (!mpRaw || !fstype) continue;
|
||||
|
||||
result.push({ mountPoint: unescapeMount(mpRaw), fstype });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function getMountForPath(p: string): Promise<MountInfo | undefined> {
|
||||
const mounts = await readMountInfo();
|
||||
|
||||
let best: MountInfo | undefined;
|
||||
for (const m of mounts) {
|
||||
if (!isPathWithin(m.mountPoint, p)) continue;
|
||||
if (!best || m.mountPoint.length > best.mountPoint.length) {
|
||||
best = m;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
export async function getStatFs(mountPoint: string) {
|
||||
const s = await fs.statfs(mountPoint, { bigint: true });
|
||||
|
||||
const unit = s.bsize > 0n ? s.bsize : 1n;
|
||||
|
||||
const blocks = s.blocks > 0n ? s.blocks : 0n;
|
||||
|
||||
let bfree = s.bfree > 0n ? s.bfree : 0n;
|
||||
if (bfree > blocks) bfree = blocks;
|
||||
|
||||
const bavail = s.bavail > 0n ? s.bavail : 0n;
|
||||
|
||||
const totalB = blocks * unit;
|
||||
const usedB = (blocks - bfree) * unit;
|
||||
const freeB = bavail * unit;
|
||||
|
||||
const MAX = BigInt(Number.MAX_SAFE_INTEGER);
|
||||
const toNumber = (x: bigint) => (x > MAX ? Number.MAX_SAFE_INTEGER : Number(x));
|
||||
|
||||
return {
|
||||
total: toNumber(totalB),
|
||||
used: toNumber(usedB),
|
||||
free: toNumber(freeB),
|
||||
};
|
||||
}
|
||||
76
app/server/utils/rclone.ts
Normal file
76
app/server/utils/rclone.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import { $ } from "bun";
|
||||
import { logger } from "./logger";
|
||||
|
||||
/**
|
||||
* List all configured rclone remotes
|
||||
* @returns Array of remote names
|
||||
*/
|
||||
export async function listRcloneRemotes(): Promise<string[]> {
|
||||
const result = await $`rclone listremotes`.nothrow();
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
logger.error(`Failed to list rclone remotes: ${result.stderr}`);
|
||||
return [];
|
||||
}
|
||||
|
||||
// Parse output - each line is a remote name ending with ":"
|
||||
const remotes = result.stdout
|
||||
.toString()
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.endsWith(":"))
|
||||
.map((line) => line.slice(0, -1)); // Remove trailing ":"
|
||||
|
||||
return remotes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get information about a specific rclone remote
|
||||
* @param remote Remote name
|
||||
* @returns Remote type and configuration info
|
||||
*/
|
||||
export async function getRcloneRemoteInfo(
|
||||
remote: string,
|
||||
): Promise<{ type: string; config: Record<string, string> } | null> {
|
||||
try {
|
||||
const result = await $`rclone config show ${remote}`.quiet();
|
||||
|
||||
if (result.exitCode !== 0) {
|
||||
logger.error(`Failed to get info for remote ${remote}: ${result.stderr}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// Parse the output to extract type and config
|
||||
const output = result.stdout.toString();
|
||||
const lines = output
|
||||
.split("\n")
|
||||
.map((l) => l.trim())
|
||||
.filter((l) => l);
|
||||
|
||||
const config: Record<string, string> = {};
|
||||
let type = "unknown";
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.includes("=")) {
|
||||
const parts = line.split("=");
|
||||
const key = parts[0];
|
||||
if (!key) continue;
|
||||
|
||||
const valueParts = parts.slice(1);
|
||||
const value = valueParts.join("=").trim();
|
||||
const cleanKey = key.trim();
|
||||
|
||||
if (cleanKey === "type") {
|
||||
type = value;
|
||||
}
|
||||
|
||||
config[cleanKey] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return { type, config };
|
||||
} catch (error) {
|
||||
logger.error(`Error getting remote info for ${remote}: ${error}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
589
app/server/utils/restic.ts
Normal file
589
app/server/utils/restic.ts
Normal file
@@ -0,0 +1,589 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { throttle } from "es-toolkit";
|
||||
import { type } from "arktype";
|
||||
import { $ } from "bun";
|
||||
import { REPOSITORY_BASE, RESTIC_PASS_FILE } from "../core/constants";
|
||||
import { logger } from "./logger";
|
||||
import { cryptoUtils } from "./crypto";
|
||||
import type { RetentionPolicy } from "../modules/backups/backups.dto";
|
||||
import { safeSpawn } from "./spawn";
|
||||
import type { RepositoryConfig } from "~/schemas/restic";
|
||||
|
||||
const backupOutputSchema = type({
|
||||
message_type: "'summary'",
|
||||
files_new: "number",
|
||||
files_changed: "number",
|
||||
files_unmodified: "number",
|
||||
dirs_new: "number",
|
||||
dirs_changed: "number",
|
||||
dirs_unmodified: "number",
|
||||
data_blobs: "number",
|
||||
tree_blobs: "number",
|
||||
data_added: "number",
|
||||
total_files_processed: "number",
|
||||
total_bytes_processed: "number",
|
||||
total_duration: "number",
|
||||
snapshot_id: "string",
|
||||
});
|
||||
|
||||
const snapshotInfoSchema = type({
|
||||
gid: "number?",
|
||||
hostname: "string",
|
||||
id: "string",
|
||||
parent: "string?",
|
||||
paths: "string[]",
|
||||
program_version: "string?",
|
||||
short_id: "string",
|
||||
time: "string",
|
||||
uid: "number?",
|
||||
username: "string",
|
||||
summary: type({
|
||||
backup_end: "string",
|
||||
backup_start: "string",
|
||||
data_added: "number",
|
||||
data_added_packed: "number",
|
||||
data_blobs: "number",
|
||||
dirs_changed: "number",
|
||||
dirs_new: "number",
|
||||
dirs_unmodified: "number",
|
||||
files_changed: "number",
|
||||
files_new: "number",
|
||||
files_unmodified: "number",
|
||||
total_bytes_processed: "number",
|
||||
total_files_processed: "number",
|
||||
tree_blobs: "number",
|
||||
}).optional(),
|
||||
});
|
||||
|
||||
const ensurePassfile = async () => {
|
||||
await fs.mkdir(path.dirname(RESTIC_PASS_FILE), { recursive: true });
|
||||
|
||||
try {
|
||||
await fs.access(RESTIC_PASS_FILE);
|
||||
} catch {
|
||||
logger.info("Restic passfile not found, creating a new one...");
|
||||
await fs.writeFile(RESTIC_PASS_FILE, crypto.randomBytes(32).toString("hex"), { mode: 0o600 });
|
||||
}
|
||||
};
|
||||
|
||||
const buildRepoUrl = (config: RepositoryConfig): string => {
|
||||
switch (config.backend) {
|
||||
case "local":
|
||||
return `${REPOSITORY_BASE}/${config.name}`;
|
||||
case "s3":
|
||||
return `s3:${config.endpoint}/${config.bucket}`;
|
||||
case "gcs":
|
||||
return `gs:${config.bucket}:/`;
|
||||
case "azure":
|
||||
return `azure:${config.container}:/`;
|
||||
case "rclone":
|
||||
return `rclone:${config.remote}:${config.path}`;
|
||||
default: {
|
||||
throw new Error(`Unsupported repository backend: ${JSON.stringify(config)}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const buildEnv = async (config: RepositoryConfig) => {
|
||||
const env: Record<string, string> = {
|
||||
RESTIC_CACHE_DIR: "/var/lib/ironmount/restic/cache",
|
||||
RESTIC_PASSWORD_FILE: RESTIC_PASS_FILE,
|
||||
PATH: process.env.PATH || "/usr/local/bin:/usr/bin:/bin",
|
||||
};
|
||||
|
||||
switch (config.backend) {
|
||||
case "s3":
|
||||
env.AWS_ACCESS_KEY_ID = await cryptoUtils.decrypt(config.accessKeyId);
|
||||
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.decrypt(config.secretAccessKey);
|
||||
break;
|
||||
case "gcs": {
|
||||
const decryptedCredentials = await cryptoUtils.decrypt(config.credentialsJson);
|
||||
const credentialsPath = path.join("/tmp", `gcs-credentials-${crypto.randomBytes(8).toString("hex")}.json`);
|
||||
await fs.writeFile(credentialsPath, decryptedCredentials, { mode: 0o600 });
|
||||
env.GOOGLE_PROJECT_ID = config.projectId;
|
||||
env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;
|
||||
break;
|
||||
}
|
||||
case "azure": {
|
||||
env.AZURE_ACCOUNT_NAME = config.accountName;
|
||||
env.AZURE_ACCOUNT_KEY = await cryptoUtils.decrypt(config.accountKey);
|
||||
if (config.endpointSuffix) {
|
||||
env.AZURE_ENDPOINT_SUFFIX = config.endpointSuffix;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return env;
|
||||
};
|
||||
|
||||
const init = async (config: RepositoryConfig) => {
|
||||
await ensurePassfile();
|
||||
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const res = await $`restic init --repo ${repoUrl} --json`.env(env).nothrow();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic init failed: ${res.stderr}`);
|
||||
return { success: false, error: res.stderr };
|
||||
}
|
||||
|
||||
logger.info(`Restic repository initialized: ${repoUrl}`);
|
||||
return { success: true, error: null };
|
||||
};
|
||||
|
||||
const backupProgressSchema = type({
|
||||
message_type: "'status'",
|
||||
seconds_elapsed: "number",
|
||||
percent_done: "number",
|
||||
total_files: "number",
|
||||
files_done: "number",
|
||||
total_bytes: "number",
|
||||
bytes_done: "number",
|
||||
current_files: "string[]",
|
||||
});
|
||||
|
||||
export type BackupProgress = typeof backupProgressSchema.infer;
|
||||
|
||||
const backup = async (
|
||||
config: RepositoryConfig,
|
||||
source: string,
|
||||
options?: {
|
||||
exclude?: string[];
|
||||
include?: string[];
|
||||
tags?: string[];
|
||||
signal?: AbortSignal;
|
||||
onProgress?: (progress: BackupProgress) => void;
|
||||
},
|
||||
) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "backup", "--one-file-system"];
|
||||
|
||||
if (options?.tags && options.tags.length > 0) {
|
||||
for (const tag of options.tags) {
|
||||
args.push("--tag", tag);
|
||||
}
|
||||
}
|
||||
|
||||
let includeFile: string | null = null;
|
||||
if (options?.include && options.include.length > 0) {
|
||||
const tmp = await fs.mkdtemp("restic-include");
|
||||
includeFile = path.join(tmp, `include.txt`);
|
||||
const includePaths = options.include.map((p) => path.join(source, p));
|
||||
|
||||
await fs.writeFile(includeFile, includePaths.join("\n"), "utf-8");
|
||||
|
||||
args.push("--files-from", includeFile);
|
||||
} else {
|
||||
args.push(source);
|
||||
}
|
||||
|
||||
if (options?.exclude && options.exclude.length > 0) {
|
||||
for (const pattern of options.exclude) {
|
||||
args.push("--exclude", pattern);
|
||||
}
|
||||
}
|
||||
|
||||
args.push("--json");
|
||||
|
||||
const logData = throttle((data: string) => {
|
||||
logger.info(data.trim());
|
||||
}, 5000);
|
||||
|
||||
const streamProgress = throttle((data: string) => {
|
||||
if (options?.onProgress) {
|
||||
try {
|
||||
const jsonData = JSON.parse(data);
|
||||
const progress = backupProgressSchema(jsonData);
|
||||
if (!(progress instanceof type.errors)) {
|
||||
options.onProgress(progress);
|
||||
}
|
||||
} catch (_) {
|
||||
// Ignore JSON parse errors for non-JSON lines
|
||||
}
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
let stdout = "";
|
||||
|
||||
const res = await safeSpawn({
|
||||
command: "restic",
|
||||
args,
|
||||
env,
|
||||
signal: options?.signal,
|
||||
onStdout: (data) => {
|
||||
stdout = data;
|
||||
logData(data);
|
||||
|
||||
if (options?.onProgress) {
|
||||
streamProgress(data);
|
||||
}
|
||||
},
|
||||
onStderr: (error) => {
|
||||
logger.error(error.trim());
|
||||
},
|
||||
finally: async () => {
|
||||
includeFile && (await fs.unlink(includeFile).catch(() => {}));
|
||||
},
|
||||
});
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic backup failed: ${res.stderr}`);
|
||||
logger.error(`Command executed: restic ${args.join(" ")}`);
|
||||
|
||||
throw new Error(`Restic backup failed: ${res.stderr}`);
|
||||
}
|
||||
|
||||
const lastLine = stdout.trim();
|
||||
const resSummary = JSON.parse(lastLine ?? "{}");
|
||||
|
||||
const result = backupOutputSchema(resSummary);
|
||||
|
||||
if (result instanceof type.errors) {
|
||||
logger.error(`Restic backup output validation failed: ${result}`);
|
||||
|
||||
throw new Error(`Restic backup output validation failed: ${result}`);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const restoreOutputSchema = type({
|
||||
message_type: "'summary'",
|
||||
total_files: "number",
|
||||
files_restored: "number",
|
||||
files_skipped: "number",
|
||||
total_bytes: "number?",
|
||||
bytes_restored: "number?",
|
||||
bytes_skipped: "number",
|
||||
});
|
||||
|
||||
const restore = async (
|
||||
config: RepositoryConfig,
|
||||
snapshotId: string,
|
||||
target: string,
|
||||
options?: {
|
||||
include?: string[];
|
||||
exclude?: string[];
|
||||
path?: string;
|
||||
delete?: boolean;
|
||||
},
|
||||
) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "restore", snapshotId, "--target", target];
|
||||
|
||||
if (options?.path) {
|
||||
args[args.length - 4] = `${snapshotId}:${options.path}`;
|
||||
}
|
||||
|
||||
if (options?.delete) {
|
||||
args.push("--delete");
|
||||
}
|
||||
|
||||
if (options?.include?.length) {
|
||||
for (const pattern of options.include) {
|
||||
args.push("--include", pattern);
|
||||
}
|
||||
}
|
||||
|
||||
if (options?.exclude && options.exclude.length > 0) {
|
||||
for (const pattern of options.exclude) {
|
||||
args.push("--exclude", pattern);
|
||||
}
|
||||
}
|
||||
|
||||
args.push("--json");
|
||||
|
||||
console.log("Restic restore command:", ["restic", ...args].join(" "));
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic restore failed: ${res.stderr}`);
|
||||
throw new Error(`Restic restore failed: ${res.stderr}`);
|
||||
}
|
||||
|
||||
const stdout = res.text();
|
||||
const outputLines = stdout.trim().split("\n");
|
||||
const lastLine = outputLines[outputLines.length - 1];
|
||||
|
||||
if (!lastLine) {
|
||||
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`);
|
||||
return {
|
||||
message_type: "summary" as const,
|
||||
total_files: 0,
|
||||
files_restored: 0,
|
||||
files_skipped: 0,
|
||||
bytes_skipped: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const resSummary = JSON.parse(lastLine);
|
||||
const result = restoreOutputSchema(resSummary);
|
||||
|
||||
if (result instanceof type.errors) {
|
||||
logger.warn(`Restic restore output validation failed: ${result}`);
|
||||
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`);
|
||||
return {
|
||||
message_type: "summary" as const,
|
||||
total_files: 0,
|
||||
files_restored: 0,
|
||||
files_skipped: 0,
|
||||
bytes_skipped: 0,
|
||||
};
|
||||
}
|
||||
|
||||
logger.info(
|
||||
`Restic restore completed for snapshot ${snapshotId} to target ${target}: ${result.files_restored} restored, ${result.files_skipped} skipped`,
|
||||
);
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const snapshots = async (config: RepositoryConfig, options: { tags?: string[] } = {}) => {
|
||||
const { tags } = options;
|
||||
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args = ["--repo", repoUrl, "snapshots"];
|
||||
|
||||
if (tags && tags.length > 0) {
|
||||
for (const tag of tags) {
|
||||
args.push("--tag", tag);
|
||||
}
|
||||
}
|
||||
|
||||
args.push("--json");
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow().quiet();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic snapshots retrieval failed: ${res.stderr}`);
|
||||
throw new Error(`Restic snapshots retrieval failed: ${res.stderr}`);
|
||||
}
|
||||
|
||||
const result = snapshotInfoSchema.array()(res.json());
|
||||
|
||||
if (result instanceof type.errors) {
|
||||
logger.error(`Restic snapshots output validation failed: ${result}`);
|
||||
throw new Error(`Restic snapshots output validation failed: ${result}`);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra: { tag: string }) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag];
|
||||
|
||||
if (options.keepLast) {
|
||||
args.push("--keep-last", String(options.keepLast));
|
||||
}
|
||||
if (options.keepHourly) {
|
||||
args.push("--keep-hourly", String(options.keepHourly));
|
||||
}
|
||||
if (options.keepDaily) {
|
||||
args.push("--keep-daily", String(options.keepDaily));
|
||||
}
|
||||
if (options.keepWeekly) {
|
||||
args.push("--keep-weekly", String(options.keepWeekly));
|
||||
}
|
||||
if (options.keepMonthly) {
|
||||
args.push("--keep-monthly", String(options.keepMonthly));
|
||||
}
|
||||
if (options.keepYearly) {
|
||||
args.push("--keep-yearly", String(options.keepYearly));
|
||||
}
|
||||
if (options.keepWithinDuration) {
|
||||
args.push("--keep-within-duration", options.keepWithinDuration);
|
||||
}
|
||||
|
||||
args.push("--prune");
|
||||
args.push("--json");
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic forget failed: ${res.stderr}`);
|
||||
throw new Error(`Restic forget failed: ${res.stderr}`);
|
||||
}
|
||||
|
||||
return { success: true };
|
||||
};
|
||||
|
||||
const lsNodeSchema = type({
|
||||
name: "string",
|
||||
type: "string",
|
||||
path: "string",
|
||||
uid: "number?",
|
||||
gid: "number?",
|
||||
size: "number?",
|
||||
mode: "number?",
|
||||
mtime: "string?",
|
||||
atime: "string?",
|
||||
ctime: "string?",
|
||||
struct_type: "'node'",
|
||||
});
|
||||
|
||||
const lsSnapshotInfoSchema = type({
|
||||
time: "string",
|
||||
parent: "string?",
|
||||
tree: "string",
|
||||
paths: "string[]",
|
||||
hostname: "string",
|
||||
username: "string?",
|
||||
id: "string",
|
||||
short_id: "string",
|
||||
struct_type: "'snapshot'",
|
||||
message_type: "'snapshot'",
|
||||
});
|
||||
|
||||
const ls = async (config: RepositoryConfig, snapshotId: string, path?: string) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "ls", snapshotId, "--json", "--long"];
|
||||
|
||||
if (path) {
|
||||
args.push(path);
|
||||
}
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow().quiet();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic ls failed: ${res.stderr}`);
|
||||
throw new Error(`Restic ls failed: ${res.stderr}`);
|
||||
}
|
||||
|
||||
// The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes
|
||||
const stdout = res.text();
|
||||
const lines = stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter((line) => line.trim());
|
||||
|
||||
if (lines.length === 0) {
|
||||
return { snapshot: null, nodes: [] };
|
||||
}
|
||||
|
||||
// First line is snapshot info
|
||||
const snapshotLine = JSON.parse(lines[0] ?? "{}");
|
||||
const snapshot = lsSnapshotInfoSchema(snapshotLine);
|
||||
|
||||
if (snapshot instanceof type.errors) {
|
||||
logger.error(`Restic ls snapshot info validation failed: ${snapshot}`);
|
||||
throw new Error(`Restic ls snapshot info validation failed: ${snapshot}`);
|
||||
}
|
||||
|
||||
const nodes: Array<typeof lsNodeSchema.infer> = [];
|
||||
for (let i = 1; i < lines.length; i++) {
|
||||
const nodeLine = JSON.parse(lines[i] ?? "{}");
|
||||
const nodeValidation = lsNodeSchema(nodeLine);
|
||||
|
||||
if (nodeValidation instanceof type.errors) {
|
||||
logger.warn(`Skipping invalid node: ${nodeValidation}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
nodes.push(nodeValidation);
|
||||
}
|
||||
|
||||
return { snapshot, nodes };
|
||||
};
|
||||
|
||||
const unlock = async (config: RepositoryConfig) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const res = await $`restic unlock --repo ${repoUrl} --remove-all --json`.env(env).nothrow();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic unlock failed: ${res.stderr}`);
|
||||
throw new Error(`Restic unlock failed: ${res.stderr}`);
|
||||
}
|
||||
|
||||
logger.info(`Restic unlock succeeded for repository: ${repoUrl}`);
|
||||
return { success: true, message: "Repository unlocked successfully" };
|
||||
};
|
||||
|
||||
const check = async (config: RepositoryConfig, options?: { readData?: boolean }) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "check"];
|
||||
|
||||
if (options?.readData) {
|
||||
args.push("--read-data");
|
||||
}
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
|
||||
const stdout = res.text();
|
||||
const stderr = res.stderr.toString();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic check failed: ${stderr}`);
|
||||
return {
|
||||
success: false,
|
||||
hasErrors: true,
|
||||
output: stdout,
|
||||
error: stderr,
|
||||
};
|
||||
}
|
||||
|
||||
const hasErrors = stdout.includes("Fatal");
|
||||
|
||||
logger.info(`Restic check completed for repository: ${repoUrl}`);
|
||||
return {
|
||||
success: !hasErrors,
|
||||
hasErrors,
|
||||
output: stdout,
|
||||
error: hasErrors ? "Repository contains errors" : null,
|
||||
};
|
||||
};
|
||||
|
||||
const repairIndex = async (config: RepositoryConfig) => {
|
||||
const repoUrl = buildRepoUrl(config);
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const res = await $`restic repair index --repo ${repoUrl}`.env(env).nothrow();
|
||||
|
||||
const stdout = res.text();
|
||||
const stderr = res.stderr.toString();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic repair index failed: ${stderr}`);
|
||||
throw new Error(`Restic repair index failed: ${stderr}`);
|
||||
}
|
||||
|
||||
logger.info(`Restic repair index completed for repository: ${repoUrl}`);
|
||||
return {
|
||||
success: true,
|
||||
output: stdout,
|
||||
message: "Index repaired successfully",
|
||||
};
|
||||
};
|
||||
|
||||
export const restic = {
|
||||
ensurePassfile,
|
||||
init,
|
||||
backup,
|
||||
restore,
|
||||
snapshots,
|
||||
forget,
|
||||
unlock,
|
||||
ls,
|
||||
check,
|
||||
repairIndex,
|
||||
};
|
||||
18
app/server/utils/sanitize.ts
Normal file
18
app/server/utils/sanitize.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* Sanitizes sensitive information from strings
|
||||
* This removes passwords and credentials from logs and error messages
|
||||
*/
|
||||
export const sanitizeSensitiveData = (text: string): string => {
|
||||
let sanitized = text.replace(/\b(pass|password)=([^\s,]+)/gi, "$1=***");
|
||||
|
||||
sanitized = sanitized.replace(/\/\/([^:@\s]+):([^@\s]+)@/g, "//$1:***@");
|
||||
|
||||
sanitized = sanitized.replace(/(\S+)\s+(\S+)\s+(\S+)/g, (match, url, user, _pass) => {
|
||||
if (url.startsWith("http://") || url.startsWith("https://")) {
|
||||
return `${url} ${user} ***`;
|
||||
}
|
||||
return match;
|
||||
});
|
||||
|
||||
return sanitized;
|
||||
};
|
||||
79
app/server/utils/spawn.ts
Normal file
79
app/server/utils/spawn.ts
Normal file
@@ -0,0 +1,79 @@
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
interface Params {
|
||||
command: string;
|
||||
args: string[];
|
||||
env?: NodeJS.ProcessEnv;
|
||||
signal?: AbortSignal;
|
||||
onStdout?: (data: string) => void;
|
||||
onStderr?: (error: string) => void;
|
||||
onError?: (error: Error) => Promise<void> | void;
|
||||
onClose?: (code: number | null) => Promise<void> | void;
|
||||
finally?: () => Promise<void> | void;
|
||||
}
|
||||
|
||||
type SpawnResult = {
|
||||
exitCode: number;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
};
|
||||
|
||||
export const safeSpawn = (params: Params) => {
|
||||
const { command, args, env = {}, signal, ...callbacks } = params;
|
||||
|
||||
return new Promise<SpawnResult>((resolve) => {
|
||||
let stdoutData = "";
|
||||
let stderrData = "";
|
||||
|
||||
const child = spawn(command, args, {
|
||||
env: { ...process.env, ...env },
|
||||
signal: signal,
|
||||
});
|
||||
|
||||
child.stdout.on("data", (data) => {
|
||||
if (callbacks.onStdout) {
|
||||
callbacks.onStdout(data.toString());
|
||||
} else {
|
||||
stdoutData += data.toString();
|
||||
}
|
||||
});
|
||||
|
||||
child.stderr.on("data", (data) => {
|
||||
if (callbacks.onStderr) {
|
||||
callbacks.onStderr(data.toString());
|
||||
} else {
|
||||
stderrData += data.toString();
|
||||
}
|
||||
});
|
||||
|
||||
child.on("error", async (error) => {
|
||||
if (callbacks.onError) {
|
||||
await callbacks.onError(error);
|
||||
}
|
||||
if (callbacks.finally) {
|
||||
await callbacks.finally();
|
||||
}
|
||||
|
||||
resolve({
|
||||
exitCode: -1,
|
||||
stdout: stdoutData,
|
||||
stderr: stderrData,
|
||||
});
|
||||
});
|
||||
|
||||
child.on("close", async (code) => {
|
||||
if (callbacks.onClose) {
|
||||
await callbacks.onClose(code);
|
||||
}
|
||||
if (callbacks.finally) {
|
||||
await callbacks.finally();
|
||||
}
|
||||
|
||||
resolve({
|
||||
exitCode: code === null ? -1 : code,
|
||||
stdout: stdoutData,
|
||||
stderr: stderrData,
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
17
app/server/utils/timeout.ts
Normal file
17
app/server/utils/timeout.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
class TimeoutError extends Error {
|
||||
code = "ETIMEOUT";
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "TimeoutError";
|
||||
}
|
||||
}
|
||||
|
||||
export async function withTimeout<T>(promise: Promise<T>, ms: number, label = "operation"): Promise<T> {
|
||||
let timer: NodeJS.Timeout | undefined;
|
||||
const timeout = new Promise<T>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new TimeoutError(`${label} timed out after ${ms}ms`)), ms);
|
||||
});
|
||||
return Promise.race([promise, timeout]).finally(() => {
|
||||
if (timer) clearTimeout(timer);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user