Files
ironmount/app/server/utils/restic.ts
2025-11-22 17:39:08 +01:00

731 lines
19 KiB
TypeScript

import crypto from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { throttle } from "es-toolkit";
import { type } from "arktype";
import { $ } from "bun";
import { REPOSITORY_BASE, RESTIC_PASS_FILE } from "../core/constants";
import { logger } from "./logger";
import { cryptoUtils } from "./crypto";
import type { RetentionPolicy } from "../modules/backups/backups.dto";
import { safeSpawn } from "./spawn";
import type { RepositoryConfig } from "~/schemas/restic";
const backupOutputSchema = type({
message_type: "'summary'",
files_new: "number",
files_changed: "number",
files_unmodified: "number",
dirs_new: "number",
dirs_changed: "number",
dirs_unmodified: "number",
data_blobs: "number",
tree_blobs: "number",
data_added: "number",
total_files_processed: "number",
total_bytes_processed: "number",
total_duration: "number",
snapshot_id: "string",
});
const snapshotInfoSchema = type({
gid: "number?",
hostname: "string",
id: "string",
parent: "string?",
paths: "string[]",
program_version: "string?",
short_id: "string",
time: "string",
uid: "number?",
username: "string",
summary: type({
backup_end: "string",
backup_start: "string",
data_added: "number",
data_added_packed: "number",
data_blobs: "number",
dirs_changed: "number",
dirs_new: "number",
dirs_unmodified: "number",
files_changed: "number",
files_new: "number",
files_unmodified: "number",
total_bytes_processed: "number",
total_files_processed: "number",
tree_blobs: "number",
}).optional(),
});
const ensurePassfile = async () => {
await fs.mkdir(path.dirname(RESTIC_PASS_FILE), { recursive: true });
try {
await fs.access(RESTIC_PASS_FILE);
} catch {
logger.info("Restic passfile not found, creating a new one...");
await fs.writeFile(RESTIC_PASS_FILE, crypto.randomBytes(32).toString("hex"), { mode: 0o600 });
}
};
const buildRepoUrl = (config: RepositoryConfig): string => {
switch (config.backend) {
case "local":
return config.path ? `${config.path}/${config.name}` : `${REPOSITORY_BASE}/${config.name}`;
case "s3":
return `s3:${config.endpoint}/${config.bucket}`;
case "r2": {
const endpoint = config.endpoint.replace(/^https?:\/\//, "");
return `s3:${endpoint}/${config.bucket}`;
}
case "gcs":
return `gs:${config.bucket}:/`;
case "azure":
return `azure:${config.container}:/`;
case "rclone":
return `rclone:${config.remote}:${config.path}`;
case "rest": {
const path = config.path ? `/${config.path}` : "";
return `rest:${config.url}${path}`;
}
case "sftp":
return `sftp:${config.user}@${config.host}:${config.path}`;
default: {
throw new Error(`Unsupported repository backend: ${JSON.stringify(config)}`);
}
}
};
const buildEnv = async (config: RepositoryConfig) => {
const env: Record<string, string> = {
RESTIC_CACHE_DIR: "/var/lib/zerobyte/restic/cache",
PATH: process.env.PATH || "/usr/local/bin:/usr/bin:/bin",
};
if (config.isExistingRepository && config.customPassword) {
const decryptedPassword = await cryptoUtils.decrypt(config.customPassword);
const passwordFilePath = path.join("/tmp", `zerobyte-pass-${crypto.randomBytes(8).toString("hex")}.txt`);
await fs.writeFile(passwordFilePath, decryptedPassword, { mode: 0o600 });
env.RESTIC_PASSWORD_FILE = passwordFilePath;
} else {
env.RESTIC_PASSWORD_FILE = RESTIC_PASS_FILE;
}
switch (config.backend) {
case "s3":
env.AWS_ACCESS_KEY_ID = await cryptoUtils.decrypt(config.accessKeyId);
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.decrypt(config.secretAccessKey);
break;
case "r2":
env.AWS_ACCESS_KEY_ID = await cryptoUtils.decrypt(config.accessKeyId);
env.AWS_SECRET_ACCESS_KEY = await cryptoUtils.decrypt(config.secretAccessKey);
env.AWS_REGION = "auto";
env.AWS_S3_FORCE_PATH_STYLE = "true";
break;
case "gcs": {
const decryptedCredentials = await cryptoUtils.decrypt(config.credentialsJson);
const credentialsPath = path.join("/tmp", `zerobyte-gcs-${crypto.randomBytes(8).toString("hex")}.json`);
await fs.writeFile(credentialsPath, decryptedCredentials, { mode: 0o600 });
env.GOOGLE_PROJECT_ID = config.projectId;
env.GOOGLE_APPLICATION_CREDENTIALS = credentialsPath;
break;
}
case "azure": {
env.AZURE_ACCOUNT_NAME = config.accountName;
env.AZURE_ACCOUNT_KEY = await cryptoUtils.decrypt(config.accountKey);
if (config.endpointSuffix) {
env.AZURE_ENDPOINT_SUFFIX = config.endpointSuffix;
}
break;
}
case "rest": {
if (config.username) {
env.RESTIC_REST_USERNAME = await cryptoUtils.decrypt(config.username);
}
if (config.password) {
env.RESTIC_REST_PASSWORD = await cryptoUtils.decrypt(config.password);
}
break;
}
case "sftp": {
const decryptedKey = await cryptoUtils.decrypt(config.privateKey);
const keyPath = path.join("/tmp", `ironmount-ssh-${crypto.randomBytes(8).toString("hex")}`);
let normalizedKey = decryptedKey.replace(/\r\n/g, "\n");
if (!normalizedKey.endsWith("\n")) {
normalizedKey += "\n";
}
if (normalizedKey.includes("ENCRYPTED")) {
logger.error("SFTP: Private key appears to be passphrase-protected. Please use an unencrypted key.");
throw new Error("Passphrase-protected SSH keys are not supported. Please provide an unencrypted private key.");
}
await fs.writeFile(keyPath, normalizedKey, { mode: 0o600 });
env._SFTP_KEY_PATH = keyPath;
const sshArgs = [
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"LogLevel=VERBOSE",
"-i",
keyPath,
];
if (config.port && config.port !== 22) {
sshArgs.push("-p", String(config.port));
}
env._SFTP_SSH_ARGS = sshArgs.join(" ");
logger.info(`SFTP: SSH args: ${env._SFTP_SSH_ARGS}`);
break;
}
}
return env;
};
const init = async (config: RepositoryConfig) => {
await ensurePassfile();
const repoUrl = buildRepoUrl(config);
logger.info(`Initializing restic repository at ${repoUrl}...`);
const env = await buildEnv(config);
const args = ["init", "--repo", repoUrl, "--json"];
addRepoSpecificArgs(args, config, env);
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic init failed: ${res.stderr}`);
return { success: false, error: res.stderr.toString() };
}
logger.info(`Restic repository initialized: ${repoUrl}`);
return { success: true, error: null };
};
const backupProgressSchema = type({
message_type: "'status'",
seconds_elapsed: "number",
percent_done: "number",
total_files: "number",
files_done: "number",
total_bytes: "number",
bytes_done: "number",
current_files: "string[]",
});
export type BackupProgress = typeof backupProgressSchema.infer;
const backup = async (
config: RepositoryConfig,
source: string,
options?: {
exclude?: string[];
include?: string[];
tags?: string[];
signal?: AbortSignal;
onProgress?: (progress: BackupProgress) => void;
},
) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args: string[] = ["--repo", repoUrl, "backup", "--one-file-system"];
if (options?.tags && options.tags.length > 0) {
for (const tag of options.tags) {
args.push("--tag", tag);
}
}
let includeFile: string | null = null;
if (options?.include && options.include.length > 0) {
const tmp = await fs.mkdtemp("restic-include");
includeFile = path.join(tmp, `include.txt`);
const includePaths = options.include.map((p) => path.join(source, p));
await fs.writeFile(includeFile, includePaths.join("\n"), "utf-8");
args.push("--files-from", includeFile);
} else {
args.push(source);
}
if (options?.exclude && options.exclude.length > 0) {
for (const pattern of options.exclude) {
args.push("--exclude", pattern);
}
}
addRepoSpecificArgs(args, config, env);
args.push("--json");
const logData = throttle((data: string) => {
logger.info(data.trim());
}, 5000);
const streamProgress = throttle((data: string) => {
if (options?.onProgress) {
try {
const jsonData = JSON.parse(data);
const progress = backupProgressSchema(jsonData);
if (!(progress instanceof type.errors)) {
options.onProgress(progress);
}
} catch (_) {
// Ignore JSON parse errors for non-JSON lines
}
}
}, 1000);
let stdout = "";
const res = await safeSpawn({
command: "restic",
args,
env,
signal: options?.signal,
onStdout: (data) => {
stdout = data;
logData(data);
if (options?.onProgress) {
streamProgress(data);
}
},
onStderr: (error) => {
logger.error(error.trim());
},
finally: async () => {
includeFile && (await fs.unlink(includeFile).catch(() => {}));
await cleanupTemporaryKeys(config, env);
},
});
if (res.exitCode !== 0) {
logger.error(`Restic backup failed: ${res.stderr}`);
logger.error(`Command executed: restic ${args.join(" ")}`);
throw new Error(`Restic backup failed: ${res.stderr}`);
}
const lastLine = stdout.trim();
const resSummary = JSON.parse(lastLine ?? "{}");
const result = backupOutputSchema(resSummary);
if (result instanceof type.errors) {
logger.error(`Restic backup output validation failed: ${result}`);
throw new Error(`Restic backup output validation failed: ${result}`);
}
return result;
};
const restoreOutputSchema = type({
message_type: "'summary'",
total_files: "number",
files_restored: "number",
files_skipped: "number",
total_bytes: "number?",
bytes_restored: "number?",
bytes_skipped: "number",
});
const restore = async (
config: RepositoryConfig,
snapshotId: string,
target: string,
options?: {
include?: string[];
exclude?: string[];
excludeXattr?: string[];
path?: string;
delete?: boolean;
},
) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args: string[] = ["--repo", repoUrl, "restore", snapshotId, "--target", target];
if (options?.path) {
args[args.length - 4] = `${snapshotId}:${options.path}`;
}
if (options?.delete) {
args.push("--delete");
}
if (options?.include?.length) {
for (const pattern of options.include) {
args.push("--include", pattern);
}
}
if (options?.exclude && options.exclude.length > 0) {
for (const pattern of options.exclude) {
args.push("--exclude", pattern);
}
}
if (options?.excludeXattr && options.excludeXattr.length > 0) {
for (const xattr of options.excludeXattr) {
args.push("--exclude-xattr", xattr);
}
}
addRepoSpecificArgs(args, config, env);
args.push("--json");
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic restore failed: ${res.stderr}`);
throw new Error(`Restic restore failed: ${res.stderr}`);
}
const stdout = res.text();
const outputLines = stdout.trim().split("\n");
const lastLine = outputLines[outputLines.length - 1];
if (!lastLine) {
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`);
return {
message_type: "summary" as const,
total_files: 0,
files_restored: 0,
files_skipped: 0,
bytes_skipped: 0,
};
}
logger.debug(`Restic restore output last line: ${lastLine}`);
const resSummary = JSON.parse(lastLine);
const result = restoreOutputSchema(resSummary);
if (result instanceof type.errors) {
logger.warn(`Restic restore output validation failed: ${result}`);
logger.info(`Restic restore completed for snapshot ${snapshotId} to target ${target}`);
return {
message_type: "summary" as const,
total_files: 0,
files_restored: 0,
files_skipped: 0,
bytes_skipped: 0,
};
}
logger.info(
`Restic restore completed for snapshot ${snapshotId} to target ${target}: ${result.files_restored} restored, ${result.files_skipped} skipped`,
);
return result;
};
const snapshots = async (config: RepositoryConfig, options: { tags?: string[] } = {}) => {
const { tags } = options;
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args = ["--repo", repoUrl, "snapshots"];
if (tags && tags.length > 0) {
for (const tag of tags) {
args.push("--tag", tag);
}
}
addRepoSpecificArgs(args, config, env);
args.push("--json");
const res = await $`restic ${args}`.env(env).nothrow().quiet();
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic snapshots retrieval failed: ${res.stderr}`);
throw new Error(`Restic snapshots retrieval failed: ${res.stderr}`);
}
const result = snapshotInfoSchema.array()(res.json());
if (result instanceof type.errors) {
logger.error(`Restic snapshots output validation failed: ${result}`);
throw new Error(`Restic snapshots output validation failed: ${result}`);
}
return result;
};
const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra: { tag: string }) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args: string[] = ["--repo", repoUrl, "forget", "--group-by", "tags", "--tag", extra.tag];
if (options.keepLast) {
args.push("--keep-last", String(options.keepLast));
}
if (options.keepHourly) {
args.push("--keep-hourly", String(options.keepHourly));
}
if (options.keepDaily) {
args.push("--keep-daily", String(options.keepDaily));
}
if (options.keepWeekly) {
args.push("--keep-weekly", String(options.keepWeekly));
}
if (options.keepMonthly) {
args.push("--keep-monthly", String(options.keepMonthly));
}
if (options.keepYearly) {
args.push("--keep-yearly", String(options.keepYearly));
}
if (options.keepWithinDuration) {
args.push("--keep-within-duration", options.keepWithinDuration);
}
args.push("--prune");
addRepoSpecificArgs(args, config, env);
args.push("--json");
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic forget failed: ${res.stderr}`);
throw new Error(`Restic forget failed: ${res.stderr}`);
}
return { success: true };
};
const deleteSnapshot = async (config: RepositoryConfig, snapshotId: string) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args: string[] = ["--repo", repoUrl, "forget", snapshotId, "--prune"];
addRepoSpecificArgs(args, config, env);
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic snapshot deletion failed: ${res.stderr}`);
throw new Error(`Failed to delete snapshot: ${res.stderr}`);
}
return { success: true };
};
const lsNodeSchema = type({
name: "string",
type: "string",
path: "string",
uid: "number?",
gid: "number?",
size: "number?",
mode: "number?",
mtime: "string?",
atime: "string?",
ctime: "string?",
struct_type: "'node'",
});
const lsSnapshotInfoSchema = type({
time: "string",
parent: "string?",
tree: "string",
paths: "string[]",
hostname: "string",
username: "string?",
id: "string",
short_id: "string",
struct_type: "'snapshot'",
message_type: "'snapshot'",
});
const ls = async (config: RepositoryConfig, snapshotId: string, path?: string) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args: string[] = ["--repo", repoUrl, "ls", snapshotId, "--json", "--long"];
if (path) {
args.push(path);
}
addRepoSpecificArgs(args, config, env);
const res = await safeSpawn({ command: "restic", args, env });
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic ls failed: ${res.stderr}`);
throw new Error(`Restic ls failed: ${res.stderr}`);
}
// The output is a stream of JSON objects, first is snapshot info, rest are file/dir nodes
const stdout = res.stdout;
const lines = stdout
.trim()
.split("\n")
.filter((line) => line.trim());
if (lines.length === 0) {
return { snapshot: null, nodes: [] };
}
// First line is snapshot info
const snapshotLine = JSON.parse(lines[0] ?? "{}");
const snapshot = lsSnapshotInfoSchema(snapshotLine);
if (snapshot instanceof type.errors) {
logger.error(`Restic ls snapshot info validation failed: ${snapshot}`);
throw new Error(`Restic ls snapshot info validation failed: ${snapshot}`);
}
const nodes: Array<typeof lsNodeSchema.infer> = [];
for (let i = 1; i < lines.length; i++) {
const nodeLine = JSON.parse(lines[i] ?? "{}");
const nodeValidation = lsNodeSchema(nodeLine);
if (nodeValidation instanceof type.errors) {
logger.warn(`Skipping invalid node: ${nodeValidation}`);
continue;
}
nodes.push(nodeValidation);
}
return { snapshot, nodes };
};
const unlock = async (config: RepositoryConfig) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args = ["unlock", "--repo", repoUrl, "--remove-all", "--json"];
addRepoSpecificArgs(args, config, env);
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
if (res.exitCode !== 0) {
logger.error(`Restic unlock failed: ${res.stderr}`);
throw new Error(`Restic unlock failed: ${res.stderr}`);
}
logger.info(`Restic unlock succeeded for repository: ${repoUrl}`);
return { success: true, message: "Repository unlocked successfully" };
};
const check = async (config: RepositoryConfig, options?: { readData?: boolean }) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args: string[] = ["--repo", repoUrl, "check"];
if (options?.readData) {
args.push("--read-data");
}
addRepoSpecificArgs(args, config, env);
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
const stdout = res.text();
const stderr = res.stderr.toString();
if (res.exitCode !== 0) {
logger.error(`Restic check failed: ${stderr}`);
return {
success: false,
hasErrors: true,
output: stdout,
error: stderr,
};
}
const hasErrors = stdout.includes("Fatal");
logger.info(`Restic check completed for repository: ${repoUrl}`);
return {
success: !hasErrors,
hasErrors,
output: stdout,
error: hasErrors ? "Repository contains errors" : null,
};
};
const repairIndex = async (config: RepositoryConfig) => {
const repoUrl = buildRepoUrl(config);
const env = await buildEnv(config);
const args = ["repair", "index", "--repo", repoUrl];
addRepoSpecificArgs(args, config, env);
const res = await $`restic ${args}`.env(env).nothrow();
await cleanupTemporaryKeys(config, env);
const stdout = res.text();
const stderr = res.stderr.toString();
if (res.exitCode !== 0) {
logger.error(`Restic repair index failed: ${stderr}`);
throw new Error(`Restic repair index failed: ${stderr}`);
}
logger.info(`Restic repair index completed for repository: ${repoUrl}`);
return {
success: true,
output: stdout,
message: "Index repaired successfully",
};
};
const addRepoSpecificArgs = (args: string[], config: RepositoryConfig, env: Record<string, string>) => {
if (config.backend === "sftp" && env._SFTP_SSH_ARGS) {
args.push("-o", `sftp.args=${env._SFTP_SSH_ARGS}`);
}
};
const cleanupTemporaryKeys = async (config: RepositoryConfig, env: Record<string, string>) => {
if (config.backend === "sftp" && env._SFTP_KEY_PATH) {
await fs.unlink(env._SFTP_KEY_PATH).catch(() => {});
} else if (config.isExistingRepository && config.customPassword && env.RESTIC_PASSWORD_FILE) {
await fs.unlink(env.RESTIC_PASSWORD_FILE).catch(() => {});
} else if (config.backend === "gcs" && env.GOOGLE_APPLICATION_CREDENTIALS) {
await fs.unlink(env.GOOGLE_APPLICATION_CREDENTIALS).catch(() => {});
}
};
export const restic = {
ensurePassfile,
init,
backup,
restore,
snapshots,
forget,
deleteSnapshot,
unlock,
ls,
check,
repairIndex,
};