mirror of
https://github.com/nicotsx/ironmount.git
synced 2025-12-10 12:10:51 +01:00
feat: mirror repositories (#95)
* feat: mirror repositories feat: mirror backup repositories * chore: pr feedbacks
This commit is contained in:
Nico
GitHub
148
app/server/utils/backend-compatibility.ts
Normal file
148
app/server/utils/backend-compatibility.ts
Normal file
@@ -0,0 +1,148 @@
|
||||
import type { RepositoryConfig } from "~/schemas/restic";
|
||||
import { cryptoUtils } from "./crypto";
|
||||
|
||||
type BackendConflictGroup = "s3" | "gcs" | "azure" | "rest" | "sftp" | null;
|
||||
|
||||
export const getBackendConflictGroup = (backend: string): BackendConflictGroup => {
|
||||
switch (backend) {
|
||||
case "s3":
|
||||
case "r2":
|
||||
return "s3";
|
||||
case "gcs":
|
||||
return "gcs";
|
||||
case "azure":
|
||||
return "azure";
|
||||
case "rest":
|
||||
return "rest";
|
||||
case "sftp":
|
||||
return "sftp";
|
||||
case "local":
|
||||
case "rclone":
|
||||
return null;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
export const hasCompatibleCredentials = async (
|
||||
config1: RepositoryConfig,
|
||||
config2: RepositoryConfig,
|
||||
): Promise<boolean> => {
|
||||
const group1 = getBackendConflictGroup(config1.backend);
|
||||
const group2 = getBackendConflictGroup(config2.backend);
|
||||
|
||||
if (!group1 || !group2 || group1 !== group2) {
|
||||
return true;
|
||||
}
|
||||
|
||||
switch (group1) {
|
||||
case "s3": {
|
||||
if (
|
||||
(config1.backend === "s3" || config1.backend === "r2") &&
|
||||
(config2.backend === "s3" || config2.backend === "r2")
|
||||
) {
|
||||
const accessKey1 = await cryptoUtils.decrypt(config1.accessKeyId);
|
||||
const secretKey1 = await cryptoUtils.decrypt(config1.secretAccessKey);
|
||||
|
||||
const accessKey2 = await cryptoUtils.decrypt(config2.accessKeyId);
|
||||
const secretKey2 = await cryptoUtils.decrypt(config2.secretAccessKey);
|
||||
|
||||
return accessKey1 === accessKey2 && secretKey1 === secretKey2;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
case "gcs": {
|
||||
if (config1.backend === "gcs" && config2.backend === "gcs") {
|
||||
const credentials1 = await cryptoUtils.decrypt(config1.credentialsJson);
|
||||
const credentials2 = await cryptoUtils.decrypt(config2.credentialsJson);
|
||||
|
||||
return credentials1 === credentials2 && config1.projectId === config2.projectId;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
case "azure": {
|
||||
if (config1.backend === "azure" && config2.backend === "azure") {
|
||||
const config1Accountkey = await cryptoUtils.decrypt(config1.accountKey);
|
||||
const config2Accountkey = await cryptoUtils.decrypt(config2.accountKey);
|
||||
|
||||
return config1.accountName === config2.accountName && config1Accountkey === config2Accountkey;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
case "rest": {
|
||||
if (config1.backend === "rest" && config2.backend === "rest") {
|
||||
if (!config1.username && !config2.username && !config1.password && !config2.password) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const config1Username = await cryptoUtils.decrypt(config1.username || "");
|
||||
const config1Password = await cryptoUtils.decrypt(config1.password || "");
|
||||
const config2Username = await cryptoUtils.decrypt(config2.username || "");
|
||||
const config2Password = await cryptoUtils.decrypt(config2.password || "");
|
||||
|
||||
return config1Username === config2Username && config1Password === config2Password;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
case "sftp": {
|
||||
return false;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
export interface CompatibilityResult {
|
||||
repositoryId: string;
|
||||
compatible: boolean;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
export const checkMirrorCompatibility = async (
|
||||
primaryConfig: RepositoryConfig,
|
||||
mirrorConfig: RepositoryConfig,
|
||||
mirrorRepositoryId: string,
|
||||
): Promise<CompatibilityResult> => {
|
||||
const primaryConflictGroup = getBackendConflictGroup(primaryConfig.backend);
|
||||
const mirrorConflictGroup = getBackendConflictGroup(mirrorConfig.backend);
|
||||
|
||||
if (!primaryConflictGroup || !mirrorConflictGroup) {
|
||||
return {
|
||||
repositoryId: mirrorRepositoryId,
|
||||
compatible: true,
|
||||
reason: null,
|
||||
};
|
||||
}
|
||||
|
||||
if (primaryConflictGroup !== mirrorConflictGroup) {
|
||||
return {
|
||||
repositoryId: mirrorRepositoryId,
|
||||
compatible: true,
|
||||
reason: null,
|
||||
};
|
||||
}
|
||||
|
||||
const compatible = await hasCompatibleCredentials(primaryConfig, mirrorConfig);
|
||||
|
||||
if (compatible) {
|
||||
return {
|
||||
repositoryId: mirrorRepositoryId,
|
||||
compatible: true,
|
||||
reason: null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
repositoryId: mirrorRepositoryId,
|
||||
compatible: false,
|
||||
reason: `Both use ${primaryConflictGroup.toUpperCase()} backends with different credentials`,
|
||||
};
|
||||
};
|
||||
|
||||
export const getIncompatibleMirrorError = (mirrorRepoName: string, primaryBackend: string, mirrorBackend: string) => {
|
||||
return (
|
||||
`Cannot mirror to ${mirrorRepoName}: both repositories use the same backend type (${primaryBackend}/${mirrorBackend}) with different credentials. ` +
|
||||
"Restic cannot use different credentials for the same backend in a copy operation. " +
|
||||
"Consider creating a new backup scheduler with the desired destination instead."
|
||||
);
|
||||
};
|
||||
@@ -40,6 +40,7 @@ const snapshotInfoSchema = type({
|
||||
time: "string",
|
||||
uid: "number?",
|
||||
username: "string",
|
||||
tags: "string[]?",
|
||||
summary: type({
|
||||
backup_end: "string",
|
||||
backup_start: "string",
|
||||
@@ -201,7 +202,7 @@ const init = async (config: RepositoryConfig) => {
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args = ["init", "--repo", repoUrl];
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -277,7 +278,7 @@ const backup = async (
|
||||
}
|
||||
}
|
||||
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const logData = throttle((data: string) => {
|
||||
logger.info(data.trim());
|
||||
@@ -403,7 +404,7 @@ const restore = async (
|
||||
}
|
||||
}
|
||||
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
@@ -466,7 +467,7 @@ const snapshots = async (config: RepositoryConfig, options: { tags?: string[] }
|
||||
}
|
||||
}
|
||||
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow().quiet();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -515,7 +516,7 @@ const forget = async (config: RepositoryConfig, options: RetentionPolicy, extra:
|
||||
}
|
||||
|
||||
args.push("--prune");
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -533,7 +534,7 @@ const deleteSnapshot = async (config: RepositoryConfig, snapshotId: string) => {
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args: string[] = ["--repo", repoUrl, "forget", snapshotId, "--prune"];
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -583,7 +584,7 @@ const ls = async (config: RepositoryConfig, snapshotId: string, path?: string) =
|
||||
args.push(path);
|
||||
}
|
||||
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await safeSpawn({ command: "restic", args, env });
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -634,7 +635,7 @@ const unlock = async (config: RepositoryConfig) => {
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args = ["unlock", "--repo", repoUrl, "--remove-all"];
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -658,7 +659,7 @@ const check = async (config: RepositoryConfig, options?: { readData?: boolean })
|
||||
args.push("--read-data");
|
||||
}
|
||||
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -692,7 +693,7 @@ const repairIndex = async (config: RepositoryConfig) => {
|
||||
const env = await buildEnv(config);
|
||||
|
||||
const args = ["repair", "index", "--repo", repoUrl];
|
||||
addCommonArgs(args, config, env);
|
||||
addCommonArgs(args, env);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
await cleanupTemporaryKeys(config, env);
|
||||
@@ -713,12 +714,65 @@ const repairIndex = async (config: RepositoryConfig) => {
|
||||
};
|
||||
};
|
||||
|
||||
const addCommonArgs = (args: string[], config: RepositoryConfig, env: Record<string, string>) => {
|
||||
args.push("--retry-lock", "1m", "--json");
|
||||
const copy = async (
|
||||
sourceConfig: RepositoryConfig,
|
||||
destConfig: RepositoryConfig,
|
||||
options: {
|
||||
tag?: string;
|
||||
snapshotId?: string;
|
||||
},
|
||||
) => {
|
||||
const sourceRepoUrl = buildRepoUrl(sourceConfig);
|
||||
const destRepoUrl = buildRepoUrl(destConfig);
|
||||
|
||||
if (config.backend === "sftp" && env._SFTP_SSH_ARGS) {
|
||||
args.push("-o", `sftp.args=${env._SFTP_SSH_ARGS}`);
|
||||
const sourceEnv = await buildEnv(sourceConfig);
|
||||
const destEnv = await buildEnv(destConfig);
|
||||
|
||||
const env: Record<string, string> = {
|
||||
...sourceEnv,
|
||||
...destEnv,
|
||||
RESTIC_FROM_PASSWORD_FILE: sourceEnv.RESTIC_PASSWORD_FILE,
|
||||
};
|
||||
|
||||
const args: string[] = ["--repo", destRepoUrl, "copy", "--from-repo", sourceRepoUrl];
|
||||
|
||||
if (options.tag) {
|
||||
args.push("--tag", options.tag);
|
||||
}
|
||||
|
||||
if (options.snapshotId) {
|
||||
args.push(options.snapshotId);
|
||||
} else {
|
||||
args.push("latest");
|
||||
}
|
||||
|
||||
addCommonArgs(args, env);
|
||||
|
||||
if (sourceConfig.backend === "sftp" && sourceEnv._SFTP_SSH_ARGS) {
|
||||
args.push("-o", `sftp.args=${sourceEnv._SFTP_SSH_ARGS}`);
|
||||
}
|
||||
|
||||
logger.info(`Copying snapshots from ${sourceRepoUrl} to ${destRepoUrl}...`);
|
||||
logger.debug(`Executing: restic ${args.join(" ")}`);
|
||||
|
||||
const res = await $`restic ${args}`.env(env).nothrow();
|
||||
|
||||
await cleanupTemporaryKeys(sourceConfig, sourceEnv);
|
||||
await cleanupTemporaryKeys(destConfig, destEnv);
|
||||
|
||||
const stdout = res.text();
|
||||
const stderr = res.stderr.toString();
|
||||
|
||||
if (res.exitCode !== 0) {
|
||||
logger.error(`Restic copy failed: ${stderr}`);
|
||||
throw new ResticError(res.exitCode, stderr);
|
||||
}
|
||||
|
||||
logger.info(`Restic copy completed from ${sourceRepoUrl} to ${destRepoUrl}`);
|
||||
return {
|
||||
success: true,
|
||||
output: stdout,
|
||||
};
|
||||
};
|
||||
|
||||
const cleanupTemporaryKeys = async (config: RepositoryConfig, env: Record<string, string>) => {
|
||||
@@ -731,6 +785,14 @@ const cleanupTemporaryKeys = async (config: RepositoryConfig, env: Record<string
|
||||
}
|
||||
};
|
||||
|
||||
const addCommonArgs = (args: string[], env: Record<string, string>) => {
|
||||
args.push("--retry-lock", "1m", "--json");
|
||||
|
||||
if (env._SFTP_SSH_ARGS) {
|
||||
args.push("-o", `sftp.args=${env._SFTP_SSH_ARGS}`);
|
||||
}
|
||||
};
|
||||
|
||||
export const restic = {
|
||||
ensurePassfile,
|
||||
init,
|
||||
@@ -743,4 +805,5 @@ export const restic = {
|
||||
ls,
|
||||
check,
|
||||
repairIndex,
|
||||
copy,
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user