feat: mirror repositories

feat: mirror backup repositories

app/server/modules/backups/backups.controller.ts

@@ -12,6 +12,10 @@ import {
 	stopBackupDto,
 	updateBackupScheduleDto,
 	updateBackupScheduleBody,
+	getScheduleMirrorsDto,
+	updateScheduleMirrorsDto,
+	updateScheduleMirrorsBody,
+	getMirrorCompatibilityDto,
 	type CreateBackupScheduleDto,
 	type DeleteBackupScheduleDto,
 	type GetBackupScheduleDto,
@@ -21,6 +25,9 @@ import {
 	type RunForgetDto,
 	type StopBackupDto,
 	type UpdateBackupScheduleDto,
+	type GetScheduleMirrorsDto,
+	type UpdateScheduleMirrorsDto,
+	type GetMirrorCompatibilityDto,
 } from "./backups.dto";
 import { backupsService } from "./backups.service";
 import {
@@ -113,4 +120,23 @@ export const backupScheduleController = new Hono()
 
 			return c.json<UpdateScheduleNotificationsDto>(assignments, 200);
 		},
-	);
+	)
+	.get("/:scheduleId/mirrors", getScheduleMirrorsDto, async (c) => {
+		const scheduleId = Number.parseInt(c.req.param("scheduleId"), 10);
+		const mirrors = await backupsService.getMirrors(scheduleId);
+
+		return c.json<GetScheduleMirrorsDto>(mirrors, 200);
+	})
+	.put("/:scheduleId/mirrors", updateScheduleMirrorsDto, validator("json", updateScheduleMirrorsBody), async (c) => {
+		const scheduleId = Number.parseInt(c.req.param("scheduleId"), 10);
+		const body = c.req.valid("json");
+		const mirrors = await backupsService.updateMirrors(scheduleId, body);
+
+		return c.json<UpdateScheduleMirrorsDto>(mirrors, 200);
+	})
+	.get("/:scheduleId/mirrors/compatibility", getMirrorCompatibilityDto, async (c) => {
+		const scheduleId = Number.parseInt(c.req.param("scheduleId"), 10);
+		const compatibility = await backupsService.getMirrorCompatibility(scheduleId);
+
+		return c.json<GetMirrorCompatibilityDto>(compatibility, 200);
+	});

app/server/modules/backups/backups.dto.ts

@@ -37,6 +37,19 @@ const backupScheduleSchema = type({
 	}),
 );
 
+const scheduleMirrorSchema = type({
+	scheduleId: "number",
+	repositoryId: "string",
+	enabled: "boolean",
+	lastCopyAt: "number | null",
+	lastCopyStatus: "'success' | 'error' | null",
+	lastCopyError: "string | null",
+	createdAt: "number",
+	repository: repositorySchema,
+});
+
+export type ScheduleMirrorDto = typeof scheduleMirrorSchema.infer;
+
 /**
  * List all backup schedules
  */
@@ -276,3 +289,75 @@ export const runForgetDto = describeRoute({
 		},
 	},
 });
+
+export const getScheduleMirrorsResponse = scheduleMirrorSchema.array();
+export type GetScheduleMirrorsDto = typeof getScheduleMirrorsResponse.infer;
+
+export const getScheduleMirrorsDto = describeRoute({
+	description: "Get mirror repository assignments for a backup schedule",
+	operationId: "getScheduleMirrors",
+	tags: ["Backups"],
+	responses: {
+		200: {
+			description: "List of mirror repository assignments for the schedule",
+			content: {
+				"application/json": {
+					schema: resolver(getScheduleMirrorsResponse),
+				},
+			},
+		},
+	},
+});
+
+export const updateScheduleMirrorsBody = type({
+	mirrors: type({
+		repositoryId: "string",
+		enabled: "boolean",
+	}).array(),
+});
+
+export type UpdateScheduleMirrorsBody = typeof updateScheduleMirrorsBody.infer;
+
+export const updateScheduleMirrorsResponse = scheduleMirrorSchema.array();
+export type UpdateScheduleMirrorsDto = typeof updateScheduleMirrorsResponse.infer;
+
+export const updateScheduleMirrorsDto = describeRoute({
+	description: "Update mirror repository assignments for a backup schedule",
+	operationId: "updateScheduleMirrors",
+	tags: ["Backups"],
+	responses: {
+		200: {
+			description: "Mirror assignments updated successfully",
+			content: {
+				"application/json": {
+					schema: resolver(updateScheduleMirrorsResponse),
+				},
+			},
+		},
+	},
+});
+
+const mirrorCompatibilitySchema = type({
+	repositoryId: "string",
+	compatible: "boolean",
+	reason: "string | null",
+});
+
+export const getMirrorCompatibilityResponse = mirrorCompatibilitySchema.array();
+export type GetMirrorCompatibilityDto = typeof getMirrorCompatibilityResponse.infer;
+
+export const getMirrorCompatibilityDto = describeRoute({
+	description: "Get mirror compatibility info for all repositories relative to a backup schedule's primary repository",
+	operationId: "getMirrorCompatibility",
+	tags: ["Backups"],
+	responses: {
+		200: {
+			description: "List of repositories with their mirror compatibility status",
+			content: {
+				"application/json": {
+					schema: resolver(getMirrorCompatibilityResponse),
+				},
+			},
+		},
+	},
+});

app/server/modules/backups/backups.service.ts

@@ -3,15 +3,16 @@ import cron from "node-cron";
 import { CronExpressionParser } from "cron-parser";
 import { NotFoundError, BadRequestError, ConflictError } from "http-errors-enhanced";
 import { db } from "../../db/db";
-import { backupSchedulesTable, repositoriesTable, volumesTable } from "../../db/schema";
+import { backupSchedulesTable, backupScheduleMirrorsTable, repositoriesTable, volumesTable } from "../../db/schema";
 import { restic } from "../../utils/restic";
 import { logger } from "../../utils/logger";
 import { getVolumePath } from "../volumes/helpers";
-import type { CreateBackupScheduleBody, UpdateBackupScheduleBody } from "./backups.dto";
+import type { CreateBackupScheduleBody, UpdateBackupScheduleBody, UpdateScheduleMirrorsBody } from "./backups.dto";
 import { toMessage } from "../../utils/errors";
 import { serverEvents } from "../../core/events";
 import { notificationsService } from "../notifications/notifications.service";
 import { repoMutex } from "../../core/repository-mutex";
+import { checkMirrorCompatibility, getIncompatibleMirrorError } from "~/server/utils/backend-compatibility";
 
 const runningBackups = new Map<number, AbortController>();
 
@@ -266,19 +267,25 @@ const executeBackup = async (scheduleId: number, manual = false) => {
 			void runForget(schedule.id);
 		}
 
+		copyToMirrors(scheduleId, repository, schedule.retentionPolicy).catch((error) => {
+			logger.error(`Background mirror copy failed for schedule ${scheduleId}: ${toMessage(error)}`);
+		});
+
+		const finalStatus = exitCode === 0 ? "success" : "warning";
+
 		const nextBackupAt = calculateNextRun(schedule.cronExpression);
 		await db
 			.update(backupSchedulesTable)
 			.set({
 				lastBackupAt: Date.now(),
-				lastBackupStatus: exitCode === 0 ? "success" : "warning",
+				lastBackupStatus: finalStatus,
 				lastBackupError: null,
 				nextBackupAt: nextBackupAt,
 				updatedAt: Date.now(),
 			})
 			.where(eq(backupSchedulesTable.id, scheduleId));
 
-		if (exitCode !== 0) {
+		if (finalStatus === "warning") {
 			logger.warn(`Backup completed with warnings for volume ${volume.name} to repository ${repository.name}`);
 		} else {
 			logger.info(`Backup completed successfully for volume ${volume.name} to repository ${repository.name}`);
@@ -288,11 +295,11 @@ const executeBackup = async (scheduleId: number, manual = false) => {
 			scheduleId,
 			volumeName: volume.name,
 			repositoryName: repository.name,
-			status: exitCode === 0 ? "success" : "warning",
+			status: finalStatus,
 		});
 
 		notificationsService
-			.sendBackupNotification(scheduleId, exitCode === 0 ? "success" : "warning", {
+			.sendBackupNotification(scheduleId, finalStatus === "success" ? "success" : "warning", {
 				volumeName: volume.name,
 				repositoryName: repository.name,
 			})
@@ -421,6 +428,162 @@ const runForget = async (scheduleId: number) => {
 	logger.info(`Retention policy applied successfully for schedule ${scheduleId}`);
 };
 
+const getMirrors = async (scheduleId: number) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.id, scheduleId),
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	const mirrors = await db.query.backupScheduleMirrorsTable.findMany({
+		where: eq(backupScheduleMirrorsTable.scheduleId, scheduleId),
+		with: { repository: true },
+	});
+
+	return mirrors;
+};
+
+const updateMirrors = async (scheduleId: number, data: UpdateScheduleMirrorsBody) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.id, scheduleId),
+		with: { repository: true },
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	for (const mirror of data.mirrors) {
+		if (mirror.repositoryId === schedule.repositoryId) {
+			throw new BadRequestError("Cannot add the primary repository as a mirror");
+		}
+
+		const repo = await db.query.repositoriesTable.findFirst({
+			where: eq(repositoriesTable.id, mirror.repositoryId),
+		});
+
+		if (!repo) {
+			throw new NotFoundError(`Repository ${mirror.repositoryId} not found`);
+		}
+
+		const compatibility = await checkMirrorCompatibility(schedule.repository.config, repo.config, repo.id);
+
+		if (!compatibility.compatible) {
+			throw new BadRequestError(
+				getIncompatibleMirrorError(repo.name, schedule.repository.config.backend, repo.config.backend),
+			);
+		}
+	}
+
+	await db.delete(backupScheduleMirrorsTable).where(eq(backupScheduleMirrorsTable.scheduleId, scheduleId));
+
+	if (data.mirrors.length > 0) {
+		await db.insert(backupScheduleMirrorsTable).values(
+			data.mirrors.map((mirror) => ({
+				scheduleId,
+				repositoryId: mirror.repositoryId,
+				enabled: mirror.enabled,
+			})),
+		);
+	}
+
+	return getMirrors(scheduleId);
+};
+
+const copyToMirrors = async (
+	scheduleId: number,
+	sourceRepository: { id: string; config: (typeof repositoriesTable.$inferSelect)["config"] },
+	retentionPolicy: (typeof backupSchedulesTable.$inferSelect)["retentionPolicy"],
+) => {
+	const mirrors = await db.query.backupScheduleMirrorsTable.findMany({
+		where: eq(backupScheduleMirrorsTable.scheduleId, scheduleId),
+		with: { repository: true },
+	});
+
+	const enabledMirrors = mirrors.filter((m) => m.enabled);
+
+	if (enabledMirrors.length === 0) {
+		return;
+	}
+
+	logger.info(
+		`[Background] Copying snapshots to ${enabledMirrors.length} mirror repositories for schedule ${scheduleId}`,
+	);
+
+	for (const mirror of enabledMirrors) {
+		try {
+			logger.info(`[Background] Copying to mirror repository: ${mirror.repository.name}`);
+
+			serverEvents.emit("mirror:started", {
+				scheduleId,
+				repositoryId: mirror.repositoryId,
+				repositoryName: mirror.repository.name,
+			});
+
+			await restic.copy(sourceRepository.config, mirror.repository.config, {
+				tag: scheduleId.toString(),
+			});
+
+			if (retentionPolicy) {
+				logger.info(`[Background] Applying retention policy to mirror repository: ${mirror.repository.name}`);
+				await restic.forget(mirror.repository.config, retentionPolicy, { tag: scheduleId.toString() });
+			}
+
+			await db
+				.update(backupScheduleMirrorsTable)
+				.set({ lastCopyAt: Date.now(), lastCopyStatus: "success", lastCopyError: null })
+				.where(eq(backupScheduleMirrorsTable.id, mirror.id));
+
+			logger.info(`[Background] Successfully copied to mirror repository: ${mirror.repository.name}`);
+
+			serverEvents.emit("mirror:completed", {
+				scheduleId,
+				repositoryId: mirror.repositoryId,
+				repositoryName: mirror.repository.name,
+				status: "success",
+			});
+		} catch (error) {
+			const errorMessage = toMessage(error);
+			logger.error(`[Background] Failed to copy to mirror repository ${mirror.repository.name}: ${errorMessage}`);
+
+			await db
+				.update(backupScheduleMirrorsTable)
+				.set({ lastCopyAt: Date.now(), lastCopyStatus: "error", lastCopyError: errorMessage })
+				.where(eq(backupScheduleMirrorsTable.id, mirror.id));
+
+			serverEvents.emit("mirror:completed", {
+				scheduleId,
+				repositoryId: mirror.repositoryId,
+				repositoryName: mirror.repository.name,
+				status: "error",
+				error: errorMessage,
+			});
+		}
+	}
+};
+
+const getMirrorCompatibility = async (scheduleId: number) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.id, scheduleId),
+		with: { repository: true },
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	const allRepositories = await db.query.repositoriesTable.findMany();
+	const repos = allRepositories.filter((repo) => repo.id !== schedule.repositoryId);
+
+	const compatibility = await Promise.all(
+		repos.map((repo) => checkMirrorCompatibility(schedule.repository.config, repo.config, repo.id)),
+	);
+
+	return compatibility;
+};
+
 export const backupsService = {
 	listSchedules,
 	getSchedule,
@@ -432,4 +595,7 @@ export const backupsService = {
 	getScheduleForVolume,
 	stopBackup,
 	runForget,
+	getMirrors,
+	updateMirrors,
+	getMirrorCompatibility,
 };