feat(backend): backup service with retention policy

apps/server/src/db/schema.ts

@@ -58,3 +58,35 @@ export const repositoriesTable = sqliteTable("repositories_table", {
 });
 
 export type Repository = typeof repositoriesTable.$inferSelect;
+
+export const backupSchedulesTable = sqliteTable("backup_schedules_table", {
+	id: int().primaryKey({ autoIncrement: true }),
+	volumeId: int("volume_id")
+		.notNull()
+		.unique()
+		.references(() => volumesTable.id, { onDelete: "cascade" }),
+	repositoryId: text("repository_id")
+		.notNull()
+		.references(() => repositoriesTable.id, { onDelete: "cascade" }),
+	enabled: int("enabled", { mode: "boolean" }).notNull().default(true),
+	cronExpression: text("cron_expression").notNull(),
+	retentionPolicy: text("retention_policy", { mode: "json" }).$type<{
+		keepLast?: number;
+		keepHourly?: number;
+		keepDaily?: number;
+		keepWeekly?: number;
+		keepMonthly?: number;
+		keepYearly?: number;
+		keepWithinDuration?: string;
+	}>(),
+	excludePatterns: text("exclude_patterns", { mode: "json" }).$type<string[]>().default([]),
+	includePatterns: text("include_patterns", { mode: "json" }).$type<string[]>().default([]),
+	lastBackupAt: int("last_backup_at", { mode: "timestamp" }),
+	lastBackupStatus: text("last_backup_status").$type<"success" | "error">(),
+	lastBackupError: text("last_backup_error"),
+	nextBackupAt: int("next_backup_at", { mode: "timestamp" }),
+	createdAt: int("created_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`),
+	updatedAt: int("updated_at", { mode: "timestamp" }).notNull().default(sql`(unixepoch())`),
+});
+
+export type BackupSchedule = typeof backupSchedulesTable.$inferSelect;

apps/server/src/index.ts

@@ -11,6 +11,7 @@ import { driverController } from "./modules/driver/driver.controller";
 import { startup } from "./modules/lifecycle/startup";
 import { repositoriesController } from "./modules/repositories/repositories.controller";
 import { volumeController } from "./modules/volumes/volume.controller";
+import { backupScheduleController } from "./modules/backups/backups.controller";
 import { handleServiceError } from "./utils/errors";
 import { logger } from "./utils/logger";
 
@@ -39,6 +40,7 @@ const app = new Hono()
 	.route("/api/v1/auth", authController.basePath("/api/v1"))
 	.route("/api/v1/volumes", volumeController.use(requireAuth))
 	.route("/api/v1/repositories", repositoriesController.use(requireAuth))
+	.route("/api/v1/backups", backupScheduleController.use(requireAuth))
 	.get("/assets/*", serveStatic({ root: "./assets/frontend" }))
 	.get("*", serveStatic({ path: "./assets/frontend/index.html" }));
 

apps/server/src/jobs/backup-execution.ts

@@ -0,0 +1,29 @@
+import { Job } from "../core/scheduler";
+import { backupsService } from "../modules/backups/backups.service";
+import { toMessage } from "../utils/errors";
+import { logger } from "../utils/logger";
+
+export class BackupExecutionJob extends Job {
+	async run() {
+		logger.debug("Checking for backup schedules to execute...");
+
+		const scheduleIds = await backupsService.getSchedulesToExecute();
+
+		if (scheduleIds.length === 0) {
+			logger.debug("No backup schedules to execute");
+			return { done: true, timestamp: new Date(), executed: 0 };
+		}
+
+		logger.info(`Found ${scheduleIds.length} backup schedule(s) to execute`);
+
+		for (const scheduleId of scheduleIds) {
+			try {
+				await backupsService.executeBackup(scheduleId);
+			} catch (error) {
+				logger.error(`Failed to execute backup for schedule ${scheduleId}: ${toMessage(error)}`);
+			}
+		}
+
+		return { done: true, timestamp: new Date(), executed: scheduleIds.length };
+	}
+}

apps/server/src/modules/backups/backups.controller.ts

@@ -0,0 +1,64 @@
+import { Hono } from "hono";
+import { validator } from "hono-openapi";
+import {
+	createBackupScheduleBody,
+	createBackupScheduleDto,
+	deleteBackupScheduleDto,
+	getBackupScheduleDto,
+	listBackupSchedulesDto,
+	runBackupNowDto,
+	updateBackupScheduleBody,
+	updateBackupScheduleDto,
+} from "./backups.dto";
+import { backupsService } from "./backups.service";
+
+export const backupScheduleController = new Hono()
+	.get("/", listBackupSchedulesDto, async (c) => {
+		const schedules = await backupsService.listSchedules();
+
+		return c.json({ schedules }, 200);
+	})
+	.get("/:scheduleId", getBackupScheduleDto, async (c) => {
+		const scheduleId = c.req.param("scheduleId");
+
+		const schedule = await backupsService.getSchedule(Number(scheduleId));
+
+		return c.json({ schedule }, 200);
+	})
+	.post("/", createBackupScheduleDto, validator("json", createBackupScheduleBody), async (c) => {
+		const body = c.req.valid("json");
+
+		const schedule = await backupsService.createSchedule(body);
+
+		return c.json({ message: "Backup schedule created successfully", schedule }, 201);
+	})
+	.patch("/:scheduleId", updateBackupScheduleDto, validator("json", updateBackupScheduleBody), async (c) => {
+		const scheduleId = c.req.param("scheduleId");
+		const body = c.req.valid("json");
+
+		const schedule = await backupsService.updateSchedule(Number(scheduleId), body);
+
+		return c.json({ message: "Backup schedule updated successfully", schedule }, 200);
+	})
+	.delete("/:scheduleId", deleteBackupScheduleDto, async (c) => {
+		const scheduleId = c.req.param("scheduleId");
+
+		await backupsService.deleteSchedule(Number(scheduleId));
+
+		return c.json({ message: "Backup schedule deleted successfully" }, 200);
+	})
+	.post("/:scheduleId/run", runBackupNowDto, async (c) => {
+		const scheduleId = c.req.param("scheduleId");
+
+		backupsService.executeBackup(Number(scheduleId)).catch((error) => {
+			console.error("Backup execution failed:", error);
+		});
+
+		return c.json(
+			{
+				message: "Backup started",
+				backupStarted: true,
+			},
+			200,
+		);
+	});

apps/server/src/modules/backups/backups.dto.ts

@@ -0,0 +1,205 @@
+import { type } from "arktype";
+import { describeRoute, resolver } from "hono-openapi";
+
+const retentionPolicySchema = type({
+	keepLast: "number?",
+	keepHourly: "number?",
+	keepDaily: "number?",
+	keepWeekly: "number?",
+	keepMonthly: "number?",
+	keepYearly: "number?",
+	keepWithinDuration: "string?",
+});
+
+export type RetentionPolicy = typeof retentionPolicySchema.infer;
+
+const backupScheduleSchema = type({
+	id: "number",
+	volumeId: "number",
+	volumeName: "string",
+	repositoryId: "string",
+	repositoryName: "string",
+	enabled: "boolean",
+	cronExpression: "string",
+	retentionPolicy: retentionPolicySchema.or("null"),
+	excludePatterns: "string[]",
+	includePatterns: "string[]",
+	lastBackupAt: "number | null",
+	lastBackupStatus: "'success' | 'error' | null",
+	lastBackupError: "string | null",
+	nextBackupAt: "number | null",
+	createdAt: "number",
+	updatedAt: "number",
+});
+
+export type BackupScheduleDto = typeof backupScheduleSchema.infer;
+
+/**
+ * List all backup schedules
+ */
+export const listBackupSchedulesResponse = type({
+	schedules: backupScheduleSchema.array(),
+});
+
+export type ListBackupSchedulesResponseDto = typeof listBackupSchedulesResponse.infer;
+
+export const listBackupSchedulesDto = describeRoute({
+	description: "List all backup schedules",
+	tags: ["Backups"],
+	operationId: "listBackupSchedules",
+	responses: {
+		200: {
+			description: "List of backup schedules",
+			content: {
+				"application/json": {
+					schema: resolver(listBackupSchedulesResponse),
+				},
+			},
+		},
+	},
+});
+
+/**
+ * Get a single backup schedule
+ */
+export const getBackupScheduleResponse = type({
+	schedule: backupScheduleSchema,
+});
+
+export type GetBackupScheduleResponseDto = typeof getBackupScheduleResponse.infer;
+
+export const getBackupScheduleDto = describeRoute({
+	description: "Get a backup schedule by ID",
+	tags: ["Backups"],
+	operationId: "getBackupSchedule",
+	responses: {
+		200: {
+			description: "Backup schedule details",
+			content: {
+				"application/json": {
+					schema: resolver(getBackupScheduleResponse),
+				},
+			},
+		},
+	},
+});
+
+/**
+ * Create a new backup schedule
+ */
+export const createBackupScheduleBody = type({
+	volumeId: "number",
+	repositoryId: "string",
+	enabled: "boolean",
+	cronExpression: "string",
+	retentionPolicy: retentionPolicySchema.optional(),
+	excludePatterns: "string[]?",
+	includePatterns: "string[]?",
+	tags: "string[]?",
+});
+
+export type CreateBackupScheduleBody = typeof createBackupScheduleBody.infer;
+
+export const createBackupScheduleResponse = type({
+	message: "string",
+	schedule: backupScheduleSchema,
+});
+
+export const createBackupScheduleDto = describeRoute({
+	description: "Create a new backup schedule for a volume",
+	operationId: "createBackupSchedule",
+	tags: ["Backups"],
+	responses: {
+		201: {
+			description: "Backup schedule created successfully",
+			content: {
+				"application/json": {
+					schema: resolver(createBackupScheduleResponse),
+				},
+			},
+		},
+	},
+});
+
+/**
+ * Update a backup schedule
+ */
+export const updateBackupScheduleBody = type({
+	repositoryId: "string?",
+	enabled: "boolean?",
+	cronExpression: "string?",
+	retentionPolicy: retentionPolicySchema.optional(),
+	excludePatterns: "string[]?",
+	includePatterns: "string[]?",
+	tags: "string[]?",
+});
+
+export type UpdateBackupScheduleBody = typeof updateBackupScheduleBody.infer;
+
+export const updateBackupScheduleResponse = type({
+	message: "string",
+	schedule: backupScheduleSchema,
+});
+
+export const updateBackupScheduleDto = describeRoute({
+	description: "Update a backup schedule",
+	operationId: "updateBackupSchedule",
+	tags: ["Backups"],
+	responses: {
+		200: {
+			description: "Backup schedule updated successfully",
+			content: {
+				"application/json": {
+					schema: resolver(updateBackupScheduleResponse),
+				},
+			},
+		},
+	},
+});
+
+/**
+ * Delete a backup schedule
+ */
+export const deleteBackupScheduleResponse = type({
+	message: "string",
+});
+
+export const deleteBackupScheduleDto = describeRoute({
+	description: "Delete a backup schedule",
+	operationId: "deleteBackupSchedule",
+	tags: ["Backups"],
+	responses: {
+		200: {
+			description: "Backup schedule deleted successfully",
+			content: {
+				"application/json": {
+					schema: resolver(deleteBackupScheduleResponse),
+				},
+			},
+		},
+	},
+});
+
+/**
+ * Run a backup immediately
+ */
+export const runBackupNowResponse = type({
+	message: "string",
+	backupStarted: "boolean",
+});
+
+export const runBackupNowDto = describeRoute({
+	description: "Trigger a backup immediately for a schedule",
+	operationId: "runBackupNow",
+	tags: ["Backups"],
+	responses: {
+		200: {
+			description: "Backup started successfully",
+			content: {
+				"application/json": {
+					schema: resolver(runBackupNowResponse),
+				},
+			},
+		},
+	},
+});

apps/server/src/modules/backups/backups.service.ts

@@ -0,0 +1,258 @@
+import { eq } from "drizzle-orm";
+import cron from "node-cron";
+import { CronExpressionParser } from "cron-parser";
+import { NotFoundError, BadRequestError } from "http-errors-enhanced";
+import { db } from "../../db/db";
+import { backupSchedulesTable, repositoriesTable, volumesTable } from "../../db/schema";
+import { restic } from "../../utils/restic";
+import { logger } from "../../utils/logger";
+import { getVolumePath } from "../volumes/helpers";
+import type { CreateBackupScheduleBody, UpdateBackupScheduleBody } from "./backups.dto";
+import { toMessage } from "../../utils/errors";
+
+const calculateNextRun = (cronExpression: string): Date => {
+	try {
+		const interval = CronExpressionParser.parse(cronExpression, {
+			currentDate: new Date(),
+			tz: "UTC",
+		});
+
+		return interval.next().toDate();
+	} catch (error) {
+		logger.error(`Failed to parse cron expression "${cronExpression}": ${error}`);
+		const fallback = new Date();
+		fallback.setMinutes(fallback.getMinutes() + 1);
+		return fallback;
+	}
+};
+
+const listSchedules = async () => {
+	const schedules = await db.query.backupSchedulesTable.findMany({});
+	return schedules;
+};
+
+const getSchedule = async (scheduleId: number) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(volumesTable.id, scheduleId),
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	return schedule;
+};
+
+const createSchedule = async (data: CreateBackupScheduleBody) => {
+	if (!cron.validate(data.cronExpression)) {
+		throw new BadRequestError("Invalid cron expression");
+	}
+
+	const volume = await db.query.volumesTable.findFirst({
+		where: eq(volumesTable.id, data.volumeId),
+	});
+
+	if (!volume) {
+		throw new NotFoundError("Volume not found");
+	}
+
+	const repository = await db.query.repositoriesTable.findFirst({
+		where: eq(repositoriesTable.id, data.repositoryId),
+	});
+
+	if (!repository) {
+		throw new NotFoundError("Repository not found");
+	}
+
+	const existingSchedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.volumeId, data.volumeId),
+	});
+
+	if (existingSchedule) {
+		throw new BadRequestError("Volume already has a backup schedule");
+	}
+
+	const nextBackupAt = calculateNextRun(data.cronExpression);
+
+	const [newSchedule] = await db
+		.insert(backupSchedulesTable)
+		.values({
+			volumeId: data.volumeId,
+			repositoryId: data.repositoryId,
+			enabled: data.enabled,
+			cronExpression: data.cronExpression,
+			retentionPolicy: data.retentionPolicy ?? null,
+			excludePatterns: data.excludePatterns ?? [],
+			includePatterns: data.includePatterns ?? [],
+			nextBackupAt: nextBackupAt,
+		})
+		.returning();
+
+	if (!newSchedule) {
+		throw new Error("Failed to create backup schedule");
+	}
+
+	return newSchedule;
+};
+
+const updateSchedule = async (scheduleId: number, data: UpdateBackupScheduleBody) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.id, scheduleId),
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	if (data.cronExpression && !cron.validate(data.cronExpression)) {
+		throw new BadRequestError("Invalid cron expression");
+	}
+
+	if (data.repositoryId) {
+		const repository = await db.query.repositoriesTable.findFirst({
+			where: eq(repositoriesTable.id, data.repositoryId),
+		});
+
+		if (!repository) {
+			throw new NotFoundError("Repository not found");
+		}
+	}
+
+	const cronExpression = data.cronExpression ?? schedule.cronExpression;
+	const nextBackupAt = data.cronExpression ? calculateNextRun(cronExpression) : schedule.nextBackupAt;
+
+	const [updated] = await db
+		.update(backupSchedulesTable)
+		.set({ ...data, nextBackupAt, updatedAt: new Date() })
+		.where(eq(backupSchedulesTable.id, scheduleId))
+		.returning();
+
+	if (!updated) {
+		throw new Error("Failed to update backup schedule");
+	}
+
+	return updated;
+};
+
+const deleteSchedule = async (scheduleId: number) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.id, scheduleId),
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	await db.delete(backupSchedulesTable).where(eq(backupSchedulesTable.id, scheduleId));
+};
+
+const executeBackup = async (scheduleId: number) => {
+	const schedule = await db.query.backupSchedulesTable.findFirst({
+		where: eq(backupSchedulesTable.id, scheduleId),
+	});
+
+	if (!schedule) {
+		throw new NotFoundError("Backup schedule not found");
+	}
+
+	const volume = await db.query.volumesTable.findFirst({
+		where: eq(volumesTable.id, schedule.volumeId),
+	});
+
+	if (!volume) {
+		throw new NotFoundError("Volume not found");
+	}
+
+	const repository = await db.query.repositoriesTable.findFirst({
+		where: eq(repositoriesTable.id, schedule.repositoryId),
+	});
+
+	if (!repository) {
+		throw new NotFoundError("Repository not found");
+	}
+
+	if (volume.status !== "mounted") {
+		throw new BadRequestError("Volume is not mounted");
+	}
+
+	logger.info(`Starting backup for volume ${volume.name} to repository ${repository.name}`);
+
+	try {
+		const volumePath = getVolumePath(volume.name);
+
+		const backupOptions: {
+			exclude?: string[];
+			include?: string[];
+			tags?: string[];
+		} = {};
+
+		if (schedule.excludePatterns && schedule.excludePatterns.length > 0) {
+			backupOptions.exclude = schedule.excludePatterns;
+		}
+
+		if (schedule.includePatterns && schedule.includePatterns.length > 0) {
+			backupOptions.include = schedule.includePatterns;
+		}
+
+		await restic.backup(repository.config, volumePath, backupOptions);
+
+		if (schedule.retentionPolicy) {
+			await restic.forget(repository.config, schedule.retentionPolicy);
+		}
+
+		const nextBackupAt = calculateNextRun(schedule.cronExpression);
+		await db
+			.update(backupSchedulesTable)
+			.set({
+				lastBackupAt: new Date(),
+				lastBackupStatus: "success",
+				lastBackupError: null,
+				nextBackupAt: nextBackupAt,
+				updatedAt: new Date(),
+			})
+			.where(eq(backupSchedulesTable.id, scheduleId));
+
+		logger.info(`Backup completed successfully for volume ${volume.name} to repository ${repository.name}`);
+	} catch (error) {
+		logger.error(`Backup failed for volume ${volume.name} to repository ${repository.name}: ${toMessage(error)}`);
+
+		await db
+			.update(backupSchedulesTable)
+			.set({
+				lastBackupAt: new Date(),
+				lastBackupStatus: "error",
+				lastBackupError: toMessage(error),
+				updatedAt: new Date(),
+			})
+			.where(eq(backupSchedulesTable.id, scheduleId));
+
+		throw error;
+	}
+};
+
+const getSchedulesToExecute = async () => {
+	const now = new Date();
+	const schedules = await db.query.backupSchedulesTable.findMany({
+		where: eq(backupSchedulesTable.enabled, true),
+	});
+
+	const schedulesToRun: number[] = [];
+
+	for (const schedule of schedules) {
+		if (!schedule.nextBackupAt || schedule.nextBackupAt <= now) {
+			schedulesToRun.push(schedule.id);
+		}
+	}
+
+	return schedulesToRun;
+};
+
+export const backupsService = {
+	listSchedules,
+	getSchedule,
+	createSchedule,
+	updateSchedule,
+	deleteSchedule,
+	executeBackup,
+	getSchedulesToExecute,
+};

apps/server/src/modules/lifecycle/startup.ts

@@ -7,6 +7,7 @@ import { restic } from "../../utils/restic";
 import { volumeService } from "../volumes/volume.service";
 import { CleanupDanglingMountsJob } from "../../jobs/cleanup-dangling";
 import { VolumeHealthCheckJob } from "../../jobs/healthchecks";
+import { BackupExecutionJob } from "../../jobs/backup-execution";
 
 export const startup = async () => {
 	await Scheduler.start();
@@ -30,4 +31,5 @@ export const startup = async () => {
 
 	Scheduler.build(CleanupDanglingMountsJob).schedule("0 * * * *");
 	Scheduler.build(VolumeHealthCheckJob).schedule("* * * * *");
+	Scheduler.build(BackupExecutionJob).schedule("* * * * *");
 };

apps/server/src/utils/restic.ts

@@ -7,6 +7,7 @@ import { $ } from "bun";
 import { RESTIC_PASS_FILE } from "../core/constants";
 import { logger } from "./logger";
 import { cryptoUtils } from "./crypto";
+import type { RetentionPolicy } from "../modules/backups/backups.dto";
 
 const backupOutputSchema = type({
 	message_type: "'summary'",
@@ -110,18 +111,44 @@ const init = async (config: RepositoryConfig) => {
 	return { success: true, error: null };
 };
 
-const backup = async (config: RepositoryConfig, source: string) => {
+const backup = async (
+	config: RepositoryConfig,
+	source: string,
+	options?: { exclude?: string[]; include?: string[] },
+) => {
 	const repoUrl = buildRepoUrl(config);
 	const env = await buildEnv(config);
 
-	const res = await $`restic --repo ${repoUrl} backup ${source} --json`.env(env).nothrow();
+	const args: string[] = ["--repo", repoUrl, "backup", source];
+
+	if (options?.exclude && options.exclude.length > 0) {
+		for (const pattern of options.exclude) {
+			args.push("--exclude", pattern);
+		}
+	}
+
+	if (options?.include && options.include.length > 0) {
+		for (const pattern of options.include) {
+			args.push("--include", pattern);
+		}
+	}
+
+	args.push("--json");
+
+	const res = await $`restic ${args}`.env(env).nothrow();
 
 	if (res.exitCode !== 0) {
 		logger.error(`Restic backup failed: ${res.stderr}`);
 		throw new Error(`Restic backup failed: ${res.stderr}`);
 	}
 
-	const result = backupOutputSchema(res.json());
+	// res is a succession of JSON objects, we need to parse the last one which contains the summary
+	const stdout = res.text();
+	const outputLines = stdout.trim().split("\n");
+	const lastLine = outputLines[outputLines.length - 1];
+	const resSummary = JSON.parse(lastLine ?? "{}");
+
+	const result = backupOutputSchema(resSummary);
 
 	if (result instanceof type.errors) {
 		logger.error(`Restic backup output validation failed: ${result}`);
@@ -166,10 +193,53 @@ const snapshots = async (config: RepositoryConfig) => {
 	return result;
 };
 
+const forget = async (config: RepositoryConfig, options: RetentionPolicy) => {
+	const repoUrl = buildRepoUrl(config);
+	const env = await buildEnv(config);
+
+	const args: string[] = ["--repo", repoUrl, "forget"];
+
+	if (options.keepLast) {
+		args.push("--keep-last", String(options.keepLast));
+	}
+	if (options.keepHourly) {
+		args.push("--keep-hourly", String(options.keepHourly));
+	}
+	if (options.keepDaily) {
+		args.push("--keep-daily", String(options.keepDaily));
+	}
+	if (options.keepWeekly) {
+		args.push("--keep-weekly", String(options.keepWeekly));
+	}
+	if (options.keepMonthly) {
+		args.push("--keep-monthly", String(options.keepMonthly));
+	}
+	if (options.keepYearly) {
+		args.push("--keep-yearly", String(options.keepYearly));
+	}
+	if (options.keepWithinDuration) {
+		args.push("--keep-within-duration", options.keepWithinDuration);
+	}
+
+	args.push("--prune");
+	args.push("--json");
+
+	const res = await $`restic ${args}`.env(env).nothrow();
+
+	if (res.exitCode !== 0) {
+		logger.error(`Restic forget failed: ${res.stderr}`);
+		throw new Error(`Restic forget failed: ${res.stderr}`);
+	}
+
+	logger.info("Restic forget completed successfully");
+	return { success: true };
+};
+
 export const restic = {
 	ensurePassfile,
 	init,
 	backup,
 	restore,
 	snapshots,
+	forget,
 };