From 8f9873148a7740a48a79e8c5d30ae24113ea30de Mon Sep 17 00:00:00 2001 From: Nicolas Meienberger Date: Tue, 11 Nov 2025 20:42:44 +0100 Subject: [PATCH] feat(repositories): rclone backends --- Dockerfile | 8 ++ .../repositories/repositories.controller.ts | 17 ++++ .../modules/repositories/repositories.dto.ts | 26 ++++++ apps/server/src/utils/rclone.ts | 81 +++++++++++++++++++ apps/server/src/utils/restic.ts | 2 + packages/schemas/src/restic.ts | 10 ++- 6 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 apps/server/src/utils/rclone.ts diff --git a/Dockerfile b/Dockerfile index c1ffe9a..0a7ad18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,11 +21,17 @@ RUN apk add --no-cache curl bzip2 RUN echo "Building for ${TARGETARCH}" RUN if [ "${TARGETARCH}" = "arm64" ]; then \ curl -L -o restic.bz2 "https://github.com/restic/restic/releases/download/v$RESTIC_VERSION/restic_$RESTIC_VERSION"_linux_arm64.bz2; \ + curl -O https://downloads.rclone.org/rclone-current-linux-arm64.zip; \ + unzip rclone-current-linux-arm64.zip; \ elif [ "${TARGETARCH}" = "amd64" ]; then \ curl -L -o restic.bz2 "https://github.com/restic/restic/releases/download/v$RESTIC_VERSION/restic_$RESTIC_VERSION"_linux_amd64.bz2; \ + curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip; \ + unzip rclone-current-linux-amd64.zip; \ fi RUN bzip2 -d restic.bz2 && chmod +x restic +RUN mv rclone-*-linux-*/rclone /deps/rclone && chmod +x /deps/rclone + # ------------------------------ # DEVELOPMENT @@ -37,6 +43,7 @@ ENV NODE_ENV="development" WORKDIR /app COPY --from=deps /deps/restic /usr/local/bin/restic +COPY --from=deps /deps/rclone /usr/local/bin/rclone COPY ./package.json ./bun.lock ./ COPY ./packages/schemas/package.json ./packages/schemas/package.json COPY ./apps/client/package.json ./apps/client/package.json @@ -76,6 +83,7 @@ ENV NODE_ENV="production" WORKDIR /app COPY --from=deps /deps/restic /usr/local/bin/restic +COPY --from=deps /deps/rclone /usr/local/bin/rclone COPY --from=builder /app/apps/server/dist ./ COPY --from=builder /app/apps/server/drizzle ./assets/migrations COPY --from=builder /app/apps/client/dist/client ./assets/frontend diff --git a/apps/server/src/modules/repositories/repositories.controller.ts b/apps/server/src/modules/repositories/repositories.controller.ts index 36eee2c..7731612 100644 --- a/apps/server/src/modules/repositories/repositories.controller.ts +++ b/apps/server/src/modules/repositories/repositories.controller.ts @@ -7,6 +7,7 @@ import { doctorRepositoryDto, getRepositoryDto, getSnapshotDetailsDto, + listRcloneRemotesDto, listRepositoriesDto, listSnapshotFilesDto, listSnapshotFilesQuery, @@ -24,6 +25,7 @@ import { type RestoreSnapshotDto, } from "./repositories.dto"; import { repositoriesService } from "./repositories.service"; +import { getRcloneRemoteInfo, listRcloneRemotes } from "../../utils/rclone"; export const repositoriesController = new Hono() .get("/", listRepositoriesDto, async (c) => { @@ -125,4 +127,19 @@ export const repositoriesController = new Hono() const result = await repositoriesService.doctorRepository(name); return c.json(result, 200); + }) + .get("/rclone-remotes", listRcloneRemotesDto, async (c) => { + const remoteNames = await listRcloneRemotes(); + + const remotes = await Promise.all( + remoteNames.map(async (name) => { + const info = await getRcloneRemoteInfo(name); + return { + name, + type: info?.type ?? "unknown", + }; + }), + ); + + return c.json(remotes); }); diff --git a/apps/server/src/modules/repositories/repositories.dto.ts b/apps/server/src/modules/repositories/repositories.dto.ts index 18d4f98..f42dd0f 100644 --- a/apps/server/src/modules/repositories/repositories.dto.ts +++ b/apps/server/src/modules/repositories/repositories.dto.ts @@ -305,3 +305,29 @@ export const doctorRepositoryDto = describeRoute({ }, }, }); + +/** + * List rclone available remotes + */ +const rcloneRemoteSchema = type({ + name: "string", + type: "string", +}); + +const listRcloneRemotesResponse = rcloneRemoteSchema.array(); + +export const listRcloneRemotesDto = describeRoute({ + description: "List all configured rclone remotes on the host system", + tags: ["Rclone"], + operationId: "listRcloneRemotes", + responses: { + 200: { + description: "List of rclone remotes", + content: { + "application/json": { + schema: resolver(listRcloneRemotesResponse), + }, + }, + }, + }, +}); diff --git a/apps/server/src/utils/rclone.ts b/apps/server/src/utils/rclone.ts new file mode 100644 index 0000000..7fea88f --- /dev/null +++ b/apps/server/src/utils/rclone.ts @@ -0,0 +1,81 @@ +import { $ } from "bun"; +import { logger } from "./logger"; + +/** + * List all configured rclone remotes + * @returns Array of remote names + */ +export async function listRcloneRemotes(): Promise { + try { + const result = await $`rclone listremotes`.quiet(); + + if (result.exitCode !== 0) { + logger.error(`Failed to list rclone remotes: ${result.stderr}`); + return []; + } + + // Parse output - each line is a remote name ending with ":" + const remotes = result.stdout + .toString() + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.endsWith(":")) + .map((line) => line.slice(0, -1)); // Remove trailing ":" + + return remotes; + } catch (error) { + logger.error(`Error listing rclone remotes: ${error}`); + return []; + } +} + +/** + * Get information about a specific rclone remote + * @param remote Remote name + * @returns Remote type and configuration info + */ +export async function getRcloneRemoteInfo( + remote: string, +): Promise<{ type: string; config: Record } | null> { + try { + const result = await $`rclone config show ${remote}`.quiet(); + + if (result.exitCode !== 0) { + logger.error(`Failed to get info for remote ${remote}: ${result.stderr}`); + return null; + } + + // Parse the output to extract type and config + const output = result.stdout.toString(); + const lines = output + .split("\n") + .map((l) => l.trim()) + .filter((l) => l); + + const config: Record = {}; + let type = "unknown"; + + for (const line of lines) { + if (line.includes("=")) { + const parts = line.split("="); + const key = parts[0]; + if (!key) continue; + + const valueParts = parts.slice(1); + const value = valueParts.join("=").trim(); + const cleanKey = key.trim(); + + if (cleanKey === "type") { + type = value; + } + + config[cleanKey] = value; + } + } + + return { type, config }; + } catch (error) { + logger.error(`Error getting remote info for ${remote}: ${error}`); + return null; + } +} diff --git a/apps/server/src/utils/restic.ts b/apps/server/src/utils/restic.ts index 7526c95..b0f22b5 100644 --- a/apps/server/src/utils/restic.ts +++ b/apps/server/src/utils/restic.ts @@ -78,6 +78,8 @@ const buildRepoUrl = (config: RepositoryConfig): string => { return `gs:${config.bucket}:/`; case "azure": return `azure:${config.container}:/`; + case "rclone": + return `rclone:${config.remote}:${config.path}`; default: { throw new Error(`Unsupported repository backend: ${JSON.stringify(config)}`); } diff --git a/packages/schemas/src/restic.ts b/packages/schemas/src/restic.ts index 717858a..6d72fcc 100644 --- a/packages/schemas/src/restic.ts +++ b/packages/schemas/src/restic.ts @@ -5,6 +5,7 @@ export const REPOSITORY_BACKENDS = { s3: "s3", gcs: "gcs", azure: "azure", + rclone: "rclone", } as const; export type RepositoryBackend = keyof typeof REPOSITORY_BACKENDS; @@ -37,10 +38,17 @@ export const azureRepositoryConfigSchema = type({ endpointSuffix: "string?", }); +export const rcloneRepositoryConfigSchema = type({ + backend: "'rclone'", + remote: "string", + path: "string", +}); + export const repositoryConfigSchema = s3RepositoryConfigSchema .or(localRepositoryConfigSchema) .or(gcsRepositoryConfigSchema) - .or(azureRepositoryConfigSchema); + .or(azureRepositoryConfigSchema) + .or(rcloneRepositoryConfigSchema); export type RepositoryConfig = typeof repositoryConfigSchema.infer;