add mysql, mariadb, postgresql, sqlite volumes support

This commit is contained in:
Renan Bernordi
2025-11-15 23:32:26 -03:00
parent c0bef7f65e
commit eb28667d90
15 changed files with 1022 additions and 25 deletions

View File

@@ -5,6 +5,10 @@ import { makeDirectoryBackend } from "./directory/directory-backend";
import { makeNfsBackend } from "./nfs/nfs-backend";
import { makeSmbBackend } from "./smb/smb-backend";
import { makeWebdavBackend } from "./webdav/webdav-backend";
import { makeMariaDBBackend } from "./mariadb/mariadb-backend";
import { makeMySQLBackend } from "./mysql/mysql-backend";
import { makePostgresBackend } from "./postgres/postgres-backend";
import { makeSQLiteBackend } from "./sqlite/sqlite-backend";
type OperationResult = {
error?: string;
@@ -33,5 +37,20 @@ export const createVolumeBackend = (volume: Volume): VolumeBackend => {
case "webdav": {
return makeWebdavBackend(volume.config, path);
}
case "mariadb": {
return makeMariaDBBackend(volume.config, path);
}
case "mysql": {
return makeMySQLBackend(volume.config, path);
}
case "postgres": {
return makePostgresBackend(volume.config, path);
}
case "sqlite": {
return makeSQLiteBackend(volume.config, path);
}
default: {
throw new Error(`Unsupported backend type: ${(volume.config as any).backend}`);
}
}
};

View File

@@ -0,0 +1,57 @@
import * as fs from "node:fs/promises";
import { toMessage } from "../../../utils/errors";
import { logger } from "../../../utils/logger";
import { testMariaDBConnection } from "../../../utils/database-dump";
import type { VolumeBackend } from "../backend";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const mount = async (config: BackendConfig, volumePath: string) => {
if (config.backend !== "mariadb") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
logger.info(`Testing MariaDB connection to: ${config.host}:${config.port}`);
try {
await testMariaDBConnection(config);
await fs.mkdir(volumePath, { recursive: true });
logger.info("MariaDB connection successful");
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("Failed to connect to MariaDB:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const unmount = async (volumePath: string) => {
logger.info("Cleaning up MariaDB dump directory");
try {
await fs.rm(volumePath, { recursive: true, force: true });
return { status: BACKEND_STATUS.unmounted };
} catch (error) {
logger.warn(`Failed to clean up MariaDB dump directory: ${toMessage(error)}`);
return { status: BACKEND_STATUS.unmounted };
}
};
const checkHealth = async (config: BackendConfig) => {
if (config.backend !== "mariadb") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
try {
await testMariaDBConnection(config);
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("MariaDB health check failed:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
export const makeMariaDBBackend = (config: BackendConfig, volumePath: string): VolumeBackend => ({
mount: () => mount(config, volumePath),
unmount: () => unmount(volumePath),
checkHealth: () => checkHealth(config),
});

View File

@@ -0,0 +1,57 @@
import * as fs from "node:fs/promises";
import { toMessage } from "../../../utils/errors";
import { logger } from "../../../utils/logger";
import { testMySQLConnection } from "../../../utils/database-dump";
import type { VolumeBackend } from "../backend";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const mount = async (config: BackendConfig, volumePath: string) => {
if (config.backend !== "mysql") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
logger.info(`Testing MySQL connection to: ${config.host}:${config.port}`);
try {
await testMySQLConnection(config);
await fs.mkdir(volumePath, { recursive: true });
logger.info("MySQL connection successful");
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("Failed to connect to MySQL:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const unmount = async (volumePath: string) => {
logger.info("Cleaning up MySQL dump directory");
try {
await fs.rm(volumePath, { recursive: true, force: true });
return { status: BACKEND_STATUS.unmounted };
} catch (error) {
logger.warn(`Failed to clean up MySQL dump directory: ${toMessage(error)}`);
return { status: BACKEND_STATUS.unmounted };
}
};
const checkHealth = async (config: BackendConfig) => {
if (config.backend !== "mysql") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
try {
await testMySQLConnection(config);
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("MySQL health check failed:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
export const makeMySQLBackend = (config: BackendConfig, volumePath: string): VolumeBackend => ({
mount: () => mount(config, volumePath),
unmount: () => unmount(volumePath),
checkHealth: () => checkHealth(config),
});

View File

@@ -0,0 +1,57 @@
import * as fs from "node:fs/promises";
import { toMessage } from "../../../utils/errors";
import { logger } from "../../../utils/logger";
import { testPostgresConnection } from "../../../utils/database-dump";
import type { VolumeBackend } from "../backend";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const mount = async (config: BackendConfig, volumePath: string) => {
if (config.backend !== "postgres") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
logger.info(`Testing PostgreSQL connection to: ${config.host}:${config.port}`);
try {
await testPostgresConnection(config);
await fs.mkdir(volumePath, { recursive: true });
logger.info("PostgreSQL connection successful");
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("Failed to connect to PostgreSQL:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const unmount = async (volumePath: string) => {
logger.info("Cleaning up PostgreSQL dump directory");
try {
await fs.rm(volumePath, { recursive: true, force: true });
return { status: BACKEND_STATUS.unmounted };
} catch (error) {
logger.warn(`Failed to clean up PostgreSQL dump directory: ${toMessage(error)}`);
return { status: BACKEND_STATUS.unmounted };
}
};
const checkHealth = async (config: BackendConfig) => {
if (config.backend !== "postgres") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
try {
await testPostgresConnection(config);
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("PostgreSQL health check failed:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
export const makePostgresBackend = (config: BackendConfig, volumePath: string): VolumeBackend => ({
mount: () => mount(config, volumePath),
unmount: () => unmount(volumePath),
checkHealth: () => checkHealth(config),
});

View File

@@ -0,0 +1,57 @@
import * as fs from "node:fs/promises";
import { toMessage } from "../../../utils/errors";
import { logger } from "../../../utils/logger";
import { testSQLiteConnection } from "../../../utils/database-dump";
import type { VolumeBackend } from "../backend";
import { BACKEND_STATUS, type BackendConfig } from "~/schemas/volumes";
const mount = async (config: BackendConfig, volumePath: string) => {
if (config.backend !== "sqlite") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
logger.info(`Testing SQLite connection to: ${config.path}`);
try {
await testSQLiteConnection(config);
await fs.mkdir(volumePath, { recursive: true });
logger.info("SQLite connection successful");
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("Failed to access SQLite database:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
const unmount = async (volumePath: string) => {
logger.info("Cleaning up SQLite dump directory");
try {
await fs.rm(volumePath, { recursive: true, force: true });
return { status: BACKEND_STATUS.unmounted };
} catch (error) {
logger.warn(`Failed to clean up SQLite dump directory: ${toMessage(error)}`);
return { status: BACKEND_STATUS.unmounted };
}
};
const checkHealth = async (config: BackendConfig) => {
if (config.backend !== "sqlite") {
return { status: BACKEND_STATUS.error, error: "Invalid backend type" };
}
try {
await testSQLiteConnection(config);
return { status: BACKEND_STATUS.mounted };
} catch (error) {
logger.error("SQLite health check failed:", error);
return { status: BACKEND_STATUS.error, error: toMessage(error) };
}
};
export const makeSQLiteBackend = (config: BackendConfig, volumePath: string): VolumeBackend => ({
mount: () => mount(config, volumePath),
unmount: () => unmount(volumePath),
checkHealth: () => checkHealth(config),
});

View File

@@ -2,14 +2,16 @@ import { eq } from "drizzle-orm";
import cron from "node-cron";
import { CronExpressionParser } from "cron-parser";
import { NotFoundError, BadRequestError, ConflictError } from "http-errors-enhanced";
import * as fs from "node:fs/promises";
import { db } from "../../db/db";
import { backupSchedulesTable, repositoriesTable, volumesTable } from "../../db/schema";
import { restic } from "../../utils/restic";
import { logger } from "../../utils/logger";
import { getVolumePath } from "../volumes/helpers";
import { getVolumePath, isDatabaseVolume, getDumpFilePath } from "../volumes/helpers";
import type { CreateBackupScheduleBody, UpdateBackupScheduleBody } from "./backups.dto";
import { toMessage } from "../../utils/errors";
import { serverEvents } from "../../core/events";
import { executeDatabaseDump, type DatabaseConfig } from "../../utils/database-dump";
const runningBackups = new Map<number, AbortController>();
@@ -206,7 +208,28 @@ const executeBackup = async (scheduleId: number, manual = false) => {
runningBackups.set(scheduleId, abortController);
try {
const volumePath = getVolumePath(volume);
let backupPath: string;
let dumpFilePath: string | null = null;
const isDatabase = isDatabaseVolume(volume);
if (isDatabase) {
logger.info(`Creating database dump for volume ${volume.name}`);
const timestamp = Date.now();
dumpFilePath = getDumpFilePath(volume, timestamp);
try {
await executeDatabaseDump(volume.config as DatabaseConfig, dumpFilePath);
logger.info(`Database dump created at: ${dumpFilePath}`);
} catch (error) {
logger.error(`Failed to create database dump: ${toMessage(error)}`);
throw error;
}
backupPath = dumpFilePath;
} else {
backupPath = getVolumePath(volume);
}
const backupOptions: {
exclude?: string[];
@@ -226,7 +249,7 @@ const executeBackup = async (scheduleId: number, manual = false) => {
backupOptions.include = schedule.includePatterns;
}
await restic.backup(repository.config, volumePath, {
await restic.backup(repository.config, backupPath, {
...backupOptions,
onProgress: (progress) => {
serverEvents.emit("backup:progress", {
@@ -242,6 +265,16 @@ const executeBackup = async (scheduleId: number, manual = false) => {
await restic.forget(repository.config, schedule.retentionPolicy, { tag: schedule.id.toString() });
}
// Clean up dump file if it was created
if (dumpFilePath) {
try {
await fs.unlink(dumpFilePath);
logger.info(`Cleaned up dump file: ${dumpFilePath}`);
} catch (error) {
logger.warn(`Failed to clean up dump file: ${toMessage(error)}`);
}
}
const nextBackupAt = calculateNextRun(schedule.cronExpression);
await db
.update(backupSchedulesTable)

View File

@@ -1,5 +1,6 @@
import { VOLUME_MOUNT_BASE } from "../../core/constants";
import type { Volume } from "../../db/schema";
import type { BackendConfig } from "~/schemas/volumes";
export const getVolumePath = (volume: Volume) => {
if (volume.config.backend === "directory") {
@@ -8,3 +9,37 @@ export const getVolumePath = (volume: Volume) => {
return `${VOLUME_MOUNT_BASE}/${volume.name}/_data`;
};
/**
* Check if a volume is a database volume
*/
export const isDatabaseVolume = (volume: Volume): boolean => {
return ["mariadb", "mysql", "postgres", "sqlite"].includes(volume.config.backend);
};
/**
* Check if a backend config is a database backend
*/
export const isDatabaseBackend = (config: BackendConfig): boolean => {
return ["mariadb", "mysql", "postgres", "sqlite"].includes(config.backend);
};
/**
* Get the dump directory path for a database volume
*/
export const getDumpPath = (volume: Volume): string => {
return `${VOLUME_MOUNT_BASE}/${volume.name}/dumps`;
};
/**
* Get the dump file path for a database volume backup
*/
export const getDumpFilePath = (volume: Volume, timestamp: number): string => {
const dumpDir = getDumpPath(volume);
const extension = volume.config.backend === "postgres" &&
volume.config.backend === "postgres" &&
(volume.config as Extract<BackendConfig, { backend: "postgres" }>).dumpFormat !== "plain"
? "dump"
: "sql";
return `${dumpDir}/${volume.name}-${timestamp}.${extension}`;
};

View File

@@ -0,0 +1,282 @@
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { safeSpawn } from "./spawn";
import { logger } from "./logger";
import { toMessage } from "./errors";
import type { BackendConfig } from "~/schemas/volumes";
export type DatabaseConfig = Extract<
BackendConfig,
{ backend: "mariadb" | "mysql" | "postgres" | "sqlite" }
>;
// MariaDB
export const dumpMariaDB = async (config: DatabaseConfig, outputPath: string): Promise<void> => {
if (config.backend !== "mariadb") {
throw new Error("Invalid backend type for MariaDB dump");
}
logger.info(`Starting MariaDB dump for database: ${config.database}`);
const args = [
`--host=${config.host}`,
`--port=${config.port}`,
`--user=${config.username}`,
`--skip-ssl`,
`--single-transaction`,
`--quick`,
`--lock-tables=false`,
...(config.dumpOptions || []),
config.database,
];
const env = {
...process.env,
MYSQL_PWD: config.password,
};
try {
const result = await safeSpawn({ command: "mariadb-dump", args, env });
await fs.writeFile(outputPath, result.stdout);
logger.info(`MariaDB dump completed: ${outputPath}`);
} catch (error) {
logger.error(`MariaDB dump failed: ${toMessage(error)}`);
throw error;
}
};
export const testMariaDBConnection = async (config: DatabaseConfig): Promise<void> => {
if (config.backend !== "mariadb") {
throw new Error("Invalid backend type for MariaDB connection test");
}
logger.debug(`Testing MariaDB connection to: ${config.host}:${config.port}`);
const args = [
`--host=${config.host}`,
`--port=${config.port}`,
`--user=${config.username}`,
`--database=${config.database}`,
"--skip-ssl",
"--execute=SELECT 1",
];
const env = {
...process.env,
MYSQL_PWD: config.password,
};
try {
await safeSpawn({ command: "mariadb", args, env, timeout: 10000 });
logger.debug("MariaDB connection test successful");
} catch (error) {
logger.error(`MariaDB connection test failed: ${toMessage(error)}`);
throw error;
}
};
// MySQL
export const dumpMySQL = async (config: DatabaseConfig, outputPath: string): Promise<void> => {
if (config.backend !== "mysql") {
throw new Error("Invalid backend type for MySQL dump");
}
logger.info(`Starting MySQL dump for database: ${config.database}`);
const args = [
`--host=${config.host}`,
`--port=${config.port}`,
`--user=${config.username}`,
`--skip-ssl`,
`--single-transaction`,
`--quick`,
`--lock-tables=false`,
...(config.dumpOptions || []),
config.database,
];
const env = {
...process.env,
MYSQL_PWD: config.password,
};
try {
const result = await safeSpawn({ command: "mysqldump", args, env });
await fs.writeFile(outputPath, result.stdout);
logger.info(`MySQL dump completed: ${outputPath}`);
} catch (error) {
logger.error(`MySQL dump failed: ${toMessage(error)}`);
throw error;
}
};
export const testMySQLConnection = async (config: DatabaseConfig): Promise<void> => {
if (config.backend !== "mysql") {
throw new Error("Invalid backend type for MySQL connection test");
}
logger.debug(`Testing MySQL connection to: ${config.host}:${config.port}`);
const args = [
`--host=${config.host}`,
`--port=${config.port}`,
`--user=${config.username}`,
`--database=${config.database}`,
"--skip-ssl",
"--execute=SELECT 1",
];
const env = {
...process.env,
MYSQL_PWD: config.password,
};
try {
await safeSpawn({ command: "mysql", args, env, timeout: 10000 });
logger.debug("MySQL connection test successful");
} catch (error) {
logger.error(`MySQL connection test failed: ${toMessage(error)}`);
throw error;
}
};
// PostgreSQL
export const dumpPostgres = async (config: DatabaseConfig, outputPath: string): Promise<void> => {
if (config.backend !== "postgres") {
throw new Error("Invalid backend type for PostgreSQL dump");
}
logger.info(`Starting PostgreSQL dump for database: ${config.database}`);
const args = [
`--host=${config.host}`,
`--port=${config.port}`,
`--username=${config.username}`,
`--dbname=${config.database}`,
`--format=${config.dumpFormat}`,
`--file=${outputPath}`,
"--no-password",
...(config.dumpOptions || []),
];
const env = {
...process.env,
PGPASSWORD: config.password,
PGSSLMODE: "disable",
};
try {
await safeSpawn({ command: "pg_dump", args, env });
logger.info(`PostgreSQL dump completed: ${outputPath}`);
} catch (error) {
logger.error(`PostgreSQL dump failed: ${toMessage(error)}`);
throw error;
}
};
export const testPostgresConnection = async (config: DatabaseConfig): Promise<void> => {
if (config.backend !== "postgres") {
throw new Error("Invalid backend type for PostgreSQL connection test");
}
logger.debug(`Testing PostgreSQL connection to: ${config.host}:${config.port}`);
const args = [
`--host=${config.host}`,
`--port=${config.port}`,
`--username=${config.username}`,
`--dbname=${config.database}`,
"--command=SELECT 1",
"--no-password",
];
const env = {
...process.env,
PGPASSWORD: config.password,
PGSSLMODE: "disable",
};
try {
await safeSpawn({ command: "psql", args, env, timeout: 10000 });
logger.debug("PostgreSQL connection test successful");
} catch (error) {
logger.error(`PostgreSQL connection test failed: ${toMessage(error)}`);
throw error;
}
};
// SQLite
export const dumpSQLite = async (config: DatabaseConfig, outputPath: string): Promise<void> => {
if (config.backend !== "sqlite") {
throw new Error("Invalid backend type for SQLite dump");
}
logger.info(`Starting SQLite dump for database: ${config.path}`);
try {
await fs.access(config.path);
const result = await safeSpawn({ command: "sqlite3", args: [config.path, ".dump"] });
await fs.writeFile(outputPath, result.stdout);
logger.info(`SQLite dump completed: ${outputPath}`);
} catch (error) {
logger.error(`SQLite dump failed: ${toMessage(error)}`);
throw error;
}
};
export const testSQLiteConnection = async (config: DatabaseConfig): Promise<void> => {
if (config.backend !== "sqlite") {
throw new Error("Invalid backend type for SQLite connection test");
}
logger.debug(`Testing SQLite connection to: ${config.path}`);
try {
await fs.access(config.path, fs.constants.R_OK);
const result = await safeSpawn({ command: "sqlite3", args: [config.path, "SELECT 1"] });
if (!result.stdout.includes("1")) {
throw new Error("SQLite database query failed");
}
logger.debug("SQLite connection test successful");
} catch (error) {
logger.error(`SQLite connection test failed: ${toMessage(error)}`);
throw error;
}
};
// Utils
export const executeDatabaseDump = async (config: DatabaseConfig, outputPath: string): Promise<void> => {
const outputDir = path.dirname(outputPath);
await fs.mkdir(outputDir, { recursive: true });
switch (config.backend) {
case "mariadb":
return dumpMariaDB(config, outputPath);
case "mysql":
return dumpMySQL(config, outputPath);
case "postgres":
return dumpPostgres(config, outputPath);
case "sqlite":
return dumpSQLite(config, outputPath);
default:
throw new Error(`Unsupported database backend: ${(config as any).backend}`);
}
};
export const testDatabaseConnection = async (config: DatabaseConfig): Promise<void> => {
switch (config.backend) {
case "mariadb":
return testMariaDBConnection(config);
case "mysql":
return testMySQLConnection(config);
case "postgres":
return testPostgresConnection(config);
case "sqlite":
return testSQLiteConnection(config);
default:
throw new Error(`Unsupported database backend: ${(config as any).backend}`);
}
};

View File

@@ -5,6 +5,8 @@ interface Params {
args: string[];
env?: NodeJS.ProcessEnv;
signal?: AbortSignal;
stdin?: string;
timeout?: number;
onStdout?: (data: string) => void;
onStderr?: (error: string) => void;
onError?: (error: Error) => Promise<void> | void;
@@ -19,17 +21,32 @@ type SpawnResult = {
};
export const safeSpawn = (params: Params) => {
const { command, args, env = {}, signal, ...callbacks } = params;
const { command, args, env = {}, signal, stdin, timeout, ...callbacks } = params;
return new Promise<SpawnResult>((resolve) => {
return new Promise<SpawnResult>((resolve, reject) => {
let stdoutData = "";
let stderrData = "";
let timeoutId: NodeJS.Timeout | undefined;
const child = spawn(command, args, {
env: { ...process.env, ...env },
signal: signal,
});
// Handle timeout if specified
if (timeout) {
timeoutId = setTimeout(() => {
child.kill("SIGTERM");
reject(new Error(`Command timed out after ${timeout}ms`));
}, timeout);
}
// Write stdin if provided
if (stdin && child.stdin) {
child.stdin.write(stdin);
child.stdin.end();
}
child.stdout.on("data", (data) => {
if (callbacks.onStdout) {
callbacks.onStdout(data.toString());
@@ -47,6 +64,7 @@ export const safeSpawn = (params: Params) => {
});
child.on("error", async (error) => {
if (timeoutId) clearTimeout(timeoutId);
if (callbacks.onError) {
await callbacks.onError(error);
}
@@ -54,14 +72,11 @@ export const safeSpawn = (params: Params) => {
await callbacks.finally();
}
resolve({
exitCode: -1,
stdout: stdoutData,
stderr: stderrData,
});
reject(error);
});
child.on("close", async (code) => {
if (timeoutId) clearTimeout(timeoutId);
if (callbacks.onClose) {
await callbacks.onClose(code);
}
@@ -69,11 +84,15 @@ export const safeSpawn = (params: Params) => {
await callbacks.finally();
}
resolve({
exitCode: code === null ? -1 : code,
stdout: stdoutData,
stderr: stderrData,
});
if (code !== 0 && code !== null) {
reject(new Error(`Command failed with exit code ${code}: ${stderrData || stdoutData}`));
} else {
resolve({
exitCode: code === null ? -1 : code,
stdout: stdoutData,
stderr: stderrData,
});
}
});
});
};