From 2880fb974864c46db65be6a140aefaf6d4a27fb3 Mon Sep 17 00:00:00 2001
From: Mauricio Siu
Date: Wed, 15 Apr 2026 12:29:07 -0600
Subject: [PATCH] feat: enhance transfer service with auto-deployment and
logging
Refactor the TransferService component to include automatic deployment after successful transfers for various service types (application, compose, postgres, mysql, mariadb, mongo, redis). Implement logging functionality to capture transfer progress and errors, improving user feedback during the transfer process. Update related API routers to support these enhancements, ensuring a seamless transfer and deployment experience.
---
.../dashboard/shared/transfer-service.tsx | 307 ++++++++-----
.../dokploy/server/api/routers/application.ts | 44 +-
apps/dokploy/server/api/routers/compose.ts | 43 +-
apps/dokploy/server/api/routers/mariadb.ts | 7 +-
apps/dokploy/server/api/routers/mongo.ts | 7 +-
apps/dokploy/server/api/routers/mysql.ts | 7 +-
apps/dokploy/server/api/routers/postgres.ts | 7 +-
apps/dokploy/server/api/routers/redis.ts | 7 +-
packages/server/src/services/transfer.ts | 420 ++++++++++--------
packages/server/src/utils/transfer/scanner.ts | 318 +++++++------
packages/server/src/utils/transfer/sync.ts | 354 ++++++++++++---
11 files changed, 1042 insertions(+), 479 deletions(-)
diff --git a/apps/dokploy/components/dashboard/shared/transfer-service.tsx b/apps/dokploy/components/dashboard/shared/transfer-service.tsx
index 575b7dbd2..85726a0c0 100644
--- a/apps/dokploy/components/dashboard/shared/transfer-service.tsx
+++ b/apps/dokploy/components/dashboard/shared/transfer-service.tsx
@@ -1,6 +1,13 @@
-import { AlertTriangle, ArrowRightLeft, Loader2, Server } from "lucide-react";
+import {
+ AlertTriangle,
+ ArrowRightLeft,
+ Loader2,
+ Server,
+} from "lucide-react";
import { useState } from "react";
import { toast } from "sonner";
+import { DrawerLogs } from "@/components/shared/drawer-logs";
+import type { LogLine } from "@/components/dashboard/docker/logs/utils";
import {
AlertDialog,
AlertDialogAction,
@@ -90,41 +97,16 @@ const formatBytes = (bytes: number): string => {
return `${Number.parseFloat((bytes / k ** i).toFixed(2))} ${sizes[i]}`;
};
-const useTransferMutations = (serviceType: ServiceType) => {
- const appScan = api.application.transferScan.useMutation();
- const appTransfer = api.application.transfer.useMutation();
- const composeScan = api.compose.transferScan.useMutation();
- const composeTransfer = api.compose.transfer.useMutation();
- const postgresScan = api.postgres.transferScan.useMutation();
- const postgresTransfer = api.postgres.transfer.useMutation();
- const mysqlScan = api.mysql.transferScan.useMutation();
- const mysqlTransfer = api.mysql.transfer.useMutation();
- const mariadbScan = api.mariadb.transferScan.useMutation();
- const mariadbTransfer = api.mariadb.transfer.useMutation();
- const mongoScan = api.mongo.transferScan.useMutation();
- const mongoTransfer = api.mongo.transfer.useMutation();
- const redisScan = api.redis.transferScan.useMutation();
- const redisTransfer = api.redis.transfer.useMutation();
-
- const mutations: Record<
- ServiceType,
- {
- scan: { mutateAsync: (input: any) => Promise; isPending: boolean };
- transfer: {
- mutateAsync: (input: any) => Promise;
- isPending: boolean;
- };
- }
- > = {
- application: { scan: appScan, transfer: appTransfer },
- compose: { scan: composeScan, transfer: composeTransfer },
- postgres: { scan: postgresScan, transfer: postgresTransfer },
- mysql: { scan: mysqlScan, transfer: mysqlTransfer },
- mariadb: { scan: mariadbScan, transfer: mariadbTransfer },
- mongo: { scan: mongoScan, transfer: mongoTransfer },
- redis: { scan: redisScan, transfer: redisTransfer },
+const useScanMutation = (serviceType: ServiceType) => {
+ const mutations = {
+ application: api.application.transferScan.useMutation(),
+ compose: api.compose.transferScan.useMutation(),
+ postgres: api.postgres.transferScan.useMutation(),
+ mysql: api.mysql.transferScan.useMutation(),
+ mariadb: api.mariadb.transferScan.useMutation(),
+ mongo: api.mongo.transferScan.useMutation(),
+ redis: api.redis.transferScan.useMutation(),
};
-
return mutations[serviceType];
};
@@ -148,15 +130,17 @@ export const TransferService = ({
}: TransferServiceProps) => {
const [targetServerId, setTargetServerId] = useState("");
const [scanResult, setScanResult] = useState(null);
- const [step, setStep] = useState<"select" | "scan" | "confirm" | "transfer">(
- "select",
- );
+ const [step, setStep] = useState<"select" | "scan" | "confirm">("select");
const [showConfirm, setShowConfirm] = useState(false);
- const [transferLogs, setTransferLogs] = useState([]);
+
+ // Drawer logs state
+ const [isDrawerOpen, setIsDrawerOpen] = useState(false);
+ const [filteredLogs, setFilteredLogs] = useState([]);
+ const [isTransferring, setIsTransferring] = useState(false);
const { data: servers } = api.server.all.useQuery();
const utils = api.useUtils();
- const { scan, transfer } = useTransferMutations(serviceType);
+ const scan = useScanMutation(serviceType);
const idKey = getServiceIdKey(serviceType);
@@ -166,6 +150,111 @@ export const TransferService = ({
const selectedServer = servers?.find((s) => s.serverId === targetServerId);
+ // Subscription for transfer with logs
+ const subscriptionInput = {
+ [idKey]: serviceId,
+ targetServerId: targetServerId || "placeholder",
+ decisions: {},
+ };
+
+ const useTransferSubscription = (sType: ServiceType) => {
+ api.application.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "application",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ api.compose.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "compose",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ api.postgres.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "postgres",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ api.mysql.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "mysql",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ api.mariadb.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "mariadb",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ api.mongo.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "mongo",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ api.redis.transferWithLogs.useSubscription(subscriptionInput as any, {
+ enabled: isTransferring && sType === "redis",
+ onData: handleLogData,
+ onError: handleLogError,
+ });
+ };
+
+ const handleLogData = (log: string) => {
+ if (!isDrawerOpen) {
+ setIsDrawerOpen(true);
+ }
+
+ // Try to parse as JSON progress
+ try {
+ const progress = JSON.parse(log);
+ if (progress.message) {
+ const logLine: LogLine = {
+ rawTimestamp: new Date().toISOString(),
+ timestamp: new Date(),
+ message: `[${progress.phase || "transfer"}] ${progress.message}`,
+ };
+ setFilteredLogs((prev) => [...prev, logLine]);
+ }
+ return;
+ } catch {
+ // Not JSON, treat as plain text
+ }
+
+ const logLine: LogLine = {
+ rawTimestamp: new Date().toISOString(),
+ timestamp: new Date(),
+ message: log,
+ };
+ setFilteredLogs((prev) => [...prev, logLine]);
+
+ if (
+ log.includes("completed successfully") ||
+ log.includes("Deployment queued") ||
+ log.includes("Deployment started")
+ ) {
+ setTimeout(() => {
+ setIsTransferring(false);
+ utils.invalidate();
+ toast.success("Transfer and deployment completed!");
+ }, 2000);
+ }
+
+ if (log.includes("Transfer failed") || log.includes("Transfer error")) {
+ setIsTransferring(false);
+ toast.error("Transfer failed");
+ }
+ };
+
+ const handleLogError = (error: unknown) => {
+ console.error("Transfer subscription error:", error);
+ setIsTransferring(false);
+ const logLine: LogLine = {
+ rawTimestamp: new Date().toISOString(),
+ timestamp: new Date(),
+ message: `Error: ${error instanceof Error ? error.message : String(error)}`,
+ };
+ setFilteredLogs((prev) => [...prev, logLine]);
+ };
+
+ // Register the subscription hooks (must be called unconditionally)
+ useTransferSubscription(serviceType);
+
const handleScan = async () => {
if (!targetServerId) {
toast.error("Please select a target server");
@@ -177,7 +266,7 @@ export const TransferService = ({
const result = await scan.mutateAsync({
[idKey]: serviceId,
targetServerId,
- });
+ } as any);
setScanResult(result as ScanResult);
setStep("confirm");
} catch (error) {
@@ -190,38 +279,27 @@ export const TransferService = ({
const handleTransfer = async () => {
setShowConfirm(false);
- setStep("transfer");
- setTransferLogs([]);
+ setFilteredLogs([]);
+ setIsTransferring(true);
+ setIsDrawerOpen(true);
- try {
- await transfer.mutateAsync({
- [idKey]: serviceId,
- targetServerId,
- decisions: {},
- });
-
- toast.success("Transfer completed successfully!");
- setTransferLogs((prev) => [...prev, "Transfer completed successfully!"]);
-
- await utils.invalidate();
-
- setTimeout(() => {
- setStep("select");
- setScanResult(null);
- setTargetServerId("");
- }, 3000);
- } catch (error) {
- const message =
- error instanceof Error ? error.message : "Unknown error";
- toast.error(`Transfer failed: ${message}`);
- setTransferLogs((prev) => [...prev, `Transfer failed: ${message}`]);
- setStep("confirm");
- }
+ // Add initial log
+ setFilteredLogs([
+ {
+ rawTimestamp: new Date().toISOString(),
+ timestamp: new Date(),
+ message: `Starting transfer to ${selectedServer?.name} (${selectedServer?.ipAddress})...`,
+ },
+ ]);
};
- const isDbService = ["postgres", "mysql", "mariadb", "mongo", "redis"].includes(
- serviceType,
- );
+ const isDbService = [
+ "postgres",
+ "mysql",
+ "mariadb",
+ "mongo",
+ "redis",
+ ].includes(serviceType);
return (
@@ -247,7 +325,7 @@ export const TransferService = ({
<>
{/* Step 1: Select target server */}
- Target Server
+ Target Server
{
@@ -255,7 +333,7 @@ export const TransferService = ({
setScanResult(null);
setStep("select");
}}
- disabled={step === "transfer"}
+ disabled={isTransferring}
>
@@ -365,6 +443,36 @@ export const TransferService = ({
Will be synced
)}
+ {scanResult.mounts.length > 0 && (
+
+
+ Docker Volumes:
+
+
+ {scanResult.mounts.map((m) => (
+
+ {m.mount.volumeName ||
+ m.mount.hostPath ||
+ m.mount.mountPath}
+ {m.totalSize > 0 && (
+
+ ({formatBytes(m.totalSize)})
+
+ )}
+ {m.files.length > 0 && (
+
+ {m.files.length} files
+
+ )}
+
+ ))}
+
+
+ )}
{/* Conflict list */}
@@ -379,7 +487,10 @@ export const TransferService = ({
key={conflict.path}
className="text-xs font-mono flex items-center gap-2"
>
-
+
{conflict.status}
@@ -401,8 +512,8 @@ export const TransferService = ({
{isDbService
- ? "Stop the database service before transferring to avoid data corruption. The service will be unavailable until deployed on the target server."
- : "The service will be unavailable during transfer. After transfer completes, deploy the service on the target server to start it."}
+ ? "Stop the database service before transferring to avoid data corruption. After transfer completes, the service will be automatically deployed on the target server."
+ : "The service will be unavailable during transfer. After transfer completes, the service will be automatically deployed on the target server."}
@@ -419,7 +530,7 @@ export const TransferService = ({
setShowConfirm(true)}
- disabled={transfer.isPending}
+ disabled={isTransferring}
>
Transfer to {selectedServer?.name}
@@ -427,30 +538,6 @@ export const TransferService = ({
)}
-
- {/* Step 4: Transfer in progress */}
- {step === "transfer" && (
-
-
-
-
- Transferring service...
-
-
- {transferLogs.length > 0 && (
-
- {transferLogs.map((log, i) => (
-
- {log}
-
- ))}
-
- )}
-
- )}
>
)}
@@ -470,12 +557,14 @@ export const TransferService = ({
{scanResult.totalFiles} files (
{formatBytes(scanResult.totalTransferSize)}) will be
copied.
+ {scanResult.mounts.length > 0 &&
+ ` ${scanResult.mounts.length} volume(s) will be transferred.`}
)}
The service will experience downtime during this
- process. After transfer, you must deploy the service on
- the target server.
+ process. After transfer, the service will be
+ automatically deployed on the target server.
@@ -487,6 +576,20 @@ export const TransferService = ({
+
+ {/* Drawer for transfer logs */}
+ {
+ setIsDrawerOpen(false);
+ if (!isTransferring) {
+ setFilteredLogs([]);
+ setStep("select");
+ setScanResult(null);
+ }
+ }}
+ filteredLogs={filteredLogs}
+ />
);
diff --git a/apps/dokploy/server/api/routers/application.ts b/apps/dokploy/server/api/routers/application.ts
index 34701d9a6..2ec0d362c 100644
--- a/apps/dokploy/server/api/routers/application.ts
+++ b/apps/dokploy/server/api/routers/application.ts
@@ -1206,7 +1206,29 @@ export const applicationRouter = createTRPCRouter({
.update(applications)
.set({ serverId: input.targetServerId })
.where(eq(applications.applicationId, input.applicationId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+
+ // Auto-deploy on target server
+ const jobData: DeploymentJob = {
+ applicationId: input.applicationId,
+ titleLog: "Transfer deployment",
+ type: "deploy",
+ applicationType: "application",
+ descriptionLog: "Auto-deploy after transfer to new server",
+ server: true,
+ };
+
+ if (IS_CLOUD) {
+ jobData.serverId = input.targetServerId;
+ deploy(jobData).catch(() => {});
+ } else {
+ await myQueue.add("deployments", jobData, {
+ removeOnComplete: true,
+ removeOnFail: true,
+ });
+ }
+
+ queue.push("Deployment queued successfully!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -1272,6 +1294,26 @@ export const applicationRouter = createTRPCRouter({
.set({ serverId: input.targetServerId })
.where(eq(applications.applicationId, input.applicationId));
+ // Auto-deploy on target server
+ const jobData: DeploymentJob = {
+ applicationId: input.applicationId,
+ titleLog: "Transfer deployment",
+ type: "deploy",
+ applicationType: "application",
+ descriptionLog: "Auto-deploy after transfer to new server",
+ server: true,
+ };
+
+ if (IS_CLOUD) {
+ jobData.serverId = input.targetServerId;
+ deploy(jobData).catch(() => {});
+ } else {
+ await myQueue.add("deployments", jobData, {
+ removeOnComplete: true,
+ removeOnFail: true,
+ });
+ }
+
return { success: true };
}),
});
diff --git a/apps/dokploy/server/api/routers/compose.ts b/apps/dokploy/server/api/routers/compose.ts
index 1384a225f..beb4a1416 100644
--- a/apps/dokploy/server/api/routers/compose.ts
+++ b/apps/dokploy/server/api/routers/compose.ts
@@ -1240,7 +1240,28 @@ export const composeRouter = createTRPCRouter({
.update(composeTable)
.set({ serverId: input.targetServerId })
.where(eq(composeTable.composeId, input.composeId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+
+ const jobData: DeploymentJob = {
+ composeId: input.composeId,
+ titleLog: "Transfer deployment",
+ type: "deploy",
+ applicationType: "compose",
+ descriptionLog: "Auto-deploy after transfer to new server",
+ server: true,
+ };
+
+ if (IS_CLOUD) {
+ jobData.serverId = input.targetServerId;
+ deploy(jobData).catch(() => {});
+ } else {
+ await myQueue.add("deployments", jobData, {
+ removeOnComplete: true,
+ removeOnFail: true,
+ });
+ }
+
+ queue.push("Deployment queued successfully!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -1306,6 +1327,26 @@ export const composeRouter = createTRPCRouter({
.set({ serverId: input.targetServerId })
.where(eq(composeTable.composeId, input.composeId));
+ // Auto-deploy on target server
+ const jobData: DeploymentJob = {
+ composeId: input.composeId,
+ titleLog: "Transfer deployment",
+ type: "deploy",
+ applicationType: "compose",
+ descriptionLog: "Auto-deploy after transfer to new server",
+ server: true,
+ };
+
+ if (IS_CLOUD) {
+ jobData.serverId = input.targetServerId;
+ deploy(jobData).catch(() => {});
+ } else {
+ await myQueue.add("deployments", jobData, {
+ removeOnComplete: true,
+ removeOnFail: true,
+ });
+ }
+
return { success: true };
}),
});
diff --git a/apps/dokploy/server/api/routers/mariadb.ts b/apps/dokploy/server/api/routers/mariadb.ts
index e0d3a2b80..7aa0c9368 100644
--- a/apps/dokploy/server/api/routers/mariadb.ts
+++ b/apps/dokploy/server/api/routers/mariadb.ts
@@ -690,7 +690,9 @@ export const mariadbRouter = createTRPCRouter({
.update(mariadbTable)
.set({ serverId: input.targetServerId })
.where(eq(mariadbTable.mariadbId, input.mariadbId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+ await deployMariadb(input.mariadbId).catch(() => {});
+ queue.push("Deployment started!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -743,6 +745,9 @@ export const mariadbRouter = createTRPCRouter({
.update(mariadbTable)
.set({ serverId: input.targetServerId })
.where(eq(mariadbTable.mariadbId, input.mariadbId));
+
+ await deployMariadb(input.mariadbId).catch(() => {});
+
return { success: true };
}),
});
diff --git a/apps/dokploy/server/api/routers/mongo.ts b/apps/dokploy/server/api/routers/mongo.ts
index 11cc115f3..34e942d72 100644
--- a/apps/dokploy/server/api/routers/mongo.ts
+++ b/apps/dokploy/server/api/routers/mongo.ts
@@ -701,7 +701,9 @@ export const mongoRouter = createTRPCRouter({
.update(mongoTable)
.set({ serverId: input.targetServerId })
.where(eq(mongoTable.mongoId, input.mongoId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+ await deployMongo(input.mongoId).catch(() => {});
+ queue.push("Deployment started!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -754,6 +756,9 @@ export const mongoRouter = createTRPCRouter({
.update(mongoTable)
.set({ serverId: input.targetServerId })
.where(eq(mongoTable.mongoId, input.mongoId));
+
+ await deployMongo(input.mongoId).catch(() => {});
+
return { success: true };
}),
});
diff --git a/apps/dokploy/server/api/routers/mysql.ts b/apps/dokploy/server/api/routers/mysql.ts
index 40b519f56..ee6918813 100644
--- a/apps/dokploy/server/api/routers/mysql.ts
+++ b/apps/dokploy/server/api/routers/mysql.ts
@@ -704,7 +704,9 @@ export const mysqlRouter = createTRPCRouter({
.update(mysqlTable)
.set({ serverId: input.targetServerId })
.where(eq(mysqlTable.mysqlId, input.mysqlId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+ await deployMySql(input.mysqlId).catch(() => {});
+ queue.push("Deployment started!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -757,6 +759,9 @@ export const mysqlRouter = createTRPCRouter({
.update(mysqlTable)
.set({ serverId: input.targetServerId })
.where(eq(mysqlTable.mysqlId, input.mysqlId));
+
+ await deployMySql(input.mysqlId).catch(() => {});
+
return { success: true };
}),
});
diff --git a/apps/dokploy/server/api/routers/postgres.ts b/apps/dokploy/server/api/routers/postgres.ts
index d7cf295dd..63202ef61 100644
--- a/apps/dokploy/server/api/routers/postgres.ts
+++ b/apps/dokploy/server/api/routers/postgres.ts
@@ -714,7 +714,9 @@ export const postgresRouter = createTRPCRouter({
.update(postgresTable)
.set({ serverId: input.targetServerId })
.where(eq(postgresTable.postgresId, input.postgresId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+ await deployPostgres(input.postgresId).catch(() => {});
+ queue.push("Deployment started!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -767,6 +769,9 @@ export const postgresRouter = createTRPCRouter({
.update(postgresTable)
.set({ serverId: input.targetServerId })
.where(eq(postgresTable.postgresId, input.postgresId));
+
+ await deployPostgres(input.postgresId).catch(() => {});
+
return { success: true };
}),
});
diff --git a/apps/dokploy/server/api/routers/redis.ts b/apps/dokploy/server/api/routers/redis.ts
index 0635a8225..6c5b3a329 100644
--- a/apps/dokploy/server/api/routers/redis.ts
+++ b/apps/dokploy/server/api/routers/redis.ts
@@ -687,7 +687,9 @@ export const redisRouter = createTRPCRouter({
.update(redisTable)
.set({ serverId: input.targetServerId })
.where(eq(redisTable.redisId, input.redisId));
- queue.push("Transfer completed successfully!");
+ queue.push("Transfer completed! Starting deployment on target server...");
+ await deployRedis(input.redisId).catch(() => {});
+ queue.push("Deployment started!");
} else {
queue.push(`Transfer failed: ${result.errors.join(", ")}`);
}
@@ -740,6 +742,9 @@ export const redisRouter = createTRPCRouter({
.update(redisTable)
.set({ serverId: input.targetServerId })
.where(eq(redisTable.redisId, input.redisId));
+
+ await deployRedis(input.redisId).catch(() => {});
+
return { success: true };
}),
});
diff --git a/packages/server/src/services/transfer.ts b/packages/server/src/services/transfer.ts
index 2a0ae03b9..aec52fceb 100644
--- a/packages/server/src/services/transfer.ts
+++ b/packages/server/src/services/transfer.ts
@@ -3,7 +3,12 @@ import path from "node:path";
import { findMountsByApplicationId } from "./mount";
import {
compareFileLists,
+ getDirectorySize,
+ getVolumeSize,
+ listComposeVolumes,
+ listVolumesByPrefix,
scanDirectory,
+ scanDockerVolume,
scanMount,
} from "../utils/transfer/scanner";
import { runPreflightChecks } from "../utils/transfer/preflight";
@@ -57,6 +62,42 @@ const getAutoDataVolumeName = (
return null;
};
+/**
+ * Discover all Docker volumes for a service.
+ * For compose: uses Docker labels + prefix matching.
+ * For databases: uses the auto {appName}-data convention.
+ * For applications: uses user-defined mounts only.
+ */
+const discoverServiceVolumes = async (
+ serverId: string | null,
+ serviceType: ServiceType,
+ appName: string,
+): Promise => {
+ const volumes: Set = new Set();
+
+ if (serviceType === "compose") {
+ // Get volumes by compose project label
+ const labelVolumes = await listComposeVolumes(serverId, appName);
+ for (const v of labelVolumes) {
+ volumes.add(v);
+ }
+
+ // Also try prefix matching (compose uses {projectName}_{volumeName} pattern)
+ const prefixVolumes = await listVolumesByPrefix(serverId, `${appName}_`);
+ for (const v of prefixVolumes) {
+ volumes.add(v);
+ }
+ }
+
+ // Auto data volume for databases
+ const autoVolume = getAutoDataVolumeName(serviceType, appName);
+ if (autoVolume) {
+ volumes.add(autoVolume);
+ }
+
+ return Array.from(volumes);
+};
+
export const scanServiceForTransfer = async (
opts: TransferOptions,
): Promise => {
@@ -80,94 +121,70 @@ export const scanServiceForTransfer = async (
);
const targetPath = getServiceBasePath(serviceType, appName, true);
- try {
- const sourceFiles = await scanDirectory(sourceServerId, sourcePath);
- const targetFiles = await scanDirectory(targetServerId, targetPath);
+ const sourceFiles = await scanDirectory(sourceServerId, sourcePath);
+ const targetFiles = await scanDirectory(targetServerId, targetPath);
+ const dirSize = await getDirectorySize(sourceServerId, sourcePath);
- const fileConflicts = await compareFileLists(
- sourceFiles,
- targetFiles,
- sourceServerId,
- targetServerId,
- sourcePath,
- );
+ const fileConflicts = compareFileLists(sourceFiles, targetFiles);
- result.serviceDirectory = {
- files: fileConflicts,
- totalSize: sourceFiles.reduce((sum, f) => sum + f.size, 0),
- };
- } catch {
- // Directory may not exist yet, that's ok
- }
+ result.serviceDirectory = {
+ files: fileConflicts,
+ totalSize: dirSize || sourceFiles.reduce((sum, f) => sum + f.size, 0),
+ };
}
// 2. Check Traefik config
if (serviceType === "application" || serviceType === "compose") {
- const configPath = "/etc/dokploy/traefik/dynamic";
- const configFile = `${configPath}/${appName}.yml`;
+ const { DYNAMIC_TRAEFIK_PATH } = paths(!!sourceServerId);
+ const configFile = `${appName}.yml`;
+ const sourceConfigFiles = await scanDirectory(
+ sourceServerId,
+ DYNAMIC_TRAEFIK_PATH,
+ );
+ const hasSourceConfig = sourceConfigFiles.some(
+ (f) => f.path === configFile,
+ );
- try {
- const sourceFiles = await scanDirectory(sourceServerId, configPath);
- const sourceConfig = sourceFiles.find(
- (f) => f.path === `${appName}.yml`,
- );
- if (sourceConfig) {
- result.traefikConfig.exists = true;
- const targetFiles = await scanDirectory(targetServerId, configPath);
- const targetConfig = targetFiles.find(
- (f) => f.path === `${appName}.yml`,
- );
- if (targetConfig) {
- result.traefikConfig.hasConflict = true;
- }
- }
- } catch {
- // Config may not exist
- }
- }
-
- // 3. Scan auto data volume for databases
- const autoVolume = getAutoDataVolumeName(serviceType, appName);
- if (autoVolume) {
- try {
- const sourceFiles = await scanMount(sourceServerId, {
- mountId: "auto",
- type: "volume",
- volumeName: autoVolume,
- mountPath: "/data",
- });
- const targetFiles = await scanMount(targetServerId, {
- mountId: "auto",
- type: "volume",
- volumeName: autoVolume,
- mountPath: "/data",
- });
-
- const fileConflicts = await compareFileLists(
- sourceFiles,
- targetFiles,
- sourceServerId,
+ if (hasSourceConfig) {
+ result.traefikConfig.exists = true;
+ const { DYNAMIC_TRAEFIK_PATH: targetTraefikPath } = paths(true);
+ const targetConfigFiles = await scanDirectory(
targetServerId,
- undefined,
- autoVolume,
+ targetTraefikPath,
+ );
+ result.traefikConfig.hasConflict = targetConfigFiles.some(
+ (f) => f.path === configFile,
);
-
- result.mounts.push({
- mount: {
- mountId: "auto",
- type: "volume",
- volumeName: autoVolume,
- mountPath: "/data",
- },
- files: fileConflicts,
- totalSize: sourceFiles.reduce((sum, f) => sum + f.size, 0),
- });
- } catch {
- // Volume may not exist
}
}
- // 4. Scan user-defined mounts
+ // 3. Discover and scan ALL Docker volumes for the service
+ const discoveredVolumes = await discoverServiceVolumes(
+ sourceServerId,
+ serviceType,
+ appName,
+ );
+
+ for (const volumeName of discoveredVolumes) {
+ const sourceFiles = await scanDockerVolume(sourceServerId, volumeName);
+ const targetFiles = await scanDockerVolume(targetServerId, volumeName);
+ const volSize = await getVolumeSize(sourceServerId, volumeName);
+
+ const fileConflicts = compareFileLists(sourceFiles, targetFiles);
+
+ result.mounts.push({
+ mount: {
+ mountId: `docker-${volumeName}`,
+ type: "volume",
+ volumeName,
+ mountPath: "/data",
+ },
+ files: fileConflicts,
+ totalSize: volSize || sourceFiles.reduce((sum, f) => sum + f.size, 0),
+ });
+ }
+
+ // 4. Scan user-defined mounts from Dokploy DB
const serviceTypeForMount = serviceType as
| "application"
| "postgres"
@@ -176,49 +193,51 @@ export const scanServiceForTransfer = async (
| "mongo"
| "redis"
| "compose";
- try {
- const userMounts = await findMountsByApplicationId(
- opts.serviceId,
- serviceTypeForMount,
- );
- for (const mount of userMounts) {
- const mountConfig: MountTransferConfig = {
- mountId: mount.mountId,
- type: mount.type,
- hostPath: mount.hostPath,
- volumeName: mount.volumeName,
- mountPath: mount.mountPath,
- content: mount.content,
- filePath: mount.filePath,
- };
+ const userMounts = await findMountsByApplicationId(
+ opts.serviceId,
+ serviceTypeForMount,
+ );
- if (mount.type === "file") continue; // File mounts are DB-stored
+ for (const mount of userMounts) {
+ if (mount.type === "file") continue;
- try {
- const sourceFiles = await scanMount(sourceServerId, mountConfig);
- const targetFiles = await scanMount(targetServerId, mountConfig);
-
- const fileConflicts = await compareFileLists(
- sourceFiles,
- targetFiles,
- sourceServerId,
- targetServerId,
- mount.type === "bind" ? mount.hostPath || undefined : undefined,
- mount.type === "volume" ? mount.volumeName || undefined : undefined,
- );
-
- result.mounts.push({
- mount: mountConfig,
- files: fileConflicts,
- totalSize: sourceFiles.reduce((sum, f) => sum + f.size, 0),
- });
- } catch {
- // Individual mount scan failure shouldn't stop entire scan
- }
+ // Skip if already discovered as Docker volume
+ if (
+ mount.type === "volume" &&
+ mount.volumeName &&
+ discoveredVolumes.includes(mount.volumeName)
+ ) {
+ continue;
}
- } catch {
- // No mounts found
+
+ const mountConfig: MountTransferConfig = {
+ mountId: mount.mountId,
+ type: mount.type,
+ hostPath: mount.hostPath,
+ volumeName: mount.volumeName,
+ mountPath: mount.mountPath,
+ content: mount.content,
+ filePath: mount.filePath,
+ };
+
+ const sourceFiles = await scanMount(sourceServerId, mountConfig);
+ const targetFiles = await scanMount(targetServerId, mountConfig);
+
+ let mountSize = 0;
+ if (mount.type === "volume" && mount.volumeName) {
+ mountSize = await getVolumeSize(sourceServerId, mount.volumeName);
+ } else if (mount.type === "bind" && mount.hostPath) {
+ mountSize = await getDirectorySize(sourceServerId, mount.hostPath);
+ }
+
+ const fileConflicts = compareFileLists(sourceFiles, targetFiles);
+
+ result.mounts.push({
+ mount: mountConfig,
+ files: fileConflicts,
+ totalSize: mountSize || sourceFiles.reduce((sum, f) => sum + f.size, 0),
+ });
}
// Calculate totals
@@ -233,11 +252,7 @@ export const scanServiceForTransfer = async (
result.conflicts = [
...result.serviceDirectory.files,
...result.mounts.flatMap((m) => m.files),
- ].filter(
- (f) =>
- f.status !== "match" &&
- f.status !== "missing_target",
- );
+ ].filter((f) => f.status !== "match" && f.status !== "missing_target");
return result;
};
@@ -249,58 +264,90 @@ export const executeTransfer = async (
): Promise => {
const { serviceType, appName, sourceServerId, targetServerId } = opts;
const errors: string[] = [];
- let processedFiles = 0;
- let transferredBytes = 0;
-
- const scan = await scanServiceForTransfer(opts);
- const totalFiles = scan.totalFiles;
- const totalBytes = scan.totalTransferSize;
+ const processedFiles = 0;
+ const transferredBytes = 0;
const reportProgress = (
phase: TransferProgress["phase"],
message?: string,
currentFile?: string,
) => {
- if (processedFiles > 0) {
- const percentage = totalFiles > 0 ? Math.round((processedFiles / totalFiles) * 100) : 0;
- onProgress?.({
- phase,
- currentFile,
- processedFiles,
- totalFiles,
- transferredBytes,
- totalBytes,
- percentage,
- message,
- });
- } else {
- onProgress?.({
- phase,
- currentFile,
- processedFiles,
- totalFiles,
- transferredBytes,
- totalBytes,
- percentage: 0,
- message,
- });
- }
+ onProgress?.({
+ phase,
+ currentFile,
+ processedFiles,
+ totalFiles: 0,
+ transferredBytes,
+ totalBytes: 0,
+ percentage: 0,
+ message,
+ });
};
try {
- // Phase 1: Preflight checks
+ // Phase 1: Preflight
reportProgress("preparing", "Running preflight checks...");
- const mountConfigs: MountTransferConfig[] = scan.mounts.map(
- (m) => m.mount,
+ // Discover all volumes
+ const discoveredVolumes = await discoverServiceVolumes(
+ sourceServerId,
+ serviceType,
+ appName,
);
+
+ // User-defined mounts
+ const mountConfigs: MountTransferConfig[] = [];
+ const serviceTypeForMount = serviceType as
+ | "application"
+ | "postgres"
+ | "mysql"
+ | "mariadb"
+ | "mongo"
+ | "redis"
+ | "compose";
+
+ const userMounts = await findMountsByApplicationId(
+ opts.serviceId,
+ serviceTypeForMount,
+ );
+
+ for (const mount of userMounts) {
+ if (mount.type === "file") continue;
+ if (
+ mount.type === "volume" &&
+ mount.volumeName &&
+ discoveredVolumes.includes(mount.volumeName)
+ ) {
+ continue; // Will be handled as discovered volume
+ }
+ mountConfigs.push({
+ mountId: mount.mountId,
+ type: mount.type,
+ hostPath: mount.hostPath,
+ volumeName: mount.volumeName,
+ mountPath: mount.mountPath,
+ content: mount.content,
+ filePath: mount.filePath,
+ });
+ }
+
+ const allVolumeConfigs: MountTransferConfig[] = [
+ ...discoveredVolumes.map((v) => ({
+ mountId: `docker-${v}`,
+ type: "volume" as const,
+ volumeName: v,
+ mountPath: "/data",
+ })),
+ ...mountConfigs,
+ ];
+
const targetBasePath = getServiceBasePath(serviceType, appName, true);
const preflight = await runPreflightChecks(
targetServerId,
targetBasePath,
- totalBytes,
- mountConfigs,
+ 0,
+ allVolumeConfigs,
(msg) => reportProgress("preparing", msg),
);
@@ -326,18 +373,16 @@ export const executeTransfer = async (
targetBasePath,
(msg) => reportProgress("syncing_directory", msg),
);
- processedFiles += scan.serviceDirectory.files.length;
- transferredBytes += scan.serviceDirectory.totalSize;
reportProgress("syncing_directory", "Service directory synced");
} catch (error) {
- errors.push(
- `Failed to sync service directory: ${error instanceof Error ? error.message : String(error)}`,
- );
+ const msg = error instanceof Error ? error.message : String(error);
+ errors.push(`Failed to sync service directory: ${msg}`);
+ reportProgress("syncing_directory", `Error: ${msg}`);
}
}
// Phase 3: Sync Traefik config
- if (scan.traefikConfig.exists) {
+ if (serviceType === "application" || serviceType === "compose") {
reportProgress("syncing_traefik", "Syncing Traefik configuration...");
try {
await syncTraefikConfig(
@@ -346,43 +391,58 @@ export const executeTransfer = async (
appName,
(msg) => reportProgress("syncing_traefik", msg),
);
- reportProgress("syncing_traefik", "Traefik config synced");
} catch (error) {
- errors.push(
- `Failed to sync Traefik config: ${error instanceof Error ? error.message : String(error)}`,
- );
+ const msg = error instanceof Error ? error.message : String(error);
+ errors.push(`Failed to sync Traefik config: ${msg}`);
+ reportProgress("syncing_traefik", `Error: ${msg}`);
}
}
- // Phase 4: Sync mounts
- reportProgress("syncing_mounts", "Syncing mounts and volumes...");
- for (const mountScan of scan.mounts) {
+ // Phase 4: Sync all discovered Docker volumes
+ reportProgress("syncing_mounts", "Syncing Docker volumes...");
+
+ for (const volumeName of discoveredVolumes) {
+ reportProgress("syncing_mounts", `Syncing volume: ${volumeName}`);
+ try {
+ await syncDockerVolume(
+ sourceServerId,
+ targetServerId,
+ volumeName,
+ (msg) => reportProgress("syncing_mounts", msg),
+ );
+ } catch (error) {
+ const msg = error instanceof Error ? error.message : String(error);
+ errors.push(`Failed to sync volume ${volumeName}: ${msg}`);
+ reportProgress("syncing_mounts", `Error: ${msg}`);
+ }
+ }
+
+ // Phase 5: Sync user-defined mounts (bind mounts, etc.)
+ for (const mountConfig of mountConfigs) {
const mountLabel =
- mountScan.mount.volumeName ||
- mountScan.mount.hostPath ||
- mountScan.mount.mountPath;
- reportProgress("syncing_mounts", `Syncing: ${mountLabel}`, mountLabel);
+ mountConfig.volumeName || mountConfig.hostPath || mountConfig.mountPath;
+ reportProgress("syncing_mounts", `Syncing: ${mountLabel}`);
try {
await syncMount(
sourceServerId,
targetServerId,
- mountScan.mount,
+ mountConfig,
decisions,
(msg) => reportProgress("syncing_mounts", msg),
);
- processedFiles += mountScan.files.length;
- transferredBytes += mountScan.totalSize;
- reportProgress("syncing_mounts", `Completed: ${mountLabel}`);
} catch (error) {
- errors.push(
- `Failed to sync mount ${mountLabel}: ${error instanceof Error ? error.message : String(error)}`,
- );
+ const msg = error instanceof Error ? error.message : String(error);
+ errors.push(`Failed to sync mount ${mountLabel}: ${msg}`);
+ reportProgress("syncing_mounts", `Error: ${msg}`);
}
}
if (errors.length > 0) {
- reportProgress("failed", `Transfer completed with errors: ${errors.join(", ")}`);
+ reportProgress(
+ "failed",
+ `Transfer completed with errors: ${errors.join(", ")}`,
+ );
return { success: false, errors };
}
diff --git a/packages/server/src/utils/transfer/scanner.ts b/packages/server/src/utils/transfer/scanner.ts
index 6595eadfc..3489d4665 100644
--- a/packages/server/src/utils/transfer/scanner.ts
+++ b/packages/server/src/utils/transfer/scanner.ts
@@ -6,103 +6,215 @@ import type {
MountTransferConfig,
} from "./types";
+const execOnServer = async (
+ serverId: string | null,
+ command: string,
+): Promise<{ stdout: string; stderr: string }> => {
+ if (serverId) {
+ return execAsyncRemote(serverId, command);
+ }
+ return execAsync(command);
+};
+
export const scanDirectory = async (
serverId: string | null,
dirPath: string,
): Promise => {
- const command = `find ${dirPath} -type f -exec stat --format='%n|%s|%Y' {} + 2>/dev/null || true`;
-
- let stdout: string;
- if (serverId) {
- const result = await execAsyncRemote(serverId, command);
- stdout = result.stdout;
- } else {
- const result = await execAsync(command);
- stdout = result.stdout;
+ // Check if directory exists first
+ try {
+ const { stdout: exists } = await execOnServer(
+ serverId,
+ `test -d "${dirPath}" && echo "yes" || echo "no"`,
+ );
+ if (exists.trim() !== "yes") {
+ return [];
+ }
+ } catch {
+ return [];
}
- if (!stdout.trim()) return [];
+ // Use find + stat -c (POSIX-compatible on Linux)
+ // stat -c works on GNU coreutils (Debian, Ubuntu, etc.)
+ const command = `find "${dirPath}" -type f -printf '%p|%s|%T@\\n' 2>/dev/null`;
- return stdout
- .trim()
- .split("\n")
- .filter(Boolean)
- .map((line) => {
- const [filePath, size, modifiedAt] = line.split("|");
- return {
- path: filePath!.replace(dirPath, "").replace(/^\//, ""),
- size: Number.parseInt(size || "0", 10),
- modifiedAt: Number.parseInt(modifiedAt || "0", 10),
- };
- });
+ try {
+ const { stdout } = await execOnServer(serverId, command);
+ if (!stdout.trim()) return [];
+
+ return stdout
+ .trim()
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => {
+ const parts = line.split("|");
+ const filePath = parts[0] || "";
+ const size = parts[1] || "0";
+ const modifiedAt = parts[2] || "0";
+ return {
+ path: filePath.replace(dirPath, "").replace(/^\//, ""),
+ size: Number.parseInt(size, 10),
+ modifiedAt: Math.floor(Number.parseFloat(modifiedAt)),
+ };
+ })
+ .filter((f) => f.path);
+ } catch {
+ // Fallback: try simpler ls-based approach
+ try {
+ const { stdout } = await execOnServer(
+ serverId,
+ `find "${dirPath}" -type f 2>/dev/null`,
+ );
+ if (!stdout.trim()) return [];
+
+ return stdout
+ .trim()
+ .split("\n")
+ .filter(Boolean)
+ .map((filePath) => ({
+ path: filePath.replace(dirPath, "").replace(/^\//, ""),
+ size: 0,
+ modifiedAt: 0,
+ }))
+ .filter((f) => f.path);
+ } catch {
+ return [];
+ }
+ }
};
export const scanDockerVolume = async (
serverId: string | null,
volumeName: string,
): Promise => {
- const command = `docker run --rm -v ${volumeName}:/volume alpine find /volume -type f -exec stat -c '%n|%s|%Y' {} + 2>/dev/null || true`;
-
- let stdout: string;
- if (serverId) {
- const result = await execAsyncRemote(serverId, command);
- stdout = result.stdout;
- } else {
- const result = await execAsync(command);
- stdout = result.stdout;
+ // First check if volume exists
+ try {
+ const { stdout: exists } = await execOnServer(
+ serverId,
+ `docker volume inspect "${volumeName}" >/dev/null 2>&1 && echo "yes" || echo "no"`,
+ );
+ if (exists.trim() !== "yes") {
+ return [];
+ }
+ } catch {
+ return [];
}
- if (!stdout.trim()) return [];
+ // Use busybox/alpine stat format (-c '%n|%s|%Y')
+ const command = `docker run --rm -v "${volumeName}":/volume:ro alpine sh -c 'find /volume -type f -exec stat -c "%n|%s|%Y" {} + 2>/dev/null || find /volume -type f 2>/dev/null'`;
- return stdout
- .trim()
- .split("\n")
- .filter(Boolean)
- .map((line) => {
- const [filePath, size, modifiedAt] = line.split("|");
- return {
- path: (filePath || "").replace("/volume/", ""),
- size: Number.parseInt(size || "0", 10),
- modifiedAt: Number.parseInt(modifiedAt || "0", 10),
- };
- });
+ try {
+ const { stdout } = await execOnServer(serverId, command);
+ if (!stdout.trim()) return [];
+
+ return stdout
+ .trim()
+ .split("\n")
+ .filter(Boolean)
+ .map((line) => {
+ const parts = line.split("|");
+ if (parts.length >= 3) {
+ return {
+ path: (parts[0] || "").replace(/^\/volume\/?/, ""),
+ size: Number.parseInt(parts[1] || "0", 10),
+ modifiedAt: Number.parseInt(parts[2] || "0", 10),
+ };
+ }
+ // Fallback: just file path
+ return {
+ path: line.replace(/^\/volume\/?/, ""),
+ size: 0,
+ modifiedAt: 0,
+ };
+ })
+ .filter((f) => f.path);
+ } catch {
+ return [];
+ }
+};
+
+export const getDirectorySize = async (
+ serverId: string | null,
+ dirPath: string,
+): Promise => {
+ try {
+ const { stdout } = await execOnServer(
+ serverId,
+ `du -sb "${dirPath}" 2>/dev/null | awk '{print $1}'`,
+ );
+ return Number.parseInt(stdout.trim(), 10) || 0;
+ } catch {
+ return 0;
+ }
+};
+
+export const getVolumeSize = async (
+ serverId: string | null,
+ volumeName: string,
+): Promise => {
+ try {
+ const { stdout } = await execOnServer(
+ serverId,
+ `docker run --rm -v "${volumeName}":/volume:ro alpine du -sb /volume 2>/dev/null | awk '{print $1}'`,
+ );
+ return Number.parseInt(stdout.trim(), 10) || 0;
+ } catch {
+ return 0;
+ }
+};
+
+/**
+ * List all Docker volumes belonging to a compose project.
+ * Docker compose automatically labels volumes with com.docker.compose.project
+ */
+export const listComposeVolumes = async (
+ serverId: string | null,
+ projectName: string,
+): Promise => {
+ try {
+ const { stdout } = await execOnServer(
+ serverId,
+ `docker volume ls --filter "label=com.docker.compose.project=${projectName}" --format "{{.Name}}" 2>/dev/null`,
+ );
+ if (!stdout.trim()) return [];
+ return stdout.trim().split("\n").filter(Boolean);
+ } catch {
+ return [];
+ }
+};
+
+/**
+ * List all Docker volumes that match a prefix pattern (appName_*).
+ * Fallback for when compose labels are not available.
+ */
+export const listVolumesByPrefix = async (
+ serverId: string | null,
+ prefix: string,
+): Promise => {
+ try {
+ const { stdout } = await execOnServer(
+ serverId,
+ `docker volume ls --format "{{.Name}}" 2>/dev/null | grep "^${prefix}" || true`,
+ );
+ if (!stdout.trim()) return [];
+ return stdout.trim().split("\n").filter(Boolean);
+ } catch {
+ return [];
+ }
};
export const computeFileHash = async (
serverId: string | null,
filePath: string,
): Promise => {
- const command = `md5sum "${filePath}" | awk '{print $1}'`;
-
- let stdout: string;
- if (serverId) {
- const result = await execAsyncRemote(serverId, command);
- stdout = result.stdout;
- } else {
- const result = await execAsync(command);
- stdout = result.stdout;
+ try {
+ const { stdout } = await execOnServer(
+ serverId,
+ `md5sum "${filePath}" 2>/dev/null | awk '{print $1}'`,
+ );
+ return stdout.trim();
+ } catch {
+ return "";
}
-
- return stdout.trim();
-};
-
-export const computeVolumeFileHash = async (
- serverId: string | null,
- volumeName: string,
- filePath: string,
-): Promise => {
- const command = `docker run --rm -v ${volumeName}:/volume alpine md5sum "/volume/${filePath}" | awk '{print $1}'`;
-
- let stdout: string;
- if (serverId) {
- const result = await execAsyncRemote(serverId, command);
- stdout = result.stdout;
- } else {
- const result = await execAsync(command);
- stdout = result.stdout;
- }
-
- return stdout.trim();
};
export const scanMount = async (
@@ -115,20 +227,13 @@ export const scanMount = async (
if (mount.type === "bind" && mount.hostPath) {
return scanDirectory(serverId, mount.hostPath);
}
- if (mount.type === "file") {
- return [];
- }
return [];
};
-export const compareFileLists = async (
+export const compareFileLists = (
sourceFiles: FileInfo[],
targetFiles: FileInfo[],
- sourceServerId: string | null,
- targetServerId: string,
- basePath?: string,
- volumeName?: string,
-): Promise => {
+): FileConflict[] => {
const targetMap = new Map();
for (const f of targetFiles) {
targetMap.set(f.path, f);
@@ -161,44 +266,7 @@ export const compareFileLists = async (
continue;
}
- let sourceHash: string;
- let targetHash: string;
-
- if (volumeName) {
- sourceHash = await computeVolumeFileHash(
- sourceServerId,
- volumeName,
- sourceFile.path,
- );
- targetHash = await computeVolumeFileHash(
- targetServerId,
- volumeName,
- targetFile.path,
- );
- } else if (basePath) {
- sourceHash = await computeFileHash(
- sourceServerId,
- `${basePath}/${sourceFile.path}`,
- );
- targetHash = await computeFileHash(
- targetServerId,
- `${basePath}/${targetFile.path}`,
- );
- } else {
- sourceHash = "";
- targetHash = "";
- }
-
- if (sourceHash && targetHash && sourceHash === targetHash) {
- conflicts.push({
- path: sourceFile.path,
- status: "match",
- sourceFile: { ...sourceFile, hash: sourceHash },
- targetFile: { ...targetFile, hash: targetHash },
- });
- continue;
- }
-
+ // Different size or time = conflict
let status: ConflictStatus;
if (sourceFile.modifiedAt > targetFile.modifiedAt) {
status = "newer_source";
@@ -211,14 +279,14 @@ export const compareFileLists = async (
conflicts.push({
path: sourceFile.path,
status,
- sourceFile: { ...sourceFile, hash: sourceHash || undefined },
- targetFile: { ...targetFile, hash: targetHash || undefined },
+ sourceFile,
+ targetFile,
});
}
+ // Files only on target
for (const targetFile of targetFiles) {
- const exists = sourceFiles.some((sf) => sf.path === targetFile.path);
- if (!exists) {
+ if (!sourceFiles.some((sf) => sf.path === targetFile.path)) {
conflicts.push({
path: targetFile.path,
status: "newer_target",
diff --git a/packages/server/src/utils/transfer/sync.ts b/packages/server/src/utils/transfer/sync.ts
index 0bdd8e733..661a4798b 100644
--- a/packages/server/src/utils/transfer/sync.ts
+++ b/packages/server/src/utils/transfer/sync.ts
@@ -1,17 +1,193 @@
+import { spawn } from "node:child_process";
+import { findServerById } from "../../services/server";
+import { Client } from "ssh2";
import { execAsync, execAsyncRemote } from "../process/execAsync";
import type { ConflictDecision, MountTransferConfig } from "./types";
const execOnServer = async (
serverId: string | null,
command: string,
- onData?: (data: string) => void,
): Promise<{ stdout: string; stderr: string }> => {
if (serverId) {
- return execAsyncRemote(serverId, command, onData);
+ return execAsyncRemote(serverId, command);
}
return execAsync(command);
};
+/**
+ * Get a direct SSH connection to a server.
+ * Used for streaming binary data (tar pipes) that can't go through execAsyncRemote.
+ */
+const getSSHConnection = async (
+ serverId: string,
+): Promise<{ conn: Client }> => {
+ const server = await findServerById(serverId);
+ if (!server.sshKeyId) {
+ throw new Error(`No SSH key configured for server ${server.name}`);
+ }
+
+ return new Promise((resolve, reject) => {
+ const conn = new Client();
+ conn
+ .on("ready", () => {
+ resolve({ conn });
+ })
+ .on("error", (err) => {
+ reject(
+ new Error(
+ `SSH connection failed to ${server.name} (${server.ipAddress}): ${err.message}`,
+ ),
+ );
+ })
+ .connect({
+ host: server.ipAddress,
+ port: server.port,
+ username: server.username,
+ privateKey: server.sshKey?.privateKey,
+ });
+ });
+};
+
+/**
+ * Pipe a tar stream from source SSH connection to target SSH connection.
+ */
+const pipeSSH = (
+ sourceConn: Client,
+ targetConn: Client,
+ sourceCmd: string,
+ targetCmd: string,
+ onLog?: (message: string) => void,
+): Promise => {
+ return new Promise((resolve, reject) => {
+ sourceConn.exec(sourceCmd, (err, sourceStream) => {
+ if (err) return reject(new Error(`Source exec failed: ${err.message}`));
+
+ targetConn.exec(targetCmd, (err2, targetStream) => {
+ if (err2)
+ return reject(new Error(`Target exec failed: ${err2.message}`));
+
+ let totalBytes = 0;
+
+ sourceStream.on("data", (chunk: Buffer) => {
+ totalBytes += chunk.length;
+ targetStream.write(chunk);
+ });
+
+ sourceStream.on("end", () => {
+ targetStream.end();
+ });
+
+ targetStream.on("close", () => {
+ onLog?.(
+ `Transferred ${(totalBytes / 1024 / 1024).toFixed(2)} MB`,
+ );
+ resolve();
+ });
+
+ sourceStream.on("error", (e: Error) =>
+ reject(new Error(`Source stream error: ${e.message}`)),
+ );
+ targetStream.on("error", (e: Error) =>
+ reject(new Error(`Target stream error: ${e.message}`)),
+ );
+ });
+ });
+ });
+};
+
+/**
+ * Stream data from local tar command into a remote SSH command.
+ */
+const pipeLocalToRemote = (
+ targetConn: Client,
+ localCmd: string,
+ localArgs: string[],
+ remoteCmd: string,
+ onLog?: (message: string) => void,
+): Promise => {
+
+ return new Promise((resolve, reject) => {
+ const localProcess = spawn(localCmd, localArgs, {
+ stdio: ["ignore", "pipe", "pipe"],
+ });
+
+ targetConn.exec(remoteCmd, (err, targetStream) => {
+ if (err) {
+ localProcess.kill();
+ return reject(new Error(`Remote exec failed: ${err.message}`));
+ }
+
+ let totalBytes = 0;
+
+ localProcess.stdout.on("data", (chunk: Buffer) => {
+ totalBytes += chunk.length;
+ targetStream.write(chunk);
+ });
+
+ localProcess.stdout.on("end", () => {
+ targetStream.end();
+ });
+
+ targetStream.on("close", () => {
+ onLog?.(
+ `Transferred ${(totalBytes / 1024 / 1024).toFixed(2)} MB`,
+ );
+ resolve();
+ });
+
+ localProcess.on("error", (e) => reject(e));
+ targetStream.on("error", (e: Error) => reject(e));
+ });
+ });
+};
+
+/**
+ * Stream data from a remote SSH command into a local tar command.
+ */
+const pipeRemoteToLocal = (
+ sourceConn: Client,
+ remoteCmd: string,
+ localCmd: string,
+ localArgs: string[],
+ onLog?: (message: string) => void,
+): Promise => {
+
+ return new Promise((resolve, reject) => {
+ const localProcess = spawn(localCmd, localArgs, {
+ stdio: ["pipe", "pipe", "pipe"],
+ });
+
+ sourceConn.exec(remoteCmd, (err, sourceStream) => {
+ if (err) {
+ localProcess.kill();
+ return reject(new Error(`Remote exec failed: ${err.message}`));
+ }
+
+ let totalBytes = 0;
+
+ sourceStream.on("data", (chunk: Buffer) => {
+ totalBytes += chunk.length;
+ localProcess.stdin.write(chunk);
+ });
+
+ sourceStream.on("end", () => {
+ localProcess.stdin.end();
+ });
+
+ localProcess.on("close", (code: number) => {
+ onLog?.(
+ `Transferred ${(totalBytes / 1024 / 1024).toFixed(2)} MB`,
+ );
+ if (code === 0) resolve();
+ else reject(new Error(`Local process exited with code ${code}`));
+ });
+
+ sourceStream.on("error", (e: Error) => reject(e));
+ localProcess.on("error", (e) => reject(e));
+ });
+ });
+};
+
export const syncDirectory = async (
sourceServerId: string | null,
targetServerId: string,
@@ -21,47 +197,59 @@ export const syncDirectory = async (
): Promise => {
onLog?.(`Syncing directory: ${sourcePath} → ${targetPath}`);
+ // Ensure target directory exists
await execOnServer(targetServerId, `mkdir -p "${targetPath}"`);
- if (!sourceServerId && targetServerId) {
- // Local → Remote: use rsync over SSH
- const { stdout: sshKeyInfo } = await execAsyncRemote(
- targetServerId,
- "echo connected",
- );
- // Tar from local, pipe to remote via SSH
- await execAsync(
- `tar czf - -C "${sourcePath}" . 2>/dev/null | ssh -o StrictHostKeyChecking=no -i /tmp/transfer_key_${targetServerId} "tar xzf - -C ${targetPath}"`,
- ).catch(async () => {
- // Fallback: read from local, write to remote via tar through dokploy
- const { stdout: tarData } = await execAsync(
- `tar czf - -C "${sourcePath}" . | base64`,
+ if (sourceServerId && targetServerId) {
+ // Remote → Remote: pipe tar directly between SSH connections
+ onLog?.("Using direct SSH pipe for remote-to-remote transfer...");
+ const [source, target] = await Promise.all([
+ getSSHConnection(sourceServerId),
+ getSSHConnection(targetServerId),
+ ]);
+ try {
+ await pipeSSH(
+ source.conn,
+ target.conn,
+ `tar czf - -C "${sourcePath}" . 2>/dev/null`,
+ `tar xzf - -C "${targetPath}"`,
+ onLog,
);
- await execAsyncRemote(
- targetServerId,
- `echo "${tarData}" | base64 -d | tar xzf - -C "${targetPath}"`,
+ } finally {
+ source.conn.end();
+ target.conn.end();
+ }
+ } else if (!sourceServerId && targetServerId) {
+ // Local → Remote
+ onLog?.("Transferring from local to remote...");
+ const { conn } = await getSSHConnection(targetServerId);
+ try {
+ await pipeLocalToRemote(
+ conn,
+ "tar",
+ ["czf", "-", "-C", sourcePath, "."],
+ `tar xzf - -C "${targetPath}"`,
+ onLog,
);
- });
- } else if (sourceServerId && targetServerId) {
- // Remote → Remote: tar pipeline through Dokploy server
- onLog?.("Using tar pipeline for remote-to-remote transfer...");
- const { stdout: tarData } = await execAsyncRemote(
- sourceServerId,
- `tar czf - -C "${sourcePath}" . | base64`,
- );
- await execAsyncRemote(
- targetServerId,
- `echo "${tarData}" | base64 -d | tar xzf - -C "${targetPath}"`,
- );
+ } finally {
+ conn.end();
+ }
} else if (sourceServerId && !targetServerId) {
// Remote → Local
- const { stdout: tarData } = await execAsyncRemote(
- sourceServerId,
- `tar czf - -C "${sourcePath}" . | base64`,
- );
- await execAsync(
- `echo "${tarData}" | base64 -d | tar xzf - -C "${targetPath}"`,
- );
+ onLog?.("Transferring from remote to local...");
+ await execAsync(`mkdir -p "${targetPath}"`);
+ const { conn } = await getSSHConnection(sourceServerId);
+ try {
+ await pipeRemoteToLocal(
+ conn,
+ `tar czf - -C "${sourcePath}" . 2>/dev/null`,
+ "tar",
+ ["xzf", "-", "-C", targetPath],
+ onLog,
+ );
+ } finally {
+ conn.end();
+ }
}
onLog?.(`Directory synced successfully: ${targetPath}`);
@@ -75,27 +263,68 @@ export const syncDockerVolume = async (
): Promise => {
onLog?.(`Syncing Docker volume: ${volumeName}`);
+ // Ensure volume exists on target
await execOnServer(
targetServerId,
- `docker volume inspect ${volumeName} > /dev/null 2>&1 || docker volume create ${volumeName}`,
+ `docker volume inspect "${volumeName}" > /dev/null 2>&1 || docker volume create "${volumeName}"`,
);
- // Export volume from source as tar
- const exportCommand = `docker run --rm -v ${volumeName}:/volume alpine tar czf - -C /volume . | base64`;
- let tarData: string;
+ const srcTarCmd = `docker run --rm -v "${volumeName}":/volume:ro alpine tar czf - -C /volume . 2>/dev/null`;
+ const dstTarCmd = `docker run --rm -i -v "${volumeName}":/volume alpine tar xzf - -C /volume`;
- if (sourceServerId) {
- const result = await execAsyncRemote(sourceServerId, exportCommand);
- tarData = result.stdout;
- } else {
- const result = await execAsync(exportCommand);
- tarData = result.stdout;
+ if (sourceServerId && targetServerId) {
+ // Remote → Remote
+ onLog?.("Using direct SSH pipe for volume transfer...");
+ const [source, target] = await Promise.all([
+ getSSHConnection(sourceServerId),
+ getSSHConnection(targetServerId),
+ ]);
+ try {
+ await pipeSSH(source.conn, target.conn, srcTarCmd, dstTarCmd, onLog);
+ } finally {
+ source.conn.end();
+ target.conn.end();
+ }
+ } else if (!sourceServerId && targetServerId) {
+ // Local → Remote
+ onLog?.("Transferring volume from local to remote...");
+ const { conn } = await getSSHConnection(targetServerId);
+ try {
+ await pipeLocalToRemote(
+ conn,
+ "docker",
+ [
+ "run", "--rm",
+ "-v", `${volumeName}:/volume:ro`,
+ "alpine", "tar", "czf", "-", "-C", "/volume", ".",
+ ],
+ dstTarCmd,
+ onLog,
+ );
+ } finally {
+ conn.end();
+ }
+ } else if (sourceServerId && !targetServerId) {
+ // Remote → Local
+ onLog?.("Transferring volume from remote to local...");
+ const { conn } = await getSSHConnection(sourceServerId);
+ try {
+ await pipeRemoteToLocal(
+ conn,
+ srcTarCmd,
+ "docker",
+ [
+ "run", "--rm", "-i",
+ "-v", `${volumeName}:/volume`,
+ "alpine", "tar", "xzf", "-", "-C", "/volume",
+ ],
+ onLog,
+ );
+ } finally {
+ conn.end();
+ }
}
- // Import volume on target
- const importCommand = `echo "${tarData}" | base64 -d | docker run --rm -i -v ${volumeName}:/volume alpine tar xzf - -C /volume`;
-
- await execOnServer(targetServerId, importCommand);
onLog?.(`Volume synced successfully: ${volumeName}`);
};
@@ -122,9 +351,6 @@ export const syncMount = async (
onLog,
);
} else if (mount.type === "file" && mount.content) {
- onLog?.(`Syncing file mount: ${mount.mountPath}`);
- // File mounts are stored in the database, they get created during deploy
- // No file transfer needed, the content is in the DB
onLog?.("File mount will be recreated from database content during deploy");
}
};
@@ -141,30 +367,28 @@ export const syncTraefikConfig = async (
const configFile = `${configPath}/${appName}.yml`;
let configContent: string;
- if (sourceServerId) {
- const { stdout } = await execAsyncRemote(
+ try {
+ const { stdout } = await execOnServer(
sourceServerId,
- `cat "${configFile}" 2>/dev/null || echo ""`,
- );
- configContent = stdout;
- } else {
- const { stdout } = await execAsync(
- `cat "${configFile}" 2>/dev/null || echo ""`,
+ `cat "${configFile}" 2>/dev/null`,
);
configContent = stdout;
+ } catch {
+ onLog?.("No Traefik config found on source, skipping");
+ return;
}
if (!configContent.trim()) {
- onLog?.("No Traefik config found on source, skipping");
+ onLog?.("Empty Traefik config on source, skipping");
return;
}
await execOnServer(targetServerId, `mkdir -p "${configPath}"`);
- const escapedContent = configContent.replace(/'/g, "'\\''");
+ const b64 = Buffer.from(configContent).toString("base64");
await execOnServer(
targetServerId,
- `echo '${escapedContent}' > "${configFile}"`,
+ `echo "${b64}" | base64 -d > "${configFile}"`,
);
onLog?.("Traefik config synced successfully");