fix: multiple tasks

This commit is contained in:
2026-02-26 17:15:31 +01:00
parent 1666e6bba9
commit d138bd88b4
4 changed files with 358 additions and 375 deletions

View File

@@ -12,10 +12,8 @@ export interface PublishCredentials {
sshMode: 'scp' | 'rsync';
}
export interface PublishResult {
htmlFilesUploaded: number;
thumbnailFilesUploaded: number;
mediaFilesUploaded: number;
export interface DirectoryUploadResult {
filesUploaded: number;
filesSkipped: number;
}
@@ -37,49 +35,101 @@ export class PublishEngine extends EventEmitter {
this.dataDir = dataDir;
}
async uploadSite(
// ── Public upload methods (one per directory, run as parallel tasks) ───
/**
* Upload html/ → remote root.
* Requires html/ to exist (site must be rendered first).
*/
async uploadHtml(
credentials: PublishCredentials,
onProgress: ProgressCallback,
): Promise<PublishResult> {
if (!this.dataDir || !this.projectId) {
throw new Error('No project context set');
}
): Promise<DirectoryUploadResult> {
this.ensureProjectContext();
this.validateCredentials(credentials);
const htmlDir = path.join(this.dataDir, 'html');
const thumbnailsDir = path.join(this.dataDir, 'thumbnails');
const mediaDir = path.join(this.dataDir, 'media');
// Verify the generated site exists
const htmlDir = path.join(this.dataDir!, 'html');
await this.ensureDirectoryExists(htmlDir, 'Generated site not found. Please render the site first.');
const result: PublishResult = {
htmlFilesUploaded: 0,
thumbnailFilesUploaded: 0,
mediaFilesUploaded: 0,
filesSkipped: 0,
};
if (credentials.sshMode === 'rsync') {
await this.uploadViaRsync(credentials, htmlDir, thumbnailsDir, mediaDir, result, onProgress);
} else {
await this.uploadViaScp(credentials, htmlDir, thumbnailsDir, mediaDir, result, onProgress);
return this.rsyncDirectory(
htmlDir + '/',
this.rsyncDest(credentials, '/'),
onProgress,
);
}
onProgress(100, 'Upload complete');
return result;
return this.scpUploadDir(credentials, htmlDir, credentials.sshRemotePath, onProgress);
}
// ── SCP mode ──────────────────────────────────────────────────────────────
private async uploadViaScp(
/**
* Upload thumbnails/ → remote/thumbnails/.
* Silently returns zero counts if thumbnails/ does not exist.
*/
async uploadThumbnails(
credentials: PublishCredentials,
htmlDir: string,
thumbnailsDir: string,
mediaDir: string,
result: PublishResult,
onProgress: ProgressCallback,
): Promise<void> {
): Promise<DirectoryUploadResult> {
this.ensureProjectContext();
this.validateCredentials(credentials);
const thumbnailsDir = path.join(this.dataDir!, 'thumbnails');
if (!(await this.directoryExists(thumbnailsDir))) {
onProgress(100, 'No thumbnails to upload');
return { filesUploaded: 0, filesSkipped: 0 };
}
const remotePath = path.posix.join(credentials.sshRemotePath, 'thumbnails');
if (credentials.sshMode === 'rsync') {
return this.rsyncDirectory(
thumbnailsDir + '/',
this.rsyncDest(credentials, '/thumbnails/'),
onProgress,
);
}
return this.scpUploadDir(credentials, thumbnailsDir, remotePath, onProgress);
}
/**
* Upload media/ → remote/media/, excluding .meta sidecar files.
* Silently returns zero counts if media/ does not exist.
*/
async uploadMedia(
credentials: PublishCredentials,
onProgress: ProgressCallback,
): Promise<DirectoryUploadResult> {
this.ensureProjectContext();
this.validateCredentials(credentials);
const mediaDir = path.join(this.dataDir!, 'media');
if (!(await this.directoryExists(mediaDir))) {
onProgress(100, 'No media to upload');
return { filesUploaded: 0, filesSkipped: 0 };
}
const remotePath = path.posix.join(credentials.sshRemotePath, 'media');
if (credentials.sshMode === 'rsync') {
return this.rsyncDirectory(
mediaDir + '/',
this.rsyncDest(credentials, '/media/'),
onProgress,
['*.meta'],
);
}
return this.scpUploadDir(
credentials, mediaDir, remotePath, onProgress,
(name) => !name.endsWith(META_EXTENSION),
);
}
// ── SCP mode ──────────────────────────────────────────────────────────
private async scpUploadDir(
credentials: PublishCredentials,
localDir: string,
remoteDir: string,
onProgress: ProgressCallback,
fileFilter?: (name: string) => boolean,
): Promise<DirectoryUploadResult> {
const client = await scpClient({
host: credentials.sshHost,
username: credentials.sshUser,
@@ -87,109 +137,53 @@ export class PublishEngine extends EventEmitter {
});
try {
// Phase 1: html/ → remote root (033%)
onProgress(0, 'Uploading HTML files...');
const htmlResult = await this.scpUploadDirectory(
client,
htmlDir,
credentials.sshRemotePath,
(p, msg) => onProgress(Math.round(p * 0.33), msg),
);
result.htmlFilesUploaded = htmlResult.uploaded;
result.filesSkipped += htmlResult.skipped;
const files = await this.collectFiles(localDir, '', fileFilter);
let uploaded = 0;
let skipped = 0;
// Phase 2: thumbnails/ → remote/thumbnails/ (3366%)
onProgress(33, 'Uploading thumbnails...');
if (await this.directoryExists(thumbnailsDir)) {
const thumbResult = await this.scpUploadDirectory(
client,
thumbnailsDir,
path.posix.join(credentials.sshRemotePath, 'thumbnails'),
(p, msg) => onProgress(33 + Math.round(p * 0.33), msg),
);
result.thumbnailFilesUploaded = thumbResult.uploaded;
result.filesSkipped += thumbResult.skipped;
if (files.length === 0) {
onProgress(100, 'No files to upload');
return { filesUploaded: 0, filesSkipped: 0 };
}
// Phase 3: media/ → remote/media/ (6699%), excluding .meta files
onProgress(66, 'Uploading media files...');
if (await this.directoryExists(mediaDir)) {
const mediaResult = await this.scpUploadDirectory(
client,
mediaDir,
path.posix.join(credentials.sshRemotePath, 'media'),
(p, msg) => onProgress(66 + Math.round(p * 0.33), msg),
(name) => !name.endsWith(META_EXTENSION),
);
result.mediaFilesUploaded = mediaResult.uploaded;
result.filesSkipped += mediaResult.skipped;
await this.scpEnsureDir(client, remoteDir);
const createdDirs = new Set<string>();
for (let i = 0; i < files.length; i++) {
const relativePath = files[i];
const localPath = path.join(localDir, relativePath);
const remotePath = path.posix.join(remoteDir, relativePath.split(path.sep).join('/'));
// Ensure parent directory exists on remote
const remoteParent = path.posix.dirname(remotePath);
if (!createdDirs.has(remoteParent)) {
await this.scpEnsureDir(client, remoteParent);
createdDirs.add(remoteParent);
}
// Check if we need to upload (compare mtime)
const localStat = await fs.stat(localPath);
const needsUpload = await this.scpNeedsUpload(client, remotePath, localStat.mtimeMs);
if (needsUpload) {
await client.uploadFile(localPath, remotePath);
uploaded++;
const progress = Math.round(((i + 1) / files.length) * 100);
onProgress(progress, `Uploaded ${relativePath} (${uploaded}/${files.length})`);
} else {
skipped++;
const progress = Math.round(((i + 1) / files.length) * 100);
onProgress(progress, `Skipped ${relativePath} (${i + 1}/${files.length})`);
}
}
onProgress(100, `Done: ${uploaded} uploaded, ${skipped} unchanged`);
return { filesUploaded: uploaded, filesSkipped: skipped };
} finally {
client.close();
}
}
/**
* Recursively upload a local directory to a remote path via SCP/SFTP.
* Only uploads files that are newer than the remote version.
*/
private async scpUploadDirectory(
client: ScpClient,
localDir: string,
remoteDir: string,
onProgress: ProgressCallback,
fileFilter?: (name: string) => boolean,
): Promise<{ uploaded: number; skipped: number }> {
// Collect all files first for progress tracking
const files = await this.collectFiles(localDir, '', fileFilter);
let uploaded = 0;
let skipped = 0;
if (files.length === 0) {
onProgress(100, 'No files to upload');
return { uploaded: 0, skipped: 0 };
}
// Ensure remote directory exists
await this.scpEnsureDir(client, remoteDir);
// Track created directories to avoid redundant mkdir calls
const createdDirs = new Set<string>();
for (let i = 0; i < files.length; i++) {
const relativePath = files[i];
const localPath = path.join(localDir, relativePath);
const remotePath = path.posix.join(remoteDir, relativePath.split(path.sep).join('/'));
// Ensure parent directory exists on remote
const remoteParent = path.posix.dirname(remotePath);
if (!createdDirs.has(remoteParent)) {
await this.scpEnsureDir(client, remoteParent);
createdDirs.add(remoteParent);
}
// Check if we need to upload (compare mtime)
const localStat = await fs.stat(localPath);
const needsUpload = await this.scpNeedsUpload(client, remotePath, localStat.mtimeMs);
if (needsUpload) {
await client.uploadFile(localPath, remotePath);
uploaded++;
} else {
skipped++;
}
const progress = ((i + 1) / files.length) * 100;
onProgress(progress, `${uploaded} uploaded, ${skipped} skipped (${i + 1}/${files.length})`);
}
return { uploaded, skipped };
}
/**
* Check if a local file is newer than the remote file.
* Returns true if upload is needed.
*/
private async scpNeedsUpload(
client: ScpClient,
remotePath: string,
@@ -197,11 +191,9 @@ export class PublishEngine extends EventEmitter {
): Promise<boolean> {
try {
const remoteStat = await client.stat(remotePath);
// SSH2 Stats.mtime is in seconds, localMtimeMs is in milliseconds
const remoteMtimeMs = remoteStat.mtime * 1000;
return localMtimeMs > remoteMtimeMs;
} catch {
// File doesn't exist on remote → needs upload
return true;
}
}
@@ -214,61 +206,20 @@ export class PublishEngine extends EventEmitter {
}
}
// ── rsync mode ────────────────────────────────────────────────────────────
// ── rsync mode ────────────────────────────────────────────────────────
private async uploadViaRsync(
credentials: PublishCredentials,
htmlDir: string,
thumbnailsDir: string,
mediaDir: string,
result: PublishResult,
onProgress: ProgressCallback,
): Promise<void> {
const remoteDest = `${credentials.sshUser}@${credentials.sshHost}:${credentials.sshRemotePath}`;
// Phase 1: html/ → remote root (033%)
onProgress(0, 'Syncing HTML files via rsync...');
const htmlCount = await this.rsyncDirectory(
htmlDir + '/',
remoteDest + '/',
);
result.htmlFilesUploaded = htmlCount;
onProgress(33, 'HTML sync complete');
// Phase 2: thumbnails/ → remote/thumbnails/ (3366%)
if (await this.directoryExists(thumbnailsDir)) {
onProgress(33, 'Syncing thumbnails via rsync...');
const thumbCount = await this.rsyncDirectory(
thumbnailsDir + '/',
remoteDest + '/thumbnails/',
);
result.thumbnailFilesUploaded = thumbCount;
}
onProgress(66, 'Thumbnails sync complete');
// Phase 3: media/ → remote/media/ (6699%), excluding .meta files
if (await this.directoryExists(mediaDir)) {
onProgress(66, 'Syncing media files via rsync...');
const mediaCount = await this.rsyncDirectory(
mediaDir + '/',
remoteDest + '/media/',
['*.meta'],
);
result.mediaFilesUploaded = mediaCount;
}
onProgress(99, 'Media sync complete');
private rsyncDest(credentials: PublishCredentials, suffix: string): string {
return `${credentials.sshUser}@${credentials.sshHost}:${credentials.sshRemotePath}${suffix}`;
}
/**
* Run rsync for a directory with incremental (--update --times) and recursive transfer.
* Returns estimated file count from stdout.
*/
private rsyncDirectory(
src: string,
dest: string,
onProgress: ProgressCallback,
exclude?: string[],
): Promise<number> {
): Promise<DirectoryUploadResult> {
return new Promise((resolve, reject) => {
onProgress(0, 'Starting rsync...');
rsync(
{
src,
@@ -283,20 +234,18 @@ export class PublishEngine extends EventEmitter {
if (error) {
reject(error);
} else {
// Count uploaded files from rsync stdout (each transferred file gets a line)
const lines = stdout.trim().split('\n').filter((l: string) => l.length > 0);
resolve(lines.length);
const count = lines.length;
onProgress(100, `rsync complete: ${count} files transferred`);
resolve({ filesUploaded: count, filesSkipped: 0 });
}
},
);
});
}
// ── Helpers ───────────────────────────────────────────────────────────────
// ── Helpers ───────────────────────────────────────────────────────────
/**
* Recursively collect all file paths relative to baseDir, with optional filter.
*/
private async collectFiles(
baseDir: string,
prefix: string,
@@ -329,6 +278,12 @@ export class PublishEngine extends EventEmitter {
return files;
}
private ensureProjectContext(): void {
if (!this.dataDir || !this.projectId) {
throw new Error('No project context set');
}
}
private validateCredentials(credentials: PublishCredentials): void {
if (!credentials.sshHost?.trim()) {
throw new Error('SSH host is required');

View File

@@ -112,5 +112,5 @@ export {
PublishEngine,
getPublishEngine,
type PublishCredentials,
type PublishResult,
type DirectoryUploadResult,
} from './PublishEngine';

View File

@@ -15,16 +15,42 @@ export function registerPublishHandlers(safeHandle: SafeHandle): void {
const publishEngine = getPublishEngine();
publishEngine.setProjectContext(project.id, project.dataPath!);
return taskManager.runTask({
id: `publish-upload-${Date.now()}`,
name: 'Upload Site',
groupId: 'publish',
groupName: 'Site Publishing',
execute: async (onProgress) => {
return publishEngine.uploadSite(credentials, (progress, message) => {
onProgress(progress, message);
});
},
const ts = Date.now();
const groupId = `publish-${ts}`;
const groupName = 'Site Publishing';
// Launch three parallel tasks, one per directory
const htmlTask = taskManager.runTask({
id: `publish-html-${ts}`,
name: 'Upload HTML',
groupId,
groupName,
execute: (onProgress) => publishEngine.uploadHtml(credentials, onProgress),
});
const thumbsTask = taskManager.runTask({
id: `publish-thumbnails-${ts}`,
name: 'Upload Thumbnails',
groupId,
groupName,
execute: (onProgress) => publishEngine.uploadThumbnails(credentials, onProgress),
});
const mediaTask = taskManager.runTask({
id: `publish-media-${ts}`,
name: 'Upload Media',
groupId,
groupName,
execute: (onProgress) => publishEngine.uploadMedia(credentials, onProgress),
});
const [html, thumbnails, media] = await Promise.all([htmlTask, thumbsTask, mediaTask]);
return {
htmlFilesUploaded: html.filesUploaded,
thumbnailFilesUploaded: thumbnails.filesUploaded,
mediaFilesUploaded: media.filesUploaded,
filesSkipped: html.filesSkipped + thumbnails.filesSkipped + media.filesSkipped,
};
});
}