import fs from "node:fs/promises"; import path from "node:path"; import { buildExportDataJson } from "#server/service/export/build-data"; import { buildExportMediaFiles } from "#server/service/export/build-files"; import { buildExportManifest } from "#server/service/export/build-manifest"; import { markExportTaskFailed, markExportTaskRunning, markExportTaskSucceeded, } from "#server/service/export/jobs"; import { sha256File } from "#server/utils/export-hash"; import { RELATIVE_TMP_DIR } from "#server/constants/media"; const EXPORT_RESULT_TTL_MS = 24 * 60 * 60 * 1000; function resolveExportRootDir() { return path.resolve(process.cwd(), RELATIVE_TMP_DIR, "exports"); } async function calcChecksums(baseDir: string, files: Array<{ file: string }>) { return Promise.all( files.map(async (item) => ({ file: item.file, sha256: await sha256File(path.resolve(baseDir, item.file)), })), ); } export async function runExportTask(taskId: number) { const runningTask = await markExportTaskRunning(taskId); return runExportTaskWithRunning(taskId, runningTask); } export async function runExportTaskWithRunning( taskId: number, runningTask: Awaited>, ) { const cutoffAt = runningTask.exportCutoffAt ?? new Date(); const exportRootDir = resolveExportRootDir(); const outputName = `export-task-${taskId}`; const outputDir = path.resolve(exportRootDir, outputName); const outputDataDir = path.resolve(outputDir, "data"); const outputFilesDir = path.resolve(outputDir, "files"); try { await fs.mkdir(outputDir, { recursive: true }); const dataResult = await buildExportDataJson({ userId: runningTask.userId, maskPolicy: runningTask.maskPolicy as "masked" | "raw", cutoffAt, outputDataDir, }); const filesResult = await buildExportMediaFiles({ userId: runningTask.userId, cutoffAt, outputFilesDir, }); const dataChecksums = await calcChecksums(outputDir, dataResult.dataFiles); const fileChecksums = await calcChecksums(outputDir, filesResult.files); const filesBytes = filesResult.files.reduce((sum, item) => sum + item.bytes, 0); const manifest = buildExportManifest({ schemaVersion: 1, userId: runningTask.userId, maskPolicy: runningTask.maskPolicy as "masked" | "raw", exportedAt: new Date(), exportCutoffAt: cutoffAt, stats: { dataRows: dataResult.totalRows, files: filesResult.files.length, bytes: filesBytes, }, dataChecksums, fileChecksums, }); const manifestPath = path.resolve(outputDir, "manifest.json"); await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); const dataBytes = await Promise.all( dataResult.dataFiles.map(async (item) => { const stat = await fs.stat(path.resolve(outputDir, item.file)); return stat.size; }), ); const manifestBytes = (await fs.stat(manifestPath)).size; const totalBytes = dataBytes.reduce((sum, size) => sum + size, 0) + filesBytes + manifestBytes; await markExportTaskSucceeded(taskId, { outputDir, outputName, totalBytes, expiresAt: new Date(Date.now() + EXPORT_RESULT_TTL_MS), }); } catch (error) { await markExportTaskFailed(taskId, { errorCode: "EXPORT_BUILD_FAILED", errorMessage: error instanceof Error ? error.message : "unknown export error", }); throw error; } }