Browse Source
Add export job lifecycle services, artifact builders, task processor, and authenticated me endpoints to request, monitor, and download export results. Made-with: Cursormain
16 changed files with 972 additions and 0 deletions
@ -0,0 +1,18 @@ |
|||||
|
import { createExportTask } from "#server/service/export/jobs"; |
||||
|
import { runExportTask } from "#server/service/export/run"; |
||||
|
import { parseMeExportRequestBody } from "#server/utils/me-export-request-body"; |
||||
|
import { R } from "#server/utils/response"; |
||||
|
|
||||
|
export default defineWrappedResponseHandler(async (event) => { |
||||
|
const user = await event.context.auth.requireUser(); |
||||
|
const body = await readBody(event); |
||||
|
const { maskPolicy } = parseMeExportRequestBody(body); |
||||
|
const task = await createExportTask({ userId: user.id, maskPolicy }); |
||||
|
void runExportTask(task.id).catch(() => { |
||||
|
// 后台执行失败会在任务状态中可见,此处不影响请求返回。
|
||||
|
}); |
||||
|
return R.success({ |
||||
|
taskId: task.id, |
||||
|
status: task.status, |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,21 @@ |
|||||
|
import { listExportTasksByUser } from "#server/service/export/jobs"; |
||||
|
import { R } from "#server/utils/response"; |
||||
|
|
||||
|
export default defineWrappedResponseHandler(async (event) => { |
||||
|
const user = await event.context.auth.requireUser(); |
||||
|
const tasks = await listExportTasksByUser(user.id); |
||||
|
return R.success({ |
||||
|
items: tasks.map((task) => ({ |
||||
|
id: task.id, |
||||
|
status: task.status, |
||||
|
maskPolicy: task.maskPolicy, |
||||
|
outputName: task.outputName, |
||||
|
totalBytes: task.totalBytes, |
||||
|
errorCode: task.errorCode, |
||||
|
errorMessage: task.errorMessage, |
||||
|
createdAt: task.createdAt.toISOString(), |
||||
|
updatedAt: task.updatedAt.toISOString(), |
||||
|
expiresAt: task.expiresAt ? task.expiresAt.toISOString() : null, |
||||
|
})), |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,36 @@ |
|||||
|
import fs from "node:fs"; |
||||
|
import path from "node:path"; |
||||
|
import { sendStream, setHeader } from "h3"; |
||||
|
import { getExportTaskForUser } from "#server/service/export/jobs"; |
||||
|
|
||||
|
export default defineWrappedResponseHandler(async (event) => { |
||||
|
const user = await event.context.auth.requireUser(); |
||||
|
const idRaw = getRouterParam(event, "id"); |
||||
|
const taskId = Number(idRaw); |
||||
|
if (!Number.isInteger(taskId) || taskId < 1) { |
||||
|
throw createError({ statusCode: 400, statusMessage: "无效的任务 id" }); |
||||
|
} |
||||
|
|
||||
|
const task = await getExportTaskForUser(taskId, user.id); |
||||
|
if (!task) { |
||||
|
throw createError({ statusCode: 404, statusMessage: "导出任务不存在" }); |
||||
|
} |
||||
|
if (task.status !== "succeeded") { |
||||
|
throw createError({ statusCode: 409, statusMessage: "导出任务尚未完成" }); |
||||
|
} |
||||
|
if (!task.expiresAt || task.expiresAt.getTime() <= Date.now()) { |
||||
|
throw createError({ statusCode: 410, statusMessage: "导出结果已过期" }); |
||||
|
} |
||||
|
if (!task.outputDir || !task.outputName) { |
||||
|
throw createError({ statusCode: 500, statusMessage: "导出结果缺失" }); |
||||
|
} |
||||
|
|
||||
|
const manifestPath = path.resolve(task.outputDir, "manifest.json"); |
||||
|
if (!fs.existsSync(manifestPath)) { |
||||
|
throw createError({ statusCode: 404, statusMessage: "导出文件不存在" }); |
||||
|
} |
||||
|
|
||||
|
setHeader(event, "Content-Type", "application/json; charset=utf-8"); |
||||
|
setHeader(event, "Content-Disposition", `attachment; filename="${task.outputName}-manifest.json"`); |
||||
|
return sendStream(event, fs.createReadStream(manifestPath)); |
||||
|
}); |
||||
@ -0,0 +1,25 @@ |
|||||
|
export const EXPORT_MASK_POLICIES = ["masked", "raw"] as const; |
||||
|
|
||||
|
export type ExportMaskPolicy = (typeof EXPORT_MASK_POLICIES)[number]; |
||||
|
|
||||
|
/** |
||||
|
* 标准化后(小写并移除 `_` / `-`)的永不导出字段精确匹配名单。 |
||||
|
*/ |
||||
|
export const NEVER_EXPORT_FIELD_EXACT_NAMES = [ |
||||
|
"password", |
||||
|
"passwordhash", |
||||
|
"resettoken", |
||||
|
"resetpasswordtoken", |
||||
|
"sessionid", |
||||
|
"sessiontoken", |
||||
|
"accesstoken", |
||||
|
"refreshtoken", |
||||
|
"apikey", |
||||
|
"secretkey", |
||||
|
"credential", |
||||
|
] as const; |
||||
|
|
||||
|
/** |
||||
|
* masked 策略下优先脱敏的字段关键词(可扩展)。 |
||||
|
*/ |
||||
|
export const DEFAULT_MASK_FIELD_KEYWORDS = ["email", "phone", "mobile"] as const; |
||||
@ -0,0 +1,113 @@ |
|||||
|
import fs from "node:fs/promises"; |
||||
|
import path from "node:path"; |
||||
|
import { dbGlobal } from "drizzle-pkg/lib/db"; |
||||
|
import { users } from "drizzle-pkg/lib/schema/auth"; |
||||
|
import { userConfigs } from "drizzle-pkg/lib/schema/config"; |
||||
|
import { mediaAssets, mediaRefs, postComments, posts, timelineEvents } from "drizzle-pkg/lib/schema/content"; |
||||
|
import { and, eq, inArray, lte, or } from "drizzle-orm"; |
||||
|
import { MEDIA_REF_OWNER_POST } from "../../constants/media-refs"; |
||||
|
import type { ExportMaskPolicy } from "../../constants/export"; |
||||
|
import { sanitizeUserForExport } from "../../utils/export-mask"; |
||||
|
|
||||
|
export type BuildExportDataResult = { |
||||
|
dataFiles: Array<{ file: string; rowCount: number }>; |
||||
|
totalRows: number; |
||||
|
}; |
||||
|
|
||||
|
async function writeDataFile(baseDir: string, fileName: string, payload: unknown): Promise<{ file: string; rowCount: number }> { |
||||
|
const file = `data/${fileName}`; |
||||
|
const fullPath = path.resolve(baseDir, fileName); |
||||
|
await fs.writeFile(fullPath, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); |
||||
|
const rowCount = Array.isArray(payload) ? payload.length : payload ? 1 : 0; |
||||
|
return { file, rowCount }; |
||||
|
} |
||||
|
|
||||
|
export async function buildExportDataJson(params: { |
||||
|
userId: number; |
||||
|
maskPolicy: ExportMaskPolicy; |
||||
|
cutoffAt: Date; |
||||
|
outputDataDir: string; |
||||
|
}): Promise<BuildExportDataResult> { |
||||
|
await fs.mkdir(params.outputDataDir, { recursive: true }); |
||||
|
|
||||
|
const [userRow] = await dbGlobal.select().from(users).where(eq(users.id, params.userId)).limit(1); |
||||
|
if (!userRow) { |
||||
|
throw new Error(`user not found: ${params.userId}`); |
||||
|
} |
||||
|
|
||||
|
const sanitizedUser = sanitizeUserForExport(userRow as unknown as Record<string, unknown>, params.maskPolicy); |
||||
|
|
||||
|
const [postRows, timelineRows, mediaRows, configRows] = await Promise.all([ |
||||
|
dbGlobal |
||||
|
.select() |
||||
|
.from(posts) |
||||
|
.where(and(eq(posts.userId, params.userId), lte(posts.createdAt, params.cutoffAt))), |
||||
|
dbGlobal |
||||
|
.select() |
||||
|
.from(timelineEvents) |
||||
|
.where(and(eq(timelineEvents.userId, params.userId), lte(timelineEvents.createdAt, params.cutoffAt))), |
||||
|
dbGlobal |
||||
|
.select() |
||||
|
.from(mediaAssets) |
||||
|
.where(and(eq(mediaAssets.userId, params.userId), lte(mediaAssets.createdAt, params.cutoffAt))), |
||||
|
dbGlobal |
||||
|
.select() |
||||
|
.from(userConfigs) |
||||
|
.where(and(eq(userConfigs.userId, params.userId), lte(userConfigs.updatedAt, params.cutoffAt))), |
||||
|
]); |
||||
|
|
||||
|
const postIds = postRows.map((p) => p.id); |
||||
|
const commentCondition = |
||||
|
postIds.length === 0 |
||||
|
? and(lte(postComments.createdAt, params.cutoffAt), eq(postComments.authorUserId, params.userId)) |
||||
|
: and( |
||||
|
lte(postComments.createdAt, params.cutoffAt), |
||||
|
or(eq(postComments.authorUserId, params.userId), inArray(postComments.postId, postIds)), |
||||
|
); |
||||
|
const commentRowsRaw = await dbGlobal.select().from(postComments).where(commentCondition); |
||||
|
|
||||
|
const mediaRefRows = |
||||
|
postIds.length === 0 |
||||
|
? [] |
||||
|
: await dbGlobal |
||||
|
.select() |
||||
|
.from(mediaRefs) |
||||
|
.where(and(eq(mediaRefs.ownerType, MEDIA_REF_OWNER_POST), inArray(mediaRefs.ownerId, postIds))); |
||||
|
const commentRows = commentRowsRaw.map((row) => |
||||
|
params.maskPolicy === "masked" |
||||
|
? sanitizeUserForExport(row as unknown as Record<string, unknown>, params.maskPolicy) |
||||
|
: row, |
||||
|
); |
||||
|
const configRowsMasked = configRows.map((row) => |
||||
|
params.maskPolicy === "masked" |
||||
|
? sanitizeUserForExport(row as unknown as Record<string, unknown>, params.maskPolicy) |
||||
|
: row, |
||||
|
); |
||||
|
|
||||
|
const dataFiles = await Promise.all([ |
||||
|
writeDataFile(params.outputDataDir, "user.json", sanitizedUser), |
||||
|
writeDataFile(params.outputDataDir, "posts.json", postRows), |
||||
|
writeDataFile(params.outputDataDir, "timeline.json", timelineRows), |
||||
|
writeDataFile(params.outputDataDir, "comments.json", commentRows), |
||||
|
writeDataFile(params.outputDataDir, "media-assets.json", mediaRows), |
||||
|
writeDataFile(params.outputDataDir, "media-refs.json", mediaRefRows), |
||||
|
writeDataFile(params.outputDataDir, "user-configs.json", configRowsMasked), |
||||
|
]); |
||||
|
|
||||
|
const totalRows = dataFiles.reduce((sum, item) => sum + item.rowCount, 0); |
||||
|
|
||||
|
return { dataFiles, totalRows }; |
||||
|
} |
||||
|
|
||||
|
export async function listExportUserMediaAssets(params: { |
||||
|
userId: number; |
||||
|
cutoffAt: Date; |
||||
|
}): Promise<Array<{ storageKey: string }>> { |
||||
|
const rows = await dbGlobal |
||||
|
.select({ |
||||
|
storageKey: mediaAssets.storageKey, |
||||
|
}) |
||||
|
.from(mediaAssets) |
||||
|
.where(and(eq(mediaAssets.userId, params.userId), lte(mediaAssets.createdAt, params.cutoffAt))); |
||||
|
return rows; |
||||
|
} |
||||
@ -0,0 +1,63 @@ |
|||||
|
import fs from "node:fs/promises"; |
||||
|
import path from "node:path"; |
||||
|
import { RELATIVE_ASSETS_DIR } from "../../constants/media"; |
||||
|
import { listExportUserMediaAssets } from "./build-data"; |
||||
|
|
||||
|
type ExportFileAsset = { |
||||
|
storageKey: string; |
||||
|
}; |
||||
|
|
||||
|
export type BuildExportFilesResult = { |
||||
|
files: Array<{ file: string; bytes: number }>; |
||||
|
missingCount: number; |
||||
|
}; |
||||
|
|
||||
|
function resolveAssetsBaseDir(): string { |
||||
|
return path.resolve(process.cwd(), RELATIVE_ASSETS_DIR); |
||||
|
} |
||||
|
|
||||
|
export async function buildExportMediaFiles(params: { |
||||
|
userId: number; |
||||
|
cutoffAt: Date; |
||||
|
outputFilesDir: string; |
||||
|
}): Promise<BuildExportFilesResult> { |
||||
|
await fs.mkdir(params.outputFilesDir, { recursive: true }); |
||||
|
|
||||
|
const mediaAssets: ExportFileAsset[] = await listExportUserMediaAssets({ |
||||
|
userId: params.userId, |
||||
|
cutoffAt: params.cutoffAt, |
||||
|
}); |
||||
|
const assetsBaseDir = resolveAssetsBaseDir(); |
||||
|
let missingCount = 0; |
||||
|
const files: Array<{ file: string; bytes: number }> = []; |
||||
|
|
||||
|
for (const asset of mediaAssets) { |
||||
|
const key = asset.storageKey; |
||||
|
if (!key || key.includes("..") || path.isAbsolute(key)) { |
||||
|
missingCount += 1; |
||||
|
continue; |
||||
|
} |
||||
|
|
||||
|
const fromPath = path.resolve(assetsBaseDir, key); |
||||
|
const toPath = path.resolve(params.outputFilesDir, key); |
||||
|
const relativeOutputPath = `files/${key}`; |
||||
|
await fs.mkdir(path.dirname(toPath), { recursive: true }); |
||||
|
|
||||
|
try { |
||||
|
await fs.copyFile(fromPath, toPath); |
||||
|
const stat = await fs.stat(toPath); |
||||
|
files.push({ file: relativeOutputPath, bytes: stat.size }); |
||||
|
} catch (error) { |
||||
|
if ((error as NodeJS.ErrnoException).code === "ENOENT") { |
||||
|
missingCount += 1; |
||||
|
continue; |
||||
|
} |
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return { |
||||
|
files, |
||||
|
missingCount, |
||||
|
}; |
||||
|
} |
||||
@ -0,0 +1,47 @@ |
|||||
|
import { describe, expect, test } from "bun:test"; |
||||
|
import { buildExportManifest } from "./build-manifest"; |
||||
|
|
||||
|
describe("buildExportManifest", () => { |
||||
|
test("builds manifest with required fields and checksums", () => { |
||||
|
const exportedAt = new Date("2026-04-24T10:00:00.000Z"); |
||||
|
const cutoffAt = new Date("2026-04-24T09:00:00.000Z"); |
||||
|
|
||||
|
const manifest = buildExportManifest({ |
||||
|
schemaVersion: 1, |
||||
|
userId: 123, |
||||
|
maskPolicy: "masked", |
||||
|
exportedAt, |
||||
|
exportCutoffAt: cutoffAt, |
||||
|
stats: { |
||||
|
dataRows: 10, |
||||
|
files: 1, |
||||
|
bytes: 1024, |
||||
|
}, |
||||
|
dataChecksums: [ |
||||
|
{ file: "data/user.json", sha256: "sha256-user" }, |
||||
|
{ file: "data/media-assets.json", sha256: "sha256-media" }, |
||||
|
], |
||||
|
fileChecksums: [{ file: "files/a.webp", sha256: "sha256-file-a" }], |
||||
|
}); |
||||
|
|
||||
|
expect(manifest).toEqual({ |
||||
|
schemaVersion: 1, |
||||
|
exportedAt: exportedAt.toISOString(), |
||||
|
exportCutoffAt: cutoffAt.toISOString(), |
||||
|
userId: 123, |
||||
|
maskPolicy: "masked", |
||||
|
stats: { |
||||
|
dataRows: 10, |
||||
|
files: 1, |
||||
|
bytes: 1024, |
||||
|
}, |
||||
|
checksums: { |
||||
|
data: [ |
||||
|
{ file: "data/user.json", sha256: "sha256-user" }, |
||||
|
{ file: "data/media-assets.json", sha256: "sha256-media" }, |
||||
|
], |
||||
|
files: [{ file: "files/a.webp", sha256: "sha256-file-a" }], |
||||
|
}, |
||||
|
}); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,44 @@ |
|||||
|
import type { ExportMaskPolicy } from "../../constants/export"; |
||||
|
|
||||
|
type ExportManifestStats = { |
||||
|
dataRows: number; |
||||
|
files: number; |
||||
|
bytes: number; |
||||
|
}; |
||||
|
|
||||
|
export type ExportManifest = { |
||||
|
schemaVersion: number; |
||||
|
exportedAt: string; |
||||
|
exportCutoffAt: string; |
||||
|
userId: number; |
||||
|
maskPolicy: ExportMaskPolicy; |
||||
|
stats: ExportManifestStats; |
||||
|
checksums: { |
||||
|
data: Array<{ file: string; sha256: string }>; |
||||
|
files: Array<{ file: string; sha256: string }>; |
||||
|
}; |
||||
|
}; |
||||
|
|
||||
|
export function buildExportManifest(params: { |
||||
|
schemaVersion: number; |
||||
|
userId: number; |
||||
|
maskPolicy: ExportMaskPolicy; |
||||
|
exportedAt: Date; |
||||
|
exportCutoffAt: Date; |
||||
|
stats: ExportManifestStats; |
||||
|
dataChecksums: Array<{ file: string; sha256: string }>; |
||||
|
fileChecksums: Array<{ file: string; sha256: string }>; |
||||
|
}): ExportManifest { |
||||
|
return { |
||||
|
schemaVersion: params.schemaVersion, |
||||
|
exportedAt: params.exportedAt.toISOString(), |
||||
|
exportCutoffAt: params.exportCutoffAt.toISOString(), |
||||
|
userId: params.userId, |
||||
|
maskPolicy: params.maskPolicy, |
||||
|
stats: params.stats, |
||||
|
checksums: { |
||||
|
data: params.dataChecksums, |
||||
|
files: params.fileChecksums, |
||||
|
}, |
||||
|
}; |
||||
|
} |
||||
@ -0,0 +1,153 @@ |
|||||
|
import { beforeAll, beforeEach, describe, expect, mock, test } from "bun:test"; |
||||
|
import { eq, inArray } from "drizzle-orm"; |
||||
|
|
||||
|
process.env.DATABASE_URL ??= "file:./packages/drizzle-pkg/db.sqlite"; |
||||
|
|
||||
|
const { dbGlobal } = await import("drizzle-pkg/database/sqlite/db-bun"); |
||||
|
mock.module("drizzle-pkg/lib/db", () => ({ dbGlobal })); |
||||
|
|
||||
|
const { users } = await import("drizzle-pkg/lib/schema/auth"); |
||||
|
const { userExportTasks } = await import("drizzle-pkg/lib/schema/export"); |
||||
|
const { |
||||
|
claimNextQueuedTask, |
||||
|
createExportTask, |
||||
|
getExportTaskForUser, |
||||
|
listExportTasksByUser, |
||||
|
markExportTaskFailed, |
||||
|
markExportTaskRunning, |
||||
|
markExportTaskSucceeded, |
||||
|
} = await import("./jobs"); |
||||
|
|
||||
|
const USER_1 = { id: 910001, username: "export_jobs_u1", password: "pw1" }; |
||||
|
const USER_2 = { id: 910002, username: "export_jobs_u2", password: "pw2" }; |
||||
|
const TEST_USERS: Array<{ id: number; username: string; password: string }> = [USER_1, USER_2]; |
||||
|
|
||||
|
async function resetRows() { |
||||
|
const ids = [USER_1.id, USER_2.id]; |
||||
|
await dbGlobal.delete(userExportTasks).where(inArray(userExportTasks.userId, ids)); |
||||
|
await dbGlobal.delete(users).where(inArray(users.id, ids)); |
||||
|
} |
||||
|
|
||||
|
describe("export jobs service", () => { |
||||
|
beforeAll(async () => { |
||||
|
await resetRows(); |
||||
|
}); |
||||
|
|
||||
|
beforeEach(async () => { |
||||
|
await resetRows(); |
||||
|
await dbGlobal.insert(users).values(TEST_USERS); |
||||
|
}); |
||||
|
|
||||
|
test("createExportTask creates queued task with given policy", async () => { |
||||
|
const task = await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
|
||||
|
expect(task).toBeTruthy(); |
||||
|
expect(task.userId).toBe(USER_1.id); |
||||
|
expect(task.status).toBe("queued"); |
||||
|
expect(task.maskPolicy).toBe("masked"); |
||||
|
expect(task.exportCutoffAt).toBeNull(); |
||||
|
}); |
||||
|
|
||||
|
test("markExportTaskRunning writes exportCutoffAt timestamp", async () => { |
||||
|
const task = await createExportTask({ userId: USER_1.id, maskPolicy: "raw" }); |
||||
|
|
||||
|
const running = await markExportTaskRunning(task.id); |
||||
|
expect(running).toBeTruthy(); |
||||
|
expect(running?.status).toBe("running"); |
||||
|
expect(running?.exportCutoffAt).toBeInstanceOf(Date); |
||||
|
|
||||
|
const [fromDb] = await dbGlobal |
||||
|
.select() |
||||
|
.from(userExportTasks) |
||||
|
.where(eq(userExportTasks.id, task.id)) |
||||
|
.limit(1); |
||||
|
expect(fromDb?.exportCutoffAt).toBeInstanceOf(Date); |
||||
|
}); |
||||
|
|
||||
|
test("listExportTasksByUser only returns the requested user tasks", async () => { |
||||
|
const firstU1 = await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
await markExportTaskRunning(firstU1.id); |
||||
|
await markExportTaskSucceeded(firstU1.id, { |
||||
|
outputDir: "/tmp/export-u1-1", |
||||
|
outputName: "export-u1-1.zip", |
||||
|
totalBytes: 128, |
||||
|
expiresAt: new Date("2026-04-24T00:00:00.000Z"), |
||||
|
}); |
||||
|
await createExportTask({ userId: USER_2.id, maskPolicy: "raw" }); |
||||
|
await createExportTask({ userId: USER_1.id, maskPolicy: "raw" }); |
||||
|
|
||||
|
const rows = await listExportTasksByUser(USER_1.id); |
||||
|
expect(rows.length).toBe(2); |
||||
|
expect(rows.every((row) => row.userId === USER_1.id)).toBe(true); |
||||
|
expect(rows[0]!.id).toBeGreaterThan(rows[1]!.id); |
||||
|
}); |
||||
|
|
||||
|
test("markExportTaskSucceeded and markExportTaskFailed persist payload fields", async () => { |
||||
|
const task1 = await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
await markExportTaskRunning(task1.id); |
||||
|
|
||||
|
await markExportTaskSucceeded(task1.id, { |
||||
|
outputDir: "/tmp/export-1", |
||||
|
outputName: "export-1.zip", |
||||
|
totalBytes: 1234, |
||||
|
expiresAt: new Date("2026-04-24T00:00:00.000Z"), |
||||
|
}); |
||||
|
const task2 = await createExportTask({ userId: USER_1.id, maskPolicy: "raw" }); |
||||
|
await markExportTaskRunning(task2.id); |
||||
|
await markExportTaskFailed(task2.id, { |
||||
|
errorCode: "EXPORT_IO", |
||||
|
errorMessage: "disk full", |
||||
|
}); |
||||
|
|
||||
|
const row1 = await getExportTaskForUser(task1.id, USER_1.id); |
||||
|
const row2 = await getExportTaskForUser(task2.id, USER_1.id); |
||||
|
expect(row1?.status).toBe("succeeded"); |
||||
|
expect(row1?.outputName).toBe("export-1.zip"); |
||||
|
expect(row2?.status).toBe("failed"); |
||||
|
expect(row2?.errorCode).toBe("EXPORT_IO"); |
||||
|
}); |
||||
|
|
||||
|
test("invalid status transition is rejected", async () => { |
||||
|
const task = await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
await expect( |
||||
|
markExportTaskSucceeded(task.id, { |
||||
|
outputDir: "/tmp/export-invalid", |
||||
|
outputName: "export-invalid.zip", |
||||
|
totalBytes: 1, |
||||
|
expiresAt: new Date("2026-04-24T00:00:00.000Z"), |
||||
|
}), |
||||
|
).rejects.toThrow("invalid export task transition"); |
||||
|
}); |
||||
|
|
||||
|
test("getExportTaskForUser enforces ownership", async () => { |
||||
|
const task = await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
|
||||
|
const own = await getExportTaskForUser(task.id, USER_1.id); |
||||
|
const other = await getExportTaskForUser(task.id, USER_2.id); |
||||
|
expect(own?.id).toBe(task.id); |
||||
|
expect(other).toBeNull(); |
||||
|
}); |
||||
|
|
||||
|
test("claimNextQueuedTask returns null when no queued task", async () => { |
||||
|
const task = await claimNextQueuedTask(); |
||||
|
expect(task).toBeNull(); |
||||
|
}); |
||||
|
|
||||
|
test("claimNextQueuedTask claims the earliest queued task once", async () => { |
||||
|
const first = await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
await createExportTask({ userId: USER_2.id, maskPolicy: "raw" }); |
||||
|
|
||||
|
const claimed = await claimNextQueuedTask(); |
||||
|
const nextClaimed = await claimNextQueuedTask(); |
||||
|
expect(claimed?.id).toBe(first.id); |
||||
|
expect(claimed?.status).toBe("running"); |
||||
|
expect(nextClaimed?.id).not.toBe(first.id); |
||||
|
}); |
||||
|
|
||||
|
test("createExportTask rejects when user already has active task", async () => { |
||||
|
await createExportTask({ userId: USER_1.id, maskPolicy: "masked" }); |
||||
|
await expect(createExportTask({ userId: USER_1.id, maskPolicy: "raw" })).rejects.toMatchObject({ |
||||
|
statusCode: 409, |
||||
|
}); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,155 @@ |
|||||
|
import { dbGlobal } from "drizzle-pkg/lib/db"; |
||||
|
import { userExportTasks } from "drizzle-pkg/lib/schema/export"; |
||||
|
import { and, desc, eq, or } from "drizzle-orm"; |
||||
|
import { createError } from "h3"; |
||||
|
import { nextIntegerId } from "../../utils/sqlite-id"; |
||||
|
|
||||
|
type ExportMaskPolicy = "masked" | "raw"; |
||||
|
|
||||
|
async function getExportTaskById(taskId: number) { |
||||
|
const [row] = await dbGlobal |
||||
|
.select() |
||||
|
.from(userExportTasks) |
||||
|
.where(eq(userExportTasks.id, taskId)) |
||||
|
.limit(1); |
||||
|
return row ?? null; |
||||
|
} |
||||
|
|
||||
|
async function getRequiredExportTaskById(taskId: number) { |
||||
|
const row = await getExportTaskById(taskId); |
||||
|
if (!row) { |
||||
|
throw new Error(`export task not found: ${taskId}`); |
||||
|
} |
||||
|
return row; |
||||
|
} |
||||
|
|
||||
|
export async function createExportTask(params: { userId: number; maskPolicy: ExportMaskPolicy }) { |
||||
|
const [activeTask] = await dbGlobal |
||||
|
.select({ id: userExportTasks.id }) |
||||
|
.from(userExportTasks) |
||||
|
.where( |
||||
|
and( |
||||
|
eq(userExportTasks.userId, params.userId), |
||||
|
or(eq(userExportTasks.status, "queued"), eq(userExportTasks.status, "running")), |
||||
|
), |
||||
|
) |
||||
|
.limit(1); |
||||
|
if (activeTask) { |
||||
|
throw createError({ statusCode: 409, statusMessage: "已有导出任务在处理中,请稍后再试" }); |
||||
|
} |
||||
|
const id = await nextIntegerId(userExportTasks, userExportTasks.id); |
||||
|
await dbGlobal.insert(userExportTasks).values({ |
||||
|
id, |
||||
|
userId: params.userId, |
||||
|
maskPolicy: params.maskPolicy, |
||||
|
status: "queued", |
||||
|
}); |
||||
|
return getRequiredExportTaskById(id); |
||||
|
} |
||||
|
|
||||
|
export async function listExportTasksByUser(userId: number) { |
||||
|
return dbGlobal |
||||
|
.select() |
||||
|
.from(userExportTasks) |
||||
|
.where(eq(userExportTasks.userId, userId)) |
||||
|
.orderBy(desc(userExportTasks.id)); |
||||
|
} |
||||
|
|
||||
|
export async function markExportTaskRunning(taskId: number) { |
||||
|
const cutoffAt = new Date(); |
||||
|
await dbGlobal |
||||
|
.update(userExportTasks) |
||||
|
.set({ |
||||
|
status: "running", |
||||
|
exportCutoffAt: cutoffAt, |
||||
|
}) |
||||
|
.where(and(eq(userExportTasks.id, taskId), eq(userExportTasks.status, "queued"))); |
||||
|
const row = await getRequiredExportTaskById(taskId); |
||||
|
if (row.status !== "running" || row.exportCutoffAt?.getTime() !== cutoffAt.getTime()) { |
||||
|
throw new Error(`invalid export task transition for ${taskId}: expected queued -> running`); |
||||
|
} |
||||
|
return row; |
||||
|
} |
||||
|
|
||||
|
export async function claimNextQueuedTask() { |
||||
|
for (let i = 0; i < 5; i += 1) { |
||||
|
const [queued] = await dbGlobal |
||||
|
.select({ id: userExportTasks.id }) |
||||
|
.from(userExportTasks) |
||||
|
.where(eq(userExportTasks.status, "queued")) |
||||
|
.orderBy(userExportTasks.id) |
||||
|
.limit(1); |
||||
|
if (!queued) { |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
try { |
||||
|
return await markExportTaskRunning(queued.id); |
||||
|
} catch (error) { |
||||
|
const message = error instanceof Error ? error.message : ""; |
||||
|
if (!message.includes("invalid export task transition")) { |
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
|
||||
|
export async function markExportTaskSucceeded( |
||||
|
taskId: number, |
||||
|
payload: { |
||||
|
outputDir: string; |
||||
|
outputName: string; |
||||
|
totalBytes: number; |
||||
|
expiresAt: Date; |
||||
|
}, |
||||
|
) { |
||||
|
await dbGlobal |
||||
|
.update(userExportTasks) |
||||
|
.set({ |
||||
|
status: "succeeded", |
||||
|
outputDir: payload.outputDir, |
||||
|
outputName: payload.outputName, |
||||
|
totalBytes: payload.totalBytes, |
||||
|
expiresAt: payload.expiresAt, |
||||
|
errorCode: null, |
||||
|
errorMessage: null, |
||||
|
}) |
||||
|
.where(and(eq(userExportTasks.id, taskId), eq(userExportTasks.status, "running"))); |
||||
|
const row = await getRequiredExportTaskById(taskId); |
||||
|
if (row.status !== "succeeded") { |
||||
|
throw new Error(`invalid export task transition for ${taskId}: expected running -> succeeded`); |
||||
|
} |
||||
|
return row; |
||||
|
} |
||||
|
|
||||
|
export async function markExportTaskFailed( |
||||
|
taskId: number, |
||||
|
payload: { |
||||
|
errorCode: string; |
||||
|
errorMessage: string; |
||||
|
}, |
||||
|
) { |
||||
|
await dbGlobal |
||||
|
.update(userExportTasks) |
||||
|
.set({ |
||||
|
status: "failed", |
||||
|
errorCode: payload.errorCode, |
||||
|
errorMessage: payload.errorMessage, |
||||
|
}) |
||||
|
.where(and(eq(userExportTasks.id, taskId), eq(userExportTasks.status, "running"))); |
||||
|
const row = await getRequiredExportTaskById(taskId); |
||||
|
if (row.status !== "failed") { |
||||
|
throw new Error(`invalid export task transition for ${taskId}: expected running -> failed`); |
||||
|
} |
||||
|
return row; |
||||
|
} |
||||
|
|
||||
|
export async function getExportTaskForUser(taskId: number, userId: number) { |
||||
|
const [row] = await dbGlobal |
||||
|
.select() |
||||
|
.from(userExportTasks) |
||||
|
.where(and(eq(userExportTasks.id, taskId), eq(userExportTasks.userId, userId))) |
||||
|
.limit(1); |
||||
|
return row ?? null; |
||||
|
} |
||||
@ -0,0 +1,103 @@ |
|||||
|
import fs from "node:fs/promises"; |
||||
|
import path from "node:path"; |
||||
|
import { buildExportDataJson } from "#server/service/export/build-data"; |
||||
|
import { buildExportMediaFiles } from "#server/service/export/build-files"; |
||||
|
import { buildExportManifest } from "#server/service/export/build-manifest"; |
||||
|
import { |
||||
|
markExportTaskFailed, |
||||
|
markExportTaskRunning, |
||||
|
markExportTaskSucceeded, |
||||
|
} from "#server/service/export/jobs"; |
||||
|
import { sha256File } from "#server/utils/export-hash"; |
||||
|
|
||||
|
const EXPORT_RESULT_TTL_MS = 24 * 60 * 60 * 1000; |
||||
|
|
||||
|
function resolveExportRootDir() { |
||||
|
return path.resolve(process.cwd(), ".tmp", "exports"); |
||||
|
} |
||||
|
|
||||
|
async function calcChecksums(baseDir: string, files: Array<{ file: string }>) { |
||||
|
return Promise.all( |
||||
|
files.map(async (item) => ({ |
||||
|
file: item.file, |
||||
|
sha256: await sha256File(path.resolve(baseDir, item.file)), |
||||
|
})), |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
export async function runExportTask(taskId: number) { |
||||
|
const runningTask = await markExportTaskRunning(taskId); |
||||
|
return runExportTaskWithRunning(taskId, runningTask); |
||||
|
} |
||||
|
|
||||
|
export async function runExportTaskWithRunning( |
||||
|
taskId: number, |
||||
|
runningTask: Awaited<ReturnType<typeof markExportTaskRunning>>, |
||||
|
) { |
||||
|
const cutoffAt = runningTask.exportCutoffAt ?? new Date(); |
||||
|
const exportRootDir = resolveExportRootDir(); |
||||
|
const outputName = `export-task-${taskId}`; |
||||
|
const outputDir = path.resolve(exportRootDir, outputName); |
||||
|
const outputDataDir = path.resolve(outputDir, "data"); |
||||
|
const outputFilesDir = path.resolve(outputDir, "files"); |
||||
|
|
||||
|
try { |
||||
|
await fs.mkdir(outputDir, { recursive: true }); |
||||
|
|
||||
|
const dataResult = await buildExportDataJson({ |
||||
|
userId: runningTask.userId, |
||||
|
maskPolicy: runningTask.maskPolicy as "masked" | "raw", |
||||
|
cutoffAt, |
||||
|
outputDataDir, |
||||
|
}); |
||||
|
const filesResult = await buildExportMediaFiles({ |
||||
|
userId: runningTask.userId, |
||||
|
cutoffAt, |
||||
|
outputFilesDir, |
||||
|
}); |
||||
|
|
||||
|
const dataChecksums = await calcChecksums(outputDir, dataResult.dataFiles); |
||||
|
const fileChecksums = await calcChecksums(outputDir, filesResult.files); |
||||
|
const filesBytes = filesResult.files.reduce((sum, item) => sum + item.bytes, 0); |
||||
|
|
||||
|
const manifest = buildExportManifest({ |
||||
|
schemaVersion: 1, |
||||
|
userId: runningTask.userId, |
||||
|
maskPolicy: runningTask.maskPolicy as "masked" | "raw", |
||||
|
exportedAt: new Date(), |
||||
|
exportCutoffAt: cutoffAt, |
||||
|
stats: { |
||||
|
dataRows: dataResult.totalRows, |
||||
|
files: filesResult.files.length, |
||||
|
bytes: filesBytes, |
||||
|
}, |
||||
|
dataChecksums, |
||||
|
fileChecksums, |
||||
|
}); |
||||
|
|
||||
|
const manifestPath = path.resolve(outputDir, "manifest.json"); |
||||
|
await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); |
||||
|
|
||||
|
const dataBytes = await Promise.all( |
||||
|
dataResult.dataFiles.map(async (item) => { |
||||
|
const stat = await fs.stat(path.resolve(outputDir, item.file)); |
||||
|
return stat.size; |
||||
|
}), |
||||
|
); |
||||
|
const manifestBytes = (await fs.stat(manifestPath)).size; |
||||
|
const totalBytes = dataBytes.reduce((sum, size) => sum + size, 0) + filesBytes + manifestBytes; |
||||
|
|
||||
|
await markExportTaskSucceeded(taskId, { |
||||
|
outputDir, |
||||
|
outputName, |
||||
|
totalBytes, |
||||
|
expiresAt: new Date(Date.now() + EXPORT_RESULT_TTL_MS), |
||||
|
}); |
||||
|
} catch (error) { |
||||
|
await markExportTaskFailed(taskId, { |
||||
|
errorCode: "EXPORT_BUILD_FAILED", |
||||
|
errorMessage: error instanceof Error ? error.message : "unknown export error", |
||||
|
}); |
||||
|
throw error; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,17 @@ |
|||||
|
import { claimNextQueuedTask } from "#server/service/export/jobs"; |
||||
|
import { runExportTaskWithRunning } from "#server/service/export/run"; |
||||
|
|
||||
|
export default defineTask({ |
||||
|
meta: { |
||||
|
name: "export:process", |
||||
|
description: "Claim and process one queued export task", |
||||
|
}, |
||||
|
async run() { |
||||
|
const task = await claimNextQueuedTask(); |
||||
|
if (!task) { |
||||
|
return { result: "skipped: no queued task" }; |
||||
|
} |
||||
|
await runExportTaskWithRunning(task.id, task); |
||||
|
return { result: "ok", taskId: task.id }; |
||||
|
}, |
||||
|
}); |
||||
@ -0,0 +1,13 @@ |
|||||
|
import fs from "node:fs"; |
||||
|
import { createHash } from "node:crypto"; |
||||
|
|
||||
|
export async function sha256File(filePath: string): Promise<string> { |
||||
|
return new Promise((resolve, reject) => { |
||||
|
const hash = createHash("sha256"); |
||||
|
const stream = fs.createReadStream(filePath); |
||||
|
|
||||
|
stream.on("error", reject); |
||||
|
stream.on("data", (chunk) => hash.update(chunk)); |
||||
|
stream.on("end", () => resolve(hash.digest("hex"))); |
||||
|
}); |
||||
|
} |
||||
@ -0,0 +1,62 @@ |
|||||
|
import { |
||||
|
DEFAULT_MASK_FIELD_KEYWORDS, |
||||
|
NEVER_EXPORT_FIELD_EXACT_NAMES, |
||||
|
type ExportMaskPolicy, |
||||
|
} from "../constants/export"; |
||||
|
|
||||
|
function hitKeyword(field: string, keywords: readonly string[]): boolean { |
||||
|
const name = field.toLowerCase(); |
||||
|
return keywords.some((k) => name.includes(k.toLowerCase())); |
||||
|
} |
||||
|
|
||||
|
function normalizeFieldName(field: string): string { |
||||
|
return field.toLowerCase().replace(/[_-]/g, ""); |
||||
|
} |
||||
|
|
||||
|
function shouldNeverExportField(field: string): boolean { |
||||
|
const normalizedField = normalizeFieldName(field); |
||||
|
return NEVER_EXPORT_FIELD_EXACT_NAMES.includes( |
||||
|
normalizedField as (typeof NEVER_EXPORT_FIELD_EXACT_NAMES)[number], |
||||
|
); |
||||
|
} |
||||
|
|
||||
|
function shouldMaskField(field: string): boolean { |
||||
|
return hitKeyword(field, DEFAULT_MASK_FIELD_KEYWORDS); |
||||
|
} |
||||
|
|
||||
|
function maskValue(value: unknown): unknown { |
||||
|
if (value === null || value === undefined) { |
||||
|
return value; |
||||
|
} |
||||
|
if (typeof value === "string") { |
||||
|
return "***"; |
||||
|
} |
||||
|
if (typeof value === "number") { |
||||
|
return 0; |
||||
|
} |
||||
|
if (typeof value === "boolean") { |
||||
|
return false; |
||||
|
} |
||||
|
return "[masked]"; |
||||
|
} |
||||
|
|
||||
|
export function sanitizeUserForExport( |
||||
|
row: Record<string, unknown>, |
||||
|
maskPolicy: ExportMaskPolicy, |
||||
|
): Record<string, unknown> { |
||||
|
const out: Record<string, unknown> = {}; |
||||
|
for (const [field, value] of Object.entries(row)) { |
||||
|
if (shouldNeverExportField(field)) { |
||||
|
continue; |
||||
|
} |
||||
|
if (maskPolicy === "masked" && shouldMaskField(field)) { |
||||
|
out[field] = maskValue(value); |
||||
|
continue; |
||||
|
} |
||||
|
out[field] = value; |
||||
|
} |
||||
|
return out; |
||||
|
} |
||||
|
|
||||
|
// Backward-compatible alias used by current tests/callers.
|
||||
|
export const applyExportMask = sanitizeUserForExport; |
||||
@ -0,0 +1,82 @@ |
|||||
|
import { describe, expect, test } from "bun:test"; |
||||
|
|
||||
|
import { applyExportMask } from "./export-mask"; |
||||
|
import { parseMeExportRequestBody } from "./me-export-request-body"; |
||||
|
|
||||
|
describe("parseMeExportRequestBody", () => { |
||||
|
test("maskPolicy defaults to masked", () => { |
||||
|
expect(parseMeExportRequestBody({})).toEqual({ maskPolicy: "masked" }); |
||||
|
expect(parseMeExportRequestBody({ maskPolicy: undefined })).toEqual({ maskPolicy: "masked" }); |
||||
|
}); |
||||
|
|
||||
|
test("invalid maskPolicy throws 400", () => { |
||||
|
try { |
||||
|
parseMeExportRequestBody({ maskPolicy: "invalid-policy" }); |
||||
|
expect.unreachable(); |
||||
|
} catch (e: unknown) { |
||||
|
expect(e).toMatchObject({ statusCode: 400 }); |
||||
|
} |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
describe("applyExportMask", () => { |
||||
|
test("never-export fields are excluded", () => { |
||||
|
const input = { |
||||
|
nickname: "dash", |
||||
|
passwordHash: "hash", |
||||
|
resetToken: "rst", |
||||
|
sessionId: "sid", |
||||
|
apiKey: "k", |
||||
|
accessToken: "token", |
||||
|
tokenCount: 3, |
||||
|
}; |
||||
|
|
||||
|
expect(applyExportMask(input, "raw")).toEqual({ |
||||
|
nickname: "dash", |
||||
|
tokenCount: 3, |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
test("never-export fields match case and underscore variants", () => { |
||||
|
const input = { |
||||
|
nickname: "dash", |
||||
|
Access_Token: "token", |
||||
|
SESSION_ID: "sid", |
||||
|
}; |
||||
|
|
||||
|
expect(applyExportMask(input, "raw")).toEqual({ |
||||
|
nickname: "dash", |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
test("raw policy still filters resetToken/apiKey and naming variants", () => { |
||||
|
const input = { |
||||
|
nickname: "dash", |
||||
|
resetToken: "rst-1", |
||||
|
reset_token: "rst-2", |
||||
|
resetPasswordToken: "rst-3", |
||||
|
apiKey: "key-1", |
||||
|
API_KEY: "key-2", |
||||
|
email: "dash@example.com", |
||||
|
}; |
||||
|
|
||||
|
expect(applyExportMask(input, "raw")).toEqual({ |
||||
|
nickname: "dash", |
||||
|
email: "dash@example.com", |
||||
|
}); |
||||
|
}); |
||||
|
|
||||
|
test("masked policy masks email and phone fields", () => { |
||||
|
const input = { |
||||
|
nickname: "dash", |
||||
|
email: "dash@example.com", |
||||
|
phone: "13800138000", |
||||
|
}; |
||||
|
|
||||
|
expect(applyExportMask(input, "masked")).toEqual({ |
||||
|
nickname: "dash", |
||||
|
email: "***", |
||||
|
phone: "***", |
||||
|
}); |
||||
|
}); |
||||
|
}); |
||||
@ -0,0 +1,20 @@ |
|||||
|
import { EXPORT_MASK_POLICIES, type ExportMaskPolicy } from "../constants/export"; |
||||
|
import { createError } from "h3"; |
||||
|
|
||||
|
type MeExportRequestBody = { |
||||
|
maskPolicy?: unknown; |
||||
|
}; |
||||
|
|
||||
|
export function parseMeExportRequestBody(body: unknown): { maskPolicy: ExportMaskPolicy } { |
||||
|
const payload = (body && typeof body === "object" ? body : {}) as MeExportRequestBody; |
||||
|
const raw = payload.maskPolicy; |
||||
|
if (raw === undefined || raw === null || raw === "") { |
||||
|
return { maskPolicy: "masked" }; |
||||
|
} |
||||
|
|
||||
|
if (typeof raw !== "string" || !EXPORT_MASK_POLICIES.includes(raw as ExportMaskPolicy)) { |
||||
|
throw createError({ statusCode: 400, statusMessage: "maskPolicy 非法,仅支持 masked|raw" }); |
||||
|
} |
||||
|
|
||||
|
return { maskPolicy: raw as ExportMaskPolicy }; |
||||
|
} |
||||
Loading…
Reference in new issue