Browse Source

feat(export): implement export task limits and environment configuration

- Added environment variables to control the maximum number of running and queued export tasks, as well as the maximum retained bytes for exports.
- Implemented logic in the export job service to enforce these limits, providing appropriate error messages when limits are exceeded.
- Enhanced the export task creation process to ensure better resource management and prevent server overload.

These changes improve the stability and reliability of the export functionality by managing task concurrency and resource usage.
main
npmrun 2 weeks ago
parent
commit
b0102afd0d
  1. 4
      .env.example
  2. 74
      server/service/export/jobs.ts

4
.env.example

@ -4,6 +4,10 @@ NITRO_PORT=3399
STATIC_DIR=static
MEDIA_UPLOAD_SUBDIR=media
TMP_DIR=.tmp
# 导出任务限制(防止服务器快速占满)
EXPORT_MAX_RUNNING_TASKS=2
EXPORT_MAX_QUEUED_TASKS=30
EXPORT_MAX_RETAINED_BYTES=2147483648
# 站点对外根 URL(含协议与域名,可带端口)。用于:① 媒体库复制绝对链接 ② 文章/资料里绝对地址图片是否计为本站 /static/media/ 引用。生产环境务必设置,与浏览器访问地址一致。
NUXT_PUBLIC_SITE_URL=https://example.com
# Optional: first admin for an empty instance. Creates an admin only when no user has role=admin yet (same username/password rules as registration).

74
server/service/export/jobs.ts

@ -2,12 +2,38 @@ import fs from "node:fs/promises";
import path from "node:path";
import { dbGlobal } from "drizzle-pkg/lib/db";
import { userExportTasks } from "drizzle-pkg/lib/schema/export";
import { and, desc, eq, or } from "drizzle-orm";
import { and, desc, eq, or, sql } from "drizzle-orm";
import { nextIntegerId } from "../../utils/sqlite-id";
import { RELATIVE_TMP_DIR } from "#server/constants/media";
type ExportMaskPolicy = "masked" | "raw";
function positiveIntFromEnv(name: string, fallback: number): number {
const raw = process.env[name];
if (!raw) {
return fallback;
}
const n = Number(raw);
return Number.isInteger(n) && n > 0 ? n : fallback;
}
function positiveBigIntFromEnv(name: string, fallback: bigint): bigint {
const raw = process.env[name];
if (!raw) {
return fallback;
}
try {
const n = BigInt(raw);
return n > 0n ? n : fallback;
} catch {
return fallback;
}
}
const EXPORT_MAX_RUNNING_TASKS = positiveIntFromEnv("EXPORT_MAX_RUNNING_TASKS", 2);
const EXPORT_MAX_QUEUED_TASKS = positiveIntFromEnv("EXPORT_MAX_QUEUED_TASKS", 30);
const EXPORT_MAX_RETAINED_BYTES = positiveBigIntFromEnv("EXPORT_MAX_RETAINED_BYTES", 2n * 1024n * 1024n * 1024n);
function exportRootDir(): string {
return path.resolve(process.cwd(), RELATIVE_TMP_DIR, "exports");
}
@ -61,6 +87,52 @@ export async function createExportTask(params: { userId: number; maskPolicy: Exp
if (activeTask) {
throw { statusCode: 409, statusMessage: "已有导出任务在处理中,请稍后再试" };
}
const runningRows = await dbGlobal
.select({ runningCount: sql<number>`count(*)` })
.from(userExportTasks)
.where(eq(userExportTasks.status, "running"));
const runningCount = Number(runningRows[0]?.runningCount ?? 0);
if (runningCount >= EXPORT_MAX_RUNNING_TASKS) {
throw {
statusCode: 429,
statusMessage: "导出任务繁忙,请稍后重试",
};
}
const queuedRows = await dbGlobal
.select({ queuedCount: sql<number>`count(*)` })
.from(userExportTasks)
.where(eq(userExportTasks.status, "queued"));
const queuedCount = Number(queuedRows[0]?.queuedCount ?? 0);
if (queuedCount >= EXPORT_MAX_QUEUED_TASKS) {
throw {
statusCode: 429,
statusMessage: "导出排队过多,请稍后重试",
};
}
const now = new Date();
const retainedRows = await dbGlobal
.select({
retainedBytes: sql<string>`coalesce(sum(${userExportTasks.totalBytes}), 0)`,
})
.from(userExportTasks)
.where(
and(
eq(userExportTasks.status, "succeeded"),
sql`${userExportTasks.expiresAt} IS NOT NULL`,
sql`${userExportTasks.expiresAt} > ${now}`,
),
);
const retainedBytes = retainedRows[0]?.retainedBytes ?? "0";
if (BigInt(retainedBytes) >= EXPORT_MAX_RETAINED_BYTES) {
throw {
statusCode: 507,
statusMessage: "导出空间已达上限,请先删除旧导出后重试",
};
}
const id = await nextIntegerId(userExportTasks, userExportTasks.id);
await dbGlobal.insert(userExportTasks).values({
id,

Loading…
Cancel
Save