import { PrismaClient } from "@prisma/client"; import { createHash, randomUUID } from "node:crypto"; import { afterAll, beforeEach, describe, expect, it, vi } from "vitest"; import { assertSafeTestDatabaseUrl } from "../../../prisma/database-safety.js"; import { ModerationWorker, decideModeration } from "./moderation-worker.js"; const prisma = new PrismaClient(); describe("moderation worker with real PostgreSQL", () => { beforeEach(async () => { assertSafeTestDatabaseUrl(process.env.DATABASE_URL ?? ""); await prisma.$connect(); await prisma.outboxEvent.deleteMany(); await prisma.$executeRawUnsafe(`TRUNCATE TABLE "accounts" CASCADE`); }); afterAll(() => prisma.$disconnect()); it.each([ ["ordinary sea note", "APPROVED", "IN_POOL"], ["contains reject-word", "REJECTED", "CLOSED"], ["contains review-word", "MANUAL_REVIEW", "CLOSED"], ] as const)( "maps simulated rule %#", async (content, reviewStatus, poolStatus) => { const { bottle } = await fixture(content); expect(await new ModerationWorker(prisma).runOnce()).toBe(true); const updated = await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id }, }); expect(updated).toMatchObject({ reviewStatus, poolStatus }); expect(updated.approvedAt !== null).toBe(reviewStatus === "APPROVED"); expect((await prisma.moderationTask.findFirstOrThrow()).status).toBe( "COMPLETED", ); expect((await prisma.outboxEvent.findFirstOrThrow()).status).toBe( "PUBLISHED", ); expect(await new ModerationWorker(prisma).runOnce()).toBe(false); }, ); it.each([ ["PENDING", null, 0], ["PROCESSING", new Date(Date.now() - 60_000), 5], ] as const)( "leaves %s MESSAGE_CREATED for the API relay", async (status, lockedAt, attempts) => { await prisma.account.create({ data: { phoneCiphertext: Buffer.from("cipher"), phoneHmac: randomUUID(), }, }); const event = await prisma.outboxEvent.create({ data: { aggregateType: "MESSAGE", aggregateId: randomUUID(), eventType: "MESSAGE_CREATED", dedupeKey: randomUUID(), payload: { messageId: randomUUID(), conversationId: randomUUID(), }, status, lockedAt, lockToken: lockedAt ? randomUUID() : null, attempts, }, }); expect(await new ModerationWorker(prisma).runOnce()).toBe(false); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status, attempts, lockedAt }); }, ); it("is fail-closed and exponentially reschedules failures", async () => { const { bottle, event } = await fixture("ordinary"); const worker = new ModerationWorker(prisma, () => { throw new Error("provider unavailable"); }); await expect(worker.runOnce()).resolves.toBe(true); expect( await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }), ).toMatchObject({ reviewStatus: "REVIEWING", poolStatus: "CLOSED", }); const failed = await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id }, }); expect(failed.status).toBe("FAILED"); expect(failed.attempts).toBe(1); expect(failed.nextRetryAt.getTime()).toBeGreaterThan(Date.now()); }); it("stops retrying at max attempts while remaining fail-closed", async () => { const { bottle, event } = await fixture("ordinary"); await prisma.outboxEvent.update({ where: { id: event.id }, data: { attempts: 4 }, }); const worker = new ModerationWorker(prisma, () => { throw new Error("provider unavailable"); }); await worker.runOnce(); const failed = await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id }, }); expect(failed).toMatchObject({ status: "FAILED", attempts: 5 }); expect(failed.nextRetryAt.getUTCFullYear()).toBe(9999); expect(await worker.runOnce()).toBe(false); expect( await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }), ).toMatchObject({ reviewStatus: "REVIEWING", poolStatus: "CLOSED" }); }); it("does not publish when the task payload points at another task", async () => { const first = await fixture("ordinary"); const second = await fixture("another"); await prisma.outboxEvent.delete({ where: { id: second.event.id } }); await prisma.outboxEvent.update({ where: { id: first.event.id }, data: { payload: { bottleId: first.bottle.id, taskId: second.task.id } }, }); await new ModerationWorker(prisma).runOnce(); expect( ( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: first.event.id }, }) ).status, ).toBe("FAILED"); expect( await prisma.bottle.findUniqueOrThrow({ where: { id: first.bottle.id } }), ).toMatchObject({ reviewStatus: "REVIEWING", poolStatus: "CLOSED" }); }); it("lets two workers claim one event only once", async () => { await fixture("ordinary"); const entered = vi.fn(); const decide = async (text: string) => { entered(); await new Promise((resolve) => setTimeout(resolve, 100)); return decideModeration(text); }; const outcomes = await Promise.all([ new ModerationWorker(prisma, decide).runOnce(), new ModerationWorker(prisma, decide).runOnce(), ]); expect(outcomes.sort()).toEqual([false, true]); expect(entered).toHaveBeenCalledTimes(1); expect((await prisma.outboxEvent.findFirstOrThrow()).attempts).toBe(1); }); it("recovers an expired PROCESSING lease", async () => { const { event } = await fixture("ordinary"); await prisma.outboxEvent.update({ where: { id: event.id }, data: { status: "PROCESSING", lockedAt: new Date(Date.now() - 60_000) }, }); expect(await new ModerationWorker(prisma, undefined, 1000).runOnce()).toBe( true, ); expect( (await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } })) .status, ).toBe("PUBLISHED"); }); it("does not let an expired lease report failure after a new owner publishes", async () => { const { bottle, event } = await fixture("ordinary"); let rejectFirst!: (reason: Error) => void; const firstDecision = new Promise>( (_resolve, reject) => { rejectFirst = reject; }, ); const firstRun = new ModerationWorker( prisma, () => firstDecision, 1000, ).runOnce(); await waitForProcessing(event.id); await prisma.outboxEvent.update({ where: { id: event.id }, data: { lockedAt: new Date(Date.now() - 60_000) }, }); await new ModerationWorker(prisma, undefined, 1000).runOnce(); rejectFirst(new Error("secret-body-provider-error")); await firstRun; expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "PUBLISHED" }); expect( await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }), ).toMatchObject({ reviewStatus: "APPROVED", poolStatus: "IN_POOL" }); }); it("does not let an expired lease publish or overwrite a new owner's decision", async () => { const { bottle, event, task } = await fixture("ordinary"); let resolveFirst!: (decision: ReturnType) => void; const firstDecision = new Promise>( (resolve) => { resolveFirst = resolve; }, ); const firstRun = new ModerationWorker( prisma, () => firstDecision, 1000, ).runOnce(); await waitForProcessing(event.id); await prisma.outboxEvent.update({ where: { id: event.id }, data: { lockedAt: new Date(Date.now() - 60_000) }, }); await new ModerationWorker( prisma, () => ({ reviewStatus: "APPROVED", poolStatus: "IN_POOL", labels: ["NEW_OWNER"], }), 1000, ).runOnce(); resolveFirst({ reviewStatus: "REJECTED", poolStatus: "CLOSED", labels: ["OLD_OWNER"], }); await firstRun; expect( await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }), ).toMatchObject({ reviewStatus: "APPROVED", poolStatus: "IN_POOL" }); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }), ).toMatchObject({ decision: "APPROVED", riskLabels: ["NEW_OWNER"] }); }); it("publishes a replay when the completed bottle task matches current state", async () => { const { bottle, event, task } = await fixture("ordinary"); await prisma.$transaction([ prisma.moderationTask.update({ where: { id: task.id }, data: { status: "COMPLETED", decision: "APPROVED", result: { decision: "APPROVED" }, reviewedAt: new Date(), }, }), prisma.bottle.update({ where: { id: bottle.id }, data: { reviewStatus: "APPROVED", poolStatus: "IN_POOL" }, }), ]); const decide = vi.fn(decideModeration); expect(await new ModerationWorker(prisma, decide).runOnce()).toBe(true); expect(decide).not.toHaveBeenCalled(); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "PUBLISHED" }); }); it("fails a completed bottle event whose result drifted from current state", async () => { const { event, task } = await fixture("secret bottle text"); await prisma.moderationTask.update({ where: { id: task.id }, data: { status: "COMPLETED", decision: "APPROVED", result: { decision: "APPROVED" }, reviewedAt: new Date(), }, }); const logs: unknown[] = []; await new ModerationWorker(prisma, undefined, undefined, { error: (entry) => logs.push(entry), }).runOnce(); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "FAILED" }); expect(JSON.stringify(logs)).not.toContain("secret bottle text"); }); it("fails the event and leaves the task open when bottle state drift prevents the update", async () => { const { bottle, event, task } = await fixture("ordinary"); await prisma.bottle.update({ where: { id: bottle.id }, data: { reviewStatus: "APPROVED", poolStatus: "IN_POOL" }, }); expect(await new ModerationWorker(prisma).runOnce()).toBe(true); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "FAILED", attempts: 1 }); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }), ).toMatchObject({ status: "PENDING", decision: null }); }); it("reaps a stale PROCESSING event already at max attempts without handling it", async () => { const { event } = await fixture("ordinary"); await prisma.outboxEvent.update({ where: { id: event.id }, data: { status: "PROCESSING", attempts: 5, lockedAt: new Date(Date.now() - 60_000), }, }); const decide = vi.fn(decideModeration); expect(await new ModerationWorker(prisma, decide, 1000).runOnce()).toBe( false, ); expect(decide).not.toHaveBeenCalled(); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "FAILED", attempts: 5, lockToken: null, lockedAt: null, }); }); it.each([ ["ordinary", "APPROVED"], ["reject-word", "REJECTED"], ["review-word", "MANUAL_REVIEW"], ] as const)( "moderates normalized profile content with rule %#", async (nickname, status) => { const { profile, event, task } = await profileFixture( nickname, "profile bio", ); expect(await new ModerationWorker(prisma).runOnce()).toBe(true); expect( ( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }) ).reviewStatus, ).toBe(status); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id }, }), ).toMatchObject({ status: "COMPLETED", decision: status }); expect( ( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id }, }) ).status, ).toBe("PUBLISHED"); }, ); it("publishes a profile event that becomes stale while moderation is running", async () => { const { createHash } = await import("node:crypto"); const { profile, event, task } = await profileFixture("ordinary", "first"); const currentText = JSON.stringify(["reject-word", "#abcdef", "second"]); const decide = vi.fn(async (text: string) => { await prisma.$transaction([ prisma.anonymousProfile.update({ where: { id: profile.id }, data: { nickname: "reject-word", bio: "second", version: 2, reviewStatus: "REVIEWING", }, }), prisma.moderationTask.update({ where: { id: task.id }, data: { payloadHash: createHash("sha256").update(currentText).digest("hex"), status: "PENDING", }, }), ]); return decideModeration(text); }); expect(await new ModerationWorker(prisma, decide).runOnce()).toBe(true); expect(decide).toHaveBeenCalledTimes(1); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "PUBLISHED", lockToken: null, lockedAt: null, }); expect( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }), ).toMatchObject({ version: 2, reviewStatus: "REVIEWING" }); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }), ).toMatchObject({ status: "PENDING", decision: null }); }); it("fails a same-version profile event whose task hash drifted", async () => { const { profile, event, task } = await profileFixture("ordinary", "first"); await prisma.moderationTask.update({ where: { id: task.id }, data: { payloadHash: "0".repeat(64) }, }); expect(await new ModerationWorker(prisma).runOnce()).toBe(true); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "FAILED", attempts: 1, lockToken: null, lockedAt: null, }); expect( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }), ).toMatchObject({ version: 1, reviewStatus: "REVIEWING" }); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }), ).toMatchObject({ status: "PENDING", decision: null }); }); it("fails a replay when the completed profile task drifted from current state", async () => { const { profile, event, task } = await profileFixture("ordinary", "first"); await prisma.moderationTask.update({ where: { id: task.id }, data: { status: "COMPLETED", decision: "APPROVED", result: { decision: "APPROVED" }, reviewedAt: new Date(), }, }); const decide = vi.fn(decideModeration); expect(await new ModerationWorker(prisma, decide).runOnce()).toBe(true); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "FAILED" }); expect( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }), ).toMatchObject({ reviewStatus: "REVIEWING" }); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }), ).toMatchObject({ status: "COMPLETED", decision: "APPROVED" }); }); it("fails a completed profile task with a non-terminal reviewing decision", async () => { const { profile, event, task } = await profileFixture("ordinary", "first"); await prisma.moderationTask.update({ where: { id: task.id }, data: { status: "COMPLETED", decision: "REVIEWING", result: { decision: "REVIEWING" }, reviewedAt: new Date(), }, }); expect(await new ModerationWorker(prisma).runOnce()).toBe(true); expect( await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }), ).toMatchObject({ status: "FAILED", attempts: 1, lockToken: null, lockedAt: null, }); expect( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }), ).toMatchObject({ reviewStatus: "REVIEWING" }); expect( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }), ).toMatchObject({ status: "COMPLETED", decision: "REVIEWING" }); }); it("logs a safe error category without payload or exception message", async () => { const { event } = await fixture("secret正文"); const entries: unknown[] = []; await new ModerationWorker( prisma, () => { throw new TypeError("provider leaked secret正文"); }, undefined, { error: (entry) => entries.push(entry) }, ).runOnce(); expect(entries).toEqual([ { eventId: event.id, eventType: "BOTTLE_MODERATION_REQUESTED", attempt: 1, errorClass: "TypeError", }, ]); expect(JSON.stringify(entries)).not.toContain("secret正文"); expect(JSON.stringify(entries)).not.toContain("provider leaked"); }); it("publishes an old profile event without letting it overwrite a newer version", async () => { const { createHash } = await import("node:crypto"); const { profile, event, task } = await profileFixture("ordinary", "first"); const currentText = JSON.stringify(["reject-word", "#abcdef", "second"]); await prisma.$transaction([ prisma.anonymousProfile.update({ where: { id: profile.id }, data: { nickname: "reject-word", bio: "second", version: 2, reviewStatus: "REVIEWING", }, }), prisma.moderationTask.update({ where: { id: task.id }, data: { payloadHash: createHash("sha256").update(currentText).digest("hex"), }, }), prisma.outboxEvent.create({ data: { aggregateType: "PROFILE", aggregateId: profile.id, eventType: "PROFILE_MODERATION_REQUESTED", dedupeKey: `profile-moderation:${profile.id}:2`, payload: { profileId: profile.id, accountId: profile.accountId, taskId: task.id, }, }, }), ]); const worker = new ModerationWorker(prisma); expect(await worker.runOnce()).toBe(true); expect( (await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } })) .status, ).toBe("PUBLISHED"); expect( ( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }) ).reviewStatus, ).toBe("REVIEWING"); expect( ( await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id }, }) ).status, ).toBe("PENDING"); expect(await worker.runOnce()).toBe(true); expect( ( await prisma.anonymousProfile.findUniqueOrThrow({ where: { id: profile.id }, }) ).reviewStatus, ).toBe("REJECTED"); }); async function fixture(contentText: string) { const account = await prisma.account.create({ data: { phoneCiphertext: Buffer.from("cipher"), phoneHmac: randomUUID() }, }); const bottle = await prisma.bottle.create({ data: { authorId: account.id, clientRequestId: randomUUID(), contentText, reviewStatus: "REVIEWING", poolStatus: "CLOSED", }, }); const task = await prisma.moderationTask.create({ data: { targetType: "BOTTLE", targetId: bottle.id, provider: "SIMULATED", payloadHash: createHash("sha256").update(contentText).digest("hex"), riskLabels: [], }, }); const event = await prisma.outboxEvent.create({ data: { aggregateType: "BOTTLE", aggregateId: bottle.id, eventType: "BOTTLE_MODERATION_REQUESTED", dedupeKey: randomUUID(), payload: { bottleId: bottle.id, taskId: task.id }, }, }); return { bottle, task, event }; } async function profileFixture(nickname: string, bio: string) { const { createHash } = await import("node:crypto"); const account = await prisma.account.create({ data: { phoneCiphertext: Buffer.from("cipher"), phoneHmac: randomUUID() }, }); const profile = await prisma.anonymousProfile.create({ data: { accountId: account.id, nickname, avatarColor: "#abcdef", bio, version: 1, }, }); const text = JSON.stringify([nickname, "#abcdef", bio]); const task = await prisma.moderationTask.create({ data: { targetType: "PROFILE", targetId: profile.id, provider: "SIMULATED", payloadHash: createHash("sha256").update(text).digest("hex"), riskLabels: [], }, }); const event = await prisma.outboxEvent.create({ data: { aggregateType: "PROFILE", aggregateId: profile.id, eventType: "PROFILE_MODERATION_REQUESTED", dedupeKey: `profile-moderation:${profile.id}:1`, payload: { profileId: profile.id, accountId: account.id, taskId: task.id, }, }, }); return { profile, task, event }; } async function waitForProcessing(id: string) { for (let attempt = 0; attempt < 100; attempt += 1) { const event = await prisma.outboxEvent.findUniqueOrThrow({ where: { id }, }); if (event.status === "PROCESSING") return; await new Promise((resolve) => setTimeout(resolve, 10)); } throw new Error("worker did not claim event"); } });