Files
plp/apps/worker/src/moderation-worker.spec.ts
T

664 lines
22 KiB
TypeScript

import { PrismaClient } from "@prisma/client";
import { createHash, randomUUID } from "node:crypto";
import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
import { assertSafeTestDatabaseUrl } from "../../../prisma/database-safety.js";
import { ModerationWorker, decideModeration } from "./moderation-worker.js";
const prisma = new PrismaClient();
describe("moderation worker with real PostgreSQL", () => {
beforeEach(async () => {
assertSafeTestDatabaseUrl(process.env.DATABASE_URL ?? "");
await prisma.$connect();
await prisma.outboxEvent.deleteMany();
await prisma.$executeRawUnsafe(`TRUNCATE TABLE "accounts" CASCADE`);
});
afterAll(() => prisma.$disconnect());
it.each([
["ordinary sea note", "APPROVED", "IN_POOL"],
["contains reject-word", "REJECTED", "CLOSED"],
["contains review-word", "MANUAL_REVIEW", "CLOSED"],
] as const)(
"maps simulated rule %#",
async (content, reviewStatus, poolStatus) => {
const { bottle } = await fixture(content);
expect(await new ModerationWorker(prisma).runOnce()).toBe(true);
const updated = await prisma.bottle.findUniqueOrThrow({
where: { id: bottle.id },
});
expect(updated).toMatchObject({ reviewStatus, poolStatus });
expect(updated.approvedAt !== null).toBe(reviewStatus === "APPROVED");
expect((await prisma.moderationTask.findFirstOrThrow()).status).toBe(
"COMPLETED",
);
expect((await prisma.outboxEvent.findFirstOrThrow()).status).toBe(
"PUBLISHED",
);
expect(await new ModerationWorker(prisma).runOnce()).toBe(false);
},
);
it.each([
["PENDING", null, 0],
["PROCESSING", new Date(Date.now() - 60_000), 5],
] as const)(
"leaves %s MESSAGE_CREATED for the API relay",
async (status, lockedAt, attempts) => {
await prisma.account.create({
data: {
phoneCiphertext: Buffer.from("cipher"),
phoneHmac: randomUUID(),
},
});
const event = await prisma.outboxEvent.create({
data: {
aggregateType: "MESSAGE",
aggregateId: randomUUID(),
eventType: "MESSAGE_CREATED",
dedupeKey: randomUUID(),
payload: {
messageId: randomUUID(),
conversationId: randomUUID(),
},
status,
lockedAt,
lockToken: lockedAt ? randomUUID() : null,
attempts,
},
});
expect(await new ModerationWorker(prisma).runOnce()).toBe(false);
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({ status, attempts, lockedAt });
},
);
it("is fail-closed and exponentially reschedules failures", async () => {
const { bottle, event } = await fixture("ordinary");
const worker = new ModerationWorker(prisma, () => {
throw new Error("provider unavailable");
});
await expect(worker.runOnce()).resolves.toBe(true);
expect(
await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }),
).toMatchObject({
reviewStatus: "REVIEWING",
poolStatus: "CLOSED",
});
const failed = await prisma.outboxEvent.findUniqueOrThrow({
where: { id: event.id },
});
expect(failed.status).toBe("FAILED");
expect(failed.attempts).toBe(1);
expect(failed.nextRetryAt.getTime()).toBeGreaterThan(Date.now());
});
it("stops retrying at max attempts while remaining fail-closed", async () => {
const { bottle, event } = await fixture("ordinary");
await prisma.outboxEvent.update({
where: { id: event.id },
data: { attempts: 4 },
});
const worker = new ModerationWorker(prisma, () => {
throw new Error("provider unavailable");
});
await worker.runOnce();
const failed = await prisma.outboxEvent.findUniqueOrThrow({
where: { id: event.id },
});
expect(failed).toMatchObject({ status: "FAILED", attempts: 5 });
expect(failed.nextRetryAt.getUTCFullYear()).toBe(9999);
expect(await worker.runOnce()).toBe(false);
expect(
await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }),
).toMatchObject({ reviewStatus: "REVIEWING", poolStatus: "CLOSED" });
});
it("does not publish when the task payload points at another task", async () => {
const first = await fixture("ordinary");
const second = await fixture("another");
await prisma.outboxEvent.delete({ where: { id: second.event.id } });
await prisma.outboxEvent.update({
where: { id: first.event.id },
data: { payload: { bottleId: first.bottle.id, taskId: second.task.id } },
});
await new ModerationWorker(prisma).runOnce();
expect(
(
await prisma.outboxEvent.findUniqueOrThrow({
where: { id: first.event.id },
})
).status,
).toBe("FAILED");
expect(
await prisma.bottle.findUniqueOrThrow({ where: { id: first.bottle.id } }),
).toMatchObject({ reviewStatus: "REVIEWING", poolStatus: "CLOSED" });
});
it("lets two workers claim one event only once", async () => {
await fixture("ordinary");
const entered = vi.fn();
const decide = async (text: string) => {
entered();
await new Promise((resolve) => setTimeout(resolve, 100));
return decideModeration(text);
};
const outcomes = await Promise.all([
new ModerationWorker(prisma, decide).runOnce(),
new ModerationWorker(prisma, decide).runOnce(),
]);
expect(outcomes.sort()).toEqual([false, true]);
expect(entered).toHaveBeenCalledTimes(1);
expect((await prisma.outboxEvent.findFirstOrThrow()).attempts).toBe(1);
});
it("recovers an expired PROCESSING lease", async () => {
const { event } = await fixture("ordinary");
await prisma.outboxEvent.update({
where: { id: event.id },
data: { status: "PROCESSING", lockedAt: new Date(Date.now() - 60_000) },
});
expect(await new ModerationWorker(prisma, undefined, 1000).runOnce()).toBe(
true,
);
expect(
(await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }))
.status,
).toBe("PUBLISHED");
});
it("does not let an expired lease report failure after a new owner publishes", async () => {
const { bottle, event } = await fixture("ordinary");
let rejectFirst!: (reason: Error) => void;
const firstDecision = new Promise<ReturnType<typeof decideModeration>>(
(_resolve, reject) => {
rejectFirst = reject;
},
);
const firstRun = new ModerationWorker(
prisma,
() => firstDecision,
1000,
).runOnce();
await waitForProcessing(event.id);
await prisma.outboxEvent.update({
where: { id: event.id },
data: { lockedAt: new Date(Date.now() - 60_000) },
});
await new ModerationWorker(prisma, undefined, 1000).runOnce();
rejectFirst(new Error("secret-body-provider-error"));
await firstRun;
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({ status: "PUBLISHED" });
expect(
await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }),
).toMatchObject({ reviewStatus: "APPROVED", poolStatus: "IN_POOL" });
});
it("does not let an expired lease publish or overwrite a new owner's decision", async () => {
const { bottle, event, task } = await fixture("ordinary");
let resolveFirst!: (decision: ReturnType<typeof decideModeration>) => void;
const firstDecision = new Promise<ReturnType<typeof decideModeration>>(
(resolve) => {
resolveFirst = resolve;
},
);
const firstRun = new ModerationWorker(
prisma,
() => firstDecision,
1000,
).runOnce();
await waitForProcessing(event.id);
await prisma.outboxEvent.update({
where: { id: event.id },
data: { lockedAt: new Date(Date.now() - 60_000) },
});
await new ModerationWorker(
prisma,
() => ({
reviewStatus: "APPROVED",
poolStatus: "IN_POOL",
labels: ["NEW_OWNER"],
}),
1000,
).runOnce();
resolveFirst({
reviewStatus: "REJECTED",
poolStatus: "CLOSED",
labels: ["OLD_OWNER"],
});
await firstRun;
expect(
await prisma.bottle.findUniqueOrThrow({ where: { id: bottle.id } }),
).toMatchObject({ reviewStatus: "APPROVED", poolStatus: "IN_POOL" });
expect(
await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }),
).toMatchObject({ decision: "APPROVED", riskLabels: ["NEW_OWNER"] });
});
it("publishes a replay when the completed bottle task matches current state", async () => {
const { bottle, event, task } = await fixture("ordinary");
await prisma.$transaction([
prisma.moderationTask.update({
where: { id: task.id },
data: {
status: "COMPLETED",
decision: "APPROVED",
result: { decision: "APPROVED" },
reviewedAt: new Date(),
},
}),
prisma.bottle.update({
where: { id: bottle.id },
data: { reviewStatus: "APPROVED", poolStatus: "IN_POOL" },
}),
]);
const decide = vi.fn(decideModeration);
expect(await new ModerationWorker(prisma, decide).runOnce()).toBe(true);
expect(decide).not.toHaveBeenCalled();
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({ status: "PUBLISHED" });
});
it("fails a completed bottle event whose result drifted from current state", async () => {
const { event, task } = await fixture("secret bottle text");
await prisma.moderationTask.update({
where: { id: task.id },
data: {
status: "COMPLETED",
decision: "APPROVED",
result: { decision: "APPROVED" },
reviewedAt: new Date(),
},
});
const logs: unknown[] = [];
await new ModerationWorker(prisma, undefined, undefined, {
error: (entry) => logs.push(entry),
}).runOnce();
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({ status: "FAILED" });
expect(JSON.stringify(logs)).not.toContain("secret bottle text");
});
it("fails the event and leaves the task open when bottle state drift prevents the update", async () => {
const { bottle, event, task } = await fixture("ordinary");
await prisma.bottle.update({
where: { id: bottle.id },
data: { reviewStatus: "APPROVED", poolStatus: "IN_POOL" },
});
expect(await new ModerationWorker(prisma).runOnce()).toBe(true);
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({ status: "FAILED", attempts: 1 });
expect(
await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }),
).toMatchObject({ status: "PENDING", decision: null });
});
it("reaps a stale PROCESSING event already at max attempts without handling it", async () => {
const { event } = await fixture("ordinary");
await prisma.outboxEvent.update({
where: { id: event.id },
data: {
status: "PROCESSING",
attempts: 5,
lockedAt: new Date(Date.now() - 60_000),
},
});
const decide = vi.fn(decideModeration);
expect(await new ModerationWorker(prisma, decide, 1000).runOnce()).toBe(
false,
);
expect(decide).not.toHaveBeenCalled();
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({
status: "FAILED",
attempts: 5,
lockToken: null,
lockedAt: null,
});
});
it.each([
["ordinary", "APPROVED"],
["reject-word", "REJECTED"],
["review-word", "MANUAL_REVIEW"],
] as const)(
"moderates normalized profile content with rule %#",
async (nickname, status) => {
const { profile, event, task } = await profileFixture(
nickname,
"profile bio",
);
expect(await new ModerationWorker(prisma).runOnce()).toBe(true);
expect(
(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
})
).reviewStatus,
).toBe(status);
expect(
await prisma.moderationTask.findUniqueOrThrow({
where: { id: task.id },
}),
).toMatchObject({ status: "COMPLETED", decision: status });
expect(
(
await prisma.outboxEvent.findUniqueOrThrow({
where: { id: event.id },
})
).status,
).toBe("PUBLISHED");
},
);
it("publishes a profile event that becomes stale while moderation is running", async () => {
const { createHash } = await import("node:crypto");
const { profile, event, task } = await profileFixture("ordinary", "first");
const currentText = JSON.stringify(["reject-word", "#abcdef", "second"]);
const decide = vi.fn(async (text: string) => {
await prisma.$transaction([
prisma.anonymousProfile.update({
where: { id: profile.id },
data: {
nickname: "reject-word",
bio: "second",
version: 2,
reviewStatus: "REVIEWING",
},
}),
prisma.moderationTask.update({
where: { id: task.id },
data: {
payloadHash: createHash("sha256").update(currentText).digest("hex"),
status: "PENDING",
},
}),
]);
return decideModeration(text);
});
expect(await new ModerationWorker(prisma, decide).runOnce()).toBe(true);
expect(decide).toHaveBeenCalledTimes(1);
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({
status: "PUBLISHED",
lockToken: null,
lockedAt: null,
});
expect(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
}),
).toMatchObject({ version: 2, reviewStatus: "REVIEWING" });
expect(
await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }),
).toMatchObject({ status: "PENDING", decision: null });
});
it("fails a same-version profile event whose task hash drifted", async () => {
const { profile, event, task } = await profileFixture("ordinary", "first");
await prisma.moderationTask.update({
where: { id: task.id },
data: { payloadHash: "0".repeat(64) },
});
expect(await new ModerationWorker(prisma).runOnce()).toBe(true);
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({
status: "FAILED",
attempts: 1,
lockToken: null,
lockedAt: null,
});
expect(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
}),
).toMatchObject({ version: 1, reviewStatus: "REVIEWING" });
expect(
await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }),
).toMatchObject({ status: "PENDING", decision: null });
});
it("fails a replay when the completed profile task drifted from current state", async () => {
const { profile, event, task } = await profileFixture("ordinary", "first");
await prisma.moderationTask.update({
where: { id: task.id },
data: {
status: "COMPLETED",
decision: "APPROVED",
result: { decision: "APPROVED" },
reviewedAt: new Date(),
},
});
const decide = vi.fn(decideModeration);
expect(await new ModerationWorker(prisma, decide).runOnce()).toBe(true);
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({ status: "FAILED" });
expect(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
}),
).toMatchObject({ reviewStatus: "REVIEWING" });
expect(
await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }),
).toMatchObject({ status: "COMPLETED", decision: "APPROVED" });
});
it("fails a completed profile task with a non-terminal reviewing decision", async () => {
const { profile, event, task } = await profileFixture("ordinary", "first");
await prisma.moderationTask.update({
where: { id: task.id },
data: {
status: "COMPLETED",
decision: "REVIEWING",
result: { decision: "REVIEWING" },
reviewedAt: new Date(),
},
});
expect(await new ModerationWorker(prisma).runOnce()).toBe(true);
expect(
await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }),
).toMatchObject({
status: "FAILED",
attempts: 1,
lockToken: null,
lockedAt: null,
});
expect(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
}),
).toMatchObject({ reviewStatus: "REVIEWING" });
expect(
await prisma.moderationTask.findUniqueOrThrow({ where: { id: task.id } }),
).toMatchObject({ status: "COMPLETED", decision: "REVIEWING" });
});
it("logs a safe error category without payload or exception message", async () => {
const { event } = await fixture("secret正文");
const entries: unknown[] = [];
await new ModerationWorker(
prisma,
() => {
throw new TypeError("provider leaked secret正文");
},
undefined,
{ error: (entry) => entries.push(entry) },
).runOnce();
expect(entries).toEqual([
{
eventId: event.id,
eventType: "BOTTLE_MODERATION_REQUESTED",
attempt: 1,
errorClass: "TypeError",
},
]);
expect(JSON.stringify(entries)).not.toContain("secret正文");
expect(JSON.stringify(entries)).not.toContain("provider leaked");
});
it("publishes an old profile event without letting it overwrite a newer version", async () => {
const { createHash } = await import("node:crypto");
const { profile, event, task } = await profileFixture("ordinary", "first");
const currentText = JSON.stringify(["reject-word", "#abcdef", "second"]);
await prisma.$transaction([
prisma.anonymousProfile.update({
where: { id: profile.id },
data: {
nickname: "reject-word",
bio: "second",
version: 2,
reviewStatus: "REVIEWING",
},
}),
prisma.moderationTask.update({
where: { id: task.id },
data: {
payloadHash: createHash("sha256").update(currentText).digest("hex"),
},
}),
prisma.outboxEvent.create({
data: {
aggregateType: "PROFILE",
aggregateId: profile.id,
eventType: "PROFILE_MODERATION_REQUESTED",
dedupeKey: `profile-moderation:${profile.id}:2`,
payload: {
profileId: profile.id,
accountId: profile.accountId,
taskId: task.id,
},
},
}),
]);
const worker = new ModerationWorker(prisma);
expect(await worker.runOnce()).toBe(true);
expect(
(await prisma.outboxEvent.findUniqueOrThrow({ where: { id: event.id } }))
.status,
).toBe("PUBLISHED");
expect(
(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
})
).reviewStatus,
).toBe("REVIEWING");
expect(
(
await prisma.moderationTask.findUniqueOrThrow({
where: { id: task.id },
})
).status,
).toBe("PENDING");
expect(await worker.runOnce()).toBe(true);
expect(
(
await prisma.anonymousProfile.findUniqueOrThrow({
where: { id: profile.id },
})
).reviewStatus,
).toBe("REJECTED");
});
async function fixture(contentText: string) {
const account = await prisma.account.create({
data: { phoneCiphertext: Buffer.from("cipher"), phoneHmac: randomUUID() },
});
const bottle = await prisma.bottle.create({
data: {
authorId: account.id,
clientRequestId: randomUUID(),
contentText,
reviewStatus: "REVIEWING",
poolStatus: "CLOSED",
},
});
const task = await prisma.moderationTask.create({
data: {
targetType: "BOTTLE",
targetId: bottle.id,
provider: "SIMULATED",
payloadHash: createHash("sha256").update(contentText).digest("hex"),
riskLabels: [],
},
});
const event = await prisma.outboxEvent.create({
data: {
aggregateType: "BOTTLE",
aggregateId: bottle.id,
eventType: "BOTTLE_MODERATION_REQUESTED",
dedupeKey: randomUUID(),
payload: { bottleId: bottle.id, taskId: task.id },
},
});
return { bottle, task, event };
}
async function profileFixture(nickname: string, bio: string) {
const { createHash } = await import("node:crypto");
const account = await prisma.account.create({
data: { phoneCiphertext: Buffer.from("cipher"), phoneHmac: randomUUID() },
});
const profile = await prisma.anonymousProfile.create({
data: {
accountId: account.id,
nickname,
avatarColor: "#abcdef",
bio,
version: 1,
},
});
const text = JSON.stringify([nickname, "#abcdef", bio]);
const task = await prisma.moderationTask.create({
data: {
targetType: "PROFILE",
targetId: profile.id,
provider: "SIMULATED",
payloadHash: createHash("sha256").update(text).digest("hex"),
riskLabels: [],
},
});
const event = await prisma.outboxEvent.create({
data: {
aggregateType: "PROFILE",
aggregateId: profile.id,
eventType: "PROFILE_MODERATION_REQUESTED",
dedupeKey: `profile-moderation:${profile.id}:1`,
payload: {
profileId: profile.id,
accountId: account.id,
taskId: task.id,
},
},
});
return { profile, task, event };
}
async function waitForProcessing(id: string) {
for (let attempt = 0; attempt < 100; attempt += 1) {
const event = await prisma.outboxEvent.findUniqueOrThrow({
where: { id },
});
if (event.status === "PROCESSING") return;
await new Promise((resolve) => setTimeout(resolve, 10));
}
throw new Error("worker did not claim event");
}
});