revert(mail): roll back Phase-2 scan-internal — fixes 500 (CJS-extends bundle break)
Phase-2-Rebuild reaktivierte den bekannten imapflow/node-apn util.inherits-Bundle- Bruch → scan-internal warf 500 → Mail-Filtern (USP) down. Rollback von scan-internal.post.ts + db/mail.ts auf den funktionierenden Stand (5b57bea). Schema (folder_scan_state, last_full_sweep_at) + Migration BLEIBEN angewendet — kein Prisma-Drift; die Spalten warten ungenutzt auf den gefixten Phase-2-Retry. Root-Cause (warum der inkrementelle imap.status/search-Pfad das Bundle bricht) muss vor erneutem Phase-2-Deploy in der nitro-Externalize-Config gelöst werden. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
chahinebrini
parent
04e2979b8d
commit
0dbaac97a2
@ -7,8 +7,6 @@ import {
|
||||
upsertMailBlockedStat,
|
||||
updateMailConnectionScanStats,
|
||||
insertMailClassificationSample,
|
||||
patchFolderScanState,
|
||||
markFullSweepDone,
|
||||
} from "../../db/mail";
|
||||
import { getBlocklistedDomainsSet, getMailDisplayNamePatterns } from "../../db/domains";
|
||||
import { getProfile } from "../../db/profile";
|
||||
@ -96,21 +94,6 @@ export default defineEventHandler(async (event) => {
|
||||
let scanned = 0;
|
||||
let newlyBlocked = 0;
|
||||
|
||||
// ─── Quality Full-Sweep Wächter ──────────────────────────────────────────
|
||||
// 1×/Tag: alle Ordner werden als lastUid=0 behandelt (Full-Sweep) um sicher-
|
||||
// zustellen dass Blocklist-Updates auch ältere Mails erfassen.
|
||||
// Wichtig: wir behandeln lastUid temporär als 0 — NICHT persistieren.
|
||||
// Nach dem Sweep wird der echte maxUid gespeichert + lastFullSweepAt=NOW().
|
||||
const needsFullSweep =
|
||||
!connection.lastFullSweepAt ||
|
||||
Date.now() - new Date(connection.lastFullSweepAt).getTime() > 24 * 3_600_000;
|
||||
|
||||
// Aktueller folder_scan_state aus der DB (JSONB, Prisma liefert plain object)
|
||||
const folderScanState = (connection.folderScanState ?? {}) as Record<
|
||||
string,
|
||||
{ lastUid: number; uidvalidity: number }
|
||||
>;
|
||||
|
||||
try {
|
||||
await imap.connect();
|
||||
|
||||
@ -128,8 +111,7 @@ export default defineEventHandler(async (event) => {
|
||||
const skippedSystemFolders = mailboxes.length - scannable.length;
|
||||
console.log(
|
||||
`[scan-internal] ${connection.email} scanning ${scannable.length} folders` +
|
||||
(skippedSystemFolders > 0 ? ` (${skippedSystemFolders} system folders skipped)` : "") +
|
||||
(needsFullSweep ? " [full-sweep]" : " [incremental]"),
|
||||
(skippedSystemFolders > 0 ? ` (${skippedSystemFolders} system folders skipped)` : ""),
|
||||
);
|
||||
|
||||
for (const mb of scannable) {
|
||||
@ -141,69 +123,15 @@ export default defineEventHandler(async (event) => {
|
||||
}
|
||||
try {
|
||||
const SCAN_LIMIT = 200;
|
||||
// ─── UID-Scan-Strategie (Phase 2) ──────────────────────────────────
|
||||
// 1. Status holen: messages, uidNext, uidValidity
|
||||
const status = await imap.status(mb.path, {
|
||||
messages: true,
|
||||
uidNext: true,
|
||||
uidValidity: true,
|
||||
});
|
||||
const status = await imap.status(mb.path, { messages: true });
|
||||
const msgCount = (status as any).messages ?? 0;
|
||||
const serverUidValidity: number = (status as any).uidValidity ?? 0;
|
||||
if (msgCount === 0) continue;
|
||||
|
||||
if (msgCount === 0) {
|
||||
// Ordner leer — trotzdem Zustand für diesen Ordner persistieren
|
||||
// (verhindert endloses Re-Fetching auf leere Ordner).
|
||||
if (serverUidValidity > 0) {
|
||||
await patchFolderScanState(connection.id, mb.path, {
|
||||
lastUid: 0,
|
||||
uidvalidity: serverUidValidity,
|
||||
});
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Gespeicherten Zustand lesen + UIDVALIDITY-Wächter
|
||||
const saved = folderScanState[mb.path];
|
||||
let lastUid = saved?.lastUid ?? 0;
|
||||
|
||||
if (needsFullSweep) {
|
||||
// Quality Full-Sweep: temporär als 0 behandeln (Full-Scan), aber
|
||||
// lastUid=0 NICHT dauerhaft in folderScanState schreiben.
|
||||
// Nach dem Sweep speichern wir den echten maxUid.
|
||||
lastUid = 0;
|
||||
} else if (saved && serverUidValidity > 0 && saved.uidvalidity !== serverUidValidity) {
|
||||
// UIDVALIDITY hat sich geändert: Ordner wurde server-seitig resettet.
|
||||
// Alle bisherigen UIDs sind ungültig → Full-Scan für diesen Ordner.
|
||||
console.log(
|
||||
`[scan-internal] ${connection.email} | ${mb.path} | UIDVALIDITY changed ` +
|
||||
`(${saved.uidvalidity}→${serverUidValidity}) — forcing full sweep for this folder`,
|
||||
);
|
||||
lastUid = 0;
|
||||
}
|
||||
|
||||
// 3. Nachrichten fetchen: inkrementell oder full
|
||||
let allMessages: any[];
|
||||
|
||||
if (lastUid > 0) {
|
||||
// Inkrementell: nur UIDs > lastUid suchen
|
||||
const newUids = await (imap as any).search({ uid: `${lastUid + 1}:*` });
|
||||
if (!newUids || newUids.length === 0) {
|
||||
// Keine neuen Nachrichten → Ordner skippen, kein fetchAll nötig
|
||||
continue;
|
||||
}
|
||||
// Fetch nur die neuen UIDs
|
||||
allMessages = await imap.fetchAll(newUids.join(","), { envelope: true }, { uid: true } as any);
|
||||
} else {
|
||||
// Full-Sweep (erster Scan, UIDVALIDITY-Reset, oder Quality-Full-Sweep):
|
||||
// identische Logik wie vorher — letzten SCAN_LIMIT Nachrichten
|
||||
const fetchRange =
|
||||
msgCount > SCAN_LIMIT ? `${msgCount - SCAN_LIMIT + 1}:*` : "1:*";
|
||||
allMessages = await imap.fetchAll(fetchRange, { envelope: true });
|
||||
}
|
||||
|
||||
if (!allMessages || allMessages.length === 0) continue;
|
||||
|
||||
const allMessages = await imap.fetchAll(fetchRange, {
|
||||
envelope: true,
|
||||
});
|
||||
scanned += allMessages.length;
|
||||
totalScanned += allMessages.length;
|
||||
|
||||
@ -343,32 +271,11 @@ export default defineEventHandler(async (event) => {
|
||||
count: toInsert.length,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── UID-Zustand persistieren ─────────────────────────────────────
|
||||
// maxUid über alle gefetchten Nachrichten berechnen (numeric UID aus IMAP).
|
||||
// Nur persistieren wenn wir mindestens eine Nachricht hatten.
|
||||
const maxUid = allMessages.reduce((max: number, m: any) => {
|
||||
const u = typeof m.uid === "number" ? m.uid : parseInt(String(m.uid ?? "0"), 10);
|
||||
return u > max ? u : max;
|
||||
}, 0);
|
||||
|
||||
if (maxUid > 0 && serverUidValidity > 0) {
|
||||
await patchFolderScanState(connection.id, mb.path, {
|
||||
lastUid: maxUid,
|
||||
uidvalidity: serverUidValidity,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Full-Sweep abschließen ─────────────────────────────────────────────
|
||||
if (needsFullSweep) {
|
||||
await markFullSweepDone(connection.id);
|
||||
console.log(`[scan-internal] ${connection.email} | full-sweep complete, lastFullSweepAt updated`);
|
||||
}
|
||||
|
||||
await imap.logout();
|
||||
} catch {
|
||||
try {
|
||||
|
||||
@ -42,8 +42,15 @@ export async function getAllMailConnections(userId: string) {
|
||||
});
|
||||
}
|
||||
|
||||
// getAllActiveMailUserIds — removed (dead code after Phase-1 cron-deletion).
|
||||
// No callers remain. The cron that called it was deleted in Phase 1.
|
||||
export async function getAllActiveMailUserIds() {
|
||||
const db = usePrisma();
|
||||
const rows = await db.mailConnection.findMany({
|
||||
where: { isActive: true, nextScanAt: { lte: new Date() } },
|
||||
select: { userId: true },
|
||||
distinct: ["userId"],
|
||||
});
|
||||
return rows.map((r) => r.userId);
|
||||
}
|
||||
|
||||
export async function countMailConnections(userId: string) {
|
||||
const db = usePrisma();
|
||||
@ -134,46 +141,6 @@ export async function updateMailConnectionScanStats(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically merges a single folder's UID-scan state into folder_scan_state.
|
||||
*
|
||||
* Uses PostgreSQL JSON merge operator (||) to patch only the given folder key —
|
||||
* other folders are preserved. This is safe under concurrent multi-folder updates
|
||||
* because each folder key is independent.
|
||||
*
|
||||
* Example JSONB result after call with (id, "INBOX", {lastUid:1234, uidvalidity:5678}):
|
||||
* { "INBOX": {"lastUid":1234,"uidvalidity":5678}, "Junk Email": {"lastUid":99,...} }
|
||||
*
|
||||
* @param connectionId MailConnection.id
|
||||
* @param folderPath IMAP mailbox path (e.g. "INBOX", "Junk Email")
|
||||
* @param state { lastUid: number, uidvalidity: number }
|
||||
*/
|
||||
export async function patchFolderScanState(
|
||||
connectionId: string,
|
||||
folderPath: string,
|
||||
state: { lastUid: number; uidvalidity: number },
|
||||
): Promise<void> {
|
||||
const db = usePrisma();
|
||||
const patch = JSON.stringify({ [folderPath]: state });
|
||||
await db.$executeRaw`
|
||||
UPDATE "rebreak"."mail_connections"
|
||||
SET "folder_scan_state" = "folder_scan_state" || ${patch}::jsonb
|
||||
WHERE "id" = ${connectionId}::uuid
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a connection's last_full_sweep_at to NOW().
|
||||
* Called once per connection per day when the quality full-sweep runs.
|
||||
*/
|
||||
export async function markFullSweepDone(connectionId: string): Promise<void> {
|
||||
const db = usePrisma();
|
||||
await db.mailConnection.update({
|
||||
where: { id: connectionId },
|
||||
data: { lastFullSweepAt: new Date() },
|
||||
});
|
||||
}
|
||||
|
||||
export async function getMailBlockedStats(userId: string) {
|
||||
const db = usePrisma();
|
||||
const since7d = new Date(Date.now() - 7 * 86_400_000);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user