revert(mail): roll back Phase-2 scan-internal — fixes 500 (CJS-extends bundle break)
Phase-2-Rebuild reaktivierte den bekannten imapflow/node-apn util.inherits-Bundle- Bruch → scan-internal warf 500 → Mail-Filtern (USP) down. Rollback von scan-internal.post.ts + db/mail.ts auf den funktionierenden Stand (5b57bea). Schema (folder_scan_state, last_full_sweep_at) + Migration BLEIBEN angewendet — kein Prisma-Drift; die Spalten warten ungenutzt auf den gefixten Phase-2-Retry. Root-Cause (warum der inkrementelle imap.status/search-Pfad das Bundle bricht) muss vor erneutem Phase-2-Deploy in der nitro-Externalize-Config gelöst werden. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
04e2979b8d
commit
0dbaac97a2
@ -7,8 +7,6 @@ import {
|
|||||||
upsertMailBlockedStat,
|
upsertMailBlockedStat,
|
||||||
updateMailConnectionScanStats,
|
updateMailConnectionScanStats,
|
||||||
insertMailClassificationSample,
|
insertMailClassificationSample,
|
||||||
patchFolderScanState,
|
|
||||||
markFullSweepDone,
|
|
||||||
} from "../../db/mail";
|
} from "../../db/mail";
|
||||||
import { getBlocklistedDomainsSet, getMailDisplayNamePatterns } from "../../db/domains";
|
import { getBlocklistedDomainsSet, getMailDisplayNamePatterns } from "../../db/domains";
|
||||||
import { getProfile } from "../../db/profile";
|
import { getProfile } from "../../db/profile";
|
||||||
@ -96,21 +94,6 @@ export default defineEventHandler(async (event) => {
|
|||||||
let scanned = 0;
|
let scanned = 0;
|
||||||
let newlyBlocked = 0;
|
let newlyBlocked = 0;
|
||||||
|
|
||||||
// ─── Quality Full-Sweep Wächter ──────────────────────────────────────────
|
|
||||||
// 1×/Tag: alle Ordner werden als lastUid=0 behandelt (Full-Sweep) um sicher-
|
|
||||||
// zustellen dass Blocklist-Updates auch ältere Mails erfassen.
|
|
||||||
// Wichtig: wir behandeln lastUid temporär als 0 — NICHT persistieren.
|
|
||||||
// Nach dem Sweep wird der echte maxUid gespeichert + lastFullSweepAt=NOW().
|
|
||||||
const needsFullSweep =
|
|
||||||
!connection.lastFullSweepAt ||
|
|
||||||
Date.now() - new Date(connection.lastFullSweepAt).getTime() > 24 * 3_600_000;
|
|
||||||
|
|
||||||
// Aktueller folder_scan_state aus der DB (JSONB, Prisma liefert plain object)
|
|
||||||
const folderScanState = (connection.folderScanState ?? {}) as Record<
|
|
||||||
string,
|
|
||||||
{ lastUid: number; uidvalidity: number }
|
|
||||||
>;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await imap.connect();
|
await imap.connect();
|
||||||
|
|
||||||
@ -128,8 +111,7 @@ export default defineEventHandler(async (event) => {
|
|||||||
const skippedSystemFolders = mailboxes.length - scannable.length;
|
const skippedSystemFolders = mailboxes.length - scannable.length;
|
||||||
console.log(
|
console.log(
|
||||||
`[scan-internal] ${connection.email} scanning ${scannable.length} folders` +
|
`[scan-internal] ${connection.email} scanning ${scannable.length} folders` +
|
||||||
(skippedSystemFolders > 0 ? ` (${skippedSystemFolders} system folders skipped)` : "") +
|
(skippedSystemFolders > 0 ? ` (${skippedSystemFolders} system folders skipped)` : ""),
|
||||||
(needsFullSweep ? " [full-sweep]" : " [incremental]"),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
for (const mb of scannable) {
|
for (const mb of scannable) {
|
||||||
@ -141,69 +123,15 @@ export default defineEventHandler(async (event) => {
|
|||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const SCAN_LIMIT = 200;
|
const SCAN_LIMIT = 200;
|
||||||
// ─── UID-Scan-Strategie (Phase 2) ──────────────────────────────────
|
const status = await imap.status(mb.path, { messages: true });
|
||||||
// 1. Status holen: messages, uidNext, uidValidity
|
|
||||||
const status = await imap.status(mb.path, {
|
|
||||||
messages: true,
|
|
||||||
uidNext: true,
|
|
||||||
uidValidity: true,
|
|
||||||
});
|
|
||||||
const msgCount = (status as any).messages ?? 0;
|
const msgCount = (status as any).messages ?? 0;
|
||||||
const serverUidValidity: number = (status as any).uidValidity ?? 0;
|
if (msgCount === 0) continue;
|
||||||
|
|
||||||
if (msgCount === 0) {
|
|
||||||
// Ordner leer — trotzdem Zustand für diesen Ordner persistieren
|
|
||||||
// (verhindert endloses Re-Fetching auf leere Ordner).
|
|
||||||
if (serverUidValidity > 0) {
|
|
||||||
await patchFolderScanState(connection.id, mb.path, {
|
|
||||||
lastUid: 0,
|
|
||||||
uidvalidity: serverUidValidity,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 2. Gespeicherten Zustand lesen + UIDVALIDITY-Wächter
|
|
||||||
const saved = folderScanState[mb.path];
|
|
||||||
let lastUid = saved?.lastUid ?? 0;
|
|
||||||
|
|
||||||
if (needsFullSweep) {
|
|
||||||
// Quality Full-Sweep: temporär als 0 behandeln (Full-Scan), aber
|
|
||||||
// lastUid=0 NICHT dauerhaft in folderScanState schreiben.
|
|
||||||
// Nach dem Sweep speichern wir den echten maxUid.
|
|
||||||
lastUid = 0;
|
|
||||||
} else if (saved && serverUidValidity > 0 && saved.uidvalidity !== serverUidValidity) {
|
|
||||||
// UIDVALIDITY hat sich geändert: Ordner wurde server-seitig resettet.
|
|
||||||
// Alle bisherigen UIDs sind ungültig → Full-Scan für diesen Ordner.
|
|
||||||
console.log(
|
|
||||||
`[scan-internal] ${connection.email} | ${mb.path} | UIDVALIDITY changed ` +
|
|
||||||
`(${saved.uidvalidity}→${serverUidValidity}) — forcing full sweep for this folder`,
|
|
||||||
);
|
|
||||||
lastUid = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Nachrichten fetchen: inkrementell oder full
|
|
||||||
let allMessages: any[];
|
|
||||||
|
|
||||||
if (lastUid > 0) {
|
|
||||||
// Inkrementell: nur UIDs > lastUid suchen
|
|
||||||
const newUids = await (imap as any).search({ uid: `${lastUid + 1}:*` });
|
|
||||||
if (!newUids || newUids.length === 0) {
|
|
||||||
// Keine neuen Nachrichten → Ordner skippen, kein fetchAll nötig
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Fetch nur die neuen UIDs
|
|
||||||
allMessages = await imap.fetchAll(newUids.join(","), { envelope: true }, { uid: true } as any);
|
|
||||||
} else {
|
|
||||||
// Full-Sweep (erster Scan, UIDVALIDITY-Reset, oder Quality-Full-Sweep):
|
|
||||||
// identische Logik wie vorher — letzten SCAN_LIMIT Nachrichten
|
|
||||||
const fetchRange =
|
const fetchRange =
|
||||||
msgCount > SCAN_LIMIT ? `${msgCount - SCAN_LIMIT + 1}:*` : "1:*";
|
msgCount > SCAN_LIMIT ? `${msgCount - SCAN_LIMIT + 1}:*` : "1:*";
|
||||||
allMessages = await imap.fetchAll(fetchRange, { envelope: true });
|
const allMessages = await imap.fetchAll(fetchRange, {
|
||||||
}
|
envelope: true,
|
||||||
|
});
|
||||||
if (!allMessages || allMessages.length === 0) continue;
|
|
||||||
|
|
||||||
scanned += allMessages.length;
|
scanned += allMessages.length;
|
||||||
totalScanned += allMessages.length;
|
totalScanned += allMessages.length;
|
||||||
|
|
||||||
@ -343,32 +271,11 @@ export default defineEventHandler(async (event) => {
|
|||||||
count: toInsert.length,
|
count: toInsert.length,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── UID-Zustand persistieren ─────────────────────────────────────
|
|
||||||
// maxUid über alle gefetchten Nachrichten berechnen (numeric UID aus IMAP).
|
|
||||||
// Nur persistieren wenn wir mindestens eine Nachricht hatten.
|
|
||||||
const maxUid = allMessages.reduce((max: number, m: any) => {
|
|
||||||
const u = typeof m.uid === "number" ? m.uid : parseInt(String(m.uid ?? "0"), 10);
|
|
||||||
return u > max ? u : max;
|
|
||||||
}, 0);
|
|
||||||
|
|
||||||
if (maxUid > 0 && serverUidValidity > 0) {
|
|
||||||
await patchFolderScanState(connection.id, mb.path, {
|
|
||||||
lastUid: maxUid,
|
|
||||||
uidvalidity: serverUidValidity,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
lock.release();
|
lock.release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Full-Sweep abschließen ─────────────────────────────────────────────
|
|
||||||
if (needsFullSweep) {
|
|
||||||
await markFullSweepDone(connection.id);
|
|
||||||
console.log(`[scan-internal] ${connection.email} | full-sweep complete, lastFullSweepAt updated`);
|
|
||||||
}
|
|
||||||
|
|
||||||
await imap.logout();
|
await imap.logout();
|
||||||
} catch {
|
} catch {
|
||||||
try {
|
try {
|
||||||
|
|||||||
@ -42,8 +42,15 @@ export async function getAllMailConnections(userId: string) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// getAllActiveMailUserIds — removed (dead code after Phase-1 cron-deletion).
|
export async function getAllActiveMailUserIds() {
|
||||||
// No callers remain. The cron that called it was deleted in Phase 1.
|
const db = usePrisma();
|
||||||
|
const rows = await db.mailConnection.findMany({
|
||||||
|
where: { isActive: true, nextScanAt: { lte: new Date() } },
|
||||||
|
select: { userId: true },
|
||||||
|
distinct: ["userId"],
|
||||||
|
});
|
||||||
|
return rows.map((r) => r.userId);
|
||||||
|
}
|
||||||
|
|
||||||
export async function countMailConnections(userId: string) {
|
export async function countMailConnections(userId: string) {
|
||||||
const db = usePrisma();
|
const db = usePrisma();
|
||||||
@ -134,46 +141,6 @@ export async function updateMailConnectionScanStats(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Atomically merges a single folder's UID-scan state into folder_scan_state.
|
|
||||||
*
|
|
||||||
* Uses PostgreSQL JSON merge operator (||) to patch only the given folder key —
|
|
||||||
* other folders are preserved. This is safe under concurrent multi-folder updates
|
|
||||||
* because each folder key is independent.
|
|
||||||
*
|
|
||||||
* Example JSONB result after call with (id, "INBOX", {lastUid:1234, uidvalidity:5678}):
|
|
||||||
* { "INBOX": {"lastUid":1234,"uidvalidity":5678}, "Junk Email": {"lastUid":99,...} }
|
|
||||||
*
|
|
||||||
* @param connectionId MailConnection.id
|
|
||||||
* @param folderPath IMAP mailbox path (e.g. "INBOX", "Junk Email")
|
|
||||||
* @param state { lastUid: number, uidvalidity: number }
|
|
||||||
*/
|
|
||||||
export async function patchFolderScanState(
|
|
||||||
connectionId: string,
|
|
||||||
folderPath: string,
|
|
||||||
state: { lastUid: number; uidvalidity: number },
|
|
||||||
): Promise<void> {
|
|
||||||
const db = usePrisma();
|
|
||||||
const patch = JSON.stringify({ [folderPath]: state });
|
|
||||||
await db.$executeRaw`
|
|
||||||
UPDATE "rebreak"."mail_connections"
|
|
||||||
SET "folder_scan_state" = "folder_scan_state" || ${patch}::jsonb
|
|
||||||
WHERE "id" = ${connectionId}::uuid
|
|
||||||
`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Marks a connection's last_full_sweep_at to NOW().
|
|
||||||
* Called once per connection per day when the quality full-sweep runs.
|
|
||||||
*/
|
|
||||||
export async function markFullSweepDone(connectionId: string): Promise<void> {
|
|
||||||
const db = usePrisma();
|
|
||||||
await db.mailConnection.update({
|
|
||||||
where: { id: connectionId },
|
|
||||||
data: { lastFullSweepAt: new Date() },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getMailBlockedStats(userId: string) {
|
export async function getMailBlockedStats(userId: string) {
|
||||||
const db = usePrisma();
|
const db = usePrisma();
|
||||||
const since7d = new Date(Date.now() - 7 * 86_400_000);
|
const since7d = new Date(Date.now() - 7 * 86_400_000);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user