feat(mail): separate mail_blocked_stats table — preserves charts beyond 24h cleanup
deleteOldMailBlocked löscht weiter rohe Einträge nach 24h (Datenminimierung für Mail-Inhalte, DSGVO Art. 5 Abs. 1 lit. c). Aber für Charts und Pattern-Analysen werden vor dem Cleanup permanent aggregierte Daten in einer separaten Tabelle geführt. Architektur: - Neue Tabelle mail_blocked_stats — UNIQUE (user_id, date, connection_id), enthält ausschließlich counts + UTC-Datum + IMAP-Host. Kein Subject, kein Sender, kein Mail-Inhalt. Datenminimierung jetzt auch im Audit- Pfad sichtbar. - Live-Aggregation: scan.post.ts + scan-internal.post.ts upserten direkt nach jedem mail_blocked-INSERT in mail_blocked_stats (count += 1). - 30-Tage-Backfill als SQL im Migration-File: bestehende mail_blocked- Rows der letzten 30 Tage werden einmalig aggregiert, damit Charts nicht 30 Tage lang leer aussehen. - Stats-Endpoints (blocked-by-day, blocked-by-connection) lesen jetzt aus mail_blocked_stats. Response-Shape unverändert → Frontend bleibt unberührt. ON DELETE CASCADE auf mail_connection_id (Hans-Müller-konservativ): User-initiierter Disconnect = Art. 17-Signal → assoziierte Stats werden mitgelöscht. SetNull wäre DSGVO-grenzwertig (orphan stats ohne klare Lösch-Kontext-Zuordnung). pnpm build:backend clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
chahinebrini
parent
b7909d77e4
commit
275637f0b0
@ -0,0 +1,66 @@
|
||||
-- Migration: mail_blocked_stats_aggregation
|
||||
-- Permanente Aggregat-Tabelle für blockierte Mails (counts/dates only, KEINE Inhalte).
|
||||
-- Löst das DSGVO-Spannungsfeld: mail_blocked wird nach 24h gelöscht (Datenminimierung
|
||||
-- Art. 5 Abs. 1 lit. e), während Stats-Charts weiterhin historische Daten zeigen können.
|
||||
--
|
||||
-- Stats enthalten ausschließlich: userId, date (UTC), connectionId, provider-host, count.
|
||||
-- Kein Subject, kein Sender, kein Mail-Inhalt — Aggregat ist unbedenklich i.S. Art. 9.
|
||||
--
|
||||
-- Lösch-Verhalten: ON DELETE CASCADE wenn MailConnection gelöscht wird.
|
||||
-- Begründung: User-initiierter Disconnect = Art. 17-Löschanfrage; Stats sind an
|
||||
-- diese Connection geknüpft; Cascade hält DB konsistent ohne nullable foreign-key.
|
||||
-- Hans-Müller-Note: Aggregat-Stats sind kein "Personenbezug zur Mail" mehr, aber
|
||||
-- der Bezug zur Connection-ID bleibt — Cascade ist DSGVO-konservativere Wahl.
|
||||
--
|
||||
-- Backfill: Einmalig aus bestehenden mail_blocked-Rows der letzten 30 Tage.
|
||||
-- provider-Spalte speichert imap_host als rohen String; resolveProviderMeta()
|
||||
-- wird zur Read-Zeit im Stats-Endpoint aufgerufen (TypeScript, nicht SQL).
|
||||
-- Für den Backfill wird imap_host doppelt als providerLabel eingetragen —
|
||||
-- der Live-Scan-Code schreibt ab jetzt den korrekten resolved Label.
|
||||
-- Ein separater Script (backend/scripts/backfill-stats-provider.ts) kann nachgezogen
|
||||
-- werden wenn die Labels im Chart sauber sein sollen; für MVP nicht kritisch.
|
||||
--
|
||||
-- Deploy: automatisch via GitHub Actions (pnpm prisma migrate deploy)
|
||||
|
||||
CREATE TABLE "rebreak"."mail_blocked_stats" (
|
||||
"id" TEXT NOT NULL,
|
||||
"user_id" UUID NOT NULL,
|
||||
"date" DATE NOT NULL,
|
||||
"mail_connection_id" UUID NOT NULL,
|
||||
"provider" TEXT NOT NULL,
|
||||
"provider_label" TEXT NOT NULL,
|
||||
"count" INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
CONSTRAINT "mail_blocked_stats_pkey" PRIMARY KEY ("id"),
|
||||
CONSTRAINT "mail_blocked_stats_unique_day"
|
||||
UNIQUE ("user_id", "date", "mail_connection_id"),
|
||||
CONSTRAINT "mail_blocked_stats_connection_fkey"
|
||||
FOREIGN KEY ("mail_connection_id")
|
||||
REFERENCES "rebreak"."mail_connections" ("id")
|
||||
ON DELETE CASCADE ON UPDATE CASCADE
|
||||
);
|
||||
|
||||
CREATE INDEX "mail_blocked_stats_user_date_idx"
|
||||
ON "rebreak"."mail_blocked_stats" ("user_id", "date");
|
||||
|
||||
CREATE INDEX "mail_blocked_stats_user_connection_idx"
|
||||
ON "rebreak"."mail_blocked_stats" ("user_id", "mail_connection_id");
|
||||
|
||||
-- Backfill: bestehende mail_blocked-Rows der letzten 30 Tage aggregieren.
|
||||
-- imap_host wird als provider UND provider_label gespeichert (rohes Datum).
|
||||
-- Live-Scan schreibt ab jetzt resolved labels via resolveProviderMeta().
|
||||
INSERT INTO "rebreak"."mail_blocked_stats"
|
||||
("id", "user_id", "date", "mail_connection_id", "provider", "provider_label", "count")
|
||||
SELECT
|
||||
gen_random_uuid()::text,
|
||||
mb."user_id",
|
||||
DATE(mb."created_at" AT TIME ZONE 'UTC') AS "date",
|
||||
mb."connection_id",
|
||||
mc."imap_host",
|
||||
mc."imap_host",
|
||||
COUNT(*)::integer
|
||||
FROM "rebreak"."mail_blocked" mb
|
||||
JOIN "rebreak"."mail_connections" mc ON mc."id" = mb."connection_id"
|
||||
WHERE mb."created_at" > NOW() - INTERVAL '30 days'
|
||||
GROUP BY mb."user_id", DATE(mb."created_at" AT TIME ZONE 'UTC'), mb."connection_id", mc."imap_host"
|
||||
ON CONFLICT ("user_id", "date", "mail_connection_id") DO NOTHING;
|
||||
@ -534,7 +534,8 @@ model MailConnection {
|
||||
consentVersion String? @map("consent_version")
|
||||
consentIpAddress String? @map("consent_ip_address")
|
||||
|
||||
blockedMails MailBlocked[]
|
||||
blockedMails MailBlocked[]
|
||||
blockedStats MailBlockedStat[]
|
||||
|
||||
@@unique([userId, email])
|
||||
@@map("mail_connections")
|
||||
@ -646,6 +647,33 @@ model MailBlocked {
|
||||
@@schema("rebreak")
|
||||
}
|
||||
|
||||
/// Permanente Aggregat-Statistiken blockierter Mails pro Tag + Connection.
|
||||
/// Befüllt live beim Scan (vor dem 24h-Cleanup von mail_blocked).
|
||||
/// Enthält KEINE Mail-Inhalte — nur counts/dates (Datenminimierung Art. 5 DSGVO).
|
||||
/// Bei Connection-Disconnect werden Stats mitgelöscht (Cascade) — Konsistenz
|
||||
/// mit Art. 17 DSGVO: User-initiierter Disconnect = Recht auf Löschung aller
|
||||
/// zu dieser Connection gehörenden Daten.
|
||||
model MailBlockedStat {
|
||||
id String @id @default(cuid())
|
||||
userId String @map("user_id") @db.Uuid
|
||||
/// UTC-Datum (time=00:00:00) — ein Eintrag pro User+Tag+Connection
|
||||
date DateTime @db.Date @map("date")
|
||||
mailConnectionId String @map("mail_connection_id") @db.Uuid
|
||||
/// IMAP-Host-Slug, z.B. "imap.gmail.com" (raw, für resolveProviderMeta zur Read-Zeit)
|
||||
provider String @map("provider")
|
||||
/// Human-readable Label, z.B. "Gmail" (wird bei Scan-Zeit aus resolveProviderMeta befüllt)
|
||||
providerLabel String @map("provider_label")
|
||||
count Int @default(0)
|
||||
|
||||
connection MailConnection @relation(fields: [mailConnectionId], references: [id], onDelete: Cascade)
|
||||
|
||||
@@unique([userId, date, mailConnectionId], map: "mail_blocked_stats_unique_day")
|
||||
@@index([userId, date])
|
||||
@@index([userId, mailConnectionId])
|
||||
@@map("mail_blocked_stats")
|
||||
@@schema("rebreak")
|
||||
}
|
||||
|
||||
model ImapProxyAccount {
|
||||
id String @id @default(uuid()) @db.Uuid
|
||||
userId String @map("user_id") @db.Uuid
|
||||
|
||||
@ -4,11 +4,13 @@ import {
|
||||
deleteOldMailBlocked,
|
||||
getAlreadyBlockedUidSet,
|
||||
insertMailBlocked,
|
||||
upsertMailBlockedStat,
|
||||
updateMailConnectionScanStats,
|
||||
} from "../../db/mail";
|
||||
import { getBlocklistedDomainsSet } from "../../db/domains";
|
||||
import { getProfile } from "../../db/profile";
|
||||
import { getPlanLimits } from "../../utils/plan-features";
|
||||
import { resolveProviderMeta } from "../../utils/imap-providers";
|
||||
// Single-Source-of-Truth (Mo's Finding #4)
|
||||
// @ts-expect-error — .mjs ohne types, GAMBLING_KEYWORDS ist string[]
|
||||
import { GAMBLING_KEYWORDS } from "../../utils/gambling-keywords.mjs";
|
||||
@ -183,6 +185,18 @@ export default defineEventHandler(async (event) => {
|
||||
}
|
||||
|
||||
await insertMailBlocked(toInsert);
|
||||
|
||||
// Aggregat-Stats aktualisieren (vor 24h-Cleanup resistent)
|
||||
if (toInsert.length > 0) {
|
||||
const providerMeta = resolveProviderMeta(connection.imapHost);
|
||||
await upsertMailBlockedStat({
|
||||
userId,
|
||||
mailConnectionId: connection.id,
|
||||
provider: providerMeta.provider,
|
||||
providerLabel: providerMeta.providerLabel,
|
||||
count: toInsert.length,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
|
||||
@ -4,11 +4,13 @@ import {
|
||||
deleteOldMailBlocked,
|
||||
getAlreadyBlockedUidSet,
|
||||
insertMailBlocked,
|
||||
upsertMailBlockedStat,
|
||||
updateMailConnectionScanStats,
|
||||
} from "../../db/mail";
|
||||
import { getBlocklistedDomainsSet } from "../../db/domains";
|
||||
import { getProfile } from "../../db/profile";
|
||||
import { getPlanLimits } from "../../utils/plan-features";
|
||||
import { resolveProviderMeta } from "../../utils/imap-providers";
|
||||
// Single-Source-of-Truth (Mo's Finding #4)
|
||||
// @ts-expect-error — .mjs ohne types, GAMBLING_KEYWORDS ist string[]
|
||||
import { GAMBLING_KEYWORDS } from "../../utils/gambling-keywords.mjs";
|
||||
@ -173,6 +175,18 @@ export default defineEventHandler(async (event) => {
|
||||
}
|
||||
|
||||
await insertMailBlocked(toInsert);
|
||||
|
||||
// Aggregat-Stats aktualisieren (vor 24h-Cleanup resistent)
|
||||
if (toInsert.length > 0) {
|
||||
const providerMeta = resolveProviderMeta(connection.imapHost);
|
||||
await upsertMailBlockedStat({
|
||||
userId: user.id,
|
||||
mailConnectionId: connection.id,
|
||||
provider: providerMeta.provider,
|
||||
providerLabel: providerMeta.providerLabel,
|
||||
count: toInsert.length,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
|
||||
@ -230,6 +230,48 @@ export async function deleteOldMailBlocked(userId: string) {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* UPSERT einen Aggregat-Zähler in mail_blocked_stats.
|
||||
* Wird direkt nach insertMailBlocked pro Connection aufgerufen.
|
||||
* date ist UTC-Mitternacht des aktuellen Tages.
|
||||
* Bei Conflict (selber User+Tag+Connection): count += 1.
|
||||
*/
|
||||
export async function upsertMailBlockedStat(entry: {
|
||||
userId: string;
|
||||
mailConnectionId: string;
|
||||
provider: string;
|
||||
providerLabel: string;
|
||||
count: number;
|
||||
}) {
|
||||
const db = usePrisma();
|
||||
const today = new Date();
|
||||
today.setUTCHours(0, 0, 0, 0);
|
||||
|
||||
return db.mailBlockedStat.upsert({
|
||||
where: {
|
||||
userId_date_mailConnectionId: {
|
||||
userId: entry.userId,
|
||||
date: today,
|
||||
mailConnectionId: entry.mailConnectionId,
|
||||
},
|
||||
},
|
||||
create: {
|
||||
userId: entry.userId,
|
||||
date: today,
|
||||
mailConnectionId: entry.mailConnectionId,
|
||||
provider: entry.provider,
|
||||
providerLabel: entry.providerLabel,
|
||||
count: entry.count,
|
||||
},
|
||||
update: {
|
||||
// Neuester Label gewinnt (falls User IMAP-Host gewechselt hat)
|
||||
provider: entry.provider,
|
||||
providerLabel: entry.providerLabel,
|
||||
count: { increment: entry.count },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function getMailBlockedPaginated(
|
||||
userId: string,
|
||||
page: number,
|
||||
@ -281,6 +323,7 @@ export async function updateMailConnectionTitle(
|
||||
|
||||
/**
|
||||
* Geblockte Mails pro Tag (UTC) für die letzten N Tage — für Bar-Chart.
|
||||
* Liest aus mail_blocked_stats (permanent, kein 24h-Cleanup).
|
||||
* Fehlende Tage werden mit count=0 aufgefüllt.
|
||||
*/
|
||||
export async function getBlockedMailsByDay(
|
||||
@ -289,15 +332,16 @@ export async function getBlockedMailsByDay(
|
||||
): Promise<{ date: string; count: number }[]> {
|
||||
const db = usePrisma();
|
||||
const since = new Date(Date.now() - days * 86_400_000);
|
||||
since.setUTCHours(0, 0, 0, 0);
|
||||
|
||||
// Prisma hat kein groupBy auf DATE-Funktionen → raw query
|
||||
// Aggregiere SUM(count) pro Tag aus der permanenten Stats-Tabelle
|
||||
const rows = await db.$queryRaw<{ date: string; count: bigint }[]>`
|
||||
SELECT TO_CHAR(DATE("created_at"), 'YYYY-MM-DD') AS date, COUNT(*) AS count
|
||||
FROM "rebreak"."mail_blocked"
|
||||
SELECT TO_CHAR("date", 'YYYY-MM-DD') AS date, SUM("count")::bigint AS count
|
||||
FROM "rebreak"."mail_blocked_stats"
|
||||
WHERE "user_id" = ${userId}::uuid
|
||||
AND "created_at" >= ${since}
|
||||
GROUP BY DATE("created_at")
|
||||
ORDER BY DATE("created_at") ASC
|
||||
AND "date" >= ${since}::date
|
||||
GROUP BY "date"
|
||||
ORDER BY "date" ASC
|
||||
`;
|
||||
|
||||
const map: Record<string, number> = {};
|
||||
@ -315,20 +359,24 @@ export async function getBlockedMailsByDay(
|
||||
|
||||
/**
|
||||
* Anzahl blockierter Mails pro MailConnection — für Half-Donut-Chart.
|
||||
* Connections ohne blocked emails werden NICHT included.
|
||||
* Liest aus mail_blocked_stats (permanent).
|
||||
* Connections ohne blocked emails (stats=0) werden NICHT included.
|
||||
* Gibt imapHost zurück — resolveProviderMeta() wird im Endpoint aufgerufen.
|
||||
*/
|
||||
export async function getBlockedMailsByConnection(userId: string) {
|
||||
const db = usePrisma();
|
||||
const rows = await db.mailBlocked.groupBy({
|
||||
by: ["connectionId"],
|
||||
|
||||
// SUM(count) pro Connection aus Stats-Tabelle
|
||||
const rows = await db.mailBlockedStat.groupBy({
|
||||
by: ["mailConnectionId"],
|
||||
where: { userId },
|
||||
_count: { id: true },
|
||||
orderBy: { _count: { id: "desc" } },
|
||||
_sum: { count: true },
|
||||
orderBy: { _sum: { count: "desc" } },
|
||||
});
|
||||
|
||||
if (rows.length === 0) return [];
|
||||
|
||||
const connectionIds = rows.map((r) => r.connectionId);
|
||||
const connectionIds = rows.map((r) => r.mailConnectionId);
|
||||
const connections = await db.mailConnection.findMany({
|
||||
where: { id: { in: connectionIds } },
|
||||
select: { id: true, email: true, title: true, providerName: true, imapHost: true },
|
||||
@ -337,14 +385,14 @@ export async function getBlockedMailsByConnection(userId: string) {
|
||||
const connMap = new Map(connections.map((c) => [c.id, c]));
|
||||
|
||||
return rows.map((r) => {
|
||||
const conn = connMap.get(r.connectionId);
|
||||
const conn = connMap.get(r.mailConnectionId);
|
||||
return {
|
||||
connectionId: r.connectionId,
|
||||
connectionId: r.mailConnectionId,
|
||||
title: conn?.title ?? null,
|
||||
email: conn?.email ?? "",
|
||||
providerName: conn?.providerName ?? null,
|
||||
imapHost: conn?.imapHost ?? "",
|
||||
count: r._count.id,
|
||||
count: r._sum.count ?? 0,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user