feat(mail): separate mail_blocked_stats table — preserves charts beyond 24h cleanup
deleteOldMailBlocked löscht weiter rohe Einträge nach 24h (Datenminimierung für Mail-Inhalte, DSGVO Art. 5 Abs. 1 lit. c). Aber für Charts und Pattern-Analysen werden vor dem Cleanup permanent aggregierte Daten in einer separaten Tabelle geführt. Architektur: - Neue Tabelle mail_blocked_stats — UNIQUE (user_id, date, connection_id), enthält ausschließlich counts + UTC-Datum + IMAP-Host. Kein Subject, kein Sender, kein Mail-Inhalt. Datenminimierung jetzt auch im Audit- Pfad sichtbar. - Live-Aggregation: scan.post.ts + scan-internal.post.ts upserten direkt nach jedem mail_blocked-INSERT in mail_blocked_stats (count += 1). - 30-Tage-Backfill als SQL im Migration-File: bestehende mail_blocked- Rows der letzten 30 Tage werden einmalig aggregiert, damit Charts nicht 30 Tage lang leer aussehen. - Stats-Endpoints (blocked-by-day, blocked-by-connection) lesen jetzt aus mail_blocked_stats. Response-Shape unverändert → Frontend bleibt unberührt. ON DELETE CASCADE auf mail_connection_id (Hans-Müller-konservativ): User-initiierter Disconnect = Art. 17-Signal → assoziierte Stats werden mitgelöscht. SetNull wäre DSGVO-grenzwertig (orphan stats ohne klare Lösch-Kontext-Zuordnung). pnpm build:backend clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
b7909d77e4
commit
275637f0b0
@ -0,0 +1,66 @@
|
|||||||
|
-- Migration: mail_blocked_stats_aggregation
|
||||||
|
-- Permanente Aggregat-Tabelle für blockierte Mails (counts/dates only, KEINE Inhalte).
|
||||||
|
-- Löst das DSGVO-Spannungsfeld: mail_blocked wird nach 24h gelöscht (Datenminimierung
|
||||||
|
-- Art. 5 Abs. 1 lit. e), während Stats-Charts weiterhin historische Daten zeigen können.
|
||||||
|
--
|
||||||
|
-- Stats enthalten ausschließlich: userId, date (UTC), connectionId, provider-host, count.
|
||||||
|
-- Kein Subject, kein Sender, kein Mail-Inhalt — Aggregat ist unbedenklich i.S. Art. 9.
|
||||||
|
--
|
||||||
|
-- Lösch-Verhalten: ON DELETE CASCADE wenn MailConnection gelöscht wird.
|
||||||
|
-- Begründung: User-initiierter Disconnect = Art. 17-Löschanfrage; Stats sind an
|
||||||
|
-- diese Connection geknüpft; Cascade hält DB konsistent ohne nullable foreign-key.
|
||||||
|
-- Hans-Müller-Note: Aggregat-Stats sind kein "Personenbezug zur Mail" mehr, aber
|
||||||
|
-- der Bezug zur Connection-ID bleibt — Cascade ist DSGVO-konservativere Wahl.
|
||||||
|
--
|
||||||
|
-- Backfill: Einmalig aus bestehenden mail_blocked-Rows der letzten 30 Tage.
|
||||||
|
-- provider-Spalte speichert imap_host als rohen String; resolveProviderMeta()
|
||||||
|
-- wird zur Read-Zeit im Stats-Endpoint aufgerufen (TypeScript, nicht SQL).
|
||||||
|
-- Für den Backfill wird imap_host doppelt als providerLabel eingetragen —
|
||||||
|
-- der Live-Scan-Code schreibt ab jetzt den korrekten resolved Label.
|
||||||
|
-- Ein separater Script (backend/scripts/backfill-stats-provider.ts) kann nachgezogen
|
||||||
|
-- werden wenn die Labels im Chart sauber sein sollen; für MVP nicht kritisch.
|
||||||
|
--
|
||||||
|
-- Deploy: automatisch via GitHub Actions (pnpm prisma migrate deploy)
|
||||||
|
|
||||||
|
CREATE TABLE "rebreak"."mail_blocked_stats" (
|
||||||
|
"id" TEXT NOT NULL,
|
||||||
|
"user_id" UUID NOT NULL,
|
||||||
|
"date" DATE NOT NULL,
|
||||||
|
"mail_connection_id" UUID NOT NULL,
|
||||||
|
"provider" TEXT NOT NULL,
|
||||||
|
"provider_label" TEXT NOT NULL,
|
||||||
|
"count" INTEGER NOT NULL DEFAULT 0,
|
||||||
|
|
||||||
|
CONSTRAINT "mail_blocked_stats_pkey" PRIMARY KEY ("id"),
|
||||||
|
CONSTRAINT "mail_blocked_stats_unique_day"
|
||||||
|
UNIQUE ("user_id", "date", "mail_connection_id"),
|
||||||
|
CONSTRAINT "mail_blocked_stats_connection_fkey"
|
||||||
|
FOREIGN KEY ("mail_connection_id")
|
||||||
|
REFERENCES "rebreak"."mail_connections" ("id")
|
||||||
|
ON DELETE CASCADE ON UPDATE CASCADE
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX "mail_blocked_stats_user_date_idx"
|
||||||
|
ON "rebreak"."mail_blocked_stats" ("user_id", "date");
|
||||||
|
|
||||||
|
CREATE INDEX "mail_blocked_stats_user_connection_idx"
|
||||||
|
ON "rebreak"."mail_blocked_stats" ("user_id", "mail_connection_id");
|
||||||
|
|
||||||
|
-- Backfill: bestehende mail_blocked-Rows der letzten 30 Tage aggregieren.
|
||||||
|
-- imap_host wird als provider UND provider_label gespeichert (rohes Datum).
|
||||||
|
-- Live-Scan schreibt ab jetzt resolved labels via resolveProviderMeta().
|
||||||
|
INSERT INTO "rebreak"."mail_blocked_stats"
|
||||||
|
("id", "user_id", "date", "mail_connection_id", "provider", "provider_label", "count")
|
||||||
|
SELECT
|
||||||
|
gen_random_uuid()::text,
|
||||||
|
mb."user_id",
|
||||||
|
DATE(mb."created_at" AT TIME ZONE 'UTC') AS "date",
|
||||||
|
mb."connection_id",
|
||||||
|
mc."imap_host",
|
||||||
|
mc."imap_host",
|
||||||
|
COUNT(*)::integer
|
||||||
|
FROM "rebreak"."mail_blocked" mb
|
||||||
|
JOIN "rebreak"."mail_connections" mc ON mc."id" = mb."connection_id"
|
||||||
|
WHERE mb."created_at" > NOW() - INTERVAL '30 days'
|
||||||
|
GROUP BY mb."user_id", DATE(mb."created_at" AT TIME ZONE 'UTC'), mb."connection_id", mc."imap_host"
|
||||||
|
ON CONFLICT ("user_id", "date", "mail_connection_id") DO NOTHING;
|
||||||
@ -534,7 +534,8 @@ model MailConnection {
|
|||||||
consentVersion String? @map("consent_version")
|
consentVersion String? @map("consent_version")
|
||||||
consentIpAddress String? @map("consent_ip_address")
|
consentIpAddress String? @map("consent_ip_address")
|
||||||
|
|
||||||
blockedMails MailBlocked[]
|
blockedMails MailBlocked[]
|
||||||
|
blockedStats MailBlockedStat[]
|
||||||
|
|
||||||
@@unique([userId, email])
|
@@unique([userId, email])
|
||||||
@@map("mail_connections")
|
@@map("mail_connections")
|
||||||
@ -646,6 +647,33 @@ model MailBlocked {
|
|||||||
@@schema("rebreak")
|
@@schema("rebreak")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Permanente Aggregat-Statistiken blockierter Mails pro Tag + Connection.
|
||||||
|
/// Befüllt live beim Scan (vor dem 24h-Cleanup von mail_blocked).
|
||||||
|
/// Enthält KEINE Mail-Inhalte — nur counts/dates (Datenminimierung Art. 5 DSGVO).
|
||||||
|
/// Bei Connection-Disconnect werden Stats mitgelöscht (Cascade) — Konsistenz
|
||||||
|
/// mit Art. 17 DSGVO: User-initiierter Disconnect = Recht auf Löschung aller
|
||||||
|
/// zu dieser Connection gehörenden Daten.
|
||||||
|
model MailBlockedStat {
|
||||||
|
id String @id @default(cuid())
|
||||||
|
userId String @map("user_id") @db.Uuid
|
||||||
|
/// UTC-Datum (time=00:00:00) — ein Eintrag pro User+Tag+Connection
|
||||||
|
date DateTime @db.Date @map("date")
|
||||||
|
mailConnectionId String @map("mail_connection_id") @db.Uuid
|
||||||
|
/// IMAP-Host-Slug, z.B. "imap.gmail.com" (raw, für resolveProviderMeta zur Read-Zeit)
|
||||||
|
provider String @map("provider")
|
||||||
|
/// Human-readable Label, z.B. "Gmail" (wird bei Scan-Zeit aus resolveProviderMeta befüllt)
|
||||||
|
providerLabel String @map("provider_label")
|
||||||
|
count Int @default(0)
|
||||||
|
|
||||||
|
connection MailConnection @relation(fields: [mailConnectionId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@unique([userId, date, mailConnectionId], map: "mail_blocked_stats_unique_day")
|
||||||
|
@@index([userId, date])
|
||||||
|
@@index([userId, mailConnectionId])
|
||||||
|
@@map("mail_blocked_stats")
|
||||||
|
@@schema("rebreak")
|
||||||
|
}
|
||||||
|
|
||||||
model ImapProxyAccount {
|
model ImapProxyAccount {
|
||||||
id String @id @default(uuid()) @db.Uuid
|
id String @id @default(uuid()) @db.Uuid
|
||||||
userId String @map("user_id") @db.Uuid
|
userId String @map("user_id") @db.Uuid
|
||||||
|
|||||||
@ -4,11 +4,13 @@ import {
|
|||||||
deleteOldMailBlocked,
|
deleteOldMailBlocked,
|
||||||
getAlreadyBlockedUidSet,
|
getAlreadyBlockedUidSet,
|
||||||
insertMailBlocked,
|
insertMailBlocked,
|
||||||
|
upsertMailBlockedStat,
|
||||||
updateMailConnectionScanStats,
|
updateMailConnectionScanStats,
|
||||||
} from "../../db/mail";
|
} from "../../db/mail";
|
||||||
import { getBlocklistedDomainsSet } from "../../db/domains";
|
import { getBlocklistedDomainsSet } from "../../db/domains";
|
||||||
import { getProfile } from "../../db/profile";
|
import { getProfile } from "../../db/profile";
|
||||||
import { getPlanLimits } from "../../utils/plan-features";
|
import { getPlanLimits } from "../../utils/plan-features";
|
||||||
|
import { resolveProviderMeta } from "../../utils/imap-providers";
|
||||||
// Single-Source-of-Truth (Mo's Finding #4)
|
// Single-Source-of-Truth (Mo's Finding #4)
|
||||||
// @ts-expect-error — .mjs ohne types, GAMBLING_KEYWORDS ist string[]
|
// @ts-expect-error — .mjs ohne types, GAMBLING_KEYWORDS ist string[]
|
||||||
import { GAMBLING_KEYWORDS } from "../../utils/gambling-keywords.mjs";
|
import { GAMBLING_KEYWORDS } from "../../utils/gambling-keywords.mjs";
|
||||||
@ -183,6 +185,18 @@ export default defineEventHandler(async (event) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await insertMailBlocked(toInsert);
|
await insertMailBlocked(toInsert);
|
||||||
|
|
||||||
|
// Aggregat-Stats aktualisieren (vor 24h-Cleanup resistent)
|
||||||
|
if (toInsert.length > 0) {
|
||||||
|
const providerMeta = resolveProviderMeta(connection.imapHost);
|
||||||
|
await upsertMailBlockedStat({
|
||||||
|
userId,
|
||||||
|
mailConnectionId: connection.id,
|
||||||
|
provider: providerMeta.provider,
|
||||||
|
providerLabel: providerMeta.providerLabel,
|
||||||
|
count: toInsert.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
lock.release();
|
lock.release();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,11 +4,13 @@ import {
|
|||||||
deleteOldMailBlocked,
|
deleteOldMailBlocked,
|
||||||
getAlreadyBlockedUidSet,
|
getAlreadyBlockedUidSet,
|
||||||
insertMailBlocked,
|
insertMailBlocked,
|
||||||
|
upsertMailBlockedStat,
|
||||||
updateMailConnectionScanStats,
|
updateMailConnectionScanStats,
|
||||||
} from "../../db/mail";
|
} from "../../db/mail";
|
||||||
import { getBlocklistedDomainsSet } from "../../db/domains";
|
import { getBlocklistedDomainsSet } from "../../db/domains";
|
||||||
import { getProfile } from "../../db/profile";
|
import { getProfile } from "../../db/profile";
|
||||||
import { getPlanLimits } from "../../utils/plan-features";
|
import { getPlanLimits } from "../../utils/plan-features";
|
||||||
|
import { resolveProviderMeta } from "../../utils/imap-providers";
|
||||||
// Single-Source-of-Truth (Mo's Finding #4)
|
// Single-Source-of-Truth (Mo's Finding #4)
|
||||||
// @ts-expect-error — .mjs ohne types, GAMBLING_KEYWORDS ist string[]
|
// @ts-expect-error — .mjs ohne types, GAMBLING_KEYWORDS ist string[]
|
||||||
import { GAMBLING_KEYWORDS } from "../../utils/gambling-keywords.mjs";
|
import { GAMBLING_KEYWORDS } from "../../utils/gambling-keywords.mjs";
|
||||||
@ -173,6 +175,18 @@ export default defineEventHandler(async (event) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
await insertMailBlocked(toInsert);
|
await insertMailBlocked(toInsert);
|
||||||
|
|
||||||
|
// Aggregat-Stats aktualisieren (vor 24h-Cleanup resistent)
|
||||||
|
if (toInsert.length > 0) {
|
||||||
|
const providerMeta = resolveProviderMeta(connection.imapHost);
|
||||||
|
await upsertMailBlockedStat({
|
||||||
|
userId: user.id,
|
||||||
|
mailConnectionId: connection.id,
|
||||||
|
provider: providerMeta.provider,
|
||||||
|
providerLabel: providerMeta.providerLabel,
|
||||||
|
count: toInsert.length,
|
||||||
|
});
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
lock.release();
|
lock.release();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -230,6 +230,48 @@ export async function deleteOldMailBlocked(userId: string) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* UPSERT einen Aggregat-Zähler in mail_blocked_stats.
|
||||||
|
* Wird direkt nach insertMailBlocked pro Connection aufgerufen.
|
||||||
|
* date ist UTC-Mitternacht des aktuellen Tages.
|
||||||
|
* Bei Conflict (selber User+Tag+Connection): count += 1.
|
||||||
|
*/
|
||||||
|
export async function upsertMailBlockedStat(entry: {
|
||||||
|
userId: string;
|
||||||
|
mailConnectionId: string;
|
||||||
|
provider: string;
|
||||||
|
providerLabel: string;
|
||||||
|
count: number;
|
||||||
|
}) {
|
||||||
|
const db = usePrisma();
|
||||||
|
const today = new Date();
|
||||||
|
today.setUTCHours(0, 0, 0, 0);
|
||||||
|
|
||||||
|
return db.mailBlockedStat.upsert({
|
||||||
|
where: {
|
||||||
|
userId_date_mailConnectionId: {
|
||||||
|
userId: entry.userId,
|
||||||
|
date: today,
|
||||||
|
mailConnectionId: entry.mailConnectionId,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
create: {
|
||||||
|
userId: entry.userId,
|
||||||
|
date: today,
|
||||||
|
mailConnectionId: entry.mailConnectionId,
|
||||||
|
provider: entry.provider,
|
||||||
|
providerLabel: entry.providerLabel,
|
||||||
|
count: entry.count,
|
||||||
|
},
|
||||||
|
update: {
|
||||||
|
// Neuester Label gewinnt (falls User IMAP-Host gewechselt hat)
|
||||||
|
provider: entry.provider,
|
||||||
|
providerLabel: entry.providerLabel,
|
||||||
|
count: { increment: entry.count },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function getMailBlockedPaginated(
|
export async function getMailBlockedPaginated(
|
||||||
userId: string,
|
userId: string,
|
||||||
page: number,
|
page: number,
|
||||||
@ -281,6 +323,7 @@ export async function updateMailConnectionTitle(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Geblockte Mails pro Tag (UTC) für die letzten N Tage — für Bar-Chart.
|
* Geblockte Mails pro Tag (UTC) für die letzten N Tage — für Bar-Chart.
|
||||||
|
* Liest aus mail_blocked_stats (permanent, kein 24h-Cleanup).
|
||||||
* Fehlende Tage werden mit count=0 aufgefüllt.
|
* Fehlende Tage werden mit count=0 aufgefüllt.
|
||||||
*/
|
*/
|
||||||
export async function getBlockedMailsByDay(
|
export async function getBlockedMailsByDay(
|
||||||
@ -289,15 +332,16 @@ export async function getBlockedMailsByDay(
|
|||||||
): Promise<{ date: string; count: number }[]> {
|
): Promise<{ date: string; count: number }[]> {
|
||||||
const db = usePrisma();
|
const db = usePrisma();
|
||||||
const since = new Date(Date.now() - days * 86_400_000);
|
const since = new Date(Date.now() - days * 86_400_000);
|
||||||
|
since.setUTCHours(0, 0, 0, 0);
|
||||||
|
|
||||||
// Prisma hat kein groupBy auf DATE-Funktionen → raw query
|
// Aggregiere SUM(count) pro Tag aus der permanenten Stats-Tabelle
|
||||||
const rows = await db.$queryRaw<{ date: string; count: bigint }[]>`
|
const rows = await db.$queryRaw<{ date: string; count: bigint }[]>`
|
||||||
SELECT TO_CHAR(DATE("created_at"), 'YYYY-MM-DD') AS date, COUNT(*) AS count
|
SELECT TO_CHAR("date", 'YYYY-MM-DD') AS date, SUM("count")::bigint AS count
|
||||||
FROM "rebreak"."mail_blocked"
|
FROM "rebreak"."mail_blocked_stats"
|
||||||
WHERE "user_id" = ${userId}::uuid
|
WHERE "user_id" = ${userId}::uuid
|
||||||
AND "created_at" >= ${since}
|
AND "date" >= ${since}::date
|
||||||
GROUP BY DATE("created_at")
|
GROUP BY "date"
|
||||||
ORDER BY DATE("created_at") ASC
|
ORDER BY "date" ASC
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const map: Record<string, number> = {};
|
const map: Record<string, number> = {};
|
||||||
@ -315,20 +359,24 @@ export async function getBlockedMailsByDay(
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Anzahl blockierter Mails pro MailConnection — für Half-Donut-Chart.
|
* Anzahl blockierter Mails pro MailConnection — für Half-Donut-Chart.
|
||||||
* Connections ohne blocked emails werden NICHT included.
|
* Liest aus mail_blocked_stats (permanent).
|
||||||
|
* Connections ohne blocked emails (stats=0) werden NICHT included.
|
||||||
|
* Gibt imapHost zurück — resolveProviderMeta() wird im Endpoint aufgerufen.
|
||||||
*/
|
*/
|
||||||
export async function getBlockedMailsByConnection(userId: string) {
|
export async function getBlockedMailsByConnection(userId: string) {
|
||||||
const db = usePrisma();
|
const db = usePrisma();
|
||||||
const rows = await db.mailBlocked.groupBy({
|
|
||||||
by: ["connectionId"],
|
// SUM(count) pro Connection aus Stats-Tabelle
|
||||||
|
const rows = await db.mailBlockedStat.groupBy({
|
||||||
|
by: ["mailConnectionId"],
|
||||||
where: { userId },
|
where: { userId },
|
||||||
_count: { id: true },
|
_sum: { count: true },
|
||||||
orderBy: { _count: { id: "desc" } },
|
orderBy: { _sum: { count: "desc" } },
|
||||||
});
|
});
|
||||||
|
|
||||||
if (rows.length === 0) return [];
|
if (rows.length === 0) return [];
|
||||||
|
|
||||||
const connectionIds = rows.map((r) => r.connectionId);
|
const connectionIds = rows.map((r) => r.mailConnectionId);
|
||||||
const connections = await db.mailConnection.findMany({
|
const connections = await db.mailConnection.findMany({
|
||||||
where: { id: { in: connectionIds } },
|
where: { id: { in: connectionIds } },
|
||||||
select: { id: true, email: true, title: true, providerName: true, imapHost: true },
|
select: { id: true, email: true, title: true, providerName: true, imapHost: true },
|
||||||
@ -337,14 +385,14 @@ export async function getBlockedMailsByConnection(userId: string) {
|
|||||||
const connMap = new Map(connections.map((c) => [c.id, c]));
|
const connMap = new Map(connections.map((c) => [c.id, c]));
|
||||||
|
|
||||||
return rows.map((r) => {
|
return rows.map((r) => {
|
||||||
const conn = connMap.get(r.connectionId);
|
const conn = connMap.get(r.mailConnectionId);
|
||||||
return {
|
return {
|
||||||
connectionId: r.connectionId,
|
connectionId: r.mailConnectionId,
|
||||||
title: conn?.title ?? null,
|
title: conn?.title ?? null,
|
||||||
email: conn?.email ?? "",
|
email: conn?.email ?? "",
|
||||||
providerName: conn?.providerName ?? null,
|
providerName: conn?.providerName ?? null,
|
||||||
imapHost: conn?.imapHost ?? "",
|
imapHost: conn?.imapHost ?? "",
|
||||||
count: r._count.id,
|
count: r._sum.count ?? 0,
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user