diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index cb14b11..a525b00 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -67,6 +67,13 @@ jobs: path: backend-output.tar.gz retention-days: 7 + - name: Upload imap-idle artifact + uses: actions/upload-artifact@v4 + with: + name: imap-idle + path: backend/imap-idle/ + retention-days: 7 + # ── 2. Deploy: Artifact zum Hetzner pushen + extract + pm2 restart ────────── deploy: name: Deploy zu Hetzner @@ -79,6 +86,12 @@ jobs: with: name: backend-output + - name: Download imap-idle artifact + uses: actions/download-artifact@v4 + with: + name: imap-idle + path: imap-idle/ + - name: Setup SSH env: SSH_PRIVATE_KEY: ${{ secrets.HETZNER_SSH_KEY }} @@ -102,6 +115,14 @@ jobs: scp -i ~/.ssh/id_ed25519 backend-output.tar.gz \ "$SSH_USER@$SSH_HOST:/srv/rebreak/backend/.output-incoming.tar.gz" + - name: Upload imap-idle zu Hetzner + env: + SSH_HOST: ${{ vars.HETZNER_HOST }} + SSH_USER: ${{ vars.HETZNER_USER }} + run: | + scp -r -i ~/.ssh/id_ed25519 imap-idle/ \ + "$SSH_USER@$SSH_HOST:/srv/rebreak/backend/imap-idle/" + - name: Server-side deploy (extract + migrate + pm2 restart) env: SSH_HOST: ${{ vars.HETZNER_HOST }} diff --git a/backend/imap-idle/README.md b/backend/imap-idle/README.md new file mode 100644 index 0000000..af17b62 --- /dev/null +++ b/backend/imap-idle/README.md @@ -0,0 +1,51 @@ +# rebreak-imap-idle + +Standalone IMAP IDLE Daemon für Rebreak. + +## Was er macht + +- Hält pro aktiver `MailConnection` (DB) eine persistente IMAP-IDLE-Session +- Reagiert in Echtzeit auf `EXISTS`-Events (neue Mail im Postfach) +- Feuert bei jedem Event `POST /api/mail/scan-internal` gegen das lokale Backend +- Das Backend entscheidet ob und welche Mails gelöscht werden (Gambling-Keywords + Blocklist) +- Aktualisiert alle 5 min die Connection-Liste (neue User → neue Sessions, entfernte → geschlossen) +- IDLE wird alle 25 min erneuert (RFC 3501 Server-Timeout liegt bei 29 min) + +## Env-Vars + +| Variable | Pflicht | Beschreibung | +|---------------------|---------|-----------------------------------------------------------| +| `DATABASE_URL` | ja | Postgres-Connection-String (Supabase Pooler oder direkt) | +| `ADMIN_SECRET` | ja | Shared Secret für /api/mail/scan-internal Header | +| `ENCRYPTION_KEY` | ja | AES-256 Key (identisch zum Backend-Key, 32+ Zeichen) | +| `BACKEND_URL` | nein | Default: http://127.0.0.1:3016 (staging) / 3015 (prod) | +| `NODE_ENV` | nein | `production` → BACKEND_URL default port 3015 | + +## Lokal starten (Entwicklung) + +```bash +cd backend/imap-idle +npm install +DATABASE_URL=<...> ADMIN_SECRET=<...> ENCRYPTION_KEY=<...> node index.mjs +``` + +Via Infisical (analog zu start-staging.sh): + +```bash +infisical run --env=staging -- node index.mjs +``` + +## PM2 (Produktion) + +Wird via ecosystem.config.js gestartet — siehe `docs/internal/MAIL_DAEMON_DEPLOYMENT.md`. + +## Logs (pm2) + +``` +[idle/] connected (imap.gmail.com:993) +[idle/] exists-event received (new mail) +[idle/] scan-triggered → scanned=12 blocked=1 +[idle/] idle renewing (25min threshold) +[idle/] reconnecting in 5s (attempt 2) +[idle/db] refreshed — 47 active connections, 47 sessions +``` diff --git a/backend/imap-idle/index.mjs b/backend/imap-idle/index.mjs new file mode 100644 index 0000000..bc22c58 --- /dev/null +++ b/backend/imap-idle/index.mjs @@ -0,0 +1,353 @@ +/** + * rebreak-imap-idle — IMAP IDLE Daemon + * + * Hält pro aktivem MailConnection-Eintrag eine persistente IMAP-IDLE-Session. + * Wenn der Server "EXISTS" meldet (neue Mail), feuert der Daemon sofort + * POST /api/mail/scan-internal gegen das lokale Backend — ohne 30min-Warte. + * + * Env-Vars (via Infisical-Wrapper): + * DATABASE_URL — Postgres-Connection-String + * ADMIN_SECRET — Header-Secret für /api/mail/scan-internal + * ENCRYPTION_KEY — AES-256 Key (gleicher wie im Backend) + * BACKEND_URL — z.B. http://127.0.0.1:3016 (default: 3016) + * NODE_ENV — production / staging + * + * Starten: + * node index.mjs + * + * Prozess-Signale: + * SIGTERM / SIGINT → graceful shutdown (alle IMAP-Sessions schließen) + */ + +import { ImapFlow } from "imapflow"; +import pg from "pg"; +import { createDecipheriv } from "crypto"; + +// ─── Config ───────────────────────────────────────────────────────────────── + +const BACKEND_URL = + process.env.BACKEND_URL || + (process.env.NODE_ENV === "production" + ? "http://127.0.0.1:3015" + : "http://127.0.0.1:3016"); + +const ADMIN_SECRET = + process.env.NUXT_ADMIN_SECRET || process.env.ADMIN_SECRET || ""; + +const DB_REFRESH_INTERVAL_MS = 5 * 60 * 1000; // 5 min — neue Connections entdecken +const IDLE_RENEW_INTERVAL_MS = 25 * 60 * 1000; // 25 min — RFC 3501 max = 29min +const RECONNECT_DELAYS_MS = [1000, 5000, 30_000]; // exponential backoff, danach 60s loop +const RECONNECT_LOOP_DELAY_MS = 60 * 1000; + +// ─── DB-Pool (direktes pg, kein Prisma — Daemon ist kein Nitro-Kontext) ──── + +const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL }); + +async function loadActiveConnections() { + const { rows } = await pool.query( + `SELECT id, "userId", email, "imapHost", "imapPort", + "passwordEncrypted", "rejectUnauthorized", "useStarttls" + FROM rebreak."MailConnection" + WHERE "isActive" = true`, + ); + return rows; +} + +// ─── Crypto (analog zu server/utils/crypto.ts) ────────────────────────────── + +const AES_ALGO = "aes-256-gcm"; +const KEY_LENGTH = 32; + +function getKey() { + const raw = + process.env.NUXT_ENCRYPTION_KEY || process.env.ENCRYPTION_KEY || ""; + if (!raw || raw.length < KEY_LENGTH) { + return Buffer.alloc( + KEY_LENGTH, + raw.padEnd(KEY_LENGTH, "0").slice(0, KEY_LENGTH), + ); + } + return Buffer.from(raw.slice(0, KEY_LENGTH), "utf8"); +} + +function decrypt(stored) { + const parts = stored.split(":"); + if (parts.length !== 3) throw new Error("Invalid encrypted format"); + const [ivHex, tagHex, dataHex] = parts; + const decipher = createDecipheriv( + AES_ALGO, + getKey(), + Buffer.from(ivHex, "hex"), + ); + decipher.setAuthTag(Buffer.from(tagHex, "hex")); + return decipher.update(Buffer.from(dataHex, "hex")) + decipher.final("utf8"); +} + +// ─── Logging ──────────────────────────────────────────────────────────────── + +function log(email, msg) { + // NIEMALS password/credentials loggen. email ist safe (kein secret). + console.log(`[idle/${email}] ${msg}`); +} + +function logError(email, msg, err) { + const errMsg = err?.message ?? String(err); + // Credentials tauchen nie in err.message auf (ImapFlow maskiert sie nicht, + // aber wir liefern pass nur an ImapFlow — nicht in eigenen log-calls). + console.error(`[idle/${email}] ${msg}: ${errMsg}`); +} + +// ─── Session-Registry ──────────────────────────────────────────────────────── +// Map + +const sessions = new Map(); + +let shuttingDown = false; + +// ─── Scan-Trigger ──────────────────────────────────────────────────────────── + +async function triggerScan(conn) { + try { + const res = await fetch(`${BACKEND_URL}/api/mail/scan-internal`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-admin-secret": ADMIN_SECRET, + }, + body: JSON.stringify({ userId: conn.userId }), + }); + if (!res.ok) { + log(conn.email, `scan-trigger HTTP ${res.status}`); + } else { + const data = await res.json().catch(() => ({})); + log( + conn.email, + `scan-triggered → scanned=${data.scanned ?? "?"} blocked=${data.blocked ?? "?"}`, + ); + } + } catch (err) { + logError(conn.email, "scan-trigger failed", err); + } +} + +// ─── IDLE-Session ──────────────────────────────────────────────────────────── + +/** + * Startet eine einzelne IDLE-Session für eine MailConnection. + * Reconnect-Loop läuft intern — diese Funktion returned nie (bis shutdown). + */ +async function runSession(conn) { + let attempt = 0; + + while (!shuttingDown) { + let password; + try { + password = decrypt(conn.passwordEncrypted); + } catch (err) { + logError(conn.email, "decrypt failed — session aborted", err); + return; // Kein retry bei Decrypt-Fehler (kaputtes Credential) + } + + const useImplicitTls = !conn.useStarttls; + const imap = new ImapFlow({ + host: conn.imapHost, + port: conn.imapPort, + secure: useImplicitTls, + ...(conn.useStarttls ? { requireTLS: true } : {}), + auth: { user: conn.email, pass: password }, + logger: false, + tls: { rejectUnauthorized: conn.rejectUnauthorized ?? true }, + // Outlook schließt idle-connections aggressiv — disableCompression + // verhindert edge-cases bei partial reads nach reconnect + disableCompression: conn.imapHost.includes("office365"), + }); + + // Referenz ablegen damit shutdown() darauf zugreifen kann + const handle = sessions.get(conn.id); + if (handle) handle.imap = imap; + + try { + await imap.connect(); + log(conn.email, `connected (${conn.imapHost}:${conn.imapPort})`); + attempt = 0; // Reset nach erfolgreicher Verbindung + + await imap.getMailboxLock("INBOX"); + + // IDLE-Loop: alle IDLE_RENEW_INTERVAL_MS erneuern + while (!shuttingDown && sessions.has(conn.id)) { + let idleAbort; + + const idlePromise = new Promise((resolve, reject) => { + imap + .idle() + .then(resolve) + .catch(reject); + + idleAbort = () => { + // ImapFlow.idle() bricht ab wenn die Connection getrennt wird + imap.close(); + resolve(); + }; + }); + + // exists-event → sofort scannen + const onExists = () => { + log(conn.email, "exists-event received (new mail)"); + triggerScan(conn); // fire-and-forget + }; + imap.on("exists", onExists); + + // IDLE nach 25min erneuern (RFC 3501: Server darf nach 29min droppen) + const renewTimer = setTimeout(() => { + log(conn.email, "idle renewing (25min threshold)"); + imap.close(); // Unterbricht idle() → Loop iteriert → reconnect + }, IDLE_RENEW_INTERVAL_MS); + + try { + await idlePromise; + } finally { + clearTimeout(renewTimer); + imap.removeListener("exists", onExists); + } + + if (shuttingDown || !sessions.has(conn.id)) break; + + // Kurze Pause vor Reconnect (idle() returned auch bei normalem timeout) + await sleep(500); + } + + try { await imap.logout(); } catch { /* ignore */ } + return; // Sauberer Exit (shutdown oder Connection entfernt) + + } catch (err) { + logError(conn.email, "connection error", err); + try { imap.close(); } catch { /* ignore */ } + } + + if (shuttingDown || !sessions.has(conn.id)) return; + + // Exponential backoff + const delay = + attempt < RECONNECT_DELAYS_MS.length + ? RECONNECT_DELAYS_MS[attempt] + : RECONNECT_LOOP_DELAY_MS; + attempt++; + log(conn.email, `reconnecting in ${delay / 1000}s (attempt ${attempt})`); + await sleep(delay); + } +} + +// ─── Session-Management ────────────────────────────────────────────────────── + +function startSession(conn) { + if (sessions.has(conn.id)) return; // bereits aktiv + log(conn.email, "session starting"); + const handle = { conn, imap: null, promise: null }; + sessions.set(conn.id, handle); + handle.promise = runSession(conn).catch((err) => { + logError(conn.email, "session crashed (unhandled)", err); + sessions.delete(conn.id); + }); +} + +async function stopSession(connectionId, email) { + const handle = sessions.get(connectionId); + if (!handle) return; + log(email ?? connectionId, "session stopping"); + sessions.delete(connectionId); + if (handle.imap) { + try { handle.imap.close(); } catch { /* ignore */ } + } + await handle.promise.catch(() => {}); +} + +// ─── DB-Refresh-Loop ───────────────────────────────────────────────────────── + +async function refreshConnections() { + let rows; + try { + rows = await loadActiveConnections(); + } catch (err) { + console.error("[idle/db] loadActiveConnections failed:", err.message); + return; + } + + const activeIds = new Set(rows.map((r) => r.id)); + + // Neue Connections starten + for (const row of rows) { + if (!sessions.has(row.id)) { + startSession(row); + } + } + + // Entfernte Connections stoppen + for (const [id, handle] of sessions.entries()) { + if (!activeIds.has(id)) { + await stopSession(id, handle.conn?.email); + } + } + + console.log( + `[idle/db] refreshed — ${activeIds.size} active connections, ${sessions.size} sessions`, + ); +} + +// ─── Graceful Shutdown ─────────────────────────────────────────────────────── + +async function shutdown(signal) { + console.log(`[idle] received ${signal} — shutting down gracefully`); + shuttingDown = true; + + const stopPromises = []; + for (const [id, handle] of sessions.entries()) { + stopPromises.push(stopSession(id, handle.conn?.email)); + } + await Promise.allSettled(stopPromises); + + await pool.end().catch(() => {}); + console.log("[idle] shutdown complete"); + process.exit(0); +} + +process.on("SIGTERM", () => shutdown("SIGTERM")); +process.on("SIGINT", () => shutdown("SIGINT")); + +// ─── Startup ───────────────────────────────────────────────────────────────── + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function assertEnv() { + const missing = []; + if (!process.env.DATABASE_URL) missing.push("DATABASE_URL"); + if (!ADMIN_SECRET) missing.push("ADMIN_SECRET / NUXT_ADMIN_SECRET"); + if (missing.length > 0) { + console.error( + `[idle] FEHLER: fehlende Env-Vars: ${missing.join(", ")}. Daemon startet nicht.`, + ); + process.exit(1); + } +} + +async function main() { + assertEnv(); + + console.log( + `[idle] starting — backend=${BACKEND_URL} env=${process.env.NODE_ENV ?? "unknown"}`, + ); + + // Initialer Load + await refreshConnections(); + + // Periodischer DB-Refresh + setInterval(() => { + if (!shuttingDown) refreshConnections(); + }, DB_REFRESH_INTERVAL_MS); +} + +main().catch((err) => { + console.error("[idle] fatal startup error:", err); + process.exit(1); +}); diff --git a/backend/imap-idle/package.json b/backend/imap-idle/package.json new file mode 100644 index 0000000..55c2d7d --- /dev/null +++ b/backend/imap-idle/package.json @@ -0,0 +1,18 @@ +{ + "name": "rebreak-imap-idle", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "IMAP IDLE Daemon — Real-time Gambling-Mail-Detection für Legend-User", + "main": "index.mjs", + "scripts": { + "start": "node index.mjs" + }, + "dependencies": { + "imapflow": "^1.2.18", + "pg": "^8.16.3" + }, + "engines": { + "node": ">=20.0.0" + } +} diff --git a/backend/imap-idle/start-idle-staging.sh b/backend/imap-idle/start-idle-staging.sh new file mode 100755 index 0000000..df05da9 --- /dev/null +++ b/backend/imap-idle/start-idle-staging.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# rebreak-imap-idle Staging — Infisical-Secret-Injection +# +# Wird von pm2 als Script-Interpreter gestartet (ecosystem.config.js). +# Injiziert DATABASE_URL, ADMIN_SECRET, ENCRYPTION_KEY via Infisical-staging. +# ENCRYPTION_KEY muss identisch zum Backend-Key sein (AES-256-GCM). + +set -euo pipefail +source /etc/environment + +if [[ -z "${INFISICAL_CLIENT_ID:-}" || -z "${INFISICAL_CLIENT_SECRET:-}" ]]; then + echo "[idle] FEHLER: INFISICAL_CLIENT_ID / INFISICAL_CLIENT_SECRET nicht in /etc/environment gesetzt" >&2 + exit 1 +fi + +INFISICAL_TOKEN=$(infisical login \ + --method=universal-auth \ + --client-id="${INFISICAL_CLIENT_ID}" \ + --client-secret="${INFISICAL_CLIENT_SECRET}" \ + --silent --plain 2>/dev/null) + +[[ -z "$INFISICAL_TOKEN" ]] && { echo "[idle] Infisical login fehlgeschlagen" >&2; exit 1; } + +NODE_BIN="/root/.nvm/versions/node/v24.11.1/bin/node" +DAEMON="/srv/rebreak/backend/imap-idle/index.mjs" + +exec infisical run \ + --projectId="${INFISICAL_PROJECT_ID:-14b11b35-ef59-4b8a-a16b-398f0cc3ad93}" \ + --env=staging \ + --token="$INFISICAL_TOKEN" \ + -- bash -c ' + set -e + export DATABASE_URL="${DATABASE_URL:-${NUXT_DATABASE_URL:-}}" + export ADMIN_SECRET="${ADMIN_SECRET:-${NUXT_ADMIN_SECRET:-}}" + export ENCRYPTION_KEY="${ENCRYPTION_KEY:-${NUXT_ENCRYPTION_KEY:-}}" + export BACKEND_URL="http://127.0.0.1:3016" + exec '"$NODE_BIN"' '"$DAEMON"' + ' diff --git a/ecosystem.config.js b/ecosystem.config.js index 72d93bf..21e8b4b 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -86,6 +86,22 @@ module.exports = { max_memory_restart: "128M", }, + // ─── IMAP IDLE Daemon Staging ─────────────────────────────────────────── + // Deployment-Anleitung: docs/internal/MAIL_DAEMON_DEPLOYMENT.md + // start-idle-staging.sh injiziert Infisical-Secrets (DATABASE_URL, ADMIN_SECRET, ENCRYPTION_KEY). + // Initialer Start via: pm2 startOrReload /srv/rebreak/ecosystem.config.js + { + name: "rebreak-idle-staging", + script: `${APP_DIR}/imap-idle/start-idle-staging.sh`, + interpreter: "bash", + cwd: `${APP_DIR}/imap-idle`, + instances: 1, + autorestart: true, + watch: false, + max_memory_restart: "256M", + env: { NODE_ENV: "production" }, + }, + // ─── DNS-Blocker (auskommentiert bis DNS-Daemons aufgesetzt sind) ────── // { // name: "dns-rebreak", diff --git a/scripts/deploy-from-artifact.sh b/scripts/deploy-from-artifact.sh index 3790c88..55ec302 100755 --- a/scripts/deploy-from-artifact.sh +++ b/scripts/deploy-from-artifact.sh @@ -101,6 +101,19 @@ CI=true "${PNPM_BIN}" install --frozen-lockfile 2>&1 || { } log "pnpm install done" +# 3b. imap-idle Runtime-Deps installieren (imapflow + pg, standalone package.json) +IDLE_DIR="${APP_DIR}/imap-idle" +if [[ -d "$IDLE_DIR" && -f "$IDLE_DIR/package.json" ]]; then + log "Step 3b: npm install (imap-idle runtime-deps)..." + cd "$IDLE_DIR" + npm install --production --prefer-offline 2>&1 + # scp preserviert Permissions nicht immer -- explizit setzen + [[ -f "$IDLE_DIR/start-idle-staging.sh" ]] && chmod +x "$IDLE_DIR/start-idle-staging.sh" + log "imap-idle npm install done" +else + log "Step 3b: imap-idle-Verzeichnis nicht gefunden -- skip (wird via GH-Actions deployed)" +fi + # 4. Artifact extrahieren -- atomisches mv (gleicher Pattern wie deploy.sh) log "Step 4: Artifact extrahieren..." cd "${APP_DIR}" @@ -129,7 +142,9 @@ log "rebreak-staging restarted" # 6. Optional services (best-effort, Mo's Scope) log "Step 6: Optional services restart..." "${PM2_BIN}" restart rebreak-imap-staging 2>/dev/null || true -"${PM2_BIN}" restart rebreak-idle-staging 2>/dev/null || true +# rebreak-idle-staging: startOrReload (erster Deploy hat keinen laufenden Prozess) +"${PM2_BIN}" restart rebreak-idle-staging --update-env 2>/dev/null || \ + "${PM2_BIN}" start "${REPO_ROOT}/ecosystem.config.js" --only rebreak-idle-staging "${PM2_BIN}" restart dns-rebreak-staging 2>/dev/null || true "${PM2_BIN}" restart dns-rebreak 2>/dev/null || true