diff --git a/.gitignore b/.gitignore index c82e94c..5c27968 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,9 @@ /legacy/**/*.war # Local Claude Code permissions /.claude/settings.local.json +# Local secrets and editor swap files +.env +.env.* +!.env.example +*.swp +*.swo diff --git a/Cargo.lock b/Cargo.lock index 6d2ec0c..9a38619 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.10", +] + [[package]] name = "atoi" version = "2.0.0" @@ -417,6 +428,22 @@ dependencies = [ "serde", ] +[[package]] +name = "email-encoding" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f690ae2169f2ceb3f45b28b6fcd8fa16839f2f5c8818546774b9425f5f71004" +dependencies = [ + "base64 0.23.1", + "memchr", +] + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" + [[package]] name = "equivalent" version = "1.0.3" @@ -454,6 +481,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.15" @@ -710,6 +743,17 @@ dependencies = [ "thiserror", ] +[[package]] +name = "hostname" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "617aaa3557aef3810a6369d0a99fac8a080891b68bd9f9812a1eeda0c0730cbd" +dependencies = [ + "cfg-if", + "libc", + "windows-link", +] + [[package]] name = "http" version = "1.5.0" @@ -1011,6 +1055,34 @@ dependencies = [ "spin", ] +[[package]] +name = "lettre" +version = "0.11.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c646bd5cc763b1087b15493e29a64be6147ba8f19342004fa52048ee596eae" +dependencies = [ + "async-trait", + "base64 0.23.1", + "email-encoding", + "email_address", + "fastrand", + "futures-io", + "futures-util", + "hostname", + "httpdate", + "idna", + "mime", + "nom", + "percent-encoding", + "quoted_printable", + "rustls", + "socket2", + "tokio", + "tokio-rustls", + "url", + "webpki-roots", +] + [[package]] name = "libc" version = "0.2.191" @@ -1120,6 +1192,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1413,6 +1494,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quoted_printable" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478e0585659a122aa407eb7e3c0e1fa51b1d8a870038bd29f0cf4a8551eea972" + [[package]] name = "r-efi" version = "5.3.0" @@ -1629,6 +1716,7 @@ version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2764,6 +2852,23 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" +[[package]] +name = "worker" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "db", + "lettre", + "serde_json", + "service-kit", + "sqlx", + "time", + "tokio", + "tracing", + "uuid", +] + [[package]] name = "writeable" version = "0.6.4" diff --git a/Cargo.toml b/Cargo.toml index dbafa34..eca0d9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/horoscope-client", "services/horoscope-api", "services/backend", + "services/worker", ] [workspace.package] @@ -31,6 +32,7 @@ axum = "0.8" axum-extra = { version = "0.10", features = ["cookie"] } base64 = "0.22" jsonwebtoken = { version = "9", default-features = false } +lettre = { version = "0.11", default-features = false, features = ["builder", "hostname", "smtp-transport", "tokio1", "tokio1-rustls", "webpki-roots", "ring"] } rand = "0.9" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } sha2 = "0.10" diff --git a/README.md b/README.md index 4524722..f45f083 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Asztrológiai rendszer konténeres backendje. A frontenden kívül minden Rustba |---|---| | `services/horoscope-api` | HTTP API: születési adatokból születési horoszkóp (radix) és szinasztria | | `services/backend` | felhasználók, belépés, mentett képletek, szinasztriák, meghívók, elemzések, kreditek, megosztás. Részletek: [services/backend/README.md](services/backend/README.md) | +| `services/worker` | háttérmunkák: e-mail-küldés, takarítás (később az LLM-elemzések). Részletek: [services/worker/README.md](services/worker/README.md) | | `crates/db` | PostgreSQL-séma (migrációk) és a szolgáltatások közös adatelérése | | `crates/horoscope-client` | a horoscope-api HTTP-kliense (a backend és később a worker használja) | | `crates/service-kit` | közös alapok: naplózás, listen cím, healthcheck, leállítás | @@ -20,11 +21,17 @@ Asztrológiai rendszer konténeres backendje. A frontenden kívül minden Rustba ### Docker ```bash -docker compose up -d --build # horoscope-api :8080, backend :8081, postgres 127.0.0.1:5432 +docker compose up -d --build # horoscope-api :8080, backend :8081, worker, postgres 127.0.0.1:5432, mailpit :8025 curl -s localhost:8080/healthz curl -s localhost:8081/healthz ``` +A `docker compose up` a `docker-compose.override.yml` fájlt is betölti. Ebben csak fejlesztői dolgok vannak: a **Mailpit**, ami minden kimenő e-mailt elkap (webes felület: `http://:8025`), és a worker erre van irányítva. Élesben az override nélkül kell indítani, és az `SMTP_URL`-t meg kell adni: + +```bash +SMTP_URL=smtps://user:jelszo@smtp.example.com:465 docker compose -f docker-compose.yml up -d +``` + A horoscope-api image önálló (`scratch` alapú, kb. 70 MB, nem root felhasználóval fut). Benne van a statikus bináris és az összes adat. A container induláskor ~2 mp alatt betölti az adatokat, és ~190 MB memóriát használ. Egy horoszkóp kiszámítása 2–3 ms. Build-argumentumok (`docker compose build --build-arg …`): diff --git a/crates/db/migrations/0002_email_delivery.sql b/crates/db/migrations/0002_email_delivery.sql new file mode 100644 index 0000000..6132fc4 --- /dev/null +++ b/crates/db/migrations/0002_email_delivery.sql @@ -0,0 +1,21 @@ +-- E-mail delivery by the worker: retries with backoff, permanent failure, +-- and a notification so the worker sends new e-mails right away. + +ALTER TABLE email_outbox + ADD COLUMN next_attempt_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN failed_at timestamptz; + +DROP INDEX email_outbox_unsent_idx; +CREATE INDEX email_outbox_due_idx ON email_outbox (next_attempt_at) + WHERE sent_at IS NULL AND failed_at IS NULL; + +CREATE FUNCTION notify_email_outbox() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + PERFORM pg_notify('email_outbox', ''); + RETURN NULL; +END; +$$; + +CREATE TRIGGER email_outbox_notify + AFTER INSERT ON email_outbox + FOR EACH STATEMENT EXECUTE FUNCTION notify_email_outbox(); diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..d3d4fad --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,23 @@ +# Development only: loaded automatically by `docker compose up`, left out in +# production (`docker compose -f docker-compose.yml ...`). +# +# Mailpit catches every e-mail the worker sends; nothing leaves the machine. +# Web UI: http://:8025 + +services: + mailpit: + image: axllent/mailpit:v1.33.1 + environment: + MP_MAX_MESSAGES: 1000 + MP_SMTP_AUTH_ACCEPT_ANY: 1 + MP_SMTP_AUTH_ALLOW_INSECURE: 1 + ports: + - "8025:8025" + restart: unless-stopped + + worker: + environment: + SMTP_URL: smtp://mailpit:1025 + depends_on: + mailpit: + condition: service_healthy diff --git a/docker-compose.yml b/docker-compose.yml index 8435083..28f77f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,6 @@ +# Production: docker compose -f docker-compose.yml up -d +# Development: docker compose up -d (also loads docker-compose.override.yml: Mailpit) + services: horoscope-api: build: @@ -53,5 +56,22 @@ services: read_only: true restart: unless-stopped + worker: + build: + context: . + dockerfile: services/worker/Dockerfile + image: sorsfonalak/worker:latest + environment: + DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD:-sorsfonalak-dev}@postgres:5432/sorsfonalak + # Unset: e-mails stay in the outbox. In development the override file + # points it at Mailpit. + SMTP_URL: ${SMTP_URL:-} + MAIL_FROM: ${MAIL_FROM:-Sorsfonalak } + depends_on: + postgres: + condition: service_healthy + read_only: true + restart: unless-stopped + volumes: pgdata: diff --git a/services/backend/Dockerfile b/services/backend/Dockerfile index 12aaf4f..0414444 100644 --- a/services/backend/Dockerfile +++ b/services/backend/Dockerfile @@ -12,8 +12,8 @@ WORKDIR /app COPY Cargo.toml Cargo.lock ./ COPY crates crates COPY services services -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/app/target,sharing=locked \ cargo build --release --locked -p backend \ && cp target/release/backend /backend diff --git a/services/backend/README.md b/services/backend/README.md index 0f7aed3..0a41fa0 100644 --- a/services/backend/README.md +++ b/services/backend/README.md @@ -89,7 +89,7 @@ Hibák formátuma: `{"error": {"code", "message"}}`. A születési adatok hibái | `analyses` | szöveges elemzések: típus, nyelv, modell, promptverzió, `cache_key`, állapot (`pending` → `running` → `done`/`failed`), strukturált `content` | | `credit_ledger` | csak hozzáírható kreditnapló; az egyenleg `SUM(delta)`; havi keret lustán, egyszer havonta | | `shares` | nyilvános linkek: véletlen `slug`, mit mutat, `show_birth_data`, nézettség, visszavonás | -| `email_outbox` | kimenő e-mailek; a worker küldi ki | +| `email_outbox` | kimenő e-mailek; a worker küldi ki (újrapróbálással) | **Adatvédelem:** - A születési adatokból a `name` mező nem kerül tárolásra, és nem megy tovább a számításhoz (később az LLM-hez sem). @@ -104,9 +104,8 @@ Hibák formátuma: `{"error": {"code", "message"}}`. A születési adatok hibái - Ha ugyanarra a képletre már kértek ilyet, ugyanazt adja vissza (200). - Ha bárhol van kész elemzés ugyanezzel a kulccsal (például ugyanaz a születési adat egy másik felhasználónál), ingyen átmásolja (201, `done`). -## Ami még hátravan (a worker dolga) +## Háttérmunkák -- e-mailek kiküldése az `email_outbox`-ból; +A [worker](../worker/README.md) már kiküldi az e-maileket az `email_outbox`-ból, és óránként takarít. Még hátravan: - az elemzések megírása (OpenRouter, strukturált JSON, streaming); -- sikertelen elemzésnél a kredit visszatérítése (`refund`); -- lejárt munkamenetek és tokenek takarítása. +- sikertelen elemzésnél a kredit visszatérítése (`refund`). diff --git a/services/horoscope-api/Dockerfile b/services/horoscope-api/Dockerfile index be6faa5..ab51797 100644 --- a/services/horoscope-api/Dockerfile +++ b/services/horoscope-api/Dockerfile @@ -30,8 +30,8 @@ COPY Cargo.toml Cargo.lock ./ COPY crates crates COPY services services COPY --from=deps /out/vendor vendor -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/app/target \ +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/app/target,sharing=locked \ cargo build --release --locked -p horoscope-api \ && cp target/release/horoscope-api /horoscope-api diff --git a/services/worker/Cargo.toml b/services/worker/Cargo.toml new file mode 100644 index 0000000..205e809 --- /dev/null +++ b/services/worker/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "worker" +description = "Sorsfonalak background jobs: e-mail delivery, cleanup (later: LLM analyses)" +edition.workspace = true +version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +db.workspace = true +service-kit.workspace = true +anyhow.workspace = true +axum.workspace = true +lettre.workspace = true +serde_json.workspace = true +sqlx.workspace = true +time.workspace = true +tokio.workspace = true +tracing.workspace = true +uuid.workspace = true diff --git a/services/worker/Dockerfile b/services/worker/Dockerfile new file mode 100644 index 0000000..59c93d8 --- /dev/null +++ b/services/worker/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 +# +# Build from the repository root: +# docker build -f services/worker/Dockerfile -t sorsfonalak/worker . + +ARG RUST_VERSION=1.99 + +# --- 1. static (musl) build ---------------------------------------------------- +FROM rust:${RUST_VERSION}-alpine AS build +RUN apk add --no-cache musl-dev +WORKDIR /app +COPY Cargo.toml Cargo.lock ./ +COPY crates crates +COPY services services +RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,target=/app/target,sharing=locked \ + cargo build --release --locked -p worker \ + && cp target/release/worker /worker + +# --- 2. runtime: the binary only (TLS roots are compiled in via webpki-roots) --- +FROM scratch +COPY --from=build /worker /worker +ENV BIND_ADDR=0.0.0.0:8082 \ + LOG_FORMAT=json \ + RUST_LOG=info +USER 65532:65532 +EXPOSE 8082 +HEALTHCHECK --interval=30s --timeout=3s --start-period=10s --retries=3 \ + CMD ["/worker", "healthcheck"] +ENTRYPOINT ["/worker"] diff --git a/services/worker/README.md b/services/worker/README.md new file mode 100644 index 0000000..6a0f40c --- /dev/null +++ b/services/worker/README.md @@ -0,0 +1,35 @@ +# worker + +Háttérmunkák a közös PostgreSQL-adatbázis fölött. Nincs nyilvános API-ja, csak egy `/healthz` végpontja (8082) a Docker healthcheckhez. Induláskor lefuttatja a migrációkat, ugyanúgy, mint a backend. + +## Munkák + +**E-mail-küldés** (`email_outbox` → SMTP) +- A backend a levelet az `email_outbox` táblába teszi. Egy adatbázis-trigger (`NOTIFY email_outbox`) azonnal felébreszti a workert, így a levél másodperceken belül kimegy. Ha az értesítés elveszne, a worker `POLL_INTERVAL_SECS` másodpercenként úgyis ránéz. +- A sorokat `FOR UPDATE SKIP LOCKED` zárolással veszi fel, ezért több worker is futhat egyszerre, és nem küldik el kétszer ugyanazt. A kézbesítés „legalább egyszer” elvű: ha a worker éppen küldés közben áll le, a levél újra kimehet. +- Átmeneti hibánál (hálózat, SMTP 4xx) újrapróbálja 1, 2, 4, … percenként, legfeljebb 6 óránként. `MAX_EMAIL_ATTEMPTS` próbálkozás után a levelet sikertelennek jelöli (`failed_at`, `last_error`). +- Végleges hibánál nem próbálja újra: ilyen a hibás cím, az SMTP 5xx és az ismeretlen sablon. +- Sablonok magyarul és angolul, a felhasználó nyelve szerint (`src/templates.rs`): `verify_email`, `reset_password`, `invite_accepted`. Mindegyik szöveges és HTML-változatban megy ki; a felhasználó nevét HTML-ben escape-eli, és a vezérlőkaraktereket eltávolítja belőle. +- Ha nincs `SMTP_URL`, a worker nem küld semmit, a levelek az outboxban várnak. + +**Takarítás** (induláskor, majd óránként) +- lejárt munkamenetek és e-mail-tokenek; +- elküldött e-mailek 7 nap után (tokenes linkek vannak bennük), sikertelenek 30 nap után; +- el nem fogadott meghívók 30 nappal a lejáratuk után. + +**Később:** az elemzések megírása (OpenRouter), kredit-visszatérítés hibánál. + +## Konfiguráció + +| Változó | Alapérték | Jelentés | +|---|---|---| +| `DATABASE_URL` | – (kötelező) | PostgreSQL kapcsolat | +| `SMTP_URL` | – | `smtp://mailpit:1025` (fejlesztés), `smtp://user:pass@host:587?tls=required` (STARTTLS), `smtps://user:pass@host:465` (TLS) | +| `MAIL_FROM` | `Sorsfonalak ` | feladó | +| `POLL_INTERVAL_SECS` | `30` | tartalék lekérdezési időköz | +| `MAX_EMAIL_ATTEMPTS` | `8` | ennyi próbálkozás után sikertelen a levél | +| `BIND_ADDR`, `LOG_FORMAT`, `RUST_LOG` | `0.0.0.0:8082`, `text`, `info` | | + +## Fejlesztés: Mailpit + +A `docker-compose.override.yml` csak fejlesztéskor töltődik be. Elindítja a Mailpitet, és a workert rá irányítja. Minden levél ott landol, kifelé semmi nem megy ki. Webes felület: `http://:8025`, API: `http://:8025/api/v1/messages`. diff --git a/services/worker/src/cleanup.rs b/services/worker/src/cleanup.rs new file mode 100644 index 0000000..f5d0da2 --- /dev/null +++ b/services/worker/src/cleanup.rs @@ -0,0 +1,47 @@ +//! Hourly removal of data that is no longer needed. + +use std::time::Duration; + +use sqlx::PgPool; +use tokio::sync::watch; + +const INTERVAL: Duration = Duration::from_secs(3600); + +/// (what, statement) pairs; each returns the number of deleted rows. +const STATEMENTS: [(&str, &str); 4] = [ + ("sessions", "DELETE FROM sessions WHERE expires_at < now()"), + ("auth_tokens", "DELETE FROM auth_tokens WHERE expires_at < now()"), + // Sent e-mails contain links with tokens; keep them a week for support. + ( + "emails", + "DELETE FROM email_outbox WHERE sent_at < now() - interval '7 days' \ + OR failed_at < now() - interval '30 days'", + ), + ("invites", "DELETE FROM synastry_invites WHERE accepted_at IS NULL AND expires_at < now() - interval '30 days'"), +]; + +pub async fn run_once(db: &PgPool) -> Result, sqlx::Error> { + let mut deleted = Vec::new(); + for (what, sql) in STATEMENTS { + deleted.push((what, sqlx::query(sql).execute(db).await?.rows_affected())); + } + Ok(deleted) +} + +pub async fn run(db: PgPool, stop: watch::Receiver) { + loop { + match run_once(&db).await { + Ok(deleted) => { + let total: u64 = deleted.iter().map(|(_, n)| n).sum(); + if total > 0 { + tracing::info!(?deleted, "cleanup"); + } + } + Err(e) => tracing::error!(error = %e, "cleanup"), + } + tokio::select! { + () = tokio::time::sleep(INTERVAL) => {} + () = crate::stopped(stop.clone()) => break, + } + } +} diff --git a/services/worker/src/config.rs b/services/worker/src/config.rs new file mode 100644 index 0000000..cd7390a --- /dev/null +++ b/services/worker/src/config.rs @@ -0,0 +1,46 @@ +//! Configuration from environment variables. + +use std::time::Duration; + +use anyhow::Context; +use lettre::message::Mailbox; + +pub struct Config { + pub database_url: String, + /// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required` + /// (STARTTLS) or `smtps://...` (TLS). Unset: e-mails stay in the outbox. + pub smtp_url: Option, + /// Sender, e.g. `Sorsfonalak `. + pub mail_from: Mailbox, + /// Fallback polling interval; new e-mails are picked up at once via LISTEN/NOTIFY. + pub poll_interval: Duration, + /// After this many failed attempts an e-mail is marked as failed. + pub max_email_attempts: i32, +} + +fn var(name: &str) -> Option { + std::env::var(name).ok().map(|v| v.trim().to_owned()).filter(|v| !v.is_empty()) +} + +fn parsed(name: &str, default: T) -> anyhow::Result +where + T::Err: std::fmt::Display, +{ + match var(name) { + Some(raw) => raw.parse().map_err(|e| anyhow::anyhow!("invalid {name} '{raw}': {e}")), + None => Ok(default), + } +} + +impl Config { + pub fn from_env() -> anyhow::Result { + let mail_from = var("MAIL_FROM").unwrap_or_else(|| "Sorsfonalak ".into()); + Ok(Self { + database_url: var("DATABASE_URL").context("DATABASE_URL is required")?, + smtp_url: var("SMTP_URL"), + mail_from: mail_from.parse().with_context(|| format!("invalid MAIL_FROM '{mail_from}'"))?, + poll_interval: Duration::from_secs(parsed("POLL_INTERVAL_SECS", 30)?), + max_email_attempts: parsed("MAX_EMAIL_ATTEMPTS", 8)?, + }) + } +} diff --git a/services/worker/src/email.rs b/services/worker/src/email.rs new file mode 100644 index 0000000..1c68a67 --- /dev/null +++ b/services/worker/src/email.rs @@ -0,0 +1,216 @@ +//! Delivers queued e-mails (`email_outbox`) over SMTP. +//! +//! New rows wake the job at once (`NOTIFY email_outbox`, see migration 0002); +//! a slow poll picks up retries. Rows are claimed with `FOR UPDATE SKIP LOCKED`, +//! so several workers can run side by side. Delivery is at-least-once. + +use std::time::Duration; + +use lettre::message::{Mailbox, MultiPart}; +use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor}; +use serde_json::Value; +use sqlx::PgPool; +use sqlx::postgres::PgListener; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::config::Config; +use crate::templates::{self, Rendered}; + +const BATCH: i64 = 10; +const CHANNEL: &str = "email_outbox"; + +pub enum Mailer { + Smtp(AsyncSmtpTransport), + /// Collects messages instead of sending; fails for addresses listed in `fail`. + #[cfg(test)] + Memory { + sent: std::sync::Mutex>, + fail: Vec<(String, bool)>, + }, +} + +/// Why sending failed; permanent errors (bad address, SMTP 5xx) are not retried. +#[derive(Debug)] +pub struct SendError { + pub permanent: bool, + pub message: String, +} + +impl SendError { + fn permanent(message: impl ToString) -> Self { + Self { permanent: true, message: message.to_string() } + } +} + +/// The host part of an SMTP URL, for logs (no credentials). +pub fn smtp_host(url: &str) -> String { + let rest = url.split_once("://").map_or(url, |(_, r)| r); + let rest = rest.rsplit_once('@').map_or(rest, |(_, r)| r); + rest.split(['/', '?']).next().unwrap_or_default().to_owned() +} + +impl Mailer { + pub fn smtp(url: &str) -> anyhow::Result { + let transport = AsyncSmtpTransport::::from_url(url) + .map_err(|e| anyhow::anyhow!("invalid SMTP_URL for {}: {e}", smtp_host(url)))? + .timeout(Some(Duration::from_secs(30))) + .build(); + Ok(Mailer::Smtp(transport)) + } + + async fn send(&self, from: &Mailbox, to: &str, mail: &Rendered) -> Result<(), SendError> { + let to_mailbox: Mailbox = to.parse().map_err(SendError::permanent)?; + let message = Message::builder() + .from(from.clone()) + .to(to_mailbox) + .subject(&mail.subject) + .multipart(MultiPart::alternative_plain_html(mail.text.clone(), mail.html.clone())) + .map_err(SendError::permanent)?; + match self { + Mailer::Smtp(transport) => transport + .send(message) + .await + .map(drop) + .map_err(|e| SendError { permanent: e.is_permanent(), message: e.to_string() }), + #[cfg(test)] + Mailer::Memory { sent, fail } => { + if let Some((_, permanent)) = fail.iter().find(|(addr, _)| addr == to) { + return Err(SendError { permanent: *permanent, message: "test failure".into() }); + } + sent.lock().unwrap().push((to.to_owned(), mail.clone())); + Ok(()) + } + } + } +} + +pub struct Delivery { + pub db: PgPool, + pub mailer: Mailer, + pub from: Mailbox, + pub max_attempts: i32, +} + +#[derive(sqlx::FromRow)] +struct OutboxRow { + id: Uuid, + to_email: String, + template: String, + locale: String, + payload: Value, + attempts: i32, +} + +/// Wait before the next attempt: 1, 2, 4, ... minutes, at most 6 hours. +pub fn backoff(attempts: i32) -> Duration { + let minutes = 1u64 << attempts.clamp(1, 10).saturating_sub(1); + Duration::from_secs((minutes * 60).min(6 * 3600)) +} + +impl Delivery { + pub fn new(db: PgPool, mailer: Mailer, config: &Config) -> Self { + Self { db, mailer, from: config.mail_from.clone(), max_attempts: config.max_email_attempts } + } + + /// Sends up to one batch of due e-mails; returns how many rows were handled. + pub async fn deliver_due(&self) -> anyhow::Result { + let mut tx = self.db.begin().await?; + let rows: Vec = sqlx::query_as( + "SELECT id, to_email, template, locale, payload, attempts FROM email_outbox \ + WHERE sent_at IS NULL AND failed_at IS NULL AND next_attempt_at <= now() \ + ORDER BY next_attempt_at LIMIT $1 FOR UPDATE SKIP LOCKED", + ) + .bind(BATCH) + .fetch_all(&mut *tx) + .await?; + + for row in &rows { + let attempts = row.attempts + 1; + let result = match templates::render(&row.template, &row.locale, &row.payload) { + Ok(mail) => self.mailer.send(&self.from, &row.to_email, &mail).await, + Err(e) => Err(SendError::permanent(e)), + }; + match result { + Ok(()) => { + sqlx::query( + "UPDATE email_outbox SET sent_at = now(), attempts = $2, last_error = NULL WHERE id = $1", + ) + .bind(row.id) + .bind(attempts) + .execute(&mut *tx) + .await?; + tracing::info!(id = %row.id, template = %row.template, "e-mail sent"); + } + Err(e) if e.permanent || attempts >= self.max_attempts => { + sqlx::query( + "UPDATE email_outbox SET failed_at = now(), attempts = $2, last_error = $3 WHERE id = $1", + ) + .bind(row.id) + .bind(attempts) + .bind(&e.message) + .execute(&mut *tx) + .await?; + tracing::error!(id = %row.id, template = %row.template, attempts, error = %e.message, "e-mail failed"); + } + Err(e) => { + let wait = backoff(attempts); + sqlx::query( + "UPDATE email_outbox SET attempts = $2, last_error = $3, \ + next_attempt_at = now() + make_interval(secs => $4) WHERE id = $1", + ) + .bind(row.id) + .bind(attempts) + .bind(&e.message) + .bind(wait.as_secs_f64()) + .execute(&mut *tx) + .await?; + tracing::warn!(id = %row.id, attempts, retry_in_secs = wait.as_secs(), error = %e.message, "e-mail will be retried"); + } + } + } + tx.commit().await?; + Ok(rows.len()) + } +} + +async fn listen(db: &PgPool) -> Option { + let result = async { + let mut listener = PgListener::connect_with(db).await?; + listener.listen(CHANNEL).await?; + Ok::<_, sqlx::Error>(listener) + } + .await; + result.map_err(|e| tracing::warn!(error = %e, "LISTEN failed; polling only")).ok() +} + +pub async fn run(delivery: Delivery, poll: Duration, stop: watch::Receiver) { + let mut listener = listen(&delivery.db).await; + loop { + if *stop.borrow() { + break; + } + match delivery.deliver_due().await { + Ok(n) if n as i64 == BATCH => continue, + Ok(_) => {} + Err(e) => tracing::error!(error = %e, "e-mail delivery"), + } + let notified = async { + match listener.as_mut() { + // Reconnects by itself; an error means the connection was lost for now. + Some(l) => { + if let Err(e) = l.recv().await { + tracing::warn!(error = %e, "LISTEN connection lost"); + tokio::time::sleep(Duration::from_secs(5)).await; + } + } + None => std::future::pending().await, + } + }; + tokio::select! { + () = notified => {} + () = tokio::time::sleep(poll) => {} + () = crate::stopped(stop.clone()) => break, + } + } +} diff --git a/services/worker/src/main.rs b/services/worker/src/main.rs new file mode 100644 index 0000000..21dc453 --- /dev/null +++ b/services/worker/src/main.rs @@ -0,0 +1,84 @@ +//! Sorsfonalak worker: background jobs that read and write the shared +//! PostgreSQL database. +//! +//! * e-mail delivery from `email_outbox` over SMTP (Mailpit in development) +//! * hourly cleanup of expired sessions, tokens, invites and old e-mails +//! * later: writing the text analyses (LLM) +//! +//! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default +//! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`. +//! `worker healthcheck` performs a local health check (for Docker). + +mod cleanup; +mod config; +mod email; +mod templates; + +use std::time::Duration; + +use anyhow::Context; +use axum::Router; +use axum::extract::State; +use axum::http::StatusCode; +use axum::routing::get; +use sqlx::PgPool; +use tokio::sync::watch; + +use crate::config::Config; + +const DEFAULT_BIND_ADDR: &str = "0.0.0.0:8082"; + +/// Resolves once shutdown has been requested. +pub async fn stopped(mut stop: watch::Receiver) { + let _ = stop.wait_for(|s| *s).await; +} + +async fn healthz(State(db): State) -> StatusCode { + match sqlx::query("SELECT 1").execute(&db).await { + Ok(_) => StatusCode::OK, + Err(_) => StatusCode::SERVICE_UNAVAILABLE, + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let addr = service_kit::bind_addr(DEFAULT_BIND_ADDR)?; + if std::env::args().nth(1).as_deref() == Some("healthcheck") { + return service_kit::healthcheck(addr); + } + service_kit::init_logging(); + let config = Config::from_env()?; + let pool = db::connect(&config.database_url).await.context("connecting to the database")?; + db::MIGRATOR.run(&pool).await.context("running database migrations")?; + + let (stop_tx, stop) = watch::channel(false); + let mut jobs = tokio::task::JoinSet::new(); + match &config.smtp_url { + Some(url) => { + let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config); + tracing::info!(host = %email::smtp_host(url), "e-mail delivery enabled"); + jobs.spawn(email::run(delivery, config.poll_interval, stop.clone())); + } + None => tracing::warn!("SMTP_URL is not set: e-mails stay in the outbox"), + } + jobs.spawn(cleanup::run(pool.clone(), stop.clone())); + + let app = Router::new().route("/healthz", get(healthz)).with_state(pool); + let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?; + tracing::info!(%addr, "worker running"); + let server = axum::serve(listener, app).with_graceful_shutdown(stopped(stop.clone())); + tokio::select! { + r = server => r?, + () = service_kit::shutdown_signal() => {} + } + let _ = stop_tx.send(true); + // Let jobs finish the batch they are working on. + let finished = tokio::time::timeout(Duration::from_secs(30), async { while jobs.join_next().await.is_some() {} }); + if finished.await.is_err() { + tracing::warn!("jobs did not stop in time"); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/services/worker/src/templates.rs b/services/worker/src/templates.rs new file mode 100644 index 0000000..d403dde --- /dev/null +++ b/services/worker/src/templates.rs @@ -0,0 +1,197 @@ +//! E-mail texts (Hungarian and English), as plain text and HTML. +//! +//! The payloads are written by the backend (`queue_email`); the validity +//! periods in the texts match the token lifetimes there. + +use serde_json::Value; + +#[derive(Debug, Clone, PartialEq)] +pub struct Rendered { + pub subject: String, + pub text: String, + pub html: String, +} + +#[derive(Debug, PartialEq)] +pub enum TemplateError { + UnknownTemplate(String), + MissingField(&'static str), +} + +impl std::fmt::Display for TemplateError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TemplateError::UnknownTemplate(t) => write!(f, "unknown e-mail template '{t}'"), + TemplateError::MissingField(field) => write!(f, "e-mail payload has no '{field}'"), + } + } +} + +#[derive(Clone, Copy, PartialEq)] +enum Lang { + Hu, + En, +} + +/// One e-mail: greeting, paragraphs, a button and a closing note. +struct Content { + subject: String, + greeting: String, + intro: String, + button: &'static str, + url: String, + note: &'static str, +} + +pub fn render(template: &str, locale: &str, payload: &Value) -> Result { + let lang = if locale.eq_ignore_ascii_case("hu") { Lang::Hu } else { Lang::En }; + let url = payload["url"].as_str().filter(|u| !u.is_empty()).ok_or(TemplateError::MissingField("url"))?.to_owned(); + let greeting = |name: Option| match (lang, name) { + (Lang::Hu, Some(n)) => format!("Szia {n}!"), + (Lang::Hu, None) => "Szia!".to_owned(), + (Lang::En, Some(n)) => format!("Hi {n},"), + (Lang::En, None) => "Hi,".to_owned(), + }; + let content = match (template, lang) { + ("verify_email", Lang::Hu) => Content { + subject: "Erősítsd meg az e-mail-címed".into(), + greeting: greeting(name(payload, "display_name")), + intro: "Köszönjük, hogy regisztráltál a Sorsfonalakon! Az alábbi gombbal erősítsd meg az e-mail-címed." + .into(), + button: "E-mail-cím megerősítése", + url, + note: "A link 3 napig érvényes. Ha nem te regisztráltál, nyugodtan hagyd figyelmen kívül ezt a levelet.", + }, + ("verify_email", Lang::En) => Content { + subject: "Confirm your e-mail address".into(), + greeting: greeting(name(payload, "display_name")), + intro: "Thanks for signing up for Sorsfonalak! Please confirm your e-mail address with the button below." + .into(), + button: "Confirm e-mail address", + url, + note: "The link is valid for 3 days. If you did not sign up, you can safely ignore this e-mail.", + }, + ("reset_password", Lang::Hu) => Content { + subject: "Új jelszó megadása".into(), + greeting: greeting(name(payload, "display_name")), + intro: "Jelszó-visszaállítást kértél a Sorsfonalak-fiókodhoz. Az alábbi gombbal adhatsz meg új jelszót." + .into(), + button: "Új jelszó megadása", + url, + note: "A link 1 óráig érvényes, és csak egyszer használható. Ha nem te kérted, hagyd figyelmen kívül ezt a levelet: a jelszavad nem változik.", + }, + ("reset_password", Lang::En) => Content { + subject: "Set a new password".into(), + greeting: greeting(name(payload, "display_name")), + intro: + "You asked to reset the password of your Sorsfonalak account. Use the button below to set a new one." + .into(), + button: "Set a new password", + url, + note: "The link is valid for 1 hour and can be used once. If you did not ask for this, ignore this e-mail: your password stays the same.", + }, + ("invite_accepted", Lang::Hu) => { + let who = name(payload, "invitee_name"); + Content { + subject: match &who { + Some(n) => format!("{n} elfogadta a meghívásod"), + None => "Elfogadták a meghívásod".into(), + }, + greeting: greeting(None), + intro: format!( + "{} elfogadta a „Mennyire illünk össze?” meghívásod, és elkészült a szinasztriátok.", + who.as_deref().unwrap_or("A meghívott") + ), + button: "Szinasztria megnyitása", + url, + note: "A szinasztriát mindketten látjátok, de egymás születési adatait nem.", + } + } + ("invite_accepted", Lang::En) => { + let who = name(payload, "invitee_name"); + Content { + subject: match &who { + Some(n) => format!("{n} accepted your invitation"), + None => "Your invitation was accepted".into(), + }, + greeting: greeting(None), + intro: format!( + "{} accepted your “How compatible are we?” invitation, and your synastry is ready.", + who.as_deref().unwrap_or("Your invitee") + ), + button: "Open synastry", + url, + note: "You both see the synastry, but not each other's birth data.", + } + } + _ => return Err(TemplateError::UnknownTemplate(template.to_owned())), + }; + Ok(content.render(lang)) +} + +/// A user-supplied name, safe for a subject line. +fn name(payload: &Value, field: &str) -> Option { + let name: String = payload[field].as_str()?.chars().filter(|c| !c.is_control()).take(60).collect(); + let name = name.trim(); + (!name.is_empty()).then(|| name.to_owned()) +} + +fn escape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + _ => out.push(c), + } + } + out +} + +impl Content { + fn render(self, lang: Lang) -> Rendered { + let footer = match lang { + Lang::Hu => "Ezt a levelet a Sorsfonalak küldte.", + Lang::En => "This e-mail was sent by Sorsfonalak.", + }; + let text = format!( + "{}\n\n{}\n\n{}:\n{}\n\n{}\n\n-- \nSorsfonalak\n", + self.greeting, self.intro, self.button, self.url, self.note + ); + let html = format!( + r#" + +{subject} + + + +
+ + + + + + + +
SORSFONALAK
{greeting}
{intro}
{button}
{note}
{url_text}
+

{footer}

+
+ + +"#, + lang = if lang == Lang::Hu { "hu" } else { "en" }, + subject = escape(&self.subject), + greeting = escape(&self.greeting), + intro = escape(&self.intro), + url = escape(&self.url), + button = escape(self.button), + note = escape(self.note), + url_text = escape(&self.url), + footer = escape(footer), + ); + Rendered { subject: self.subject, text, html } + } +} diff --git a/services/worker/src/tests.rs b/services/worker/src/tests.rs new file mode 100644 index 0000000..2881673 --- /dev/null +++ b/services/worker/src/tests.rs @@ -0,0 +1,182 @@ +use serde_json::json; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::email::{Delivery, Mailer, backoff}; +use crate::templates::{TemplateError, render}; + +#[test] +fn templates_in_both_languages() { + let payload = json!({"url": "https://example.com/verify-email?token=abc", "display_name": "Anna"}); + let hu = render("verify_email", "hu", &payload).unwrap(); + assert_eq!(hu.subject, "Erősítsd meg az e-mail-címed"); + assert!(hu.text.starts_with("Szia Anna!") && hu.text.contains("token=abc")); + assert!( + hu.html.contains(r#""#) + && hu.html.contains(r#"href="https://example.com/verify-email?token=abc""#) + ); + + let en = render("reset_password", "en", &payload).unwrap(); + assert_eq!(en.subject, "Set a new password"); + assert!(en.text.starts_with("Hi Anna,")); + // Unknown locales fall back to English. + assert_eq!(render("reset_password", "de", &payload).unwrap(), en); + + let accepted = render("invite_accepted", "hu", &json!({"url": "u", "invitee_name": "Béla"})).unwrap(); + assert_eq!(accepted.subject, "Béla elfogadta a meghívásod"); + let anonymous = render("invite_accepted", "en", &json!({"url": "u", "invitee_name": null})).unwrap(); + assert_eq!(anonymous.subject, "Your invitation was accepted"); + assert!(anonymous.text.contains("Your invitee accepted")); +} + +#[test] +fn template_input_is_escaped_and_checked() { + let mail = render("verify_email", "hu", &json!({"url": "u", "display_name": "X\r\nBcc: y"})).unwrap(); + assert!(mail.html.contains("Szia <b>X</b>Bcc: y!")); + assert!(!mail.html.contains("X")); + assert_eq!(render("verify_email", "hu", &json!({})), Err(TemplateError::MissingField("url"))); + assert_eq!(render("nope", "hu", &json!({"url": "u"})), Err(TemplateError::UnknownTemplate("nope".into()))); +} + +#[test] +fn backoff_grows_and_is_capped() { + assert_eq!(backoff(1).as_secs(), 60); + assert_eq!(backoff(2).as_secs(), 120); + assert_eq!(backoff(4).as_secs(), 480); + assert_eq!(backoff(20).as_secs(), 6 * 3600); +} + +fn delivery(db: PgPool, fail: Vec<(&str, bool)>) -> Delivery { + Delivery { + db, + mailer: Mailer::Memory { + sent: Default::default(), + fail: fail.into_iter().map(|(a, p)| (a.to_owned(), p)).collect(), + }, + from: "Sorsfonalak ".parse().unwrap(), + max_attempts: 3, + } +} + +fn sent(d: &Delivery) -> Vec<(String, String)> { + let Mailer::Memory { sent, .. } = &d.mailer else { unreachable!() }; + sent.lock().unwrap().iter().map(|(to, m)| (to.clone(), m.subject.clone())).collect() +} + +async fn queue(db: &PgPool, to: &str, template: &str) -> Uuid { + let id = Uuid::now_v7(); + sqlx::query("INSERT INTO email_outbox (id, to_email, template, locale, payload) VALUES ($1, $2, $3, 'hu', $4)") + .bind(id) + .bind(to) + .bind(template) + .bind(json!({"url": "https://example.com/x"})) + .execute(db) + .await + .unwrap(); + id +} + +type State = (i32, bool, bool, Option, bool); + +async fn state(db: &PgPool, id: Uuid) -> State { + sqlx::query_as( + "SELECT attempts, sent_at IS NOT NULL, failed_at IS NOT NULL, last_error, next_attempt_at > now() \ + FROM email_outbox WHERE id = $1", + ) + .bind(id) + .fetch_one(db) + .await + .unwrap() +} + +#[sqlx::test(migrator = "db::MIGRATOR")] +async fn delivers_retries_and_gives_up(db: PgPool) { + let ok = queue(&db, "anna@example.com", "verify_email").await; + let flaky = queue(&db, "flaky@example.com", "reset_password").await; + let rejected = queue(&db, "rejected@example.com", "verify_email").await; + let broken = queue(&db, "anna@example.com", "no_such_template").await; + let d = delivery(db.clone(), vec![("flaky@example.com", false), ("rejected@example.com", true)]); + + assert_eq!(d.deliver_due().await.unwrap(), 4); + assert_eq!(sent(&d), vec![("anna@example.com".into(), "Erősítsd meg az e-mail-címed".into())]); + assert_eq!(state(&db, ok).await, (1, true, false, None, false)); + // Temporary failure: retried later, not at once. + assert_eq!(state(&db, flaky).await, (1, false, false, Some("test failure".into()), true)); + // Permanent failures are not retried. + assert_eq!(state(&db, rejected).await, (1, false, true, Some("test failure".into()), false)); + let (_, _, failed, error, _) = state(&db, broken).await; + assert!(failed && error.unwrap().contains("unknown e-mail template")); + assert_eq!(d.deliver_due().await.unwrap(), 0); + + // Retries until max_attempts (3), then gives up. + for expected in [2, 3] { + sqlx::query("UPDATE email_outbox SET next_attempt_at = now() WHERE id = $1") + .bind(flaky) + .execute(&db) + .await + .unwrap(); + assert_eq!(d.deliver_due().await.unwrap(), 1); + assert_eq!(state(&db, flaky).await.0, expected); + } + let (_, sent_ok, failed, _, _) = state(&db, flaky).await; + assert!(!sent_ok && failed); + assert_eq!(sent(&d).len(), 1); +} + +#[sqlx::test(migrator = "db::MIGRATOR")] +async fn concurrent_workers_do_not_send_twice(db: PgPool) { + for i in 0..15 { + queue(&db, &format!("u{i}@example.com"), "verify_email").await; + } + let (a, b) = (delivery(db.clone(), vec![]), delivery(db.clone(), vec![])); + let (na, nb) = tokio::join!(async { a.deliver_due().await.unwrap() }, async { b.deliver_due().await.unwrap() }); + let (ra, rb) = (a.deliver_due().await.unwrap(), b.deliver_due().await.unwrap()); + assert_eq!(na + nb + ra + rb, 15); + let mut all: Vec = sent(&a).into_iter().chain(sent(&b)).map(|(to, _)| to).collect(); + all.sort(); + all.dedup(); + assert_eq!(all.len(), 15); +} + +#[sqlx::test(migrator = "db::MIGRATOR")] +async fn insert_notifies_the_worker(db: PgPool) { + let mut listener = sqlx::postgres::PgListener::connect_with(&db).await.unwrap(); + listener.listen("email_outbox").await.unwrap(); + queue(&db, "anna@example.com", "verify_email").await; + let n = tokio::time::timeout(std::time::Duration::from_secs(5), listener.recv()).await.unwrap().unwrap(); + assert_eq!(n.channel(), "email_outbox"); +} + +#[sqlx::test(migrator = "db::MIGRATOR")] +async fn cleanup_removes_only_expired_data(db: PgPool) { + let user = Uuid::now_v7(); + sqlx::query("INSERT INTO users (id, email) VALUES ($1, 'anna@example.com')").bind(user).execute(&db).await.unwrap(); + for (hash, expires) in + [(b"live".as_slice(), "now() + interval '1 day'"), (b"dead".as_slice(), "now() - interval '1 second'")] + { + sqlx::query(&format!( + "INSERT INTO sessions (id, user_id, token_hash, expires_at) VALUES ($1, $2, $3, {expires})" + )) + .bind(Uuid::now_v7()) + .bind(user) + .bind(hash) + .execute(&db) + .await + .unwrap(); + } + let old = queue(&db, "anna@example.com", "verify_email").await; + let recent = queue(&db, "anna@example.com", "verify_email").await; + let unsent = queue(&db, "anna@example.com", "verify_email").await; + sqlx::query("UPDATE email_outbox SET sent_at = now() - interval '8 days' WHERE id = $1") + .bind(old) + .execute(&db) + .await + .unwrap(); + sqlx::query("UPDATE email_outbox SET sent_at = now() WHERE id = $1").bind(recent).execute(&db).await.unwrap(); + + let deleted = crate::cleanup::run_once(&db).await.unwrap(); + assert_eq!(deleted, vec![("sessions", 1), ("auth_tokens", 0), ("emails", 1), ("invites", 0)]); + let (left,): (Vec,) = + sqlx::query_as("SELECT array_agg(id ORDER BY id) FROM email_outbox").fetch_one(&db).await.unwrap(); + assert_eq!(left, vec![recent, unsent]); +}