Add worker service with e-mail delivery and dev Mailpit
- worker: sends email_outbox mail over SMTP (lettre), woken by LISTEN/NOTIFY, SKIP LOCKED claiming, retries with backoff, hu/en templates (text + HTML), hourly cleanup of expired data - migration 0002: next_attempt_at, failed_at, notify trigger - docker-compose.override.yml: Mailpit for development only - Dockerfiles: lock shared cargo cache mounts for parallel builds Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012vLHzh5qUhFtmJhzkcTbXL
This commit is contained in:
@@ -0,0 +1,216 @@
|
||||
//! Delivers queued e-mails (`email_outbox`) over SMTP.
|
||||
//!
|
||||
//! New rows wake the job at once (`NOTIFY email_outbox`, see migration 0002);
|
||||
//! a slow poll picks up retries. Rows are claimed with `FOR UPDATE SKIP LOCKED`,
|
||||
//! so several workers can run side by side. Delivery is at-least-once.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use lettre::message::{Mailbox, MultiPart};
|
||||
use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor};
|
||||
use serde_json::Value;
|
||||
use sqlx::PgPool;
|
||||
use sqlx::postgres::PgListener;
|
||||
use tokio::sync::watch;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::templates::{self, Rendered};
|
||||
|
||||
const BATCH: i64 = 10;
|
||||
const CHANNEL: &str = "email_outbox";
|
||||
|
||||
pub enum Mailer {
|
||||
Smtp(AsyncSmtpTransport<Tokio1Executor>),
|
||||
/// Collects messages instead of sending; fails for addresses listed in `fail`.
|
||||
#[cfg(test)]
|
||||
Memory {
|
||||
sent: std::sync::Mutex<Vec<(String, Rendered)>>,
|
||||
fail: Vec<(String, bool)>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Why sending failed; permanent errors (bad address, SMTP 5xx) are not retried.
|
||||
#[derive(Debug)]
|
||||
pub struct SendError {
|
||||
pub permanent: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl SendError {
|
||||
fn permanent(message: impl ToString) -> Self {
|
||||
Self { permanent: true, message: message.to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
/// The host part of an SMTP URL, for logs (no credentials).
|
||||
pub fn smtp_host(url: &str) -> String {
|
||||
let rest = url.split_once("://").map_or(url, |(_, r)| r);
|
||||
let rest = rest.rsplit_once('@').map_or(rest, |(_, r)| r);
|
||||
rest.split(['/', '?']).next().unwrap_or_default().to_owned()
|
||||
}
|
||||
|
||||
impl Mailer {
|
||||
pub fn smtp(url: &str) -> anyhow::Result<Self> {
|
||||
let transport = AsyncSmtpTransport::<Tokio1Executor>::from_url(url)
|
||||
.map_err(|e| anyhow::anyhow!("invalid SMTP_URL for {}: {e}", smtp_host(url)))?
|
||||
.timeout(Some(Duration::from_secs(30)))
|
||||
.build();
|
||||
Ok(Mailer::Smtp(transport))
|
||||
}
|
||||
|
||||
async fn send(&self, from: &Mailbox, to: &str, mail: &Rendered) -> Result<(), SendError> {
|
||||
let to_mailbox: Mailbox = to.parse().map_err(SendError::permanent)?;
|
||||
let message = Message::builder()
|
||||
.from(from.clone())
|
||||
.to(to_mailbox)
|
||||
.subject(&mail.subject)
|
||||
.multipart(MultiPart::alternative_plain_html(mail.text.clone(), mail.html.clone()))
|
||||
.map_err(SendError::permanent)?;
|
||||
match self {
|
||||
Mailer::Smtp(transport) => transport
|
||||
.send(message)
|
||||
.await
|
||||
.map(drop)
|
||||
.map_err(|e| SendError { permanent: e.is_permanent(), message: e.to_string() }),
|
||||
#[cfg(test)]
|
||||
Mailer::Memory { sent, fail } => {
|
||||
if let Some((_, permanent)) = fail.iter().find(|(addr, _)| addr == to) {
|
||||
return Err(SendError { permanent: *permanent, message: "test failure".into() });
|
||||
}
|
||||
sent.lock().unwrap().push((to.to_owned(), mail.clone()));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Delivery {
|
||||
pub db: PgPool,
|
||||
pub mailer: Mailer,
|
||||
pub from: Mailbox,
|
||||
pub max_attempts: i32,
|
||||
}
|
||||
|
||||
#[derive(sqlx::FromRow)]
|
||||
struct OutboxRow {
|
||||
id: Uuid,
|
||||
to_email: String,
|
||||
template: String,
|
||||
locale: String,
|
||||
payload: Value,
|
||||
attempts: i32,
|
||||
}
|
||||
|
||||
/// Wait before the next attempt: 1, 2, 4, ... minutes, at most 6 hours.
|
||||
pub fn backoff(attempts: i32) -> Duration {
|
||||
let minutes = 1u64 << attempts.clamp(1, 10).saturating_sub(1);
|
||||
Duration::from_secs((minutes * 60).min(6 * 3600))
|
||||
}
|
||||
|
||||
impl Delivery {
|
||||
pub fn new(db: PgPool, mailer: Mailer, config: &Config) -> Self {
|
||||
Self { db, mailer, from: config.mail_from.clone(), max_attempts: config.max_email_attempts }
|
||||
}
|
||||
|
||||
/// Sends up to one batch of due e-mails; returns how many rows were handled.
|
||||
pub async fn deliver_due(&self) -> anyhow::Result<usize> {
|
||||
let mut tx = self.db.begin().await?;
|
||||
let rows: Vec<OutboxRow> = sqlx::query_as(
|
||||
"SELECT id, to_email, template, locale, payload, attempts FROM email_outbox \
|
||||
WHERE sent_at IS NULL AND failed_at IS NULL AND next_attempt_at <= now() \
|
||||
ORDER BY next_attempt_at LIMIT $1 FOR UPDATE SKIP LOCKED",
|
||||
)
|
||||
.bind(BATCH)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
for row in &rows {
|
||||
let attempts = row.attempts + 1;
|
||||
let result = match templates::render(&row.template, &row.locale, &row.payload) {
|
||||
Ok(mail) => self.mailer.send(&self.from, &row.to_email, &mail).await,
|
||||
Err(e) => Err(SendError::permanent(e)),
|
||||
};
|
||||
match result {
|
||||
Ok(()) => {
|
||||
sqlx::query(
|
||||
"UPDATE email_outbox SET sent_at = now(), attempts = $2, last_error = NULL WHERE id = $1",
|
||||
)
|
||||
.bind(row.id)
|
||||
.bind(attempts)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tracing::info!(id = %row.id, template = %row.template, "e-mail sent");
|
||||
}
|
||||
Err(e) if e.permanent || attempts >= self.max_attempts => {
|
||||
sqlx::query(
|
||||
"UPDATE email_outbox SET failed_at = now(), attempts = $2, last_error = $3 WHERE id = $1",
|
||||
)
|
||||
.bind(row.id)
|
||||
.bind(attempts)
|
||||
.bind(&e.message)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tracing::error!(id = %row.id, template = %row.template, attempts, error = %e.message, "e-mail failed");
|
||||
}
|
||||
Err(e) => {
|
||||
let wait = backoff(attempts);
|
||||
sqlx::query(
|
||||
"UPDATE email_outbox SET attempts = $2, last_error = $3, \
|
||||
next_attempt_at = now() + make_interval(secs => $4) WHERE id = $1",
|
||||
)
|
||||
.bind(row.id)
|
||||
.bind(attempts)
|
||||
.bind(&e.message)
|
||||
.bind(wait.as_secs_f64())
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tracing::warn!(id = %row.id, attempts, retry_in_secs = wait.as_secs(), error = %e.message, "e-mail will be retried");
|
||||
}
|
||||
}
|
||||
}
|
||||
tx.commit().await?;
|
||||
Ok(rows.len())
|
||||
}
|
||||
}
|
||||
|
||||
async fn listen(db: &PgPool) -> Option<PgListener> {
|
||||
let result = async {
|
||||
let mut listener = PgListener::connect_with(db).await?;
|
||||
listener.listen(CHANNEL).await?;
|
||||
Ok::<_, sqlx::Error>(listener)
|
||||
}
|
||||
.await;
|
||||
result.map_err(|e| tracing::warn!(error = %e, "LISTEN failed; polling only")).ok()
|
||||
}
|
||||
|
||||
pub async fn run(delivery: Delivery, poll: Duration, stop: watch::Receiver<bool>) {
|
||||
let mut listener = listen(&delivery.db).await;
|
||||
loop {
|
||||
if *stop.borrow() {
|
||||
break;
|
||||
}
|
||||
match delivery.deliver_due().await {
|
||||
Ok(n) if n as i64 == BATCH => continue,
|
||||
Ok(_) => {}
|
||||
Err(e) => tracing::error!(error = %e, "e-mail delivery"),
|
||||
}
|
||||
let notified = async {
|
||||
match listener.as_mut() {
|
||||
// Reconnects by itself; an error means the connection was lost for now.
|
||||
Some(l) => {
|
||||
if let Err(e) = l.recv().await {
|
||||
tracing::warn!(error = %e, "LISTEN connection lost");
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
}
|
||||
}
|
||||
None => std::future::pending().await,
|
||||
}
|
||||
};
|
||||
tokio::select! {
|
||||
() = notified => {}
|
||||
() = tokio::time::sleep(poll) => {}
|
||||
() = crate::stopped(stop.clone()) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user