217 lines
7.7 KiB
Rust
217 lines
7.7 KiB
Rust
//! Delivers queued e-mails (`email_outbox`) over SMTP.
|
|||
|
|
//!
|
||
|
|
//! New rows wake the job at once (`NOTIFY email_outbox`, see migration 0002);
|
||
|
|
//! a slow poll picks up retries. Rows are claimed with `FOR UPDATE SKIP LOCKED`,
|
||
|
|
//! so several workers can run side by side. Delivery is at-least-once.
|
||
|
|
|
||
|
|
use std::time::Duration;
|
||
|
|
|
||
|
|
use lettre::message::{Mailbox, MultiPart};
|
||
|
|
use lettre::{AsyncSmtpTransport, AsyncTransport, Message, Tokio1Executor};
|
||
|
|
use serde_json::Value;
|
||
|
|
use sqlx::PgPool;
|
||
|
|
use sqlx::postgres::PgListener;
|
||
|
|
use tokio::sync::watch;
|
||
|
|
use uuid::Uuid;
|
||
|
|
|
||
|
|
use crate::config::Config;
|
||
|
|
use crate::templates::{self, Rendered};
|
||
|
|
|
||
|
|
const BATCH: i64 = 10;
|
||
|
|
const CHANNEL: &str = "email_outbox";
|
||
|
|
|
||
|
|
pub enum Mailer {
|
||
|
|
Smtp(AsyncSmtpTransport<Tokio1Executor>),
|
||
|
|
/// Collects messages instead of sending; fails for addresses listed in `fail`.
|
||
|
|
#[cfg(test)]
|
||
|
|
Memory {
|
||
|
|
sent: std::sync::Mutex<Vec<(String, Rendered)>>,
|
||
|
|
fail: Vec<(String, bool)>,
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Why sending failed; permanent errors (bad address, SMTP 5xx) are not retried.
|
||
|
|
#[derive(Debug)]
|
||
|
|
pub struct SendError {
|
||
|
|
pub permanent: bool,
|
||
|
|
pub message: String,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl SendError {
|
||
|
|
fn permanent(message: impl ToString) -> Self {
|
||
|
|
Self { permanent: true, message: message.to_string() }
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The host part of an SMTP URL, for logs (no credentials).
|
||
|
|
pub fn smtp_host(url: &str) -> String {
|
||
|
|
let rest = url.split_once("://").map_or(url, |(_, r)| r);
|
||
|
|
let rest = rest.rsplit_once('@').map_or(rest, |(_, r)| r);
|
||
|
|
rest.split(['/', '?']).next().unwrap_or_default().to_owned()
|
||
|
|
}
|
||
|
|
|
||
|
|
impl Mailer {
|
||
|
|
pub fn smtp(url: &str) -> anyhow::Result<Self> {
|
||
|
|
let transport = AsyncSmtpTransport::<Tokio1Executor>::from_url(url)
|
||
|
|
.map_err(|e| anyhow::anyhow!("invalid SMTP_URL for {}: {e}", smtp_host(url)))?
|
||
|
|
.timeout(Some(Duration::from_secs(30)))
|
||
|
|
.build();
|
||
|
|
Ok(Mailer::Smtp(transport))
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn send(&self, from: &Mailbox, to: &str, mail: &Rendered) -> Result<(), SendError> {
|
||
|
|
let to_mailbox: Mailbox = to.parse().map_err(SendError::permanent)?;
|
||
|
|
let message = Message::builder()
|
||
|
|
.from(from.clone())
|
||
|
|
.to(to_mailbox)
|
||
|
|
.subject(&mail.subject)
|
||
|
|
.multipart(MultiPart::alternative_plain_html(mail.text.clone(), mail.html.clone()))
|
||
|
|
.map_err(SendError::permanent)?;
|
||
|
|
match self {
|
||
|
|
Mailer::Smtp(transport) => transport
|
||
|
|
.send(message)
|
||
|
|
.await
|
||
|
|
.map(drop)
|
||
|
|
.map_err(|e| SendError { permanent: e.is_permanent(), message: e.to_string() }),
|
||
|
|
#[cfg(test)]
|
||
|
|
Mailer::Memory { sent, fail } => {
|
||
|
|
if let Some((_, permanent)) = fail.iter().find(|(addr, _)| addr == to) {
|
||
|
|
return Err(SendError { permanent: *permanent, message: "test failure".into() });
|
||
|
|
}
|
||
|
|
sent.lock().unwrap().push((to.to_owned(), mail.clone()));
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
pub struct Delivery {
|
||
|
|
pub db: PgPool,
|
||
|
|
pub mailer: Mailer,
|
||
|
|
pub from: Mailbox,
|
||
|
|
pub max_attempts: i32,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(sqlx::FromRow)]
|
||
|
|
struct OutboxRow {
|
||
|
|
id: Uuid,
|
||
|
|
to_email: String,
|
||
|
|
template: String,
|
||
|
|
locale: String,
|
||
|
|
payload: Value,
|
||
|
|
attempts: i32,
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Wait before the next attempt: 1, 2, 4, ... minutes, at most 6 hours.
|
||
|
|
pub fn backoff(attempts: i32) -> Duration {
|
||
|
|
let minutes = 1u64 << attempts.clamp(1, 10).saturating_sub(1);
|
||
|
|
Duration::from_secs((minutes * 60).min(6 * 3600))
|
||
|
|
}
|
||
|
|
|
||
|
|
impl Delivery {
|
||
|
|
pub fn new(db: PgPool, mailer: Mailer, config: &Config) -> Self {
|
||
|
|
Self { db, mailer, from: config.mail_from.clone(), max_attempts: config.max_email_attempts }
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Sends up to one batch of due e-mails; returns how many rows were handled.
|
||
|
|
pub async fn deliver_due(&self) -> anyhow::Result<usize> {
|
||
|
|
let mut tx = self.db.begin().await?;
|
||
|
|
let rows: Vec<OutboxRow> = sqlx::query_as(
|
||
|
|
"SELECT id, to_email, template, locale, payload, attempts FROM email_outbox \
|
||
|
|
WHERE sent_at IS NULL AND failed_at IS NULL AND next_attempt_at <= now() \
|
||
|
|
ORDER BY next_attempt_at LIMIT $1 FOR UPDATE SKIP LOCKED",
|
||
|
|
)
|
||
|
|
.bind(BATCH)
|
||
|
|
.fetch_all(&mut *tx)
|
||
|
|
.await?;
|
||
|
|
|
||
|
|
for row in &rows {
|
||
|
|
let attempts = row.attempts + 1;
|
||
|
|
let result = match templates::render(&row.template, &row.locale, &row.payload) {
|
||
|
|
Ok(mail) => self.mailer.send(&self.from, &row.to_email, &mail).await,
|
||
|
|
Err(e) => Err(SendError::permanent(e)),
|
||
|
|
};
|
||
|
|
match result {
|
||
|
|
Ok(()) => {
|
||
|
|
sqlx::query(
|
||
|
|
"UPDATE email_outbox SET sent_at = now(), attempts = $2, last_error = NULL WHERE id = $1",
|
||
|
|
)
|
||
|
|
.bind(row.id)
|
||
|
|
.bind(attempts)
|
||
|
|
.execute(&mut *tx)
|
||
|
|
.await?;
|
||
|
|
tracing::info!(id = %row.id, template = %row.template, "e-mail sent");
|
||
|
|
}
|
||
|
|
Err(e) if e.permanent || attempts >= self.max_attempts => {
|
||
|
|
sqlx::query(
|
||
|
|
"UPDATE email_outbox SET failed_at = now(), attempts = $2, last_error = $3 WHERE id = $1",
|
||
|
|
)
|
||
|
|
.bind(row.id)
|
||
|
|
.bind(attempts)
|
||
|
|
.bind(&e.message)
|
||
|
|
.execute(&mut *tx)
|
||
|
|
.await?;
|
||
|
|
tracing::error!(id = %row.id, template = %row.template, attempts, error = %e.message, "e-mail failed");
|
||
|
|
}
|
||
|
|
Err(e) => {
|
||
|
|
let wait = backoff(attempts);
|
||
|
|
sqlx::query(
|
||
|
|
"UPDATE email_outbox SET attempts = $2, last_error = $3, \
|
||
|
|
next_attempt_at = now() + make_interval(secs => $4) WHERE id = $1",
|
||
|
|
)
|
||
|
|
.bind(row.id)
|
||
|
|
.bind(attempts)
|
||
|
|
.bind(&e.message)
|
||
|
|
.bind(wait.as_secs_f64())
|
||
|
|
.execute(&mut *tx)
|
||
|
|
.await?;
|
||
|
|
tracing::warn!(id = %row.id, attempts, retry_in_secs = wait.as_secs(), error = %e.message, "e-mail will be retried");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
tx.commit().await?;
|
||
|
|
Ok(rows.len())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn listen(db: &PgPool) -> Option<PgListener> {
|
||
|
|
let result = async {
|
||
|
|
let mut listener = PgListener::connect_with(db).await?;
|
||
|
|
listener.listen(CHANNEL).await?;
|
||
|
|
Ok::<_, sqlx::Error>(listener)
|
||
|
|
}
|
||
|
|
.await;
|
||
|
|
result.map_err(|e| tracing::warn!(error = %e, "LISTEN failed; polling only")).ok()
|
||
|
|
}
|
||
|
|
|
||
|
|
pub async fn run(delivery: Delivery, poll: Duration, stop: watch::Receiver<bool>) {
|
||
|
|
let mut listener = listen(&delivery.db).await;
|
||
|
|
loop {
|
||
|
|
if *stop.borrow() {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
match delivery.deliver_due().await {
|
||
|
|
Ok(n) if n as i64 == BATCH => continue,
|
||
|
|
Ok(_) => {}
|
||
|
|
Err(e) => tracing::error!(error = %e, "e-mail delivery"),
|
||
|
|
}
|
||
|
|
let notified = async {
|
||
|
|
match listener.as_mut() {
|
||
|
|
// Reconnects by itself; an error means the connection was lost for now.
|
||
|
|
Some(l) => {
|
||
|
|
if let Err(e) = l.recv().await {
|
||
|
|
tracing::warn!(error = %e, "LISTEN connection lost");
|
||
|
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
None => std::future::pending().await,
|
||
|
|
}
|
||
|
|
};
|
||
|
|
tokio::select! {
|
||
|
|
() = notified => {}
|
||
|
|
() = tokio::time::sleep(poll) => {}
|
||
|
|
() = crate::stopped(stop.clone()) => break,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|