Main #17
@@ -20,3 +20,7 @@ SIGNUP_MODE=invite
|
|||||||
|
|
||||||
# Image version; every deployment rewrites it
|
# Image version; every deployment rewrites it
|
||||||
TAG=latest
|
TAG=latest
|
||||||
|
|
||||||
|
# worker-ai: analyses and answers written at the same time.
|
||||||
|
# QUICK_CONCURRENCY=2 # summaries and answers
|
||||||
|
# ANALYSIS_CONCURRENCY=4 # detailed analyses and forecasts
|
||||||
|
|||||||
+2
-1
@@ -82,7 +82,8 @@ A Google OAuth kliensnél a `https://sorsfonalak.hu` címet fel kell venni az en
|
|||||||
|
|
||||||
## Üzemeltetés
|
## Üzemeltetés
|
||||||
|
|
||||||
- Naplók: `docker compose logs -f backend worker`
|
- Naplók: `docker compose logs -f backend worker worker-ai`
|
||||||
|
- A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`).
|
||||||
- Visszaállás korábbi verzióra: `.env`-ben `TAG=<régebbi hash>`, utána `docker compose up -d`
|
- Visszaállás korábbi verzióra: `.env`-ben `TAG=<régebbi hash>`, utána `docker compose up -d`
|
||||||
- Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra).
|
- Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra).
|
||||||
- Visszatöltés: `gunzip -c backups/sorsfonalak-<dátum>.sql.gz | docker compose exec -T postgres psql -U sorsfonalak sorsfonalak`
|
- Visszatöltés: `gunzip -c backups/sorsfonalak-<dátum>.sql.gz | docker compose exec -T postgres psql -U sorsfonalak sorsfonalak`
|
||||||
|
|||||||
@@ -99,10 +99,12 @@ services:
|
|||||||
logging: *logging
|
logging: *logging
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# E-mail delivery, cleanup and the other background jobs.
|
||||||
worker:
|
worker:
|
||||||
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
|
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
|
||||||
environment:
|
environment:
|
||||||
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
|
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
|
||||||
|
WORKER_TASKS: email,cleanup
|
||||||
# SMTP_URL or the SES credentials come from .env.secrets (docs/levelkuldes.md);
|
# SMTP_URL or the SES credentials come from .env.secrets (docs/levelkuldes.md);
|
||||||
# listing them here with an empty default would override that file.
|
# listing them here with an empty default would override that file.
|
||||||
SES_REGION: ${SES_REGION:-eu-central-1}
|
SES_REGION: ${SES_REGION:-eu-central-1}
|
||||||
@@ -118,6 +120,26 @@ services:
|
|||||||
logging: *logging
|
logging: *logging
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
# The same image writing the analyses and answers (quick and slow lane);
|
||||||
|
# OPENROUTER_API_KEY comes from .env.secrets.
|
||||||
|
worker-ai:
|
||||||
|
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-worker:${TAG:-latest}
|
||||||
|
environment:
|
||||||
|
DATABASE_URL: postgres://sorsfonalak:${POSTGRES_PASSWORD}@postgres:5432/sorsfonalak
|
||||||
|
WORKER_TASKS: analyses
|
||||||
|
QUICK_CONCURRENCY: ${QUICK_CONCURRENCY:-2}
|
||||||
|
ANALYSIS_CONCURRENCY: ${ANALYSIS_CONCURRENCY:-4}
|
||||||
|
PUBLIC_BASE_URL: https://${DOMAIN}
|
||||||
|
env_file:
|
||||||
|
- path: .env.secrets
|
||||||
|
required: false
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
|
read_only: true
|
||||||
|
logging: *logging
|
||||||
|
restart: unless-stopped
|
||||||
|
|
||||||
frontend:
|
frontend:
|
||||||
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-frontend:${TAG:-latest}
|
image: ${REGISTRY_PREFIX:-registry.sagifer.com/sagifer/sorsfonalak}-frontend:${TAG:-latest}
|
||||||
environment:
|
environment:
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ Ugyanazt a radix-elemzést több modellel is megírja, véletlenszerű betűkkel
|
|||||||
| `OPENROUTER_API_KEY` | – | OpenRouter-kulcs (compose-ban a `.env.secrets` fájlból); nélküle az elemzések várnak |
|
| `OPENROUTER_API_KEY` | – | OpenRouter-kulcs (compose-ban a `.env.secrets` fájlból); nélküle az elemzések várnak |
|
||||||
| `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | |
|
| `OPENROUTER_BASE_URL` | `https://openrouter.ai/api/v1` | |
|
||||||
| `PUBLIC_BASE_URL` | – | `HTTP-Referer` fejléc az OpenRouter felé |
|
| `PUBLIC_BASE_URL` | – | `HTTP-Referer` fejléc az OpenRouter felé |
|
||||||
|
| `WORKER_TASKS` | `email,cleanup,analyses` | a folyamat feladatai; élesben `worker`: `email,cleanup`, `worker-ai`: `analyses` |
|
||||||
| `ANALYSIS_CONCURRENCY` | `4` | lassú sáv: egyszerre írt részletes elemzések és előrejelzések (0: kikapcsolva) |
|
| `ANALYSIS_CONCURRENCY` | `4` | lassú sáv: egyszerre írt részletes elemzések és előrejelzések (0: kikapcsolva) |
|
||||||
| `QUICK_CONCURRENCY` | `2` | gyors sáv: egyszerre írt rövid összefoglalók és válaszok (0: kikapcsolva) |
|
| `QUICK_CONCURRENCY` | `2` | gyors sáv: egyszerre írt rövid összefoglalók és válaszok (0: kikapcsolva) |
|
||||||
| `MAX_ANALYSIS_ATTEMPTS` | `3` | ennyi próbálkozás után sikertelen az elemzés (az egység visszajár) |
|
| `MAX_ANALYSIS_ATTEMPTS` | `3` | ennyi próbálkozás után sikertelen az elemzés (az egység visszajár) |
|
||||||
|
|||||||
@@ -5,7 +5,38 @@ use std::time::Duration;
|
|||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use lettre::message::Mailbox;
|
use lettre::message::Mailbox;
|
||||||
|
|
||||||
|
/// The jobs a worker process runs (`WORKER_TASKS`, comma-separated).
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct Tasks {
|
||||||
|
/// Sending the e-mails of `email_outbox`.
|
||||||
|
pub email: bool,
|
||||||
|
/// The hourly removal of expired data.
|
||||||
|
pub cleanup: bool,
|
||||||
|
/// Writing the analyses and answers with the LLM (both lanes).
|
||||||
|
pub analyses: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Tasks {
|
||||||
|
pub const ALL: Tasks = Tasks { email: true, cleanup: true, analyses: true };
|
||||||
|
|
||||||
|
pub fn parse(raw: &str) -> anyhow::Result<Self> {
|
||||||
|
let mut tasks = Tasks { email: false, cleanup: false, analyses: false };
|
||||||
|
for name in raw.split(',').map(str::trim).filter(|n| !n.is_empty()) {
|
||||||
|
match name {
|
||||||
|
"email" => tasks.email = true,
|
||||||
|
"cleanup" => tasks.cleanup = true,
|
||||||
|
"analyses" => tasks.analyses = true,
|
||||||
|
other => anyhow::bail!("unknown task '{other}' in WORKER_TASKS (email, cleanup, analyses)"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
anyhow::ensure!(tasks != Tasks { email: false, cleanup: false, analyses: false }, "WORKER_TASKS names no task");
|
||||||
|
Ok(tasks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Config {
|
pub struct Config {
|
||||||
|
/// What this process does; all of it by default.
|
||||||
|
pub tasks: Tasks,
|
||||||
pub database_url: String,
|
pub database_url: String,
|
||||||
/// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required`
|
/// `smtp://host:port` (plain, e.g. Mailpit), `smtp://user:pass@host:587?tls=required`
|
||||||
/// (STARTTLS) or `smtps://...` (TLS). Without it, Amazon SES from the AWS
|
/// (STARTTLS) or `smtps://...` (TLS). Without it, Amazon SES from the AWS
|
||||||
@@ -57,6 +88,7 @@ impl Config {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
|
tasks: var("WORKER_TASKS").map(|raw| Tasks::parse(&raw)).transpose()?.unwrap_or(Tasks::ALL),
|
||||||
database_url: var("DATABASE_URL").context("DATABASE_URL is required")?,
|
database_url: var("DATABASE_URL").context("DATABASE_URL is required")?,
|
||||||
smtp_url,
|
smtp_url,
|
||||||
smtp_about,
|
smtp_about,
|
||||||
|
|||||||
@@ -5,6 +5,9 @@
|
|||||||
//! * hourly cleanup of expired sessions, tokens, invites and old e-mails
|
//! * hourly cleanup of expired sessions, tokens, invites and old e-mails
|
||||||
//! * writing the requested text analyses with an LLM (OpenRouter)
|
//! * writing the requested text analyses with an LLM (OpenRouter)
|
||||||
//!
|
//!
|
||||||
|
//! `WORKER_TASKS` (e.g. `email,cleanup` or `analyses`) picks the jobs of one
|
||||||
|
//! process, so they can run in separate containers; all of them by default.
|
||||||
|
//!
|
||||||
//! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default
|
//! Configuration: see [`config::Config`]; plus `BIND_ADDR` (default
|
||||||
//! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`.
|
//! `0.0.0.0:8082`, only `/healthz`), `LOG_FORMAT` and `RUST_LOG`.
|
||||||
//! `worker healthcheck` performs a local health check (for Docker);
|
//! `worker healthcheck` performs a local health check (for Docker);
|
||||||
@@ -91,7 +94,10 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
|
|
||||||
let (stop_tx, stop) = watch::channel(false);
|
let (stop_tx, stop) = watch::channel(false);
|
||||||
let mut jobs = tokio::task::JoinSet::new();
|
let mut jobs = tokio::task::JoinSet::new();
|
||||||
|
let tasks = config.tasks;
|
||||||
|
tracing::info!(email = tasks.email, cleanup = tasks.cleanup, analyses = tasks.analyses, "worker tasks");
|
||||||
match &config.smtp_url {
|
match &config.smtp_url {
|
||||||
|
_ if !tasks.email => {}
|
||||||
Some(url) => {
|
Some(url) => {
|
||||||
let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config);
|
let delivery = email::Delivery::new(pool.clone(), email::Mailer::smtp(url)?, &config);
|
||||||
let via = config.smtp_about.clone().unwrap_or_else(|| "SMTP_URL".into());
|
let via = config.smtp_about.clone().unwrap_or_else(|| "SMTP_URL".into());
|
||||||
@@ -101,6 +107,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
None => tracing::warn!("neither SMTP_URL nor SES credentials are set: e-mails stay in the outbox"),
|
None => tracing::warn!("neither SMTP_URL nor SES credentials are set: e-mails stay in the outbox"),
|
||||||
}
|
}
|
||||||
match &config.openrouter_api_key {
|
match &config.openrouter_api_key {
|
||||||
|
_ if !tasks.analyses => {}
|
||||||
Some(key) => {
|
Some(key) => {
|
||||||
let llm =
|
let llm =
|
||||||
analysis::openrouter::OpenRouter::new(&config.openrouter_base_url, key, config.public_base_url.clone());
|
analysis::openrouter::OpenRouter::new(&config.openrouter_base_url, key, config.public_base_url.clone());
|
||||||
@@ -115,7 +122,9 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
||||||
}
|
}
|
||||||
|
if tasks.cleanup {
|
||||||
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
|
jobs.spawn(cleanup::run(pool.clone(), stop.clone()));
|
||||||
|
}
|
||||||
|
|
||||||
let app = Router::new().route("/healthz", get(healthz)).with_state(pool);
|
let app = Router::new().route("/healthz", get(healthz)).with_state(pool);
|
||||||
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
||||||
|
|||||||
@@ -223,6 +223,16 @@ async fn cleanup_removes_only_expired_data(db: PgPool) {
|
|||||||
|
|
||||||
// --- analyses ----------------------------------------------------------------------------
|
// --- analyses ----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn worker_tasks_are_chosen_by_name() {
|
||||||
|
use crate::config::Tasks;
|
||||||
|
let mail = Tasks::parse("email, cleanup").unwrap();
|
||||||
|
assert_eq!(mail, Tasks { email: true, cleanup: true, analyses: false });
|
||||||
|
assert_eq!(Tasks::parse("analyses").unwrap(), Tasks { email: false, cleanup: false, analyses: true });
|
||||||
|
assert!(Tasks::parse("emails").is_err());
|
||||||
|
assert!(Tasks::parse(" , ").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
mod analyses {
|
mod analyses {
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user