Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ad436f786c | ||
|
|
16c40864bb |
@@ -134,7 +134,8 @@ jobs:
|
|||||||
docker images "$IMAGE_PREFIX-$svc" --format '{{.Repository}}:{{.Tag}}' | xargs -r docker rmi >/dev/null 2>&1 || true
|
docker images "$IMAGE_PREFIX-$svc" --format '{{.Repository}}:{{.Tag}}' | xargs -r docker rmi >/dev/null 2>&1 || true
|
||||||
done
|
done
|
||||||
docker image prune -f >/dev/null
|
docker image prune -f >/dev/null
|
||||||
docker builder prune -af --filter until=72h >/dev/null || true
|
# A size cap, not an age: a busy day builds tens of gigabytes.
|
||||||
|
docker builder prune -af --max-used-space 10GB >/dev/null || true
|
||||||
|
|
||||||
deploy:
|
deploy:
|
||||||
needs: build
|
needs: build
|
||||||
|
|||||||
Generated
+2
@@ -1862,7 +1862,9 @@ name = "service-kit"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"http",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
"tower-http",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-appender",
|
"tracing-appender",
|
||||||
"tracing-subscriber",
|
"tracing-subscriber",
|
||||||
|
|||||||
@@ -12,3 +12,5 @@ tokio.workspace = true
|
|||||||
tracing.workspace = true
|
tracing.workspace = true
|
||||||
tracing-subscriber.workspace = true
|
tracing-subscriber.workspace = true
|
||||||
tracing-appender.workspace = true
|
tracing-appender.workspace = true
|
||||||
|
tower-http.workspace = true
|
||||||
|
http = "1"
|
||||||
|
|||||||
+218
-26
@@ -3,9 +3,10 @@
|
|||||||
//! Environment:
|
//! Environment:
|
||||||
//! * `BIND_ADDR` listen address (each service has its own default)
|
//! * `BIND_ADDR` listen address (each service has its own default)
|
||||||
//! * `LOG_FORMAT` `text` (default) or `json`; `RUST_LOG` sets the level
|
//! * `LOG_FORMAT` `text` (default) or `json`; `RUST_LOG` sets the level
|
||||||
//! * `LOG_DIR` also write the log there, as JSON, one file a day
|
//! * `LOG_DIR` also write the log there, as JSON, one file a UTC day
|
||||||
//! (`<LOG_NAME>.YYYY-MM-DD.log`), keeping `LOG_KEEP_DAYS` (default 30) files
|
//! (`<LOG_NAME>/<LOG_NAME>.YYYY-MM-DD.log`), keeping `LOG_KEEP_DAYS`
|
||||||
//! * `LOG_NAME` the file name prefix (default: the service's name)
|
//! (default 30) files of at most `LOG_MAX_DAY_MB` (default 50) each
|
||||||
|
//! * `LOG_NAME` the folder and file name (default: the service's name)
|
||||||
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -14,41 +15,201 @@ use anyhow::Context;
|
|||||||
|
|
||||||
/// Days of log files kept in `LOG_DIR` by default.
|
/// Days of log files kept in `LOG_DIR` by default.
|
||||||
const LOG_KEEP_DAYS: usize = 30;
|
const LOG_KEEP_DAYS: usize = 30;
|
||||||
|
/// At most this many megabytes of log a day per service by default.
|
||||||
|
const LOG_MAX_DAY_MB: u64 = 50;
|
||||||
|
|
||||||
/// Initializes `tracing` from `RUST_LOG` and `LOG_FORMAT`, and the log files
|
/// Keeps the background log writer alive: hold it until `main` returns, so
|
||||||
/// in `LOG_DIR` (named after `LOG_NAME`, else `name`), so the log outlives the
|
/// the last lines reach the file.
|
||||||
/// container.
|
#[must_use = "the log files stop being written when this is dropped"]
|
||||||
pub fn init_logging(name: &str) {
|
pub struct LogGuard(#[allow(dead_code)] Option<tracing_appender::non_blocking::WorkerGuard>);
|
||||||
|
|
||||||
|
/// Logging for a one-off command: the console only, never the log files.
|
||||||
|
pub fn init_console_logging() {
|
||||||
use tracing_subscriber::layer::SubscriberExt;
|
use tracing_subscriber::layer::SubscriberExt;
|
||||||
use tracing_subscriber::util::SubscriberInitExt;
|
use tracing_subscriber::util::SubscriberInitExt;
|
||||||
use tracing_subscriber::{EnvFilter, Layer, fmt};
|
tracing_subscriber::registry().with(env_filter()).with(console_layer()).init();
|
||||||
|
}
|
||||||
|
|
||||||
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info,tower_http=info"));
|
fn env_filter() -> tracing_subscriber::EnvFilter {
|
||||||
let json = std::env::var("LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json"));
|
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||||
let stdout = if json { fmt::layer().json().boxed() } else { fmt::layer().boxed() };
|
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info,tower_http=info"))
|
||||||
let mut problem = None;
|
}
|
||||||
|
|
||||||
|
fn console_layer<S>() -> Box<dyn tracing_subscriber::Layer<S> + Send + Sync>
|
||||||
|
where
|
||||||
|
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
||||||
|
{
|
||||||
|
use tracing_subscriber::{Layer, fmt};
|
||||||
|
if std::env::var("LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json")) {
|
||||||
|
fmt::layer().json().boxed()
|
||||||
|
} else {
|
||||||
|
fmt::layer().boxed()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A whole number from the environment; a value that is not one is reported.
|
||||||
|
fn env_number<T: std::str::FromStr>(name: &str, default: T, problems: &mut Vec<String>) -> T {
|
||||||
|
match std::env::var(name) {
|
||||||
|
Ok(raw) if !raw.trim().is_empty() => raw.trim().parse().unwrap_or_else(|_| {
|
||||||
|
problems.push(format!("{name}={raw} is not a whole number; using the default"));
|
||||||
|
default
|
||||||
|
}),
|
||||||
|
_ => default,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Initializes `tracing` from `RUST_LOG` and `LOG_FORMAT`, and the log files
|
||||||
|
/// in `LOG_DIR`, so the log outlives the container: each service writes its
|
||||||
|
/// own folder `LOG_DIR/<name>/` (`<name>` = `LOG_NAME`, else `name`), one
|
||||||
|
/// file a UTC day, keeping `LOG_KEEP_DAYS` files (at least 2), at most
|
||||||
|
/// `LOG_MAX_DAY_MB` megabytes a day.
|
||||||
|
pub fn init_logging(name: &str) -> LogGuard {
|
||||||
|
use tracing_subscriber::layer::SubscriberExt;
|
||||||
|
use tracing_subscriber::util::SubscriberInitExt;
|
||||||
|
use tracing_subscriber::{Layer, fmt};
|
||||||
|
|
||||||
|
let mut problems = Vec::new();
|
||||||
|
let mut guard = None;
|
||||||
let file = std::env::var("LOG_DIR").ok().filter(|d| !d.trim().is_empty()).and_then(|dir| {
|
let file = std::env::var("LOG_DIR").ok().filter(|d| !d.trim().is_empty()).and_then(|dir| {
|
||||||
let prefix =
|
let name = std::env::var("LOG_NAME").ok().filter(|n| !n.trim().is_empty()).unwrap_or_else(|| name.to_string());
|
||||||
std::env::var("LOG_NAME").ok().filter(|n| !n.trim().is_empty()).unwrap_or_else(|| name.to_string());
|
let keep: usize = env_number("LOG_KEEP_DAYS", LOG_KEEP_DAYS, &mut problems);
|
||||||
let keep = std::env::var("LOG_KEEP_DAYS").ok().and_then(|d| d.parse().ok()).unwrap_or(LOG_KEEP_DAYS);
|
if keep < 2 {
|
||||||
let appender = tracing_appender::rolling::Builder::new()
|
problems.push(format!("LOG_KEEP_DAYS={keep} would delete today's file on a restart; keeping 2 days"));
|
||||||
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
}
|
||||||
.filename_prefix(prefix)
|
let max_mb: u64 = env_number("LOG_MAX_DAY_MB", LOG_MAX_DAY_MB, &mut problems);
|
||||||
.filename_suffix("log")
|
// Its own folder: the old-file cleanup takes every file whose name
|
||||||
.max_log_files(keep.max(1))
|
// starts with the prefix ("worker" would take "worker-ai" files too).
|
||||||
.build(&dir);
|
let folder = std::path::Path::new(&dir).join(&name);
|
||||||
|
let appender = std::fs::create_dir_all(&folder).map_err(|e| e.to_string()).and_then(|()| {
|
||||||
|
tracing_appender::rolling::Builder::new()
|
||||||
|
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
||||||
|
.filename_prefix(&name)
|
||||||
|
.filename_suffix("log")
|
||||||
|
.max_log_files(keep.max(2))
|
||||||
|
.build(&folder)
|
||||||
|
.map_err(|e| e.to_string())
|
||||||
|
});
|
||||||
match appender {
|
match appender {
|
||||||
Ok(appender) => Some(fmt::layer().json().with_ansi(false).with_writer(appender).boxed()),
|
Ok(appender) => {
|
||||||
|
let (writer, g) = tracing_appender::non_blocking(DailyCap::new(appender, max_mb.max(1) * 1024 * 1024));
|
||||||
|
guard = Some(g);
|
||||||
|
Some(fmt::layer().json().with_ansi(false).with_writer(writer).boxed())
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
problem = Some(format!("cannot write the log files in {dir}: {e}"));
|
problems
|
||||||
|
.push(format!("cannot write the log files in {}: {e}; logging to stdout only", folder.display()));
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
tracing_subscriber::registry().with(filter).with(stdout).with(file).init();
|
tracing_subscriber::registry().with(env_filter()).with(console_layer()).with(file).init();
|
||||||
if let Some(problem) = problem {
|
for problem in problems {
|
||||||
tracing::warn!("{problem}; logging to stdout only");
|
tracing::warn!("{problem}");
|
||||||
}
|
}
|
||||||
|
LogGuard(guard)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A daily size limit in front of the log file: past `limit` bytes in a UTC
|
||||||
|
/// day the rest of that day's lines are dropped (after one note saying so),
|
||||||
|
/// so a log storm cannot fill the disk.
|
||||||
|
struct DailyCap<W> {
|
||||||
|
inner: W,
|
||||||
|
limit: u64,
|
||||||
|
day: u64,
|
||||||
|
written: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<W> DailyCap<W> {
|
||||||
|
fn new(inner: W, limit: u64) -> Self {
|
||||||
|
Self { inner, limit, day: utc_day(), written: 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn utc_day() -> u64 {
|
||||||
|
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() / 86_400)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<W: std::io::Write> std::io::Write for DailyCap<W> {
|
||||||
|
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||||
|
let today = utc_day();
|
||||||
|
if today != self.day {
|
||||||
|
self.day = today;
|
||||||
|
self.written = 0;
|
||||||
|
}
|
||||||
|
if self.written >= self.limit {
|
||||||
|
return Ok(buf.len());
|
||||||
|
}
|
||||||
|
self.written += buf.len() as u64;
|
||||||
|
if self.written >= self.limit {
|
||||||
|
let note = format!(
|
||||||
|
"{{\"level\":\"WARN\",\"fields\":{{\"message\":\"daily log limit of {} MB reached; the rest of today is only on stdout\"}}}}\n",
|
||||||
|
self.limit / 1024 / 1024
|
||||||
|
);
|
||||||
|
self.inner.write_all(buf)?;
|
||||||
|
self.inner.write_all(note.as_bytes())?;
|
||||||
|
return Ok(buf.len());
|
||||||
|
}
|
||||||
|
self.inner.write(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn flush(&mut self) -> std::io::Result<()> {
|
||||||
|
self.inner.flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The request log of an HTTP service (`TraceLayer::make_span_with` and
|
||||||
|
/// `on_response`): the Docker health check stays out of it unless it fails,
|
||||||
|
/// and e-mail addresses and tokens in the path are masked (the log is kept).
|
||||||
|
#[derive(Debug, Clone, Copy, Default)]
|
||||||
|
pub struct RequestLog;
|
||||||
|
|
||||||
|
impl<B> tower_http::trace::MakeSpan<B> for RequestLog {
|
||||||
|
fn make_span(&mut self, request: &http::Request<B>) -> tracing::Span {
|
||||||
|
let path = request.uri().path();
|
||||||
|
if path == "/healthz" {
|
||||||
|
return tracing::Span::none();
|
||||||
|
}
|
||||||
|
tracing::info_span!("request", method = %request.method(), path = %masked_path(path))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<B> tower_http::trace::OnResponse<B> for RequestLog {
|
||||||
|
fn on_response(self, response: &http::Response<B>, latency: Duration, span: &tracing::Span) {
|
||||||
|
// A health check (no span) that went well is not logged.
|
||||||
|
if span.is_none() && response.status().is_success() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let ms = latency.as_millis() as u64;
|
||||||
|
let status = response.status().as_u16();
|
||||||
|
if span.is_none() {
|
||||||
|
tracing::warn!(path = "/healthz", status, latency_ms = ms, "health check failed");
|
||||||
|
} else {
|
||||||
|
tracing::info!(status, latency_ms = ms, "finished processing request");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The path with secrets masked: a segment holding an e-mail address, and a
|
||||||
|
/// long one that is not a UUID (share slugs, invitation tokens). Queries are
|
||||||
|
/// never logged.
|
||||||
|
pub fn masked_path(path: &str) -> String {
|
||||||
|
let uuid = |s: &str| {
|
||||||
|
s.len() == 36
|
||||||
|
&& s.char_indices()
|
||||||
|
.all(|(i, c)| if [8, 13, 18, 23].contains(&i) { c == '-' } else { c.is_ascii_hexdigit() })
|
||||||
|
};
|
||||||
|
path.split('/')
|
||||||
|
.map(|segment| {
|
||||||
|
if segment.contains('@') || segment.contains("%40") {
|
||||||
|
":email"
|
||||||
|
} else if segment.len() >= 12 && !uuid(segment) {
|
||||||
|
":token"
|
||||||
|
} else {
|
||||||
|
segment
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("/")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Listen address from `BIND_ADDR`, or `default`.
|
/// Listen address from `BIND_ADDR`, or `default`.
|
||||||
@@ -92,3 +253,34 @@ pub async fn shutdown_signal() {
|
|||||||
}
|
}
|
||||||
tracing::info!("shutting down");
|
tracing::info!("shutting down");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::io::Write;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn secrets_in_paths_are_masked() {
|
||||||
|
assert_eq!(masked_path("/v1/admin/waitlist/anna%40example.com"), "/v1/admin/waitlist/:email");
|
||||||
|
assert_eq!(masked_path("/v1/public/shares/Ab3dEf6hIj9kLm2n/visit"), "/v1/public/shares/:token/visit");
|
||||||
|
assert_eq!(masked_path("/v1/public/invites/k3jh4k5j6h7k8j9h0k1j2h3k"), "/v1/public/invites/:token");
|
||||||
|
let id = "01a1f960-96cb-765f-899a-4d76b45b236e";
|
||||||
|
assert_eq!(masked_path(&format!("/v1/tarot/readings/{id}")), format!("/v1/tarot/readings/{id}"));
|
||||||
|
assert_eq!(masked_path("/v1/admin/prompts/forecast/hu"), "/v1/admin/prompts/forecast/hu");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_daily_cap_drops_the_rest_of_the_day() {
|
||||||
|
let mut cap = DailyCap::new(Vec::new(), 10);
|
||||||
|
cap.write_all(b"12345\n").unwrap();
|
||||||
|
cap.write_all(b"67890\n").unwrap();
|
||||||
|
cap.write_all(b"dropped\n").unwrap();
|
||||||
|
let text = String::from_utf8(cap.inner.clone()).unwrap();
|
||||||
|
assert!(text.starts_with("12345\n67890\n") && text.contains("daily log limit") && !text.contains("dropped"));
|
||||||
|
// A new day starts again.
|
||||||
|
cap.day -= 1;
|
||||||
|
cap.write_all(b"next\n").unwrap();
|
||||||
|
assert!(String::from_utf8(cap.inner).unwrap().ends_with("next\n"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+2
-1
@@ -83,7 +83,8 @@ A Google OAuth kliensnél a `https://sorsfonalak.hu` címet fel kell venni az en
|
|||||||
## Üzemeltetés
|
## Üzemeltetés
|
||||||
|
|
||||||
- Naplók élőben: `docker compose logs -f backend worker worker-ai` (ez a konténer újraindításáig/cseréjéig él).
|
- Naplók élőben: `docker compose logs -f backend worker worker-ai` (ez a konténer újraindításáig/cseréjéig él).
|
||||||
- Naplók utólag: a backend és a két worker JSON-sorokat ír a `logs` volume-ba is, naponta új fájlba (`backend.2026-11-20.log`, `worker…`, `worker-ai…`), 30 napig megőrizve (`LOG_DIR`, `LOG_KEEP_DAYS`, `LOG_NAME`). A fájlok helye a szerveren: `docker volume inspect -f '{{.Mountpoint}}' sorsfonalak_logs`, például `grep -h '"level":"ERROR"' /var/lib/docker/volumes/sorsfonalak_logs/_data/backend.*.log | tail`. A Docker 30 másodpercenkénti `/healthz` ellenőrzése nem kerül a naplóba.
|
- Naplók utólag: a backend és a két worker JSON-sorokat ír a `logs` volume-ba is, szolgáltatásonként külön mappába, UTC-nap szerint új fájlba (`backend/backend.2026-11-20.log`, `worker/…`, `worker-ai/…`; a nap magyar idő szerint hajnali 1-2 órakor fordul). 30 napig maradnak meg, naponta legfeljebb 50 MB (`LOG_KEEP_DAYS`, `LOG_MAX_DAY_MB`, `LOG_DIR`, `LOG_NAME`); a kérések útvonalában az e-mail-címek és a tokenek maszkolva vannak. A fájlok helye: `docker volume inspect -f '{{.Mountpoint}}' sorsfonalak_logs`, például `grep -h '"level":"ERROR"' /var/lib/docker/volumes/sorsfonalak_logs/_data/backend/*.log | tail`. A Docker 30 másodpercenkénti `/healthz` ellenőrzése csak akkor kerül a naplóba, ha hibás.
|
||||||
|
- Deploy és futó feladatok: leállításkor a worker nem vesz fel újat, a futókat befejezi (`SHUTDOWN_DRAIN_SECS`: `worker` 40 s, `worker-ai` 270 s, `stop_grace_period` 1, illetve 5 perc); ami addig nem készül el, visszakerül a sorba, a próbálkozás nem számít. Ha éppen elemzés készül, a deploy ennyivel tovább tarthat.
|
||||||
- A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`).
|
- A worker két konténerben fut, ugyanabból az image-ből: a `worker` küldi az e-maileket és takarít (`WORKER_TASKS=email,cleanup`, ide kerülnek a további háttérfeladatok is), a `worker-ai` írja az elemzéseket és a válaszokat (`WORKER_TASKS=analyses`, gyors és lassú sáv). A sávok mérete a szerver `.env` fájljában állítható (`QUICK_CONCURRENCY`, `ANALYSIS_CONCURRENCY`).
|
||||||
- Visszaállás korábbi verzióra: `.env`-ben `TAG=<régebbi hash>`, utána `docker compose up -d`
|
- Visszaállás korábbi verzióra: `.env`-ben `TAG=<régebbi hash>`, utána `docker compose up -d`
|
||||||
- Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra).
|
- Mentés: naponta egy `pg_dump` a `/opt/sorsfonalak/backups` mappába, 14 napig megmarad. A mentéseket érdemes a szerverről is elvinni (DigitalOcean backup, vagy `rsync` otthonra).
|
||||||
|
|||||||
@@ -134,6 +134,8 @@ services:
|
|||||||
MAIL_FROM: ${MAIL_FROM:-Sorsfonalak <hello@sorsfonalak.hu>}
|
MAIL_FROM: ${MAIL_FROM:-Sorsfonalak <hello@sorsfonalak.hu>}
|
||||||
PUBLIC_BASE_URL: https://${DOMAIN}
|
PUBLIC_BASE_URL: https://${DOMAIN}
|
||||||
LOG_DIR: /logs
|
LOG_DIR: /logs
|
||||||
|
# On a deploy the e-mails and invoices being sent finish first.
|
||||||
|
SHUTDOWN_DRAIN_SECS: 40
|
||||||
env_file:
|
env_file:
|
||||||
- path: .env.secrets
|
- path: .env.secrets
|
||||||
required: false
|
required: false
|
||||||
@@ -143,6 +145,7 @@ services:
|
|||||||
read_only: true
|
read_only: true
|
||||||
volumes:
|
volumes:
|
||||||
- logs:/logs
|
- logs:/logs
|
||||||
|
stop_grace_period: 1m
|
||||||
logging: *logging
|
logging: *logging
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
@@ -158,6 +161,9 @@ services:
|
|||||||
PUBLIC_BASE_URL: https://${DOMAIN}
|
PUBLIC_BASE_URL: https://${DOMAIN}
|
||||||
LOG_DIR: /logs
|
LOG_DIR: /logs
|
||||||
LOG_NAME: worker-ai
|
LOG_NAME: worker-ai
|
||||||
|
# On a deploy the analyses being written finish first (a detailed one
|
||||||
|
# takes a few minutes); what is not done by then goes back to the queue.
|
||||||
|
SHUTDOWN_DRAIN_SECS: 270
|
||||||
env_file:
|
env_file:
|
||||||
- path: .env.secrets
|
- path: .env.secrets
|
||||||
required: false
|
required: false
|
||||||
@@ -167,6 +173,7 @@ services:
|
|||||||
read_only: true
|
read_only: true
|
||||||
volumes:
|
volumes:
|
||||||
- logs:/logs
|
- logs:/logs
|
||||||
|
stop_grace_period: 5m
|
||||||
logging: *logging
|
logging: *logging
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
if std::env::args().nth(1).as_deref() == Some("healthcheck") {
|
if std::env::args().nth(1).as_deref() == Some("healthcheck") {
|
||||||
return service_kit::healthcheck(addr);
|
return service_kit::healthcheck(addr);
|
||||||
}
|
}
|
||||||
service_kit::init_logging("backend");
|
let _log = service_kit::init_logging("backend");
|
||||||
let config = Config::from_env()?;
|
let config = Config::from_env()?;
|
||||||
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
||||||
db::migrate(&pool).await.context("running database migrations")?;
|
db::migrate(&pool).await.context("running database migrations")?;
|
||||||
|
|||||||
@@ -36,8 +36,7 @@ use axum::{Json, Router};
|
|||||||
use axum_extra::extract::WithRejection;
|
use axum_extra::extract::WithRejection;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use tower_http::cors::{AllowOrigin, CorsLayer};
|
use tower_http::cors::{AllowOrigin, CorsLayer};
|
||||||
use tower_http::trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer};
|
use tower_http::trace::TraceLayer;
|
||||||
use tracing::Level;
|
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
@@ -60,6 +59,7 @@ where
|
|||||||
|
|
||||||
pub fn router(state: Arc<AppState>) -> Router {
|
pub fn router(state: Arc<AppState>) -> Router {
|
||||||
Router::new()
|
Router::new()
|
||||||
|
.route("/healthz", get(health))
|
||||||
.route("/v1/config", get(public_config))
|
.route("/v1/config", get(public_config))
|
||||||
.route("/v1/packages", get(payments::catalog))
|
.route("/v1/packages", get(payments::catalog))
|
||||||
// login
|
// login
|
||||||
@@ -140,13 +140,8 @@ pub fn router(state: Arc<AppState>) -> Router {
|
|||||||
.route("/v1/admin/analyses/{id}/regenerate", post(analyses::regenerate))
|
.route("/v1/admin/analyses/{id}/regenerate", post(analyses::regenerate))
|
||||||
.route("/v1/admin/questions/{id}/regenerate", post(questions::regenerate))
|
.route("/v1/admin/questions/{id}/regenerate", post(questions::regenerate))
|
||||||
.layer(cors(&state.config.cors_allowed_origins))
|
.layer(cors(&state.config.cors_allowed_origins))
|
||||||
.layer(
|
// The request log: masked paths, health checks only when they fail.
|
||||||
TraceLayer::new_for_http()
|
.layer(TraceLayer::new_for_http().make_span_with(service_kit::RequestLog).on_response(service_kit::RequestLog))
|
||||||
.make_span_with(DefaultMakeSpan::new().level(Level::INFO))
|
|
||||||
.on_response(DefaultOnResponse::new().level(Level::INFO)),
|
|
||||||
)
|
|
||||||
// After the layers: the Docker health check (every 30 s) stays out of the log.
|
|
||||||
.route("/healthz", get(health))
|
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -54,7 +54,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
if std::env::args().nth(1).as_deref() == Some("healthcheck") {
|
if std::env::args().nth(1).as_deref() == Some("healthcheck") {
|
||||||
return service_kit::healthcheck(addr);
|
return service_kit::healthcheck(addr);
|
||||||
}
|
}
|
||||||
service_kit::init_logging("horoscope-api");
|
let _log = service_kit::init_logging("horoscope-api");
|
||||||
let state = Arc::new(tokio::task::spawn_blocking(load_state).await??);
|
let state = Arc::new(tokio::task::spawn_blocking(load_state).await??);
|
||||||
let app = routes::router(state);
|
let app = routes::router(state);
|
||||||
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
let listener = tokio::net::TcpListener::bind(addr).await.with_context(|| format!("binding {addr}"))?;
|
||||||
|
|||||||
@@ -9,8 +9,7 @@ use axum::{Json, Router};
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
|
use tower_http::cors::{AllowOrigin, Any, CorsLayer};
|
||||||
use tower_http::trace::{DefaultMakeSpan, DefaultOnResponse, TraceLayer};
|
use tower_http::trace::TraceLayer;
|
||||||
use tracing::Level;
|
|
||||||
|
|
||||||
use crate::AppState;
|
use crate::AppState;
|
||||||
use crate::error::ApiError;
|
use crate::error::ApiError;
|
||||||
@@ -22,6 +21,7 @@ type AppResult<T> = Result<Json<T>, ApiError>;
|
|||||||
|
|
||||||
pub fn router(state: Arc<AppState>) -> Router {
|
pub fn router(state: Arc<AppState>) -> Router {
|
||||||
Router::new()
|
Router::new()
|
||||||
|
.route("/healthz", get(health))
|
||||||
.route("/v1/info", get(info))
|
.route("/v1/info", get(info))
|
||||||
.route("/v1/countries", get(countries))
|
.route("/v1/countries", get(countries))
|
||||||
.route("/v1/places", get(places))
|
.route("/v1/places", get(places))
|
||||||
@@ -30,13 +30,8 @@ pub fn router(state: Arc<AppState>) -> Router {
|
|||||||
.route("/v1/synastry", post(synastry_chart))
|
.route("/v1/synastry", post(synastry_chart))
|
||||||
.route("/v1/forecast", post(forecast_year))
|
.route("/v1/forecast", post(forecast_year))
|
||||||
.layer(cors())
|
.layer(cors())
|
||||||
.layer(
|
// The request log: masked paths, health checks only when they fail.
|
||||||
TraceLayer::new_for_http()
|
.layer(TraceLayer::new_for_http().make_span_with(service_kit::RequestLog).on_response(service_kit::RequestLog))
|
||||||
.make_span_with(DefaultMakeSpan::new().level(Level::INFO))
|
|
||||||
.on_response(DefaultOnResponse::new().level(Level::INFO)),
|
|
||||||
)
|
|
||||||
// After the layers: the Docker health check (every 30 s) stays out of the log.
|
|
||||||
.route("/healthz", get(health))
|
|
||||||
.with_state(state)
|
.with_state(state)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -611,7 +611,7 @@ impl Analyzer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<bool>) {
|
pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, drain: Duration, stop: watch::Receiver<bool>) {
|
||||||
let quick = Arc::new(Semaphore::new(analyzer.lanes.quick));
|
let quick = Arc::new(Semaphore::new(analyzer.lanes.quick));
|
||||||
let slow = Arc::new(Semaphore::new(analyzer.lanes.slow));
|
let slow = Arc::new(Semaphore::new(analyzer.lanes.slow));
|
||||||
let mut tasks = tokio::task::JoinSet::new();
|
let mut tasks = tokio::task::JoinSet::new();
|
||||||
@@ -662,8 +662,12 @@ pub async fn run(analyzer: Arc<Analyzer>, poll: Duration, stop: watch::Receiver<
|
|||||||
() = crate::stopped(stop.clone()) => break,
|
() = crate::stopped(stop.clone()) => break,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Give running analyses a moment, then hand the rest back to the queue.
|
// Let running analyses finish within the drain time, then hand the rest
|
||||||
let _ = tokio::time::timeout(Duration::from_secs(10), async { while tasks.join_next().await.is_some() {} }).await;
|
// back to the queue (without counting the attempt).
|
||||||
|
if !tasks.is_empty() {
|
||||||
|
tracing::info!(running = tasks.len(), drain_secs = drain.as_secs(), "waiting for running analyses");
|
||||||
|
}
|
||||||
|
let _ = tokio::time::timeout(drain, async { while tasks.join_next().await.is_some() {} }).await;
|
||||||
tasks.abort_all();
|
tasks.abort_all();
|
||||||
analyzer.release_running().await;
|
analyzer.release_running().await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -52,6 +52,11 @@ pub struct Config {
|
|||||||
pub mail_from: Mailbox,
|
pub mail_from: Mailbox,
|
||||||
/// Fallback polling interval; new e-mails are picked up at once via LISTEN/NOTIFY.
|
/// Fallback polling interval; new e-mails are picked up at once via LISTEN/NOTIFY.
|
||||||
pub poll_interval: Duration,
|
pub poll_interval: Duration,
|
||||||
|
/// On a stop (a deploy): how long running work may still finish, taking
|
||||||
|
/// nothing new (`SHUTDOWN_DRAIN_SECS`, default 20); analyses not done by
|
||||||
|
/// then go back to the queue. Keep it below the container's stop grace
|
||||||
|
/// period (`stop_grace_period` in compose, Docker's default is 10 s).
|
||||||
|
pub shutdown_drain: Duration,
|
||||||
/// After this many failed attempts an e-mail is marked as failed.
|
/// After this many failed attempts an e-mail is marked as failed.
|
||||||
pub max_email_attempts: i32,
|
pub max_email_attempts: i32,
|
||||||
/// Unset: analyses stay pending.
|
/// Unset: analyses stay pending.
|
||||||
@@ -107,6 +112,7 @@ impl Config {
|
|||||||
smtp_about,
|
smtp_about,
|
||||||
mail_from: mail_from.parse().with_context(|| format!("invalid MAIL_FROM '{mail_from}'"))?,
|
mail_from: mail_from.parse().with_context(|| format!("invalid MAIL_FROM '{mail_from}'"))?,
|
||||||
poll_interval: Duration::from_secs(parsed("POLL_INTERVAL_SECS", 30)?),
|
poll_interval: Duration::from_secs(parsed("POLL_INTERVAL_SECS", 30)?),
|
||||||
|
shutdown_drain: Duration::from_secs(parsed("SHUTDOWN_DRAIN_SECS", 20)?),
|
||||||
max_email_attempts: parsed("MAX_EMAIL_ATTEMPTS", 8)?,
|
max_email_attempts: parsed("MAX_EMAIL_ATTEMPTS", 8)?,
|
||||||
// The lower-case name is accepted too (as written in some secrets files).
|
// The lower-case name is accepted too (as written in some secrets files).
|
||||||
openrouter_api_key: var("OPENROUTER_API_KEY").or_else(|| var("openrouter_api_key")),
|
openrouter_api_key: var("OPENROUTER_API_KEY").or_else(|| var("openrouter_api_key")),
|
||||||
|
|||||||
@@ -84,12 +84,12 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
match std::env::args().nth(1).as_deref() {
|
match std::env::args().nth(1).as_deref() {
|
||||||
Some("healthcheck") => return service_kit::healthcheck(addr),
|
Some("healthcheck") => return service_kit::healthcheck(addr),
|
||||||
Some("compare-models") => {
|
Some("compare-models") => {
|
||||||
service_kit::init_logging("worker");
|
service_kit::init_console_logging();
|
||||||
return compare::main(std::env::args().skip(2).collect()).await;
|
return compare::main(std::env::args().skip(2).collect()).await;
|
||||||
}
|
}
|
||||||
_ => {}
|
_ => {}
|
||||||
}
|
}
|
||||||
service_kit::init_logging("worker");
|
let _log = service_kit::init_logging("worker");
|
||||||
let config = Config::from_env()?;
|
let config = Config::from_env()?;
|
||||||
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
let pool = db::connect(&config.database_url).await.context("connecting to the database")?;
|
||||||
db::migrate(&pool).await.context("running database migrations")?;
|
db::migrate(&pool).await.context("running database migrations")?;
|
||||||
@@ -125,7 +125,12 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
} else {
|
} else {
|
||||||
let analyzer = analysis::Analyzer::new(pool.clone(), llm, config.max_analysis_attempts, lanes);
|
let analyzer = analysis::Analyzer::new(pool.clone(), llm, config.max_analysis_attempts, lanes);
|
||||||
tracing::info!(quick = lanes.quick, slow = lanes.slow, "analyses enabled");
|
tracing::info!(quick = lanes.quick, slow = lanes.slow, "analyses enabled");
|
||||||
jobs.spawn(analysis::run(Arc::new(analyzer), config.poll_interval, stop.clone()));
|
jobs.spawn(analysis::run(
|
||||||
|
Arc::new(analyzer),
|
||||||
|
config.poll_interval,
|
||||||
|
config.shutdown_drain,
|
||||||
|
stop.clone(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
None => tracing::warn!("OPENROUTER_API_KEY is not set: analyses stay pending"),
|
||||||
@@ -158,10 +163,16 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
() = service_kit::shutdown_signal() => {}
|
() = service_kit::shutdown_signal() => {}
|
||||||
}
|
}
|
||||||
let _ = stop_tx.send(true);
|
let _ = stop_tx.send(true);
|
||||||
// Let jobs finish the batch they are working on.
|
// Nothing new is taken; running work finishes (analyses hand back what is
|
||||||
let finished = tokio::time::timeout(Duration::from_secs(30), async { while jobs.join_next().await.is_some() {} });
|
// not done within the drain time, a few seconds before this gives up).
|
||||||
|
tracing::info!(drain_secs = config.shutdown_drain.as_secs(), "stopping: letting running work finish");
|
||||||
|
let finished = tokio::time::timeout(config.shutdown_drain + Duration::from_secs(5), async {
|
||||||
|
while jobs.join_next().await.is_some() {}
|
||||||
|
});
|
||||||
if finished.await.is_err() {
|
if finished.await.is_err() {
|
||||||
tracing::warn!("jobs did not stop in time");
|
tracing::warn!("jobs did not stop in time");
|
||||||
|
} else {
|
||||||
|
tracing::info!("all work finished");
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -928,6 +928,49 @@ mod analyses {
|
|||||||
assert!(!user.contains("Szeged") && !user.contains("Pécs"));
|
assert!(!user.contains("Szeged") && !user.contains("Pécs"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||||
|
async fn a_stop_lets_running_analyses_finish(db: PgPool) {
|
||||||
|
let s = setup(&db).await;
|
||||||
|
let id = request_analysis(&db, &s).await;
|
||||||
|
// A model that takes a while to answer.
|
||||||
|
let body = reply(&content(Kind::Natal)).1;
|
||||||
|
let app = axum::Router::new().route(
|
||||||
|
"/chat/completions",
|
||||||
|
axum::routing::post(move || {
|
||||||
|
let body = body.clone();
|
||||||
|
async move {
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(1500)).await;
|
||||||
|
Json(body)
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||||
|
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||||
|
let a = std::sync::Arc::new(analyzer(&db, &url));
|
||||||
|
let (stop_tx, stop) = tokio::sync::watch::channel(false);
|
||||||
|
let drain = std::time::Duration::from_secs(10);
|
||||||
|
let run = tokio::spawn(crate::analysis::run(a, std::time::Duration::from_secs(30), drain, stop));
|
||||||
|
let status = || async {
|
||||||
|
sqlx::query_scalar::<_, String>("SELECT status FROM analyses WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(&db)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
for _ in 0..60 {
|
||||||
|
if status().await == "running" {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||||
|
}
|
||||||
|
assert_eq!(status().await, "running");
|
||||||
|
// A deploy stops the worker while the model is still writing.
|
||||||
|
stop_tx.send(true).unwrap();
|
||||||
|
run.await.unwrap();
|
||||||
|
assert_eq!(status().await, "done");
|
||||||
|
}
|
||||||
|
|
||||||
#[sqlx::test(migrator = "db::MIGRATOR")]
|
#[sqlx::test(migrator = "db::MIGRATOR")]
|
||||||
async fn tarot_readings_are_written_in_the_quick_lane_and_refunded_on_failure(db: PgPool) {
|
async fn tarot_readings_are_written_in_the_quick_lane_and_refunded_on_failure(db: PgPool) {
|
||||||
let s = setup(&db).await;
|
let s = setup(&db).await;
|
||||||
|
|||||||
Reference in New Issue
Block a user