2026-11-15 10:25:35 +00:00
|
|
|
//! Plumbing shared by every HTTP service.
|
|
|
|
|
//!
|
|
|
|
|
//! Environment:
|
|
|
|
|
//! * `BIND_ADDR` listen address (each service has its own default)
|
|
|
|
|
//! * `LOG_FORMAT` `text` (default) or `json`; `RUST_LOG` sets the level
|
2026-11-20 18:57:09 +00:00
|
|
|
//! * `LOG_DIR` also write the log there, as JSON, one file a UTC day
|
|
|
|
|
//! (`<LOG_NAME>/<LOG_NAME>.YYYY-MM-DD.log`), keeping `LOG_KEEP_DAYS`
|
|
|
|
|
//! (default 30) files of at most `LOG_MAX_DAY_MB` (default 50) each
|
|
|
|
|
//! * `LOG_NAME` the folder and file name (default: the service's name)
|
2026-11-15 10:25:35 +00:00
|
|
|
|
|
|
|
|
use std::net::SocketAddr;
|
|
|
|
|
use std::time::Duration;
|
|
|
|
|
|
|
|
|
|
use anyhow::Context;
|
|
|
|
|
|
2026-11-20 17:34:24 +00:00
|
|
|
/// Days of log files kept in `LOG_DIR` by default.
|
|
|
|
|
const LOG_KEEP_DAYS: usize = 30;
|
2026-11-20 18:57:09 +00:00
|
|
|
/// At most this many megabytes of log a day per service by default.
|
|
|
|
|
const LOG_MAX_DAY_MB: u64 = 50;
|
2026-11-20 17:34:24 +00:00
|
|
|
|
2026-11-20 18:57:09 +00:00
|
|
|
/// Keeps the background log writer alive: hold it until `main` returns, so
|
|
|
|
|
/// the last lines reach the file.
|
|
|
|
|
#[must_use = "the log files stop being written when this is dropped"]
|
|
|
|
|
pub struct LogGuard(#[allow(dead_code)] Option<tracing_appender::non_blocking::WorkerGuard>);
|
|
|
|
|
|
|
|
|
|
/// Logging for a one-off command: the console only, never the log files.
|
|
|
|
|
pub fn init_console_logging() {
|
2026-11-20 17:34:24 +00:00
|
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
|
|
|
use tracing_subscriber::util::SubscriberInitExt;
|
2026-11-20 18:57:09 +00:00
|
|
|
tracing_subscriber::registry().with(env_filter()).with(console_layer()).init();
|
|
|
|
|
}
|
2026-11-20 17:34:24 +00:00
|
|
|
|
2026-11-20 18:57:09 +00:00
|
|
|
fn env_filter() -> tracing_subscriber::EnvFilter {
|
|
|
|
|
tracing_subscriber::EnvFilter::try_from_default_env()
|
|
|
|
|
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info,tower_http=info"))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn console_layer<S>() -> Box<dyn tracing_subscriber::Layer<S> + Send + Sync>
|
|
|
|
|
where
|
|
|
|
|
S: tracing::Subscriber + for<'a> tracing_subscriber::registry::LookupSpan<'a>,
|
|
|
|
|
{
|
|
|
|
|
use tracing_subscriber::{Layer, fmt};
|
|
|
|
|
if std::env::var("LOG_FORMAT").is_ok_and(|f| f.eq_ignore_ascii_case("json")) {
|
|
|
|
|
fmt::layer().json().boxed()
|
|
|
|
|
} else {
|
|
|
|
|
fmt::layer().boxed()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A whole number from the environment; a value that is not one is reported.
|
|
|
|
|
fn env_number<T: std::str::FromStr>(name: &str, default: T, problems: &mut Vec<String>) -> T {
|
|
|
|
|
match std::env::var(name) {
|
|
|
|
|
Ok(raw) if !raw.trim().is_empty() => raw.trim().parse().unwrap_or_else(|_| {
|
|
|
|
|
problems.push(format!("{name}={raw} is not a whole number; using the default"));
|
|
|
|
|
default
|
|
|
|
|
}),
|
|
|
|
|
_ => default,
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Initializes `tracing` from `RUST_LOG` and `LOG_FORMAT`, and the log files
|
|
|
|
|
/// in `LOG_DIR`, so the log outlives the container: each service writes its
|
|
|
|
|
/// own folder `LOG_DIR/<name>/` (`<name>` = `LOG_NAME`, else `name`), one
|
|
|
|
|
/// file a UTC day, keeping `LOG_KEEP_DAYS` files (at least 2), at most
|
|
|
|
|
/// `LOG_MAX_DAY_MB` megabytes a day.
|
|
|
|
|
pub fn init_logging(name: &str) -> LogGuard {
|
|
|
|
|
use tracing_subscriber::layer::SubscriberExt;
|
|
|
|
|
use tracing_subscriber::util::SubscriberInitExt;
|
|
|
|
|
use tracing_subscriber::{Layer, fmt};
|
|
|
|
|
|
|
|
|
|
let mut problems = Vec::new();
|
|
|
|
|
let mut guard = None;
|
2026-11-20 17:34:24 +00:00
|
|
|
let file = std::env::var("LOG_DIR").ok().filter(|d| !d.trim().is_empty()).and_then(|dir| {
|
2026-11-20 18:57:09 +00:00
|
|
|
let name = std::env::var("LOG_NAME").ok().filter(|n| !n.trim().is_empty()).unwrap_or_else(|| name.to_string());
|
|
|
|
|
let keep: usize = env_number("LOG_KEEP_DAYS", LOG_KEEP_DAYS, &mut problems);
|
|
|
|
|
if keep < 2 {
|
|
|
|
|
problems.push(format!("LOG_KEEP_DAYS={keep} would delete today's file on a restart; keeping 2 days"));
|
|
|
|
|
}
|
|
|
|
|
let max_mb: u64 = env_number("LOG_MAX_DAY_MB", LOG_MAX_DAY_MB, &mut problems);
|
|
|
|
|
// Its own folder: the old-file cleanup takes every file whose name
|
|
|
|
|
// starts with the prefix ("worker" would take "worker-ai" files too).
|
|
|
|
|
let folder = std::path::Path::new(&dir).join(&name);
|
|
|
|
|
let appender = std::fs::create_dir_all(&folder).map_err(|e| e.to_string()).and_then(|()| {
|
|
|
|
|
tracing_appender::rolling::Builder::new()
|
|
|
|
|
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
|
|
|
|
.filename_prefix(&name)
|
|
|
|
|
.filename_suffix("log")
|
|
|
|
|
.max_log_files(keep.max(2))
|
|
|
|
|
.build(&folder)
|
|
|
|
|
.map_err(|e| e.to_string())
|
|
|
|
|
});
|
2026-11-20 17:34:24 +00:00
|
|
|
match appender {
|
2026-11-20 18:57:09 +00:00
|
|
|
Ok(appender) => {
|
|
|
|
|
let (writer, g) = tracing_appender::non_blocking(DailyCap::new(appender, max_mb.max(1) * 1024 * 1024));
|
|
|
|
|
guard = Some(g);
|
|
|
|
|
Some(fmt::layer().json().with_ansi(false).with_writer(writer).boxed())
|
|
|
|
|
}
|
2026-11-20 17:34:24 +00:00
|
|
|
Err(e) => {
|
2026-11-20 18:57:09 +00:00
|
|
|
problems
|
|
|
|
|
.push(format!("cannot write the log files in {}: {e}; logging to stdout only", folder.display()));
|
2026-11-20 17:34:24 +00:00
|
|
|
None
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
2026-11-20 18:57:09 +00:00
|
|
|
tracing_subscriber::registry().with(env_filter()).with(console_layer()).with(file).init();
|
|
|
|
|
for problem in problems {
|
|
|
|
|
tracing::warn!("{problem}");
|
2026-11-15 10:25:35 +00:00
|
|
|
}
|
2026-11-20 18:57:09 +00:00
|
|
|
LogGuard(guard)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// A daily size limit in front of the log file: past `limit` bytes in a UTC
|
|
|
|
|
/// day the rest of that day's lines are dropped (after one note saying so),
|
|
|
|
|
/// so a log storm cannot fill the disk.
|
|
|
|
|
struct DailyCap<W> {
|
|
|
|
|
inner: W,
|
|
|
|
|
limit: u64,
|
|
|
|
|
day: u64,
|
|
|
|
|
written: u64,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl<W> DailyCap<W> {
|
|
|
|
|
fn new(inner: W, limit: u64) -> Self {
|
|
|
|
|
Self { inner, limit, day: utc_day(), written: 0 }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn utc_day() -> u64 {
|
|
|
|
|
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() / 86_400)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl<W: std::io::Write> std::io::Write for DailyCap<W> {
|
|
|
|
|
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
|
|
|
|
let today = utc_day();
|
|
|
|
|
if today != self.day {
|
|
|
|
|
self.day = today;
|
|
|
|
|
self.written = 0;
|
|
|
|
|
}
|
|
|
|
|
if self.written >= self.limit {
|
|
|
|
|
return Ok(buf.len());
|
|
|
|
|
}
|
|
|
|
|
self.written += buf.len() as u64;
|
|
|
|
|
if self.written >= self.limit {
|
|
|
|
|
let note = format!(
|
|
|
|
|
"{{\"level\":\"WARN\",\"fields\":{{\"message\":\"daily log limit of {} MB reached; the rest of today is only on stdout\"}}}}\n",
|
|
|
|
|
self.limit / 1024 / 1024
|
|
|
|
|
);
|
|
|
|
|
self.inner.write_all(buf)?;
|
|
|
|
|
self.inner.write_all(note.as_bytes())?;
|
|
|
|
|
return Ok(buf.len());
|
|
|
|
|
}
|
|
|
|
|
self.inner.write(buf)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fn flush(&mut self) -> std::io::Result<()> {
|
|
|
|
|
self.inner.flush()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The request log of an HTTP service (`TraceLayer::make_span_with` and
|
|
|
|
|
/// `on_response`): the Docker health check stays out of it unless it fails,
|
|
|
|
|
/// and e-mail addresses and tokens in the path are masked (the log is kept).
|
|
|
|
|
#[derive(Debug, Clone, Copy, Default)]
|
|
|
|
|
pub struct RequestLog;
|
|
|
|
|
|
|
|
|
|
impl<B> tower_http::trace::MakeSpan<B> for RequestLog {
|
|
|
|
|
fn make_span(&mut self, request: &http::Request<B>) -> tracing::Span {
|
|
|
|
|
let path = request.uri().path();
|
|
|
|
|
if path == "/healthz" {
|
|
|
|
|
return tracing::Span::none();
|
|
|
|
|
}
|
|
|
|
|
tracing::info_span!("request", method = %request.method(), path = %masked_path(path))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl<B> tower_http::trace::OnResponse<B> for RequestLog {
|
|
|
|
|
fn on_response(self, response: &http::Response<B>, latency: Duration, span: &tracing::Span) {
|
|
|
|
|
// A health check (no span) that went well is not logged.
|
|
|
|
|
if span.is_none() && response.status().is_success() {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
let ms = latency.as_millis() as u64;
|
|
|
|
|
let status = response.status().as_u16();
|
|
|
|
|
if span.is_none() {
|
|
|
|
|
tracing::warn!(path = "/healthz", status, latency_ms = ms, "health check failed");
|
|
|
|
|
} else {
|
|
|
|
|
tracing::info!(status, latency_ms = ms, "finished processing request");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The path with secrets masked: a segment holding an e-mail address, and a
|
|
|
|
|
/// long one that is not a UUID (share slugs, invitation tokens). Queries are
|
|
|
|
|
/// never logged.
|
|
|
|
|
pub fn masked_path(path: &str) -> String {
|
|
|
|
|
let uuid = |s: &str| {
|
|
|
|
|
s.len() == 36
|
|
|
|
|
&& s.char_indices()
|
|
|
|
|
.all(|(i, c)| if [8, 13, 18, 23].contains(&i) { c == '-' } else { c.is_ascii_hexdigit() })
|
|
|
|
|
};
|
|
|
|
|
path.split('/')
|
|
|
|
|
.map(|segment| {
|
|
|
|
|
if segment.contains('@') || segment.contains("%40") {
|
|
|
|
|
":email"
|
|
|
|
|
} else if segment.len() >= 12 && !uuid(segment) {
|
|
|
|
|
":token"
|
|
|
|
|
} else {
|
|
|
|
|
segment
|
|
|
|
|
}
|
|
|
|
|
})
|
|
|
|
|
.collect::<Vec<_>>()
|
|
|
|
|
.join("/")
|
2026-11-15 10:25:35 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Listen address from `BIND_ADDR`, or `default`.
|
|
|
|
|
pub fn bind_addr(default: &str) -> anyhow::Result<SocketAddr> {
|
|
|
|
|
let raw = std::env::var("BIND_ADDR").unwrap_or_else(|_| default.into());
|
|
|
|
|
raw.parse().with_context(|| format!("invalid BIND_ADDR '{raw}'"))
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Minimal HTTP GET against `/healthz` of the local service, so scratch
|
|
|
|
|
/// images need no curl. Used by the `healthcheck` subcommand.
|
|
|
|
|
pub fn healthcheck(mut addr: SocketAddr) -> anyhow::Result<()> {
|
|
|
|
|
use std::io::{Read, Write};
|
|
|
|
|
if addr.ip().is_unspecified() {
|
|
|
|
|
addr.set_ip([127, 0, 0, 1].into());
|
|
|
|
|
}
|
|
|
|
|
let mut stream = std::net::TcpStream::connect_timeout(&addr, Duration::from_secs(2))?;
|
|
|
|
|
stream.set_read_timeout(Some(Duration::from_secs(2)))?;
|
|
|
|
|
stream.write_all(b"GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")?;
|
|
|
|
|
let mut response = String::new();
|
|
|
|
|
stream.read_to_string(&mut response)?;
|
|
|
|
|
anyhow::ensure!(response.starts_with("HTTP/1.1 200"), "unhealthy: {}", response.lines().next().unwrap_or(""));
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// Resolves on Ctrl+C or SIGTERM (graceful shutdown in Docker).
|
|
|
|
|
pub async fn shutdown_signal() {
|
|
|
|
|
let ctrl_c = async {
|
|
|
|
|
let _ = tokio::signal::ctrl_c().await;
|
|
|
|
|
};
|
|
|
|
|
#[cfg(unix)]
|
|
|
|
|
let terminate = async {
|
|
|
|
|
if let Ok(mut s) = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
|
|
|
|
|
s.recv().await;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
#[cfg(not(unix))]
|
|
|
|
|
let terminate = std::future::pending::<()>();
|
|
|
|
|
tokio::select! {
|
|
|
|
|
_ = ctrl_c => {},
|
|
|
|
|
_ = terminate => {},
|
|
|
|
|
}
|
|
|
|
|
tracing::info!("shutting down");
|
|
|
|
|
}
|
2026-11-20 18:57:09 +00:00
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
|
mod tests {
|
|
|
|
|
use std::io::Write;
|
|
|
|
|
|
|
|
|
|
use super::*;
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn secrets_in_paths_are_masked() {
|
|
|
|
|
assert_eq!(masked_path("/v1/admin/waitlist/anna%40example.com"), "/v1/admin/waitlist/:email");
|
|
|
|
|
assert_eq!(masked_path("/v1/public/shares/Ab3dEf6hIj9kLm2n/visit"), "/v1/public/shares/:token/visit");
|
|
|
|
|
assert_eq!(masked_path("/v1/public/invites/k3jh4k5j6h7k8j9h0k1j2h3k"), "/v1/public/invites/:token");
|
|
|
|
|
let id = "01a1f960-96cb-765f-899a-4d76b45b236e";
|
|
|
|
|
assert_eq!(masked_path(&format!("/v1/tarot/readings/{id}")), format!("/v1/tarot/readings/{id}"));
|
|
|
|
|
assert_eq!(masked_path("/v1/admin/prompts/forecast/hu"), "/v1/admin/prompts/forecast/hu");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
|
fn the_daily_cap_drops_the_rest_of_the_day() {
|
|
|
|
|
let mut cap = DailyCap::new(Vec::new(), 10);
|
|
|
|
|
cap.write_all(b"12345\n").unwrap();
|
|
|
|
|
cap.write_all(b"67890\n").unwrap();
|
|
|
|
|
cap.write_all(b"dropped\n").unwrap();
|
|
|
|
|
let text = String::from_utf8(cap.inner.clone()).unwrap();
|
|
|
|
|
assert!(text.starts_with("12345\n67890\n") && text.contains("daily log limit") && !text.contains("dropped"));
|
|
|
|
|
// A new day starts again.
|
|
|
|
|
cap.day -= 1;
|
|
|
|
|
cap.write_all(b"next\n").unwrap();
|
|
|
|
|
assert!(String::from_utf8(cap.inner).unwrap().ends_with("next\n"));
|
|
|
|
|
}
|
|
|
|
|
}
|