//! Object storage for uploaded photos: any S3-compatible service, addressed //! path-style (`{endpoint}/{bucket}/{key}`). In production DigitalOcean //! Spaces, in development a RustFS container. //! //! The bucket stays private: the backend reads the objects itself and serves //! them under `/v1/public/photos/...`, from where the frontend's image //! optimizer makes the sized versions (and Cloudflare caches those). //! //! Requests are signed with AWS Signature Version 4. use std::time::Duration; use hmac::{Hmac, Mac}; use sha2::{Digest, Sha256}; use time::OffsetDateTime; /// `S3_ENDPOINT` (e.g. `https://fra1.digitaloceanspaces.com`), `S3_BUCKET`, /// `S3_REGION` (default `us-east-1`; Spaces accepts it too), /// `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` (in `.env.secrets`). #[derive(Clone)] pub struct StorageConfig { pub endpoint: String, pub bucket: String, pub region: String, pub access_key: String, pub secret_key: String, } // The secret stays out of logs. impl std::fmt::Debug for StorageConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("StorageConfig") .field("endpoint", &self.endpoint) .field("bucket", &self.bucket) .field("region", &self.region) .finish() } } #[derive(Debug, thiserror::Error)] pub enum StorageError { #[error("object storage request failed: {0}")] Http(#[from] reqwest::Error), #[error("object storage answered {status}: {body}")] Status { status: u16, body: String }, } pub struct Storage { config: StorageConfig, http: reqwest::Client, } /// An object read back: its bytes and content type. pub struct Object { pub bytes: axum::body::Bytes, pub content_type: Option, } impl Storage { pub fn new(config: StorageConfig) -> Self { let http = reqwest::Client::builder().timeout(Duration::from_secs(30)).build().expect("static configuration"); Self { config, http } } fn url(&self, key: &str) -> String { let path = if key.is_empty() { String::new() } else { format!("/{}", encode_path(key)) }; format!("{}/{}{path}", self.config.endpoint, encode_segment(&self.config.bucket)) } async fn send( &self, method: reqwest::Method, key: &str, body: Vec, content_type: Option<&str>, ) -> Result { let url = self.url(key); let parsed = reqwest::Url::parse(&url).map_err(|e| StorageError::Status { status: 0, body: e.to_string() })?; let host = match parsed.port() { Some(port) => format!("{}:{port}", parsed.host_str().unwrap_or_default()), None => parsed.host_str().unwrap_or_default().to_string(), }; let payload_hash = hex(&Sha256::digest(&body)); let now = OffsetDateTime::now_utc(); let auth = authorization(&self.config, method.as_str(), parsed.path(), &host, &payload_hash, now); let mut request = self .http .request(method, url) .header("x-amz-date", amz_date(now)) .header("x-amz-content-sha256", &payload_hash) .header("authorization", auth); if let Some(ct) = content_type { request = request.header("content-type", ct); } let response = request.body(body).send().await?; Ok(response) } async fn checked(response: reqwest::Response) -> Result { if response.status().is_success() { return Ok(response); } let status = response.status().as_u16(); let body = response.text().await.unwrap_or_default().chars().take(500).collect(); Err(StorageError::Status { status, body }) } pub async fn put(&self, key: &str, body: Vec, content_type: &str) -> Result<(), StorageError> { Self::checked(self.send(reqwest::Method::PUT, key, body, Some(content_type)).await?).await?; Ok(()) } /// The object, or `None` if there is no such key. pub async fn get(&self, key: &str) -> Result, StorageError> { let response = self.send(reqwest::Method::GET, key, Vec::new(), None).await?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(None); } let response = Self::checked(response).await?; let content_type = response.headers().get("content-type").and_then(|v| v.to_str().ok()).map(str::to_owned); Ok(Some(Object { bytes: response.bytes().await?, content_type })) } /// Deletes the object; a missing one is not an error. pub async fn delete(&self, key: &str) -> Result<(), StorageError> { let response = self.send(reqwest::Method::DELETE, key, Vec::new(), None).await?; if response.status() == reqwest::StatusCode::NOT_FOUND { return Ok(()); } Self::checked(response).await?; Ok(()) } /// Creates the bucket if it does not exist yet (development: a fresh /// RustFS container has none; in production it is made in the console). pub async fn ensure_bucket(&self) -> Result<(), StorageError> { let head = self.send(reqwest::Method::HEAD, "", Vec::new(), None).await?; if head.status().is_success() { return Ok(()); } Self::checked(self.send(reqwest::Method::PUT, "", Vec::new(), None).await?).await?; tracing::info!(bucket = %self.config.bucket, "object storage bucket created"); Ok(()) } } type HmacSha256 = Hmac; fn hmac(key: &[u8], data: &str) -> Vec { let mut mac = HmacSha256::new_from_slice(key).expect("HMAC takes any key length"); mac.update(data.as_bytes()); mac.finalize().into_bytes().to_vec() } fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } fn amz_date(t: OffsetDateTime) -> String { format!( "{:04}{:02}{:02}T{:02}{:02}{:02}Z", t.year(), u8::from(t.month()), t.day(), t.hour(), t.minute(), t.second() ) } /// URI-encodes one path segment as SigV4 wants it (unreserved characters stay). fn encode_segment(s: &str) -> String { s.bytes() .map(|b| match b { b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), _ => format!("%{b:02X}"), }) .collect() } fn encode_path(key: &str) -> String { key.split('/').map(encode_segment).collect::>().join("/") } fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec { [date, region, service, "aws4_request"] .iter() .fold(format!("AWS4{secret}").into_bytes(), |key, part| hmac(&key, part)) } /// The `Authorization` header of a request (no query string; the path is /// already encoded, as in the URL). fn authorization( config: &StorageConfig, method: &str, path: &str, host: &str, payload_hash: &str, now: OffsetDateTime, ) -> String { let date_time = amz_date(now); let date = &date_time[..8]; let signed_headers = "host;x-amz-content-sha256;x-amz-date"; let canonical = format!( "{method}\n{path}\n\nhost:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{date_time}\n\n\ {signed_headers}\n{payload_hash}" ); let scope = format!("{date}/{}/s3/aws4_request", config.region); let to_sign = format!("AWS4-HMAC-SHA256\n{date_time}\n{scope}\n{}", hex(&Sha256::digest(canonical.as_bytes()))); let signature = hex(&hmac(&signing_key(&config.secret_key, date, &config.region, "s3"), &to_sign)); format!( "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed_headers}, Signature={signature}", config.access_key ) } #[cfg(test)] mod tests { use super::*; /// The signing key example of the AWS documentation ("Examples of how to /// derive a signing key for Signature Version 4"). #[test] fn derives_the_signing_key_like_aws() { let key = signing_key("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", "iam"); assert_eq!(hex(&key), "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9"); } #[test] fn builds_the_authorization_header() { let config = StorageConfig { endpoint: "http://rustfs:9000".into(), bucket: "photos".into(), region: "us-east-1".into(), access_key: "AKIAIOSFODNN7EXAMPLE".into(), secret_key: "secret".into(), }; let now = OffsetDateTime::from_unix_timestamp(1_369_353_600).unwrap(); assert_eq!(amz_date(now), "20130524T000000Z"); let empty = hex(&Sha256::digest(b"")); assert_eq!(empty, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); let auth = authorization(&config, "GET", "/photos/a.jpg", "rustfs:9000", &empty, now); assert!(auth.starts_with( "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, \ SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=" )); let storage = Storage::new(config); assert_eq!(storage.url("celebrities/a b/ő.jpg"), "http://rustfs:9000/photos/celebrities/a%20b/%C5%91.jpg"); assert_eq!(storage.url(""), "http://rustfs:9000/photos"); } /// Against a real S3 service (e.g. the RustFS container of development): /// `S3_TEST_ENDPOINT=http://127.0.0.1:9000 S3_TEST_KEY=… S3_TEST_SECRET=… /// cargo test -p backend live_round_trip -- --ignored` #[tokio::test] #[ignore] async fn live_round_trip() { let env = |name: &str| std::env::var(name).unwrap_or_else(|_| panic!("{name} is not set")); let storage = Storage::new(StorageConfig { endpoint: env("S3_TEST_ENDPOINT"), bucket: "storage-test".into(), region: "us-east-1".into(), access_key: env("S3_TEST_KEY"), secret_key: env("S3_TEST_SECRET"), }); storage.ensure_bucket().await.unwrap(); storage.ensure_bucket().await.unwrap(); let key = "dir/próba kép.jpg"; storage.put(key, b"hello".to_vec(), "image/jpeg").await.unwrap(); let object = storage.get(key).await.unwrap().unwrap(); assert_eq!((&object.bytes[..], object.content_type.as_deref()), (&b"hello"[..], Some("image/jpeg"))); storage.delete(key).await.unwrap(); assert!(storage.get(key).await.unwrap().is_none()); storage.delete(key).await.unwrap(); } }