Files
sorsfonalak/services/backend/src/storage.rs
T

277 lines
10 KiB
Rust
Raw Normal View History

//! Object storage for uploaded photos: any S3-compatible service, addressed
//! path-style (`{endpoint}/{bucket}/{key}`). In production DigitalOcean
//! Spaces, in development a RustFS container.
//!
//! The bucket stays private: the backend reads the objects itself and serves
//! them under `/v1/public/photos/...`, from where the frontend's image
//! optimizer makes the sized versions (and Cloudflare caches those).
//!
//! Requests are signed with AWS Signature Version 4.
use std::time::Duration;
use hmac::{Hmac, Mac};
use sha2::{Digest, Sha256};
use time::OffsetDateTime;
/// `S3_ENDPOINT` (e.g. `https://fra1.digitaloceanspaces.com`), `S3_BUCKET`,
/// `S3_REGION` (default `us-east-1`; Spaces accepts it too),
/// `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` (in `.env.secrets`).
#[derive(Clone)]
pub struct StorageConfig {
pub endpoint: String,
pub bucket: String,
pub region: String,
pub access_key: String,
pub secret_key: String,
}
// The secret stays out of logs.
impl std::fmt::Debug for StorageConfig {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("StorageConfig")
.field("endpoint", &self.endpoint)
.field("bucket", &self.bucket)
.field("region", &self.region)
.finish()
}
}
#[derive(Debug, thiserror::Error)]
pub enum StorageError {
#[error("object storage request failed: {0}")]
Http(#[from] reqwest::Error),
#[error("object storage answered {status}: {body}")]
Status { status: u16, body: String },
}
pub struct Storage {
config: StorageConfig,
http: reqwest::Client,
}
/// An object read back: its bytes and content type.
pub struct Object {
pub bytes: axum::body::Bytes,
pub content_type: Option<String>,
}
impl Storage {
pub fn new(config: StorageConfig) -> Self {
let http = reqwest::Client::builder().timeout(Duration::from_secs(30)).build().expect("static configuration");
Self { config, http }
}
fn url(&self, key: &str) -> String {
let path = if key.is_empty() { String::new() } else { format!("/{}", encode_path(key)) };
format!("{}/{}{path}", self.config.endpoint, encode_segment(&self.config.bucket))
}
async fn send(
&self,
method: reqwest::Method,
key: &str,
body: Vec<u8>,
content_type: Option<&str>,
) -> Result<reqwest::Response, StorageError> {
let url = self.url(key);
let parsed = reqwest::Url::parse(&url).map_err(|e| StorageError::Status { status: 0, body: e.to_string() })?;
let host = match parsed.port() {
Some(port) => format!("{}:{port}", parsed.host_str().unwrap_or_default()),
None => parsed.host_str().unwrap_or_default().to_string(),
};
let payload_hash = hex(&Sha256::digest(&body));
let now = OffsetDateTime::now_utc();
let auth = authorization(&self.config, method.as_str(), parsed.path(), &host, &payload_hash, now);
let mut request = self
.http
.request(method, url)
.header("x-amz-date", amz_date(now))
.header("x-amz-content-sha256", &payload_hash)
.header("authorization", auth);
if let Some(ct) = content_type {
request = request.header("content-type", ct);
}
let response = request.body(body).send().await?;
Ok(response)
}
async fn checked(response: reqwest::Response) -> Result<reqwest::Response, StorageError> {
if response.status().is_success() {
return Ok(response);
}
let status = response.status().as_u16();
let body = response.text().await.unwrap_or_default().chars().take(500).collect();
Err(StorageError::Status { status, body })
}
pub async fn put(&self, key: &str, body: Vec<u8>, content_type: &str) -> Result<(), StorageError> {
Self::checked(self.send(reqwest::Method::PUT, key, body, Some(content_type)).await?).await?;
Ok(())
}
/// The object, or `None` if there is no such key.
pub async fn get(&self, key: &str) -> Result<Option<Object>, StorageError> {
let response = self.send(reqwest::Method::GET, key, Vec::new(), None).await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
let response = Self::checked(response).await?;
let content_type = response.headers().get("content-type").and_then(|v| v.to_str().ok()).map(str::to_owned);
Ok(Some(Object { bytes: response.bytes().await?, content_type }))
}
/// Deletes the object; a missing one is not an error.
pub async fn delete(&self, key: &str) -> Result<(), StorageError> {
let response = self.send(reqwest::Method::DELETE, key, Vec::new(), None).await?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(());
}
Self::checked(response).await?;
Ok(())
}
/// Creates the bucket if it does not exist yet (development: a fresh
/// RustFS container has none; in production it is made in the console).
pub async fn ensure_bucket(&self) -> Result<(), StorageError> {
let head = self.send(reqwest::Method::HEAD, "", Vec::new(), None).await?;
if head.status().is_success() {
return Ok(());
}
Self::checked(self.send(reqwest::Method::PUT, "", Vec::new(), None).await?).await?;
tracing::info!(bucket = %self.config.bucket, "object storage bucket created");
Ok(())
}
}
type HmacSha256 = Hmac<Sha256>;
fn hmac(key: &[u8], data: &str) -> Vec<u8> {
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC takes any key length");
mac.update(data.as_bytes());
mac.finalize().into_bytes().to_vec()
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
fn amz_date(t: OffsetDateTime) -> String {
format!(
"{:04}{:02}{:02}T{:02}{:02}{:02}Z",
t.year(),
u8::from(t.month()),
t.day(),
t.hour(),
t.minute(),
t.second()
)
}
/// URI-encodes one path segment as SigV4 wants it (unreserved characters stay).
fn encode_segment(s: &str) -> String {
s.bytes()
.map(|b| match b {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
_ => format!("%{b:02X}"),
})
.collect()
}
fn encode_path(key: &str) -> String {
key.split('/').map(encode_segment).collect::<Vec<_>>().join("/")
}
fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec<u8> {
[date, region, service, "aws4_request"]
.iter()
.fold(format!("AWS4{secret}").into_bytes(), |key, part| hmac(&key, part))
}
/// The `Authorization` header of a request (no query string; the path is
/// already encoded, as in the URL).
fn authorization(
config: &StorageConfig,
method: &str,
path: &str,
host: &str,
payload_hash: &str,
now: OffsetDateTime,
) -> String {
let date_time = amz_date(now);
let date = &date_time[..8];
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
let canonical = format!(
"{method}\n{path}\n\nhost:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{date_time}\n\n\
{signed_headers}\n{payload_hash}"
);
let scope = format!("{date}/{}/s3/aws4_request", config.region);
let to_sign = format!("AWS4-HMAC-SHA256\n{date_time}\n{scope}\n{}", hex(&Sha256::digest(canonical.as_bytes())));
let signature = hex(&hmac(&signing_key(&config.secret_key, date, &config.region, "s3"), &to_sign));
format!(
"AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed_headers}, Signature={signature}",
config.access_key
)
}
#[cfg(test)]
mod tests {
use super::*;
/// The signing key example of the AWS documentation ("Examples of how to
/// derive a signing key for Signature Version 4").
#[test]
fn derives_the_signing_key_like_aws() {
let key = signing_key("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", "iam");
assert_eq!(hex(&key), "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9");
}
#[test]
fn builds_the_authorization_header() {
let config = StorageConfig {
endpoint: "http://rustfs:9000".into(),
bucket: "photos".into(),
region: "us-east-1".into(),
access_key: "AKIAIOSFODNN7EXAMPLE".into(),
secret_key: "secret".into(),
};
let now = OffsetDateTime::from_unix_timestamp(1_369_353_600).unwrap();
assert_eq!(amz_date(now), "20130524T000000Z");
let empty = hex(&Sha256::digest(b""));
assert_eq!(empty, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
let auth = authorization(&config, "GET", "/photos/a.jpg", "rustfs:9000", &empty, now);
assert!(auth.starts_with(
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, \
SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature="
));
let storage = Storage::new(config);
assert_eq!(storage.url("celebrities/a b/ő.jpg"), "http://rustfs:9000/photos/celebrities/a%20b/%C5%91.jpg");
assert_eq!(storage.url(""), "http://rustfs:9000/photos");
}
/// Against a real S3 service (e.g. the RustFS container of development):
/// `S3_TEST_ENDPOINT=http://127.0.0.1:9000 S3_TEST_KEY=… S3_TEST_SECRET=…
/// cargo test -p backend live_round_trip -- --ignored`
#[tokio::test]
#[ignore]
async fn live_round_trip() {
let env = |name: &str| std::env::var(name).unwrap_or_else(|_| panic!("{name} is not set"));
let storage = Storage::new(StorageConfig {
endpoint: env("S3_TEST_ENDPOINT"),
bucket: "storage-test".into(),
region: "us-east-1".into(),
access_key: env("S3_TEST_KEY"),
secret_key: env("S3_TEST_SECRET"),
});
storage.ensure_bucket().await.unwrap();
storage.ensure_bucket().await.unwrap();
let key = "dir/próba kép.jpg";
storage.put(key, b"hello".to_vec(), "image/jpeg").await.unwrap();
let object = storage.get(key).await.unwrap().unwrap();
assert_eq!((&object.bytes[..], object.content_type.as_deref()), (&b"hello"[..], Some("image/jpeg")));
storage.delete(key).await.unwrap();
assert!(storage.get(key).await.unwrap().is_none());
storage.delete(key).await.unwrap();
}
}