277 lines
10 KiB
Rust
277 lines
10 KiB
Rust
//! Object storage for uploaded photos: any S3-compatible service, addressed
|
|||
|
|
//! path-style (`{endpoint}/{bucket}/{key}`). In production DigitalOcean
|
||
|
|
//! Spaces, in development a RustFS container.
|
||
|
|
//!
|
||
|
|
//! The bucket stays private: the backend reads the objects itself and serves
|
||
|
|
//! them under `/v1/public/photos/...`, from where the frontend's image
|
||
|
|
//! optimizer makes the sized versions (and Cloudflare caches those).
|
||
|
|
//!
|
||
|
|
//! Requests are signed with AWS Signature Version 4.
|
||
|
|
|
||
|
|
use std::time::Duration;
|
||
|
|
|
||
|
|
use hmac::{Hmac, Mac};
|
||
|
|
use sha2::{Digest, Sha256};
|
||
|
|
use time::OffsetDateTime;
|
||
|
|
|
||
|
|
/// `S3_ENDPOINT` (e.g. `https://fra1.digitaloceanspaces.com`), `S3_BUCKET`,
|
||
|
|
/// `S3_REGION` (default `us-east-1`; Spaces accepts it too),
|
||
|
|
/// `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` (in `.env.secrets`).
|
||
|
|
#[derive(Clone)]
|
||
|
|
pub struct StorageConfig {
|
||
|
|
pub endpoint: String,
|
||
|
|
pub bucket: String,
|
||
|
|
pub region: String,
|
||
|
|
pub access_key: String,
|
||
|
|
pub secret_key: String,
|
||
|
|
}
|
||
|
|
|
||
|
|
// The secret stays out of logs.
|
||
|
|
impl std::fmt::Debug for StorageConfig {
|
||
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||
|
|
f.debug_struct("StorageConfig")
|
||
|
|
.field("endpoint", &self.endpoint)
|
||
|
|
.field("bucket", &self.bucket)
|
||
|
|
.field("region", &self.region)
|
||
|
|
.finish()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Debug, thiserror::Error)]
|
||
|
|
pub enum StorageError {
|
||
|
|
#[error("object storage request failed: {0}")]
|
||
|
|
Http(#[from] reqwest::Error),
|
||
|
|
#[error("object storage answered {status}: {body}")]
|
||
|
|
Status { status: u16, body: String },
|
||
|
|
}
|
||
|
|
|
||
|
|
pub struct Storage {
|
||
|
|
config: StorageConfig,
|
||
|
|
http: reqwest::Client,
|
||
|
|
}
|
||
|
|
|
||
|
|
/// An object read back: its bytes and content type.
|
||
|
|
pub struct Object {
|
||
|
|
pub bytes: axum::body::Bytes,
|
||
|
|
pub content_type: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl Storage {
|
||
|
|
pub fn new(config: StorageConfig) -> Self {
|
||
|
|
let http = reqwest::Client::builder().timeout(Duration::from_secs(30)).build().expect("static configuration");
|
||
|
|
Self { config, http }
|
||
|
|
}
|
||
|
|
|
||
|
|
fn url(&self, key: &str) -> String {
|
||
|
|
let path = if key.is_empty() { String::new() } else { format!("/{}", encode_path(key)) };
|
||
|
|
format!("{}/{}{path}", self.config.endpoint, encode_segment(&self.config.bucket))
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn send(
|
||
|
|
&self,
|
||
|
|
method: reqwest::Method,
|
||
|
|
key: &str,
|
||
|
|
body: Vec<u8>,
|
||
|
|
content_type: Option<&str>,
|
||
|
|
) -> Result<reqwest::Response, StorageError> {
|
||
|
|
let url = self.url(key);
|
||
|
|
let parsed = reqwest::Url::parse(&url).map_err(|e| StorageError::Status { status: 0, body: e.to_string() })?;
|
||
|
|
let host = match parsed.port() {
|
||
|
|
Some(port) => format!("{}:{port}", parsed.host_str().unwrap_or_default()),
|
||
|
|
None => parsed.host_str().unwrap_or_default().to_string(),
|
||
|
|
};
|
||
|
|
let payload_hash = hex(&Sha256::digest(&body));
|
||
|
|
let now = OffsetDateTime::now_utc();
|
||
|
|
let auth = authorization(&self.config, method.as_str(), parsed.path(), &host, &payload_hash, now);
|
||
|
|
let mut request = self
|
||
|
|
.http
|
||
|
|
.request(method, url)
|
||
|
|
.header("x-amz-date", amz_date(now))
|
||
|
|
.header("x-amz-content-sha256", &payload_hash)
|
||
|
|
.header("authorization", auth);
|
||
|
|
if let Some(ct) = content_type {
|
||
|
|
request = request.header("content-type", ct);
|
||
|
|
}
|
||
|
|
let response = request.body(body).send().await?;
|
||
|
|
Ok(response)
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn checked(response: reqwest::Response) -> Result<reqwest::Response, StorageError> {
|
||
|
|
if response.status().is_success() {
|
||
|
|
return Ok(response);
|
||
|
|
}
|
||
|
|
let status = response.status().as_u16();
|
||
|
|
let body = response.text().await.unwrap_or_default().chars().take(500).collect();
|
||
|
|
Err(StorageError::Status { status, body })
|
||
|
|
}
|
||
|
|
|
||
|
|
pub async fn put(&self, key: &str, body: Vec<u8>, content_type: &str) -> Result<(), StorageError> {
|
||
|
|
Self::checked(self.send(reqwest::Method::PUT, key, body, Some(content_type)).await?).await?;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The object, or `None` if there is no such key.
|
||
|
|
pub async fn get(&self, key: &str) -> Result<Option<Object>, StorageError> {
|
||
|
|
let response = self.send(reqwest::Method::GET, key, Vec::new(), None).await?;
|
||
|
|
if response.status() == reqwest::StatusCode::NOT_FOUND {
|
||
|
|
return Ok(None);
|
||
|
|
}
|
||
|
|
let response = Self::checked(response).await?;
|
||
|
|
let content_type = response.headers().get("content-type").and_then(|v| v.to_str().ok()).map(str::to_owned);
|
||
|
|
Ok(Some(Object { bytes: response.bytes().await?, content_type }))
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Deletes the object; a missing one is not an error.
|
||
|
|
pub async fn delete(&self, key: &str) -> Result<(), StorageError> {
|
||
|
|
let response = self.send(reqwest::Method::DELETE, key, Vec::new(), None).await?;
|
||
|
|
if response.status() == reqwest::StatusCode::NOT_FOUND {
|
||
|
|
return Ok(());
|
||
|
|
}
|
||
|
|
Self::checked(response).await?;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Creates the bucket if it does not exist yet (development: a fresh
|
||
|
|
/// RustFS container has none; in production it is made in the console).
|
||
|
|
pub async fn ensure_bucket(&self) -> Result<(), StorageError> {
|
||
|
|
let head = self.send(reqwest::Method::HEAD, "", Vec::new(), None).await?;
|
||
|
|
if head.status().is_success() {
|
||
|
|
return Ok(());
|
||
|
|
}
|
||
|
|
Self::checked(self.send(reqwest::Method::PUT, "", Vec::new(), None).await?).await?;
|
||
|
|
tracing::info!(bucket = %self.config.bucket, "object storage bucket created");
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
type HmacSha256 = Hmac<Sha256>;
|
||
|
|
|
||
|
|
fn hmac(key: &[u8], data: &str) -> Vec<u8> {
|
||
|
|
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC takes any key length");
|
||
|
|
mac.update(data.as_bytes());
|
||
|
|
mac.finalize().into_bytes().to_vec()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn hex(bytes: &[u8]) -> String {
|
||
|
|
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn amz_date(t: OffsetDateTime) -> String {
|
||
|
|
format!(
|
||
|
|
"{:04}{:02}{:02}T{:02}{:02}{:02}Z",
|
||
|
|
t.year(),
|
||
|
|
u8::from(t.month()),
|
||
|
|
t.day(),
|
||
|
|
t.hour(),
|
||
|
|
t.minute(),
|
||
|
|
t.second()
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// URI-encodes one path segment as SigV4 wants it (unreserved characters stay).
|
||
|
|
fn encode_segment(s: &str) -> String {
|
||
|
|
s.bytes()
|
||
|
|
.map(|b| match b {
|
||
|
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(),
|
||
|
|
_ => format!("%{b:02X}"),
|
||
|
|
})
|
||
|
|
.collect()
|
||
|
|
}
|
||
|
|
|
||
|
|
fn encode_path(key: &str) -> String {
|
||
|
|
key.split('/').map(encode_segment).collect::<Vec<_>>().join("/")
|
||
|
|
}
|
||
|
|
|
||
|
|
fn signing_key(secret: &str, date: &str, region: &str, service: &str) -> Vec<u8> {
|
||
|
|
[date, region, service, "aws4_request"]
|
||
|
|
.iter()
|
||
|
|
.fold(format!("AWS4{secret}").into_bytes(), |key, part| hmac(&key, part))
|
||
|
|
}
|
||
|
|
|
||
|
|
/// The `Authorization` header of a request (no query string; the path is
|
||
|
|
/// already encoded, as in the URL).
|
||
|
|
fn authorization(
|
||
|
|
config: &StorageConfig,
|
||
|
|
method: &str,
|
||
|
|
path: &str,
|
||
|
|
host: &str,
|
||
|
|
payload_hash: &str,
|
||
|
|
now: OffsetDateTime,
|
||
|
|
) -> String {
|
||
|
|
let date_time = amz_date(now);
|
||
|
|
let date = &date_time[..8];
|
||
|
|
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
|
||
|
|
let canonical = format!(
|
||
|
|
"{method}\n{path}\n\nhost:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{date_time}\n\n\
|
||
|
|
{signed_headers}\n{payload_hash}"
|
||
|
|
);
|
||
|
|
let scope = format!("{date}/{}/s3/aws4_request", config.region);
|
||
|
|
let to_sign = format!("AWS4-HMAC-SHA256\n{date_time}\n{scope}\n{}", hex(&Sha256::digest(canonical.as_bytes())));
|
||
|
|
let signature = hex(&hmac(&signing_key(&config.secret_key, date, &config.region, "s3"), &to_sign));
|
||
|
|
format!(
|
||
|
|
"AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={signed_headers}, Signature={signature}",
|
||
|
|
config.access_key
|
||
|
|
)
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use super::*;
|
||
|
|
|
||
|
|
/// The signing key example of the AWS documentation ("Examples of how to
|
||
|
|
/// derive a signing key for Signature Version 4").
|
||
|
|
#[test]
|
||
|
|
fn derives_the_signing_key_like_aws() {
|
||
|
|
let key = signing_key("wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", "20150830", "us-east-1", "iam");
|
||
|
|
assert_eq!(hex(&key), "c4afb1cc5771d871763a393e44b703571b55cc28424d1a5e86da6ed3c154a4b9");
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn builds_the_authorization_header() {
|
||
|
|
let config = StorageConfig {
|
||
|
|
endpoint: "http://rustfs:9000".into(),
|
||
|
|
bucket: "photos".into(),
|
||
|
|
region: "us-east-1".into(),
|
||
|
|
access_key: "AKIAIOSFODNN7EXAMPLE".into(),
|
||
|
|
secret_key: "secret".into(),
|
||
|
|
};
|
||
|
|
let now = OffsetDateTime::from_unix_timestamp(1_369_353_600).unwrap();
|
||
|
|
assert_eq!(amz_date(now), "20130524T000000Z");
|
||
|
|
let empty = hex(&Sha256::digest(b""));
|
||
|
|
assert_eq!(empty, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
|
||
|
|
let auth = authorization(&config, "GET", "/photos/a.jpg", "rustfs:9000", &empty, now);
|
||
|
|
assert!(auth.starts_with(
|
||
|
|
"AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, \
|
||
|
|
SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature="
|
||
|
|
));
|
||
|
|
let storage = Storage::new(config);
|
||
|
|
assert_eq!(storage.url("celebrities/a b/ő.jpg"), "http://rustfs:9000/photos/celebrities/a%20b/%C5%91.jpg");
|
||
|
|
assert_eq!(storage.url(""), "http://rustfs:9000/photos");
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Against a real S3 service (e.g. the RustFS container of development):
|
||
|
|
/// `S3_TEST_ENDPOINT=http://127.0.0.1:9000 S3_TEST_KEY=… S3_TEST_SECRET=…
|
||
|
|
/// cargo test -p backend live_round_trip -- --ignored`
|
||
|
|
#[tokio::test]
|
||
|
|
#[ignore]
|
||
|
|
async fn live_round_trip() {
|
||
|
|
let env = |name: &str| std::env::var(name).unwrap_or_else(|_| panic!("{name} is not set"));
|
||
|
|
let storage = Storage::new(StorageConfig {
|
||
|
|
endpoint: env("S3_TEST_ENDPOINT"),
|
||
|
|
bucket: "storage-test".into(),
|
||
|
|
region: "us-east-1".into(),
|
||
|
|
access_key: env("S3_TEST_KEY"),
|
||
|
|
secret_key: env("S3_TEST_SECRET"),
|
||
|
|
});
|
||
|
|
storage.ensure_bucket().await.unwrap();
|
||
|
|
storage.ensure_bucket().await.unwrap();
|
||
|
|
let key = "dir/próba kép.jpg";
|
||
|
|
storage.put(key, b"hello".to_vec(), "image/jpeg").await.unwrap();
|
||
|
|
let object = storage.get(key).await.unwrap().unwrap();
|
||
|
|
assert_eq!((&object.bytes[..], object.content_type.as_deref()), (&b"hello"[..], Some("image/jpeg")));
|
||
|
|
storage.delete(key).await.unwrap();
|
||
|
|
assert!(storage.get(key).await.unwrap().is_none());
|
||
|
|
storage.delete(key).await.unwrap();
|
||
|
|
}
|
||
|
|
}
|