From 6103d93f1ebfe2a153b84f8b8f0001d1c6713690 Mon Sep 17 00:00:00 2001 From: feco Date: Thu, 8 Jan 2026 18:01:20 +0100 Subject: [PATCH] parallel downloads --- Cargo.lock | 51 +++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + config.json | 9 ++++++++ src/models/cloud.rs | 2 +- src/models/s3_items.rs | 2 +- src/utils/file_utils.rs | 19 +++++++++++++++ src/utils/s3_utils.rs | 41 +++++++++++++++++++++++++-------- 7 files changed, 114 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d1a7ed..b1829c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -554,6 +554,7 @@ version = "0.1.0" dependencies = [ "aws-config", "aws-sdk-s3", + "futures", "serde", "serde_json", "tokio", @@ -818,6 +819,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -825,6 +841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -833,6 +850,34 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -851,10 +896,16 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f00a5e7..acdc008 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,3 +9,4 @@ serde_json = "1.0" aws-config = "1.1" aws-sdk-s3 = "1.13" tokio = { version = "1", features = ["full"] } +futures = "0.3" diff --git a/config.json b/config.json index c1c7be9..a0e30ea 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,13 @@ [ + { + "name": "wanderwell-prod", + "region": "us-east-1", + "bucket": "wanderwell-prod", + "accesskey": "DO801C9G7EBVLEQTMC8J", + "secretekey": "iq4OcddNiQGe72or/6xiESx7NcwNSWxb5CsCyWIO8kE", + "endpoint_url": "https://fra1.digitaloceanspaces.com", + "folder": "" + }, { "name": "rubiconintezet", "region": "us-east-1", diff --git a/src/models/cloud.rs b/src/models/cloud.rs index 341f72d..fa0ac22 100644 --- a/src/models/cloud.rs +++ b/src/models/cloud.rs @@ -1,6 +1,6 @@ use serde::Deserialize; -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Clone)] pub struct CloudS3 { pub name: String, pub region: String, diff --git a/src/models/s3_items.rs b/src/models/s3_items.rs index 17bbb62..5d644fd 100644 --- a/src/models/s3_items.rs +++ b/src/models/s3_items.rs @@ -1,4 +1,4 @@ -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct S3Item { pub key: String, pub is_folder: bool, diff --git a/src/utils/file_utils.rs b/src/utils/file_utils.rs index 56553d0..e459e95 100644 --- a/src/utils/file_utils.rs +++ b/src/utils/file_utils.rs @@ -8,6 +8,24 @@ pub fn create_directory_if_not_exists(folder_name: &str) -> std::io::Result<()> Ok(()) } +pub fn should_skip_download(root_folder: &str, bucket_item: &S3Item) -> bool { + let path_str = format!("{}/{}", root_folder, bucket_item.key); + let path = std::path::Path::new(&path_str); + + if !path.exists() { + return false; // Nem létezik, le kell tölteni + } + + // Ellenőrizzük a méretet is + if let Some(s3_size) = bucket_item.size { + if let Ok(metadata) = std::fs::metadata(path) { + return metadata.len() == s3_size as u64; // Csak ha egyezik a méret + } + } + + false // Bizonytalan esetben letöltjük +} + pub fn create_file_if_not_exist_or_differ( root_folder: &str, bucket_item: &S3Item, @@ -26,6 +44,7 @@ pub fn create_file_if_not_exist_or_differ( .size .is_some_and(|s| std::fs::metadata(path).is_ok_and(|m| m.len() == s as u64)) { + print!("\r{}, mar letezik", path_str); return Ok(()); } diff --git a/src/utils/s3_utils.rs b/src/utils/s3_utils.rs index 5af8a16..c37714a 100644 --- a/src/utils/s3_utils.rs +++ b/src/utils/s3_utils.rs @@ -1,9 +1,11 @@ use super::file_utils::create_file_if_not_exist_or_differ; +use super::file_utils::should_skip_download; use crate::models::CloudS3; use crate::models::S3Item; use aws_config::BehaviorVersion; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::{Client, Error}; +use futures::stream::{self, StreamExt}; pub fn create_config(s3: &CloudS3) -> aws_sdk_s3::Config { let credentials = Credentials::new(&s3.accesskey, &s3.secretekey, None, None, "cloudbackup"); @@ -37,9 +39,6 @@ pub async fn list_bucket(client: &Client, s3: &CloudS3) -> Result, E request = request.prefix(p); } - println!("=== Request küldése ==="); - // dbg!(&request); - if let Some(token) = &continuation_token { request = request.continuation_token(token); } @@ -131,15 +130,39 @@ pub async fn handle_bucket_contents( let root_folder = format!("./result/{}", s3.name); - for item in &items { - print!("\r{}", item.key); + // Szűrjük ki a már meglévő fájlokat + let items_to_download: Vec<_> = items + .iter() + .filter(|item| !should_skip_download(&root_folder, item)) + .collect(); - let content = download_file(client, &s3, &item).await?; + let total = items.len(); + let to_download = items_to_download.len(); + println!("Összes fájl: {}, Letöltendő: {}", total, to_download); - create_file_if_not_exist_or_differ(&root_folder, &item, &content); - } + // Párhuzamos letöltés (max 10 egyidejű) + stream::iter(items_to_download) + .map(|item| { + let client = client.clone(); + let s3 = s3.clone(); + let root = root_folder.clone(); + let key = item.key.clone(); + let item = item.clone(); + async move { + match download_file(&client, &s3, &item).await { + Ok(content) => { + let _ = create_file_if_not_exist_or_differ(&root, &item, &content); + println!("✓ {}", key); + } + Err(e) => println!("✗ {} - {}", key, e), + } + } + }) + .buffer_unordered(10) + .collect::>() + .await; - println!("Contents count: {}", items.len()); + println!("Kész!"); Ok(()) }