parallel downloads

This commit is contained in:
2026-01-08 18:01:20 +01:00
parent 135df74b1b
commit 6103d93f1e
7 changed files with 114 additions and 11 deletions
Generated
+51
View File
@@ -554,6 +554,7 @@ version = "0.1.0"
dependencies = [
"aws-config",
"aws-sdk-s3",
"futures",
"serde",
"serde_json",
"tokio",
@@ -818,6 +819,21 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "futures"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-channel"
version = "0.3.31"
@@ -825,6 +841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
dependencies = [
"futures-core",
"futures-sink",
]
[[package]]
@@ -833,6 +850,34 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
[[package]]
name = "futures-executor"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
dependencies = [
"futures-core",
"futures-task",
"futures-util",
]
[[package]]
name = "futures-io"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
[[package]]
name = "futures-macro"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "futures-sink"
version = "0.3.31"
@@ -851,10 +896,16 @@ version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
+1
View File
@@ -9,3 +9,4 @@ serde_json = "1.0"
aws-config = "1.1"
aws-sdk-s3 = "1.13"
tokio = { version = "1", features = ["full"] }
futures = "0.3"
+9
View File
@@ -1,4 +1,13 @@
[
{
"name": "wanderwell-prod",
"region": "us-east-1",
"bucket": "wanderwell-prod",
"accesskey": "DO801C9G7EBVLEQTMC8J",
"secretekey": "iq4OcddNiQGe72or/6xiESx7NcwNSWxb5CsCyWIO8kE",
"endpoint_url": "https://fra1.digitaloceanspaces.com",
"folder": ""
},
{
"name": "rubiconintezet",
"region": "us-east-1",
+1 -1
View File
@@ -1,6 +1,6 @@
use serde::Deserialize;
#[derive(Debug, Deserialize)]
#[derive(Debug, Deserialize, Clone)]
pub struct CloudS3 {
pub name: String,
pub region: String,
+1 -1
View File
@@ -1,4 +1,4 @@
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct S3Item {
pub key: String,
pub is_folder: bool,
+19
View File
@@ -8,6 +8,24 @@ pub fn create_directory_if_not_exists(folder_name: &str) -> std::io::Result<()>
Ok(())
}
pub fn should_skip_download(root_folder: &str, bucket_item: &S3Item) -> bool {
let path_str = format!("{}/{}", root_folder, bucket_item.key);
let path = std::path::Path::new(&path_str);
if !path.exists() {
return false; // Nem létezik, le kell tölteni
}
// Ellenőrizzük a méretet is
if let Some(s3_size) = bucket_item.size {
if let Ok(metadata) = std::fs::metadata(path) {
return metadata.len() == s3_size as u64; // Csak ha egyezik a méret
}
}
false // Bizonytalan esetben letöltjük
}
pub fn create_file_if_not_exist_or_differ(
root_folder: &str,
bucket_item: &S3Item,
@@ -26,6 +44,7 @@ pub fn create_file_if_not_exist_or_differ(
.size
.is_some_and(|s| std::fs::metadata(path).is_ok_and(|m| m.len() == s as u64))
{
print!("\r{}, mar letezik", path_str);
return Ok(());
}
+31 -8
View File
@@ -1,9 +1,11 @@
use super::file_utils::create_file_if_not_exist_or_differ;
use super::file_utils::should_skip_download;
use crate::models::CloudS3;
use crate::models::S3Item;
use aws_config::BehaviorVersion;
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::{Client, Error};
use futures::stream::{self, StreamExt};
pub fn create_config(s3: &CloudS3) -> aws_sdk_s3::Config {
let credentials = Credentials::new(&s3.accesskey, &s3.secretekey, None, None, "cloudbackup");
@@ -37,9 +39,6 @@ pub async fn list_bucket(client: &Client, s3: &CloudS3) -> Result<Vec<S3Item>, E
request = request.prefix(p);
}
println!("=== Request küldése ===");
// dbg!(&request);
if let Some(token) = &continuation_token {
request = request.continuation_token(token);
}
@@ -131,15 +130,39 @@ pub async fn handle_bucket_contents(
let root_folder = format!("./result/{}", s3.name);
for item in &items {
print!("\r{}", item.key);
// Szűrjük ki a már meglévő fájlokat
let items_to_download: Vec<_> = items
.iter()
.filter(|item| !should_skip_download(&root_folder, item))
.collect();
let content = download_file(client, &s3, &item).await?;
let total = items.len();
let to_download = items_to_download.len();
println!("Összes fájl: {}, Letöltendő: {}", total, to_download);
create_file_if_not_exist_or_differ(&root_folder, &item, &content);
// Párhuzamos letöltés (max 10 egyidejű)
stream::iter(items_to_download)
.map(|item| {
let client = client.clone();
let s3 = s3.clone();
let root = root_folder.clone();
let key = item.key.clone();
let item = item.clone();
async move {
match download_file(&client, &s3, &item).await {
Ok(content) => {
let _ = create_file_if_not_exist_or_differ(&root, &item, &content);
println!("✓ {}", key);
}
Err(e) => println!("✗ {} - {}", key, e),
}
}
})
.buffer_unordered(10)
.collect::<Vec<_>>()
.await;
println!("Contents count: {}", items.len());
println!("Kész!");
Ok(())
}