parallel downloads
This commit is contained in:
Generated
+51
@@ -554,6 +554,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"aws-config",
|
||||
"aws-sdk-s3",
|
||||
"futures",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
@@ -818,6 +819,21 @@ version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.31"
|
||||
@@ -825,6 +841,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -833,6 +850,34 @@ version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.31"
|
||||
@@ -851,10 +896,16 @@ version = "0.3.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -9,3 +9,4 @@ serde_json = "1.0"
|
||||
aws-config = "1.1"
|
||||
aws-sdk-s3 = "1.13"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
futures = "0.3"
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
[
|
||||
{
|
||||
"name": "wanderwell-prod",
|
||||
"region": "us-east-1",
|
||||
"bucket": "wanderwell-prod",
|
||||
"accesskey": "DO801C9G7EBVLEQTMC8J",
|
||||
"secretekey": "iq4OcddNiQGe72or/6xiESx7NcwNSWxb5CsCyWIO8kE",
|
||||
"endpoint_url": "https://fra1.digitaloceanspaces.com",
|
||||
"folder": ""
|
||||
},
|
||||
{
|
||||
"name": "rubiconintezet",
|
||||
"region": "us-east-1",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct CloudS3 {
|
||||
pub name: String,
|
||||
pub region: String,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct S3Item {
|
||||
pub key: String,
|
||||
pub is_folder: bool,
|
||||
|
||||
@@ -8,6 +8,24 @@ pub fn create_directory_if_not_exists(folder_name: &str) -> std::io::Result<()>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn should_skip_download(root_folder: &str, bucket_item: &S3Item) -> bool {
|
||||
let path_str = format!("{}/{}", root_folder, bucket_item.key);
|
||||
let path = std::path::Path::new(&path_str);
|
||||
|
||||
if !path.exists() {
|
||||
return false; // Nem létezik, le kell tölteni
|
||||
}
|
||||
|
||||
// Ellenőrizzük a méretet is
|
||||
if let Some(s3_size) = bucket_item.size {
|
||||
if let Ok(metadata) = std::fs::metadata(path) {
|
||||
return metadata.len() == s3_size as u64; // Csak ha egyezik a méret
|
||||
}
|
||||
}
|
||||
|
||||
false // Bizonytalan esetben letöltjük
|
||||
}
|
||||
|
||||
pub fn create_file_if_not_exist_or_differ(
|
||||
root_folder: &str,
|
||||
bucket_item: &S3Item,
|
||||
@@ -26,6 +44,7 @@ pub fn create_file_if_not_exist_or_differ(
|
||||
.size
|
||||
.is_some_and(|s| std::fs::metadata(path).is_ok_and(|m| m.len() == s as u64))
|
||||
{
|
||||
print!("\r{}, mar letezik", path_str);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
+32
-9
@@ -1,9 +1,11 @@
|
||||
use super::file_utils::create_file_if_not_exist_or_differ;
|
||||
use super::file_utils::should_skip_download;
|
||||
use crate::models::CloudS3;
|
||||
use crate::models::S3Item;
|
||||
use aws_config::BehaviorVersion;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Error};
|
||||
use futures::stream::{self, StreamExt};
|
||||
|
||||
pub fn create_config(s3: &CloudS3) -> aws_sdk_s3::Config {
|
||||
let credentials = Credentials::new(&s3.accesskey, &s3.secretekey, None, None, "cloudbackup");
|
||||
@@ -37,9 +39,6 @@ pub async fn list_bucket(client: &Client, s3: &CloudS3) -> Result<Vec<S3Item>, E
|
||||
request = request.prefix(p);
|
||||
}
|
||||
|
||||
println!("=== Request küldése ===");
|
||||
// dbg!(&request);
|
||||
|
||||
if let Some(token) = &continuation_token {
|
||||
request = request.continuation_token(token);
|
||||
}
|
||||
@@ -131,15 +130,39 @@ pub async fn handle_bucket_contents(
|
||||
|
||||
let root_folder = format!("./result/{}", s3.name);
|
||||
|
||||
for item in &items {
|
||||
print!("\r{}", item.key);
|
||||
// Szűrjük ki a már meglévő fájlokat
|
||||
let items_to_download: Vec<_> = items
|
||||
.iter()
|
||||
.filter(|item| !should_skip_download(&root_folder, item))
|
||||
.collect();
|
||||
|
||||
let content = download_file(client, &s3, &item).await?;
|
||||
let total = items.len();
|
||||
let to_download = items_to_download.len();
|
||||
println!("Összes fájl: {}, Letöltendő: {}", total, to_download);
|
||||
|
||||
create_file_if_not_exist_or_differ(&root_folder, &item, &content);
|
||||
}
|
||||
// Párhuzamos letöltés (max 10 egyidejű)
|
||||
stream::iter(items_to_download)
|
||||
.map(|item| {
|
||||
let client = client.clone();
|
||||
let s3 = s3.clone();
|
||||
let root = root_folder.clone();
|
||||
let key = item.key.clone();
|
||||
let item = item.clone();
|
||||
async move {
|
||||
match download_file(&client, &s3, &item).await {
|
||||
Ok(content) => {
|
||||
let _ = create_file_if_not_exist_or_differ(&root, &item, &content);
|
||||
println!("✓ {}", key);
|
||||
}
|
||||
Err(e) => println!("✗ {} - {}", key, e),
|
||||
}
|
||||
}
|
||||
})
|
||||
.buffer_unordered(10)
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
|
||||
println!("Contents count: {}", items.len());
|
||||
println!("Kész!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user