local clean

This commit is contained in:
sagiferi
2026-03-26 13:47:01 +01:00
parent a31dc704fb
commit 24b652fd3d
15 changed files with 608 additions and 4 deletions
+3 -3
View File
@@ -1,3 +1,3 @@
# [target.x86_64-unknown-linux-gnu]
# linker = "x86_64-unknown-linux-gnu-gcc"
# ar = "x86_64-unknown-linux-gnu-ar"
[target.x86_64-unknown-linux-gnu]
linker = "x86_64-unknown-linux-gnu-gcc"
ar = "x86_64-unknown-linux-gnu-ar"
+4
View File
@@ -1,5 +1,9 @@
{
"permissions": {
"allow": [
"Bash(cargo clippy:*)",
"Bash(rustup target:*)"
],
"defaultMode": "plan"
}
}
+33
View File
@@ -0,0 +1,33 @@
package main
import (
"context"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func createConfig(s3c CloudS3) (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(s3c.AccessKey, s3c.SecretKey, "")
endpointResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID && s3c.EndpointURL != "" {
return aws.Endpoint{URL: s3c.EndpointURL, SigningRegion: s3c.Region}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(
context.Background(),
config.WithRegion(s3c.Region),
config.WithCredentialsProvider(creds),
config.WithEndpointResolverWithOptions(endpointResolver),
)
if err != nil {
return aws.Config{}, err
}
return cfg, nil
}
+45
View File
@@ -0,0 +1,45 @@
package main
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func deleteOldFiles(ctx context.Context, client *s3.Client, s3c CloudS3, days int) (int, error) {
items, err := listBucket(ctx, client, s3c)
if err != nil {
return 0, err
}
cutoff := time.Now().UTC().Add(-time.Duration(days) * 24 * time.Hour)
deleted := 0
for _, item := range items {
if item.IsFolder || item.LastModified == nil {
continue
}
lm, err := time.Parse(time.RFC3339, *item.LastModified)
if err != nil {
continue
}
if lm.Before(cutoff) {
_, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s3c.Bucket),
Key: aws.String(item.Key),
})
if err != nil {
fmt.Printf("Hiba a törléskor: %s - %v\n", item.Key, err)
continue
}
fmt.Printf("🗑 Törölve: %s (%s)\n", item.Key, *item.LastModified)
deleted++
}
}
fmt.Printf("Törlés kész: %d fájl törölve (%d napnál régebbiek)\n", deleted, days)
return deleted, nil
}
+78
View File
@@ -0,0 +1,78 @@
package main
import (
"context"
"fmt"
"io"
"path/filepath"
"strings"
"sync"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func handleBucketContents(ctx context.Context, client *s3.Client, s3c CloudS3, backupDir string) error {
items, err := listBucket(ctx, client, s3c)
if err != nil {
return err
}
fmt.Printf("Bucket tartalom: %s\n", s3c.Bucket)
if s3c.Folder != nil {
fmt.Printf("Prefix: %s\n", *s3c.Folder)
}
fmt.Printf("%s\n", strings.Repeat("-", 60))
rootFolder := filepath.Join(".", backupDir, s3c.Name)
var itemsToDownload []S3Item
for _, item := range items {
if !shouldSkipDownload(rootFolder, item) {
itemsToDownload = append(itemsToDownload, item)
}
}
fmt.Printf("Összes fájl: %d, Letöltendő: %d\n", len(items), len(itemsToDownload))
sem := make(chan struct{}, 10)
var wg sync.WaitGroup
for _, item := range itemsToDownload {
item := item
wg.Add(1)
sem <- struct{}{}
go func() {
defer wg.Done()
defer func() { <-sem }()
content, err := downloadFile(ctx, client, s3c, item)
if err != nil {
fmt.Printf("✗ %s - %v\n", item.Key, err)
return
}
if err := createFileIfNotExistOrDiffer(rootFolder, item, content); err != nil {
fmt.Printf("✗ %s - %v\n", item.Key, err)
return
}
fmt.Printf("✓ %s\n", item.Key)
}()
}
wg.Wait()
fmt.Println("Kész!")
return nil
}
func downloadFile(ctx context.Context, client *s3.Client, s3c CloudS3, item S3Item) ([]byte, error) {
out, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s3c.Bucket),
Key: aws.String(item.Key),
})
if err != nil {
return nil, err
}
defer out.Body.Close()
return io.ReadAll(out.Body)
}
+36
View File
@@ -0,0 +1,36 @@
package main
import (
"fmt"
"os"
"path/filepath"
)
func shouldSkipDownload(rootFolder string, item S3Item) bool {
path := filepath.Join(rootFolder, item.Key)
info, err := os.Stat(path)
if err != nil {
return false
}
if item.Size != nil {
return info.Size() == *item.Size
}
return false
}
func createFileIfNotExistOrDiffer(rootFolder string, item S3Item, content []byte) error {
path := filepath.Join(rootFolder, item.Key)
if info, err := os.Stat(path); err == nil {
if item.Size != nil && info.Size() == *item.Size {
fmt.Printf("\r%s, mar letezik", path)
return nil
}
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, content, 0o644)
}
+25
View File
@@ -0,0 +1,25 @@
module cloudbackup-go
go 1.22
require (
github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
)
+38
View File
@@ -0,0 +1,38 @@
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+13
View File
@@ -0,0 +1,13 @@
package main
import "fmt"
func main() {
if err := runCleanAndBackup(); err != nil {
fmt.Printf("Hiba a clean_and_backup futtatásakor: %v\n", err)
}
if err := runBackupConfig(); err != nil {
fmt.Printf("Hiba a backup_config futtatásakor: %v\n", err)
}
}
+19
View File
@@ -0,0 +1,19 @@
package main
type CloudS3 struct {
Name string `json:"name"`
Region string `json:"region"`
Bucket string `json:"bucket"`
AccessKey string `json:"accesskey"`
SecretKey string `json:"secretekey"`
EndpointURL string `json:"endpoint_url"`
Folder *string `json:"folder"`
}
type S3Item struct {
Key string
IsFolder bool
FolderName *string
Size *int64
LastModified *string
}
+101
View File
@@ -0,0 +1,101 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func runBackupConfig() error {
data, err := os.ReadFile("backup_config.json")
if err != nil {
fmt.Println("backup_config.json nem található, kihagyva.")
return nil
}
var clouds []CloudS3
if err := json.Unmarshal(data, &clouds); err != nil {
return err
}
for _, cloud := range clouds {
client, err := createS3Client(cloud)
if err != nil {
fmt.Printf("Hiba a konfiguráció létrehozásakor: %v\n", err)
continue
}
fmt.Printf("Bucket tartalom listázása: %s\n", cloud.Name)
if err := handleBucketContents(context.Background(), client, cloud, "cloud_backup"); err != nil {
fmt.Printf("Hiba a bucket listázásakor: %v\n", err)
}
}
return nil
}
func runCleanAndBackup() error {
data, err := os.ReadFile("clean_and_backup.json")
if err != nil {
fmt.Println("clean_and_backup.json nem található, kihagyva.")
return nil
}
var clouds []CloudS3
if err := json.Unmarshal(data, &clouds); err != nil {
return err
}
for _, cloud := range clouds {
client, err := createS3Client(cloud)
if err != nil {
fmt.Printf("Hiba a konfiguráció létrehozásakor: %v\n", err)
continue
}
fmt.Printf("Összes bucket listázása: %s\n", cloud.Name)
buckets, err := listAllBuckets(context.Background(), client)
if err != nil {
return err
}
fmt.Printf("Talált bucket-ek: %v\n", buckets)
for _, bucketName := range buckets {
fmt.Printf("\n--- Bucket: %s ---\n", bucketName)
bucketCloud := CloudS3{
Name: fmt.Sprintf("%s/%s", cloud.Name, bucketName),
Region: cloud.Region,
Bucket: bucketName,
AccessKey: cloud.AccessKey,
SecretKey: cloud.SecretKey,
EndpointURL: cloud.EndpointURL,
Folder: nil,
}
if err := handleBucketContents(context.Background(), client, bucketCloud, "db_backup"); err != nil {
fmt.Printf("Hiba a bucket listázásakor: %v\n", err)
}
count, err := deleteOldFiles(context.Background(), client, bucketCloud, 14)
if err != nil {
fmt.Printf("Hiba a régi fájlok törlésekor: %v\n", err)
} else if count > 0 {
fmt.Printf("%d régi fájl törölve a %s bucket-ből\n", count, bucketName)
}
}
}
return nil
}
func createS3Client(cloud CloudS3) (*s3.Client, error) {
cfg, err := createConfig(cloud)
if err != nil {
return nil, err
}
return s3.NewFromConfig(cfg), nil
}
+104
View File
@@ -0,0 +1,104 @@
package main
import (
"context"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func listAllBuckets(ctx context.Context, client *s3.Client) ([]string, error) {
out, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
return nil, err
}
var buckets []string
for _, b := range out.Buckets {
if b.Name != nil {
buckets = append(buckets, *b.Name)
}
}
return buckets, nil
}
func listBucket(ctx context.Context, client *s3.Client, s3c CloudS3) ([]S3Item, error) {
var items []S3Item
var continuation *string
for {
input := &s3.ListObjectsV2Input{
Bucket: aws.String(s3c.Bucket),
}
if s3c.Folder != nil {
input.Prefix = s3c.Folder
}
if continuation != nil {
input.ContinuationToken = continuation
}
out, err := client.ListObjectsV2(ctx, input)
if err != nil {
return nil, err
}
for _, prefix := range out.CommonPrefixes {
if prefix.Prefix != nil {
p := *prefix.Prefix
items = append(items, S3Item{
Key: p,
IsFolder: true,
Size: nil,
FolderName: &p,
})
}
}
for _, obj := range out.Contents {
if obj.Key == nil {
continue
}
key := *obj.Key
if strings.HasSuffix(key, "/") {
continue
}
var folderName *string
if idx := strings.LastIndex(key, "/"); idx >= 0 {
f := key[:idx]
folderName = &f
}
var sizePtr *int64
if obj.Size != nil {
s := *obj.Size
sizePtr = &s
}
var lastModifiedPtr *string
if obj.LastModified != nil {
lm := obj.LastModified.UTC().Format(time.RFC3339)
lastModifiedPtr = &lm
}
items = append(items, S3Item{
Key: key,
IsFolder: false,
FolderName: folderName,
Size: sizePtr,
LastModified: lastModifiedPtr,
})
}
if out.NextContinuationToken != nil {
continuation = out.NextContinuationToken
} else {
break
}
}
fmt.Printf("Contents count: %d\n", len(items))
return items, nil
}
+11 -1
View File
@@ -4,7 +4,7 @@ mod utils;
use models::CloudS3;
use std::fs;
use utils::{delete_old_files, handle_bucket_contents, list_all_buckets};
use utils::{delete_old_files, delete_old_local_files, handle_bucket_contents, list_all_buckets};
use crate::utils::s3_utils::{create_client, create_config};
@@ -103,5 +103,15 @@ async fn run_clean_and_backup() -> Result<(), Box<dyn std::error::Error>> {
}
}
// Helyi régi fájlok törlése a db_backup mappából
match delete_old_local_files("./db_backup", 14) {
Ok(count) => {
if count > 0 {
println!("{} régi helyi fájl törölve a db_backup mappából", count);
}
}
Err(e) => eprintln!("Hiba a helyi fájlok törlésekor: {}", e),
}
Ok(())
}
+97
View File
@@ -1,5 +1,102 @@
use std::fs;
use std::path::Path;
use std::time::{Duration, SystemTime};
use crate::models::S3Item;
pub fn delete_old_local_files(dir: &str, days: u64) -> std::io::Result<usize> {
let cutoff = SystemTime::now() - Duration::from_secs(days * 24 * 60 * 60);
let root = Path::new(dir);
if !root.exists() {
println!("A mappa nem létezik: {}", dir);
return Ok(0);
}
let mut deleted_count = 0;
// Első menet: régi fájlok törlése
delete_old_files_recursive(root, cutoff, &mut deleted_count)?;
// Második menet: üres mappák törlése (bottom-up), de a gyökeret meghagyjuk
for entry in fs::read_dir(root)? {
let entry = entry?;
if entry.path().is_dir() {
cleanup_empty_dirs(&entry.path())?;
}
}
if deleted_count > 0 {
println!(
"Helyi törlés kész: {} fájl törölve ({} napnál régebbiek) a {} mappából",
deleted_count, days, dir
);
}
Ok(deleted_count)
}
fn delete_old_files_recursive(
dir: &Path,
cutoff: SystemTime,
deleted_count: &mut usize,
) -> std::io::Result<()> {
if !dir.is_dir() {
return Ok(());
}
for entry in fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
delete_old_files_recursive(&path, cutoff, deleted_count)?;
} else if path.is_file()
&& let Ok(metadata) = fs::metadata(&path)
&& let Ok(modified) = metadata.modified()
&& modified < cutoff
{
match fs::remove_file(&path) {
Ok(_) => {
println!("🗑 Helyi fájl törölve: {}", path.display());
*deleted_count += 1;
}
Err(e) => {
eprintln!(
"Hiba a helyi fájl törlésekor: {} - {}",
path.display(),
e
);
}
}
}
}
Ok(())
}
fn cleanup_empty_dirs(dir: &Path) -> std::io::Result<()> {
if !dir.is_dir() {
return Ok(());
}
for entry in fs::read_dir(dir)? {
let entry = entry?;
if entry.path().is_dir() {
cleanup_empty_dirs(&entry.path())?;
}
}
if fs::read_dir(dir)?.next().is_none() {
match fs::remove_dir(dir) {
Ok(_) => println!("🗑 Üres mappa törölve: {}", dir.display()),
Err(e) => eprintln!("Hiba az üres mappa törlésekor: {} - {}", dir.display(), e),
}
}
Ok(())
}
pub fn should_skip_download(root_folder: &str, bucket_item: &S3Item) -> bool {
let path_str = format!("{}/{}", root_folder, bucket_item.key);
let path = std::path::Path::new(&path_str);
+1
View File
@@ -1,5 +1,6 @@
pub mod file_utils;
pub mod s3_utils;
pub use file_utils::delete_old_local_files;
pub use s3_utils::delete_old_files;
pub use s3_utils::handle_bucket_contents;
pub use s3_utils::list_all_buckets;