local clean

This commit is contained in:
sagiferi
2026-03-26 13:47:01 +01:00
parent a31dc704fb
commit 24b652fd3d
15 changed files with 608 additions and 4 deletions
+33
View File
@@ -0,0 +1,33 @@
package main
import (
"context"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func createConfig(s3c CloudS3) (aws.Config, error) {
creds := credentials.NewStaticCredentialsProvider(s3c.AccessKey, s3c.SecretKey, "")
endpointResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, _ ...interface{}) (aws.Endpoint, error) {
if service == s3.ServiceID && s3c.EndpointURL != "" {
return aws.Endpoint{URL: s3c.EndpointURL, SigningRegion: s3c.Region}, nil
}
return aws.Endpoint{}, &aws.EndpointNotFoundError{}
})
cfg, err := config.LoadDefaultConfig(
context.Background(),
config.WithRegion(s3c.Region),
config.WithCredentialsProvider(creds),
config.WithEndpointResolverWithOptions(endpointResolver),
)
if err != nil {
return aws.Config{}, err
}
return cfg, nil
}
+45
View File
@@ -0,0 +1,45 @@
package main
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func deleteOldFiles(ctx context.Context, client *s3.Client, s3c CloudS3, days int) (int, error) {
items, err := listBucket(ctx, client, s3c)
if err != nil {
return 0, err
}
cutoff := time.Now().UTC().Add(-time.Duration(days) * 24 * time.Hour)
deleted := 0
for _, item := range items {
if item.IsFolder || item.LastModified == nil {
continue
}
lm, err := time.Parse(time.RFC3339, *item.LastModified)
if err != nil {
continue
}
if lm.Before(cutoff) {
_, err := client.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(s3c.Bucket),
Key: aws.String(item.Key),
})
if err != nil {
fmt.Printf("Hiba a törléskor: %s - %v\n", item.Key, err)
continue
}
fmt.Printf("🗑 Törölve: %s (%s)\n", item.Key, *item.LastModified)
deleted++
}
}
fmt.Printf("Törlés kész: %d fájl törölve (%d napnál régebbiek)\n", deleted, days)
return deleted, nil
}
+78
View File
@@ -0,0 +1,78 @@
package main
import (
"context"
"fmt"
"io"
"path/filepath"
"strings"
"sync"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func handleBucketContents(ctx context.Context, client *s3.Client, s3c CloudS3, backupDir string) error {
items, err := listBucket(ctx, client, s3c)
if err != nil {
return err
}
fmt.Printf("Bucket tartalom: %s\n", s3c.Bucket)
if s3c.Folder != nil {
fmt.Printf("Prefix: %s\n", *s3c.Folder)
}
fmt.Printf("%s\n", strings.Repeat("-", 60))
rootFolder := filepath.Join(".", backupDir, s3c.Name)
var itemsToDownload []S3Item
for _, item := range items {
if !shouldSkipDownload(rootFolder, item) {
itemsToDownload = append(itemsToDownload, item)
}
}
fmt.Printf("Összes fájl: %d, Letöltendő: %d\n", len(items), len(itemsToDownload))
sem := make(chan struct{}, 10)
var wg sync.WaitGroup
for _, item := range itemsToDownload {
item := item
wg.Add(1)
sem <- struct{}{}
go func() {
defer wg.Done()
defer func() { <-sem }()
content, err := downloadFile(ctx, client, s3c, item)
if err != nil {
fmt.Printf("✗ %s - %v\n", item.Key, err)
return
}
if err := createFileIfNotExistOrDiffer(rootFolder, item, content); err != nil {
fmt.Printf("✗ %s - %v\n", item.Key, err)
return
}
fmt.Printf("✓ %s\n", item.Key)
}()
}
wg.Wait()
fmt.Println("Kész!")
return nil
}
func downloadFile(ctx context.Context, client *s3.Client, s3c CloudS3, item S3Item) ([]byte, error) {
out, err := client.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(s3c.Bucket),
Key: aws.String(item.Key),
})
if err != nil {
return nil, err
}
defer out.Body.Close()
return io.ReadAll(out.Body)
}
+36
View File
@@ -0,0 +1,36 @@
package main
import (
"fmt"
"os"
"path/filepath"
)
func shouldSkipDownload(rootFolder string, item S3Item) bool {
path := filepath.Join(rootFolder, item.Key)
info, err := os.Stat(path)
if err != nil {
return false
}
if item.Size != nil {
return info.Size() == *item.Size
}
return false
}
func createFileIfNotExistOrDiffer(rootFolder string, item S3Item, content []byte) error {
path := filepath.Join(rootFolder, item.Key)
if info, err := os.Stat(path); err == nil {
if item.Size != nil && info.Size() == *item.Size {
fmt.Printf("\r%s, mar letezik", path)
return nil
}
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, content, 0o644)
}
+25
View File
@@ -0,0 +1,25 @@
module cloudbackup-go
go 1.22
require (
github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.7 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.7 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
)
+38
View File
@@ -0,0 +1,38 @@
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA=
github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+13
View File
@@ -0,0 +1,13 @@
package main
import "fmt"
func main() {
if err := runCleanAndBackup(); err != nil {
fmt.Printf("Hiba a clean_and_backup futtatásakor: %v\n", err)
}
if err := runBackupConfig(); err != nil {
fmt.Printf("Hiba a backup_config futtatásakor: %v\n", err)
}
}
+19
View File
@@ -0,0 +1,19 @@
package main
type CloudS3 struct {
Name string `json:"name"`
Region string `json:"region"`
Bucket string `json:"bucket"`
AccessKey string `json:"accesskey"`
SecretKey string `json:"secretekey"`
EndpointURL string `json:"endpoint_url"`
Folder *string `json:"folder"`
}
type S3Item struct {
Key string
IsFolder bool
FolderName *string
Size *int64
LastModified *string
}
+101
View File
@@ -0,0 +1,101 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func runBackupConfig() error {
data, err := os.ReadFile("backup_config.json")
if err != nil {
fmt.Println("backup_config.json nem található, kihagyva.")
return nil
}
var clouds []CloudS3
if err := json.Unmarshal(data, &clouds); err != nil {
return err
}
for _, cloud := range clouds {
client, err := createS3Client(cloud)
if err != nil {
fmt.Printf("Hiba a konfiguráció létrehozásakor: %v\n", err)
continue
}
fmt.Printf("Bucket tartalom listázása: %s\n", cloud.Name)
if err := handleBucketContents(context.Background(), client, cloud, "cloud_backup"); err != nil {
fmt.Printf("Hiba a bucket listázásakor: %v\n", err)
}
}
return nil
}
func runCleanAndBackup() error {
data, err := os.ReadFile("clean_and_backup.json")
if err != nil {
fmt.Println("clean_and_backup.json nem található, kihagyva.")
return nil
}
var clouds []CloudS3
if err := json.Unmarshal(data, &clouds); err != nil {
return err
}
for _, cloud := range clouds {
client, err := createS3Client(cloud)
if err != nil {
fmt.Printf("Hiba a konfiguráció létrehozásakor: %v\n", err)
continue
}
fmt.Printf("Összes bucket listázása: %s\n", cloud.Name)
buckets, err := listAllBuckets(context.Background(), client)
if err != nil {
return err
}
fmt.Printf("Talált bucket-ek: %v\n", buckets)
for _, bucketName := range buckets {
fmt.Printf("\n--- Bucket: %s ---\n", bucketName)
bucketCloud := CloudS3{
Name: fmt.Sprintf("%s/%s", cloud.Name, bucketName),
Region: cloud.Region,
Bucket: bucketName,
AccessKey: cloud.AccessKey,
SecretKey: cloud.SecretKey,
EndpointURL: cloud.EndpointURL,
Folder: nil,
}
if err := handleBucketContents(context.Background(), client, bucketCloud, "db_backup"); err != nil {
fmt.Printf("Hiba a bucket listázásakor: %v\n", err)
}
count, err := deleteOldFiles(context.Background(), client, bucketCloud, 14)
if err != nil {
fmt.Printf("Hiba a régi fájlok törlésekor: %v\n", err)
} else if count > 0 {
fmt.Printf("%d régi fájl törölve a %s bucket-ből\n", count, bucketName)
}
}
}
return nil
}
func createS3Client(cloud CloudS3) (*s3.Client, error) {
cfg, err := createConfig(cloud)
if err != nil {
return nil, err
}
return s3.NewFromConfig(cfg), nil
}
+104
View File
@@ -0,0 +1,104 @@
package main
import (
"context"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
func listAllBuckets(ctx context.Context, client *s3.Client) ([]string, error) {
out, err := client.ListBuckets(ctx, &s3.ListBucketsInput{})
if err != nil {
return nil, err
}
var buckets []string
for _, b := range out.Buckets {
if b.Name != nil {
buckets = append(buckets, *b.Name)
}
}
return buckets, nil
}
func listBucket(ctx context.Context, client *s3.Client, s3c CloudS3) ([]S3Item, error) {
var items []S3Item
var continuation *string
for {
input := &s3.ListObjectsV2Input{
Bucket: aws.String(s3c.Bucket),
}
if s3c.Folder != nil {
input.Prefix = s3c.Folder
}
if continuation != nil {
input.ContinuationToken = continuation
}
out, err := client.ListObjectsV2(ctx, input)
if err != nil {
return nil, err
}
for _, prefix := range out.CommonPrefixes {
if prefix.Prefix != nil {
p := *prefix.Prefix
items = append(items, S3Item{
Key: p,
IsFolder: true,
Size: nil,
FolderName: &p,
})
}
}
for _, obj := range out.Contents {
if obj.Key == nil {
continue
}
key := *obj.Key
if strings.HasSuffix(key, "/") {
continue
}
var folderName *string
if idx := strings.LastIndex(key, "/"); idx >= 0 {
f := key[:idx]
folderName = &f
}
var sizePtr *int64
if obj.Size != nil {
s := *obj.Size
sizePtr = &s
}
var lastModifiedPtr *string
if obj.LastModified != nil {
lm := obj.LastModified.UTC().Format(time.RFC3339)
lastModifiedPtr = &lm
}
items = append(items, S3Item{
Key: key,
IsFolder: false,
FolderName: folderName,
Size: sizePtr,
LastModified: lastModifiedPtr,
})
}
if out.NextContinuationToken != nil {
continuation = out.NextContinuationToken
} else {
break
}
}
fmt.Printf("Contents count: %d\n", len(items))
return items, nil
}