From a63d615ad2c5ea02d8a7d6a619e77ce103ce87ae Mon Sep 17 00:00:00 2001 From: VetheonGames Date: Thu, 28 Mar 2024 14:48:37 -0600 Subject: [PATCH] Implement the S3 service --- config/default.go | 2 +- go.mod | 2 ++ go.sum | 14 +++++++++ pkg/backup/s3.go | 74 +++++++++++++++++++++++++++++++++++++++-------- 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/config/default.go b/config/default.go index c453932..b154760 100644 --- a/config/default.go +++ b/config/default.go @@ -31,7 +31,7 @@ func SetDefaultConfig() error { // S3 storage configuration "S3_ENABLED": "false", // Enable S3 storage (true/false) "S3_CONNECTION_ADDRESS": "", // S3 connection address in the format {IP}:{PORT} - "S3_AUTHORIZATION_INFO": "", // S3 authorization information (e.g., access key and secret key) + "S3_AUTHORIZATION_INFO": "", // S3 authorization information (access key:secret key) "S3_REGION": "us-east-1", // AWS region for S3 "S3_BUCKET_INFO": "", // S3 bucket name diff --git a/go.mod b/go.mod index 7a35de5..956a87b 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.22.0 require ( github.com/aws/aws-sdk-go-v2/config v1.27.9 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 github.com/aws/aws-sdk-go-v2/service/s3 v1.53.0 github.com/go-sql-driver/mysql v1.8.1 github.com/hirochachacha/go-smb2 v1.1.0 @@ -29,6 +30,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 // indirect github.com/aws/smithy-go v1.20.1 // indirect github.com/geoffgarside/ber v1.1.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/text v0.14.0 // indirect ) diff --git a/go.sum b/go.sum index 4b395c3..f62c42a 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.9 h1:N8s0/7yW+h8qR8WaRlPQeJ6czVMN github.com/aws/aws-sdk-go-v2/credentials v1.17.9/go.mod h1:446YhIdmSV0Jf/SLafGZalQo+xr2iw7/fzXGDPTU1yQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0 h1:af5YzcLf80tv4Em4jWVD75lpnOHSBkPUZxZfGkrI3HI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.0/go.mod h1:nQ3how7DMnFMWiU1SpECohgC82fpn4cKZ875NDMmwtA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13 h1:F+PUZee9mlfpEJVZdgyewRumKekS9O3fftj8fEMt0rQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.13/go.mod h1:Rl7i2dEWGHGsBIJCpUxlRt7VwK/HyXxICxdvIRssQHE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4 h1:0ScVK/4qZ8CIW0k8jOeFVsyS/sAiXpYxRBLolMkuLQM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.4/go.mod h1:84KyjNZdHC6QZW08nfHI6yZgPd+qRgaWcYsyLUo3QY8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.4 h1:sHmMWWX5E7guWEFQ9SVo6A3S4xpPrWnd77a6y4WM6PU= @@ -36,12 +38,21 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.28.5 h1:J/PpTf/hllOjx8Xu9DMflff3Fajf github.com/aws/aws-sdk-go-v2/service/sts v1.28.5/go.mod h1:0ih0Z83YDH/QeQ6Ori2yGE2XvWYv/Xm+cZc01LC6oK0= github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= @@ -54,3 +65,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/backup/s3.go b/pkg/backup/s3.go index 2741442..7d12f72 100644 --- a/pkg/backup/s3.go +++ b/pkg/backup/s3.go @@ -3,8 +3,13 @@ package backup import ( "context" "fmt" - "github.com/aws/aws-sdk-go-v2/config" + "os" + "strings" + awsCreds "github.com/aws/aws-sdk-go-v2/credentials" + awsConfig "github.com/aws/aws-sdk-go-v2/config" // Aliased AWS config package "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + backGoConfig "pixelridgesoftworks.com/BackGo/config" // Aliased your config package ) // S3Client wraps the AWS S3 client @@ -13,39 +18,84 @@ type S3Client struct { Bucket string } -// NewS3Client initializes and returns a new S3 client -func NewS3Client(bucket string) (*S3Client, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion("us-west-2"), // Specify your region +// NewS3Client initializes and returns a new S3 client using the provided application configuration +func NewS3Client(appConfig *backGoConfig.Config) (*S3Client, error) { + if !appConfig.S3Enabled { + return nil, fmt.Errorf("S3 storage is not enabled in the configuration") + } + + // Split the S3AuthorizationInfo into AccessKeyID and SecretAccessKey + parts := strings.SplitN(appConfig.S3AuthorizationInfo, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("S3AuthorizationInfo format is invalid, expected 'AccessKeyID:SecretAccessKey'") + } + accessKeyID, secretAccessKey := parts[0], parts[1] + + cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), + awsConfig.WithRegion(appConfig.S3Region), + awsConfig.WithCredentialsProvider(awsCreds.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")), ) if err != nil { - return nil, fmt.Errorf("unable to load SDK config, %w", err) + return nil, fmt.Errorf("unable to load AWS SDK config, %w", err) } s3Client := s3.NewFromConfig(cfg) return &S3Client{ Client: s3Client, - Bucket: bucket, + Bucket: appConfig.S3BucketInfo, }, nil } // UploadFile uploads a file to the specified S3 bucket func (c *S3Client) UploadFile(key string, filePath string) error { - // Implementation for uploading file to S3 - // need to open the file and use PutObject API call + // Open the file for use + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("failed to open file %q, %v", filePath, err) + } + defer file.Close() + + // Upload the file to S3. + uploader := manager.NewUploader(c.Client) + _, err = uploader.Upload(context.TODO(), &s3.PutObjectInput{ + Bucket: &c.Bucket, + Key: &key, + Body: file, + }) + if err != nil { + return fmt.Errorf("failed to upload file, %v", err) + } + fmt.Printf("File uploaded successfully: %s\n", key) return nil } // DeleteFile deletes a file from the specified S3 bucket func (c *S3Client) DeleteFile(key string) error { - // Implementation for deleting file from S3 using DeleteObject API call + _, err := c.Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: &c.Bucket, + Key: &key, + }) + if err != nil { + return fmt.Errorf("failed to delete file, %v", err) + } + fmt.Printf("File deleted successfully: %s\n", key) return nil } // ListFiles lists all files (backups) in the specified S3 bucket func (c *S3Client) ListFiles() ([]string, error) { - // Implementation for listing files in S3 bucket using ListObjectsV2 API call - return nil, nil + resp, err := c.Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: &c.Bucket, + }) + if err != nil { + return nil, fmt.Errorf("failed to list files, %v", err) + } + + var files []string + for _, item := range resp.Contents { + files = append(files, *item.Key) + } + return files, nil } // SetBucket sets the S3 bucket for the client