From a85a6183de5f3190cfef449d70436b0dd6a4fdfa Mon Sep 17 00:00:00 2001 From: heedaeshin Date: Fri, 6 Sep 2024 14:06:12 +0900 Subject: [PATCH] Refactor: Profile and Task struct integration, directory reorganization, and abstracted data management. Update: Added transactions and utf8mb4 support in MySQL, improved NCP OS S3 command handling. --- .gitignore | 3 + cmd/root.go | 2 - config/config.go | 225 +++++++----------- config/profile.go | 183 +++++--------- config/provider.go | 189 +++++++++++++++ data/example/Migrate/AwsToNcp.json | 18 ++ data/example/Migrate/NcpToAws.json | 18 ++ data/example/NRDB/aws.json | 10 + data/example/NRDB/gcp.json | 10 + data/example/NRDB/ncp.json | 13 + data/example/OBJ/aws.json | 11 + data/example/OBJ/gcp.json | 11 + data/example/OBJ/ncp.json | 12 + data/var/run/data-manager/config/config.json | 3 + data/var/run/data-manager/flow/flow.json | 3 + data/var/run/data-manager/group/group.json | 3 + .../run/data-manager/operation/operation.json | 3 + data/var/run/data-manager/profile/sample.json | 54 +++++ data/var/run/data-manager/task/task.json | 3 + internal/auth/base.go | 6 +- models/basicTask.go | 4 +- pkg/csp/aws/awsc.go | 4 +- pkg/csp/aws/awsc_test.go | 69 ++++++ websrc/controllers/publicfunc.go | 10 +- 24 files changed, 599 insertions(+), 268 deletions(-) create mode 100644 config/provider.go create mode 100644 data/example/Migrate/AwsToNcp.json create mode 100644 data/example/Migrate/NcpToAws.json create mode 100644 data/example/NRDB/aws.json create mode 100644 data/example/NRDB/gcp.json create mode 100644 data/example/NRDB/ncp.json create mode 100644 data/example/OBJ/aws.json create mode 100644 data/example/OBJ/gcp.json create mode 100644 data/example/OBJ/ncp.json create mode 100644 data/var/run/data-manager/config/config.json create mode 100644 data/var/run/data-manager/flow/flow.json create mode 100644 data/var/run/data-manager/group/group.json create mode 100644 data/var/run/data-manager/operation/operation.json create mode 100644 data/var/run/data-manager/profile/sample.json create mode 100644 data/var/run/data-manager/task/task.json create mode 100644 pkg/csp/aws/awsc_test.go diff --git a/.gitignore b/.gitignore index 7cb54fc..7363eb0 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ go.work # mc-data-manager binary, built with `go build` mc-data-manager + +# Credential +profile.json \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 4308cbd..c072814 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -18,7 +18,6 @@ package cmd import ( "os" - "github.com/cloud-barista/mc-data-manager/internal/log" "github.com/cloud-barista/mc-data-manager/models" "github.com/spf13/cobra" ) @@ -32,7 +31,6 @@ var rootCmd = &cobra.Command{ Long: `It is a tool that builds an environment for verification of data migration technology and generates test data necessary for data migration.`, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - log.Info("Data Manager started") return nil }, } diff --git a/config/config.go b/config/config.go index 2af82e8..35a6c87 100644 --- a/config/config.go +++ b/config/config.go @@ -1,189 +1,146 @@ -/* -Copyright 2023 The Cloud-Barista Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ package config import ( - "context" + "encoding/json" "errors" - "fmt" - - "cloud.google.com/go/firestore" - "cloud.google.com/go/storage" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/dynamodb" - "github.com/aws/aws-sdk-go-v2/service/s3" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" - "google.golang.org/api/option" + "io" + "os" + "path/filepath" + "sync" ) -func validateInputs(username, password, host *string, port *int) error { - if username == nil || password == nil || host == nil || port == nil { - return errors.New("The input is invalid") - } - return nil +// ConfigManager structure definition +type ConfigManager struct { + DefaultProfile string + ProfileManager ProfileManager + configFilePath string + mu sync.Mutex } -func newAWSConfig(accesskey, secretkey, region string) (*aws.Config, error) { - cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accesskey, secretkey, "")), - config.WithRegion(region), - config.WithRetryMaxAttempts(5), - ) - +// NewConfigManager loads the config from the specified path +func NewConfigManager(configPath string) (*ConfigManager, error) { + configFilePath := filepath.Join(configPath, "config.json") + defaultProfile, err := loadDefaultProfile(configFilePath) if err != nil { return nil, err } - return &cfg, nil + profilePath := filepath.Join(configPath, "profile", "profile.json") + + return &ConfigManager{ + DefaultProfile: defaultProfile, + ProfileManager: NewProfileManager(profilePath), + configFilePath: configFilePath, + }, nil } -func newAWSConfigWithEndpoint(serviceID, accesskey, secretkey, region, endpoint string) (*aws.Config, error) { - customResolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { - if service == serviceID { - return aws.Endpoint{ - PartitionID: "aws", - URL: endpoint, - SigningRegion: region, - }, nil - } - - return aws.Endpoint{}, &aws.EndpointNotFoundError{} - }) - - cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accesskey, secretkey, "")), - config.WithRegion(region), - config.WithRetryMaxAttempts(5), - config.WithEndpointResolver(customResolver), - ) +// loadDefaultProfile loads the default profile from the config file +func loadDefaultProfile(configFilePath string) (string, error) { + file, err := os.Open(configFilePath) + if err != nil { + return "", err + } + defer file.Close() + data, err := io.ReadAll(file) if err != nil { - return nil, err + return "", err } - return &cfg, nil -} + var config struct { + DefaultProfile string `json:"defaultProfile"` + } -func newNCPMongoDBConfig(username, password, host string, port int) *options.ClientOptions { - dc := true - return &options.ClientOptions{ - Auth: &options.Credential{ - Username: username, - Password: password, - }, - Direct: &dc, - Hosts: []string{fmt.Sprintf("%s:%d", host, port)}, + if err := json.Unmarshal(data, &config); err != nil { + return "", err } -} -func NewNCPMongoDBClient(username, password, host string, port int) (*mongo.Client, error) { - if err := validateInputs(&username, &password, &host, &port); err != nil { - return nil, err + if config.DefaultProfile == "" { + return "", errors.New("defaultProfile not set in config.json") } - return mongo.Connect(context.Background(), newNCPMongoDBConfig(username, password, host, port)) + + return config.DefaultProfile, nil } -func NewS3Client(accesskey, secretkey, region string) (*s3.Client, error) { - cfg, err := newAWSConfig(accesskey, secretkey, region) +// CreateConfig creates a new config.json file with the given data +func (cm *ConfigManager) CreateConfig(configData map[string]interface{}) error { + cm.mu.Lock() + defer cm.mu.Unlock() + + file, err := os.Create(cm.configFilePath) if err != nil { - return nil, err + return err } + defer file.Close() - return s3.NewFromConfig(*cfg, func(o *s3.Options) { - o.UsePathStyle = true - }), nil -} - -func NewS3ClientWithEndpoint(accesskey, secretkey, region string, endpoint string) (*s3.Client, error) { - cfg, err := newAWSConfigWithEndpoint(s3.ServiceID, accesskey, secretkey, region, endpoint) + data, err := json.MarshalIndent(configData, "", " ") if err != nil { - return nil, err + return err } - return s3.NewFromConfig(*cfg, func(o *s3.Options) { - o.UsePathStyle = true - }), nil + _, err = file.Write(data) + return err } -func NewDynamoDBClient(accesskey, secretkey, region string) (*dynamodb.Client, error) { - cfg, err := newAWSConfig(accesskey, secretkey, region) +// ReadConfig reads the config.json file and returns the data +func (cm *ConfigManager) ReadConfig() (map[string]interface{}, error) { + cm.mu.Lock() + defer cm.mu.Unlock() + + file, err := os.Open(cm.configFilePath) if err != nil { return nil, err } + defer file.Close() - return dynamodb.NewFromConfig(*cfg), nil -} - -func NewDynamoDBClientWithEndpoint(accesskey, secretkey, region string, endpoint string) (*dynamodb.Client, error) { - cfg, err := newAWSConfigWithEndpoint(dynamodb.ServiceID, accesskey, secretkey, region, endpoint) + data, err := io.ReadAll(file) if err != nil { return nil, err } - return dynamodb.NewFromConfig(*cfg), nil -} - -func NewGCPClient(credentialsJson string) (*storage.Client, error) { - var client *storage.Client - var err error - ctx := context.TODO() - switch { - - case credentialsJson != "": - client, err = storage.NewClient(ctx, option.WithCredentialsJSON([]byte(credentialsJson))) - - default: - return nil, errors.New("either credentialsFile or credentialsJson must be provided") - } + var configData map[string]interface{} + err = json.Unmarshal(data, &configData) if err != nil { return nil, err } - return client, nil + + return configData, nil } -func NewFireStoreClient(credentialsJson, projectID, databaseID string) (*firestore.Client, error) { - var client *firestore.Client - var err error +// UpdateConfig updates the config.json file with the given data +func (cm *ConfigManager) UpdateConfig(updatedData map[string]interface{}) error { + cm.mu.Lock() + defer cm.mu.Unlock() - ctx := context.TODO() - if databaseID != "" { - client, err = firestore.NewClientWithDatabase(ctx, projectID, databaseID, option.WithCredentialsJSON([]byte(credentialsJson))) - } else { - client, err = firestore.NewClient(ctx, projectID, option.WithCredentialsJSON([]byte(credentialsJson))) - } + // Read the current data + currentData, err := cm.ReadConfig() if err != nil { - return nil, err + return err } - return client, nil + // Merge the updated data + for key, value := range updatedData { + currentData[key] = value + } + + // Write the merged data back to the file + return cm.CreateConfig(currentData) } -func NewFireStoreClientWithDatabase(credentialsFile, projectID, databaseID string) (*firestore.Client, error) { - client, err := firestore.NewClientWithDatabase( - context.TODO(), - projectID, - databaseID, - option.WithCredentialsFile(credentialsFile), - ) +// DeleteConfig deletes the config.json file +func (cm *ConfigManager) DeleteConfig() error { + cm.mu.Lock() + defer cm.mu.Unlock() + + err := os.Remove(cm.configFilePath) if err != nil { - return nil, err + return err } - return client, nil + return nil +} + +// GetDefaultCredentials returns the default profile credentials +func (cm *ConfigManager) GetDefaultCredentials(provider string) (interface{}, error) { + return cm.ProfileManager.LoadCredentialsByProfile(cm.DefaultProfile, provider) } diff --git a/config/profile.go b/config/profile.go index a46dd1e..067ba50 100644 --- a/config/profile.go +++ b/config/profile.go @@ -1,55 +1,51 @@ package config import ( - "context" "encoding/json" "errors" - "fmt" "io" "os" + "path/filepath" "sync" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/cloud-barista/mc-data-manager/models" ) -type Credentials = models.ProfileCredentials - -var mu sync.Mutex - -const ( - // Define the constant path for the credentials file - CredentialsFilePath = "./data/var/run/mc-data-manager/profile/auth/auth.json" -) - -// CredentialsManager interface definition -type CredentialsManager interface { - LoadAllCredentials() (map[string]Credentials, error) - SaveAllCredentials(profiles map[string]Credentials) error - CreateProfile(profileName string, credentials Credentials) error - UpdateProfile(profileName string, credentials Credentials) error +type ProfileManager interface { + LoadAllProfiles() (map[string]models.ProfileCredentials, error) + SaveAllProfiles(profiles map[string]models.ProfileCredentials) error + CreateProfile(profileName string, credentials models.ProfileCredentials) error + UpdateProfile(profileName string, credentials models.ProfileCredentials) error DeleteProfile(profileName string) error - LoadCredentialsByProfile(profileName string, provider models.Provider) (interface{}, error) + LoadCredentialsByProfile(profileName string, provider string) (interface{}, error) } -// FileCredentialsManager struct definition -type FileCredentialsManager struct { - authFilePath string +type FileProfileManager struct { + profileFilePath string + mu sync.Mutex +} + +func NewProfileManager(profileFilePath ...string) *FileProfileManager { + var path string + if len(profileFilePath) > 0 && profileFilePath[0] != "" { + path = profileFilePath[0] + } else { + path = filepath.Join(".", "data", "var", "run", "data-manager", "profile", "profile.json") + } + return &FileProfileManager{profileFilePath: path} } -// NewFileCredentialsManager constructor function -func NewFileCredentialsManager() *FileCredentialsManager { - return &FileCredentialsManager{authFilePath: CredentialsFilePath} +func NewProfileManagerDefault() *FileProfileManager { + defaultPath := filepath.Join(".", "data", "var", "run", "data-manager", "profile", "profile.json") + return &FileProfileManager{profileFilePath: defaultPath} } -// LoadAllCredentials loads all credentials from the auth.json file -func (fcm *FileCredentialsManager) LoadAllCredentials() (map[string]Credentials, error) { - mu.Lock() - defer mu.Unlock() +// R profiles +func (fpm *FileProfileManager) LoadAllProfiles() (map[string]models.ProfileCredentials, error) { + fpm.mu.Lock() + defer fpm.mu.Unlock() - file, err := os.Open(fcm.authFilePath) + file, err := os.Open(fpm.profileFilePath) if err != nil { return nil, err } @@ -61,42 +57,42 @@ func (fcm *FileCredentialsManager) LoadAllCredentials() (map[string]Credentials, } var profiles []struct { - ProfileName string `json:"profileName"` - Credentials Credentials `json:"credentials"` + ProfileName string `json:"profileName"` + Credentials models.ProfileCredentials `json:"credentials"` } if err := json.Unmarshal(data, &profiles); err != nil { return nil, err } - credentialsMap := make(map[string]Credentials) + profileMap := make(map[string]models.ProfileCredentials) for _, profile := range profiles { - credentialsMap[profile.ProfileName] = profile.Credentials + profileMap[profile.ProfileName] = profile.Credentials } - return credentialsMap, nil + return profileMap, nil } -// SaveAllCredentials saves all credentials to the auth.json file -func (fcm *FileCredentialsManager) SaveAllCredentials(profiles map[string]Credentials) error { - mu.Lock() - defer mu.Unlock() +// Save File with profiles +func (fpm *FileProfileManager) SaveAllProfiles(profiles map[string]models.ProfileCredentials) error { + fpm.mu.Lock() + defer fpm.mu.Unlock() - file, err := os.Create(fcm.authFilePath) + file, err := os.Create(fpm.profileFilePath) if err != nil { return err } defer file.Close() var profilesList []struct { - ProfileName string `json:"profileName"` - Credentials Credentials `json:"credentials"` + ProfileName string `json:"profileName"` + Credentials models.ProfileCredentials `json:"credentials"` } for name, creds := range profiles { profilesList = append(profilesList, struct { - ProfileName string `json:"profileName"` - Credentials Credentials `json:"credentials"` + ProfileName string `json:"profileName"` + Credentials models.ProfileCredentials `json:"credentials"` }{ ProfileName: name, Credentials: creds, @@ -112,11 +108,11 @@ func (fcm *FileCredentialsManager) SaveAllCredentials(profiles map[string]Creden return err } -// CreateProfile adds a new profile -func (fcm *FileCredentialsManager) CreateProfile(profileName string, credentials Credentials) error { - mu.Lock() - defer mu.Unlock() - profiles, err := fcm.LoadAllCredentials() +// C profile by name +func (fpm *FileProfileManager) CreateProfile(profileName string, credentials models.ProfileCredentials) error { + fpm.mu.Lock() + defer fpm.mu.Unlock() + profiles, err := fpm.LoadAllProfiles() if err != nil { return err } @@ -126,14 +122,14 @@ func (fcm *FileCredentialsManager) CreateProfile(profileName string, credentials } profiles[profileName] = credentials - return fcm.SaveAllCredentials(profiles) + return fpm.SaveAllProfiles(profiles) } -// UpdateProfile updates an existing profile -func (fcm *FileCredentialsManager) UpdateProfile(profileName string, credentials Credentials) error { - mu.Lock() - defer mu.Unlock() - profiles, err := fcm.LoadAllCredentials() +// U profile by name +func (fpm *FileProfileManager) UpdateProfile(profileName string, credentials models.ProfileCredentials) error { + fpm.mu.Lock() + defer fpm.mu.Unlock() + profiles, err := fpm.LoadAllProfiles() if err != nil { return err } @@ -143,14 +139,14 @@ func (fcm *FileCredentialsManager) UpdateProfile(profileName string, credentials } profiles[profileName] = credentials - return fcm.SaveAllCredentials(profiles) + return fpm.SaveAllProfiles(profiles) } -// DeleteProfile removes a profile -func (fcm *FileCredentialsManager) DeleteProfile(profileName string) error { - mu.Lock() - defer mu.Unlock() - profiles, err := fcm.LoadAllCredentials() +// D profile by name +func (fpm *FileProfileManager) DeleteProfile(profileName string) error { + fpm.mu.Lock() + defer fpm.mu.Unlock() + profiles, err := fpm.LoadAllProfiles() if err != nil { return err } @@ -160,12 +156,12 @@ func (fcm *FileCredentialsManager) DeleteProfile(profileName string) error { } delete(profiles, profileName) - return fcm.SaveAllCredentials(profiles) + return fpm.SaveAllProfiles(profiles) } -// LoadCredentialsByProfile loads credentials by profile name and provider -func (fcm *FileCredentialsManager) LoadCredentialsByProfile(profileName string, provider string) (interface{}, error) { - profiles, err := fcm.LoadAllCredentials() +// R profile by name +func (fpm *FileProfileManager) LoadCredentialsByProfile(profileName string, provider string) (interface{}, error) { + profiles, err := fpm.LoadAllProfiles() if err != nil { return nil, err } @@ -184,56 +180,3 @@ func (fcm *FileCredentialsManager) LoadCredentialsByProfile(profileName string, return nil, errors.New("unsupported provider") } } - -// AWSClient wraps aws.Config and provides additional methods -type AWSClient struct { - Config aws.Config -} - -// NewAWSClient creates a new AWSClient -func NewAWSClient(profileName string) (*AWSClient, error) { - fmt.Println("Creating CredentialsManager") - credentialsManager := NewFileCredentialsManager() - - fmt.Println("Loading credentials") - creds, err := credentialsManager.LoadCredentialsByProfile(profileName, string(models.AWS)) - if err != nil { - return nil, fmt.Errorf("Error loading credentials: %v", err) - } - - fmt.Println("Casting credentials") - awsCreds, ok := creds.(models.AWSCredentials) - if !ok { - return nil, fmt.Errorf("Invalid credentials type: %v", creds) - } - - fmt.Println("Creating AWS config") - return newAWSConfigure(awsCreds) -} - -// newAWSConfig creates a new AWSClient with the given credentials -func newAWSConfigure(params models.AWSCredentials) (*AWSClient, error) { - cfg, err := loadConfig(params) - if err != nil { - return nil, err - } - return &AWSClient{Config: cfg}, nil -} - -// loadConfig loads the AWS configuration with the given credentials -func loadConfig(params models.AWSCredentials) (aws.Config, error) { - return config.LoadDefaultConfig(context.TODO(), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(params.AccessKey, params.SecretKey, "")), - config.WithRetryMaxAttempts(5), - ) -} - -// SetRegion sets the region in the AWS configuration -func (client *AWSClient) SetRegion(region string) { - client.Config.Region = region -} - -// GetConfig returns the current AWS configuration -func (client *AWSClient) GetConfig() aws.Config { - return client.Config -} diff --git a/config/provider.go b/config/provider.go new file mode 100644 index 0000000..2af82e8 --- /dev/null +++ b/config/provider.go @@ -0,0 +1,189 @@ +/* +Copyright 2023 The Cloud-Barista Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package config + +import ( + "context" + "errors" + "fmt" + + "cloud.google.com/go/firestore" + "cloud.google.com/go/storage" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/s3" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "google.golang.org/api/option" +) + +func validateInputs(username, password, host *string, port *int) error { + if username == nil || password == nil || host == nil || port == nil { + return errors.New("The input is invalid") + } + return nil +} + +func newAWSConfig(accesskey, secretkey, region string) (*aws.Config, error) { + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accesskey, secretkey, "")), + config.WithRegion(region), + config.WithRetryMaxAttempts(5), + ) + + if err != nil { + return nil, err + } + + return &cfg, nil +} + +func newAWSConfigWithEndpoint(serviceID, accesskey, secretkey, region, endpoint string) (*aws.Config, error) { + customResolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + if service == serviceID { + return aws.Endpoint{ + PartitionID: "aws", + URL: endpoint, + SigningRegion: region, + }, nil + } + + return aws.Endpoint{}, &aws.EndpointNotFoundError{} + }) + + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accesskey, secretkey, "")), + config.WithRegion(region), + config.WithRetryMaxAttempts(5), + config.WithEndpointResolver(customResolver), + ) + + if err != nil { + return nil, err + } + + return &cfg, nil +} + +func newNCPMongoDBConfig(username, password, host string, port int) *options.ClientOptions { + dc := true + return &options.ClientOptions{ + Auth: &options.Credential{ + Username: username, + Password: password, + }, + Direct: &dc, + Hosts: []string{fmt.Sprintf("%s:%d", host, port)}, + } +} + +func NewNCPMongoDBClient(username, password, host string, port int) (*mongo.Client, error) { + if err := validateInputs(&username, &password, &host, &port); err != nil { + return nil, err + } + return mongo.Connect(context.Background(), newNCPMongoDBConfig(username, password, host, port)) +} + +func NewS3Client(accesskey, secretkey, region string) (*s3.Client, error) { + cfg, err := newAWSConfig(accesskey, secretkey, region) + if err != nil { + return nil, err + } + + return s3.NewFromConfig(*cfg, func(o *s3.Options) { + o.UsePathStyle = true + }), nil +} + +func NewS3ClientWithEndpoint(accesskey, secretkey, region string, endpoint string) (*s3.Client, error) { + cfg, err := newAWSConfigWithEndpoint(s3.ServiceID, accesskey, secretkey, region, endpoint) + if err != nil { + return nil, err + } + + return s3.NewFromConfig(*cfg, func(o *s3.Options) { + o.UsePathStyle = true + }), nil +} + +func NewDynamoDBClient(accesskey, secretkey, region string) (*dynamodb.Client, error) { + cfg, err := newAWSConfig(accesskey, secretkey, region) + if err != nil { + return nil, err + } + + return dynamodb.NewFromConfig(*cfg), nil +} + +func NewDynamoDBClientWithEndpoint(accesskey, secretkey, region string, endpoint string) (*dynamodb.Client, error) { + cfg, err := newAWSConfigWithEndpoint(dynamodb.ServiceID, accesskey, secretkey, region, endpoint) + if err != nil { + return nil, err + } + + return dynamodb.NewFromConfig(*cfg), nil +} + +func NewGCPClient(credentialsJson string) (*storage.Client, error) { + var client *storage.Client + var err error + ctx := context.TODO() + switch { + + case credentialsJson != "": + client, err = storage.NewClient(ctx, option.WithCredentialsJSON([]byte(credentialsJson))) + + default: + return nil, errors.New("either credentialsFile or credentialsJson must be provided") + } + if err != nil { + return nil, err + } + return client, nil +} + +func NewFireStoreClient(credentialsJson, projectID, databaseID string) (*firestore.Client, error) { + var client *firestore.Client + var err error + + ctx := context.TODO() + if databaseID != "" { + client, err = firestore.NewClientWithDatabase(ctx, projectID, databaseID, option.WithCredentialsJSON([]byte(credentialsJson))) + } else { + client, err = firestore.NewClient(ctx, projectID, option.WithCredentialsJSON([]byte(credentialsJson))) + } + if err != nil { + return nil, err + } + + return client, nil +} + +func NewFireStoreClientWithDatabase(credentialsFile, projectID, databaseID string) (*firestore.Client, error) { + client, err := firestore.NewClientWithDatabase( + context.TODO(), + projectID, + databaseID, + option.WithCredentialsFile(credentialsFile), + ) + if err != nil { + return nil, err + } + + return client, nil +} diff --git a/data/example/Migrate/AwsToNcp.json b/data/example/Migrate/AwsToNcp.json new file mode 100644 index 0000000..e3a4b2b --- /dev/null +++ b/data/example/Migrate/AwsToNcp.json @@ -0,0 +1,18 @@ +{ + "operationId": "opt-00735", + "sourcePoint": { + "profileName": "admin", + "provider": "aws", + "region": "ap-northeast-2" + }, + "targetPoint": { + "profileName": "admin", + "provider": "ncp", + "region": "kr", + "databaseName": "spero", + "host": "p4spq.pub-vpc.mg.naverncp.com", + "port": "17017", + "username": "manager", + "password": "N@mutech7^^7" + } +} \ No newline at end of file diff --git a/data/example/Migrate/NcpToAws.json b/data/example/Migrate/NcpToAws.json new file mode 100644 index 0000000..1eb41f1 --- /dev/null +++ b/data/example/Migrate/NcpToAws.json @@ -0,0 +1,18 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "aws", + "region": "ap-northeast-2" + }, + "sourcePoint": { + "profileName": "admin", + "provider": "ncp", + "region": "kr", + "databaseName": "spero", + "host": "p4spq.pub-vpc.mg.naverncp.com", + "port": "17017", + "username": "manager", + "password": "N@mutech7^^7" + } +} \ No newline at end of file diff --git a/data/example/NRDB/aws.json b/data/example/NRDB/aws.json new file mode 100644 index 0000000..dcb17f4 --- /dev/null +++ b/data/example/NRDB/aws.json @@ -0,0 +1,10 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "aws", + "region": "ap-northeast-2", + "CheckServerJSON": true, + "SizeServerJSON": "1" + } +} \ No newline at end of file diff --git a/data/example/NRDB/gcp.json b/data/example/NRDB/gcp.json new file mode 100644 index 0000000..f420a40 --- /dev/null +++ b/data/example/NRDB/gcp.json @@ -0,0 +1,10 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "gcp", + "region": "asia-northeast2", + "CheckServerJSON": true, + "SizeServerJSON": "1" + } +} \ No newline at end of file diff --git a/data/example/NRDB/ncp.json b/data/example/NRDB/ncp.json new file mode 100644 index 0000000..8a03f04 --- /dev/null +++ b/data/example/NRDB/ncp.json @@ -0,0 +1,13 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "ncp", + "region": "kr", + "databaseName": "spero", + "host": "p4spq.pub-vpc.mg.naverncp.com", + "port": "17017", + "username": "manager", + "password": "N@mutech7^^7" + } +} \ No newline at end of file diff --git a/data/example/OBJ/aws.json b/data/example/OBJ/aws.json new file mode 100644 index 0000000..2619f95 --- /dev/null +++ b/data/example/OBJ/aws.json @@ -0,0 +1,11 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "aws", + "region": "ap-northeast-2", + "bucket": "mc-data-manager", + "CheckServerJSON": true, + "SizeServerJSON": "1" + } +} \ No newline at end of file diff --git a/data/example/OBJ/gcp.json b/data/example/OBJ/gcp.json new file mode 100644 index 0000000..822684e --- /dev/null +++ b/data/example/OBJ/gcp.json @@ -0,0 +1,11 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "gcp", + "region": "asia-east1", + "bucket": "mc-data-manager", + "CheckServerJSON": true, + "SizeServerJSON": "1" + } +} \ No newline at end of file diff --git a/data/example/OBJ/ncp.json b/data/example/OBJ/ncp.json new file mode 100644 index 0000000..f84ce77 --- /dev/null +++ b/data/example/OBJ/ncp.json @@ -0,0 +1,12 @@ +{ + "operationId": "opt-00735", + "targetPoint": { + "profileName": "admin", + "provider": "ncp", + "region": "kr", + "endpoint": "https://kr.object.ncloudstorage.com", + "bucket": "mc-data-manager", + "CheckServerJSON": true, + "SizeServerJSON": "1" + } +} \ No newline at end of file diff --git a/data/var/run/data-manager/config/config.json b/data/var/run/data-manager/config/config.json new file mode 100644 index 0000000..ee177e8 --- /dev/null +++ b/data/var/run/data-manager/config/config.json @@ -0,0 +1,3 @@ +{ + "defaultProfile":"default" +} \ No newline at end of file diff --git a/data/var/run/data-manager/flow/flow.json b/data/var/run/data-manager/flow/flow.json new file mode 100644 index 0000000..1610ea1 --- /dev/null +++ b/data/var/run/data-manager/flow/flow.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/data/var/run/data-manager/group/group.json b/data/var/run/data-manager/group/group.json new file mode 100644 index 0000000..1610ea1 --- /dev/null +++ b/data/var/run/data-manager/group/group.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/data/var/run/data-manager/operation/operation.json b/data/var/run/data-manager/operation/operation.json new file mode 100644 index 0000000..1610ea1 --- /dev/null +++ b/data/var/run/data-manager/operation/operation.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/data/var/run/data-manager/profile/sample.json b/data/var/run/data-manager/profile/sample.json new file mode 100644 index 0000000..88efc31 --- /dev/null +++ b/data/var/run/data-manager/profile/sample.json @@ -0,0 +1,54 @@ +[ + { + "profileName": "default", + "credentials": { + "aws": { + "accessKey": "", + "secretKey": "" + }, + "ncp": { + "accessKey": "", + "secretKey": "" + }, + "gcp": { + "type": "", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "", + "token_uri": "", + "auth_provider_x509_cert_url": "", + "client_x509_cert_url": "", + "universe_domain": "" + } + } + }, + { + "profileName": "admin", + "credentials": { + "aws": { + "accessKey": "", + "secretKey": "" + }, + "ncp": { + "accessKey": "", + "secretKey": "" + }, + "gcp": { + "type": "", + "project_id": "", + "private_key_id": "", + "private_key": "", + "client_email": "", + "client_id": "", + "auth_uri": "", + "token_uri": "", + "auth_provider_x509_cert_url": "", + "client_x509_cert_url": "", + "universe_domain": "" + } + } + } +] diff --git a/data/var/run/data-manager/task/task.json b/data/var/run/data-manager/task/task.json new file mode 100644 index 0000000..c9c513b --- /dev/null +++ b/data/var/run/data-manager/task/task.json @@ -0,0 +1,3 @@ +[ + +] \ No newline at end of file diff --git a/internal/auth/base.go b/internal/auth/base.go index 066fa93..fcb926b 100644 --- a/internal/auth/base.go +++ b/internal/auth/base.go @@ -54,7 +54,7 @@ func GetConfig(credPath string, ConfigData *models.CommandTask) error { func preRunProfileE(pName, cmdName string, params *models.ProviderConfig) error { logrus.Info("initiate a profile scan") - credentailMangeer := config.NewFileCredentialsManager() + credentailMangeer := config.NewProfileManager() if srcCreds, err := credentailMangeer.LoadCredentialsByProfile(params.ProfileName, params.Provider); err != nil { return fmt.Errorf("get config error : %s", err) @@ -108,7 +108,7 @@ func GetOS(params *models.ProviderConfig) (*osc.OSController, error) { logrus.Infof("ProfileName : %s", params.ProfileName) logrus.Infof("Provider : %s", params.Provider) logrus.Info("Get Credentail") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(params.ProfileName, params.Provider) if err != nil { logrus.Errorf("credentail load failed : %v", err) @@ -203,7 +203,7 @@ func GetNRDMS(params *models.ProviderConfig) (*nrdbc.NRDBController, error) { logrus.Infof("Provider : %s", params.Provider) logrus.Info("Get Credentail") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(params.ProfileName, params.Provider) if err != nil { logrus.Errorf("credentail load failed : %v", err) diff --git a/models/basicTask.go b/models/basicTask.go index c44a08a..d3bb4cb 100644 --- a/models/basicTask.go +++ b/models/basicTask.go @@ -14,8 +14,8 @@ type TaskMeta struct { type Task struct { OperationParams - Meta TaskMeta `json:"meta,omitempty" swaggerignore:"true"` - Status Status `json:"status,omitempty" swaggerignore:"true"` // active, inactive, etc. + TaskMeta `json:"meta,omitempty" swaggerignore:"true"` + Status `json:"status,omitempty" swaggerignore:"true"` // active, inactive, etc. } type Flow struct { diff --git a/pkg/csp/aws/awsc.go b/pkg/csp/aws/awsc.go index e540337..afc4a5d 100644 --- a/pkg/csp/aws/awsc.go +++ b/pkg/csp/aws/awsc.go @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -package main +package aws import ( "context" @@ -202,7 +202,7 @@ func main() { defaultRegion := "ap-northeast-2" fmt.Println("Creating CredentialsManager") - credentialsManager := cfg.NewFileCredentialsManager() + credentialsManager := cfg.NewProfileManager() fmt.Println("Loading credentials") // Load credentials for the specified profile and provider diff --git a/pkg/csp/aws/awsc_test.go b/pkg/csp/aws/awsc_test.go new file mode 100644 index 0000000..aa92577 --- /dev/null +++ b/pkg/csp/aws/awsc_test.go @@ -0,0 +1,69 @@ +package aws + +import ( + "fmt" + "path/filepath" + "runtime" + "testing" + + cfg "github.com/cloud-barista/mc-data-manager/config" + "github.com/cloud-barista/mc-data-manager/models" + "github.com/sirupsen/logrus" +) + +func TestMain(m *testing.M) { + fmt.Println("Starting Test function") + + _, filename, _, ok := runtime.Caller(0) + if !ok { + logrus.Fatalf("Failed to get current file path") + } + rootDir := filepath.Join(filepath.Dir(filename), "../../..") + + profileName := "default" + provider := "aws" + defaultRegion := "ap-northeast-2" + + fmt.Println("Creating CredentialsManager") + + profilePath := filepath.Join(rootDir, "data", "var", "run", "data-manager", "profile", "profile.json") + credentialsManager := cfg.NewProfileManager(profilePath) + + fmt.Println("Loading credentials") + // Load credentials for the specified profile and provider + creds, err := credentialsManager.LoadCredentialsByProfile(profileName, provider) + if err != nil { + fmt.Println("Error loading credentials:", err) + return + } + + fmt.Println("Casting credentials") + awsCreds, ok := creds.(models.AWSCredentials) + if !ok { + fmt.Println(creds) + fmt.Println("Invalid credentials type") + return + } + + fmt.Println("Creating AWS config") + client, err := newAWSConfig(awsCreds) + if err != nil { + fmt.Println("Error creating AWS config:", err) + return + } + + regions, err := client.ListRegions() + if err != nil { + fmt.Println("Error listing regions:", err) + return + } + fmt.Println("Regions:", regions) + + client.Config.Region = defaultRegion + fmt.Println("Listing AWS resources") + + // List AWS resources + listResources(client) // listResources + + fmt.Println("Finished main function") +} diff --git a/websrc/controllers/publicfunc.go b/websrc/controllers/publicfunc.go index 2e908f3..4ad4bde 100644 --- a/websrc/controllers/publicfunc.go +++ b/websrc/controllers/publicfunc.go @@ -123,7 +123,7 @@ func getS3OSC(logger *logrus.Logger, startTime time.Time, jobType string, params var awsOSC *osc.OSController logger.Infof("gmaraps : %v", gparam) logger.Info("Get S3 Client") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(gparam.ProfileName, gparam.Provider) if err != nil { end := time.Now() @@ -176,7 +176,7 @@ func getS3COSC(logger *logrus.Logger, startTime time.Time, jobType string, param var OSC *osc.OSController logger.Info("Get S3 Compataible Client") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(gparam.ProfileName, gparam.Provider) if err != nil { end := time.Now() @@ -219,7 +219,7 @@ func getGCPCOSC(logger *logrus.Logger, startTime time.Time, jobType string, para var gcpOSC *osc.OSController logger.Info("Get GCP Client") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(gparam.ProfileName, gparam.Provider) if err != nil { end := time.Now() @@ -306,7 +306,7 @@ func getDynamoNRDBC(logger *logrus.Logger, startTime time.Time, jobType string, var NRDBC *nrdbc.NRDBController logger.Info("Get DynamoDB Client") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(gparam.ProfileName, gparam.Provider) if err != nil { end := time.Now() @@ -358,7 +358,7 @@ func getFirestoreNRDBC(logger *logrus.Logger, startTime time.Time, jobType strin logger.Info("Get FirestoreDB Client") - credentailManger := config.NewFileCredentialsManager() + credentailManger := config.NewProfileManager() creds, err := credentailManger.LoadCredentialsByProfile(gparam.ProfileName, gparam.Provider) if err != nil { end := time.Now()