Skip to content

Commit

Permalink
add flags for azure and aws (#3590)
Browse files Browse the repository at this point in the history 
<!-- PR description-->
Flags for all configs-

Azure cred flags- (azure-tenant-id, azure-client-id, azure-client-secret) present in -
- Backup (create, delete, details and list) and restore of Exchange, Onedrive and Sharepoint command
-  S3 repo init and connect command

AWS cred flags - (aws-access-key, aws-secret-access-key, aws-session-token) present in- 
- Backup (create, delete, details and list) and restore of Exchange, Onedrive and Sharepoint command
-  S3 repo init and connect command

Passphrase flag- (--passphrase) present in- 
- Backup (create, delete, details and list) and restore of Exchange, Onedrive and Sharepoint command
-  S3 repo init and connect command

S3 flags- 
--endpoint, --prefix, --bucket, --disable-tls, --disable-tls-verification - flags is for repo init and connect commands
all the S3 env var will also work only in case of repo init and connect command. For all other commands  user first connects to repo. Which will store the config values in config file. And then user can use that config file for other commands.

No cred configs are save in the config file by Corso. 

Config file values added- 
Azure cred - 
- azure_client_id 
- azure_secret 
- azure_tenantid

AWS cred -
- aws_access_key_id
- aws_secret_access_key
- aws_session_token

Passphrase -
- passphrase

**NOTE:** 
- in case of AWS creds all the three values should be provided from same method. Either put all values in env, config file and so on.
- all the S3 env var will also work only in case of repo init and connect command. For all other commands  user first connects to repo. Which will store the config values in config file. And then user can use that config file for other commands.



---

#### Does this PR need a docs update or release note?

- [ ] ✅ Yes, it's included
- [x] 🕐 Yes, but in a later PR
- [ ] ⛔ No

#### Type of change

<!--- Please check the type of change your PR introduces: --->
- [x] 🌻 Feature


#### Issue(s)

<!-- Can reference multiple issues. Use one of the following "magic words" - "closes, fixes" to auto-close the Github issue. -->
* #3522

#### Test Plan

<!-- How will this be tested prior to merging.-->
- [x] 💪 Manual
- [x] ⚡ Unit test
- [ ] 💚 E2E
  • Loading branch information
@neha-Gupta1
neha-Gupta1 authored Jun 29, 2023
1 parent 78f6986 commit 8c66116
Show file tree
Hide file tree
Showing 31 changed files with 689 additions and 104 deletions.
5 changes: 3 additions & 2 deletions src/cli/backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/common/idname"
"github.com/alcionai/corso/src/internal/data"
Expand Down Expand Up @@ -270,7 +271,7 @@ func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []st

ctx := clues.Add(cmd.Context(), "delete_backup_id", bID)

r, _, _, err := utils.GetAccountAndConnect(ctx)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil {
return Only(ctx, err)
}
Expand All @@ -291,7 +292,7 @@ func genericDeleteCommand(cmd *cobra.Command, bID, designation string, args []st
func genericListCommand(cmd *cobra.Command, bID string, service path.ServiceType, args []string) error {
ctx := cmd.Context()

r, _, _, err := utils.GetAccountAndConnect(ctx)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil {
return Only(ctx, err)
}
Expand Down
17 changes: 15 additions & 2 deletions src/cli/backup/exchange.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
Expand Down Expand Up @@ -84,6 +85,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
// More generic (ex: --user) and more frequently used flags take precedence.
flags.AddMailBoxFlag(c)
flags.AddDataFlag(c, []string{dataEmail, dataContacts, dataEvents}, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddFetchParallelismFlag(c)
flags.AddFailFastFlag(c)
flags.AddDisableIncrementalsFlag(c)
Expand All @@ -96,6 +100,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
fs.SortFlags = false

flags.AddBackupIDFlag(c, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
addFailedItemsFN(c)
addSkippedItemsFN(c)
addRecoveredErrorsFN(c)
Expand All @@ -112,6 +119,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
// Flags addition ordering should follow the order we want them to appear in help and docs:
// More generic (ex: --user) and more frequently used flags take precedence.
flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddExchangeDetailsAndRestoreFlags(c)

case deleteCommand:
Expand All @@ -122,6 +132,9 @@ func addExchangeCommands(cmd *cobra.Command) *cobra.Command {
c.Example = exchangeServiceCommandDeleteExamples

flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
}

return c
Expand Down Expand Up @@ -153,7 +166,7 @@ func createExchangeCmd(cmd *cobra.Command, args []string) error {
return err
}

r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx)
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
if err != nil {
return Only(ctx, err)
}
Expand Down Expand Up @@ -262,7 +275,7 @@ func detailsExchangeCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeExchangeOpts(cmd)

r, _, _, err := utils.GetAccountAndConnect(ctx)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil {
return Only(ctx, err)
}
Expand Down
178 changes: 178 additions & 0 deletions src/cli/backup/exchange_e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,184 @@ var (
events = path.EventsCategory
)

// ---------------------------------------------------------------------------
// tests with azure flags in exchange create
// ---------------------------------------------------------------------------

type ExchangeCMDWithFlagsE2ESuite struct {
tester.Suite
acct account.Account
st storage.Storage
vpr *viper.Viper
cfgFP string
repo repository.Repository
m365UserID string
recorder strings.Builder
}

func TestExchangeCMDWithFlagsE2ESuite(t *testing.T) {
suite.Run(t, &ExchangeCMDWithFlagsE2ESuite{Suite: tester.NewE2ESuite(
t,
[][]string{tester.AWSStorageCredEnvs, tester.M365AcctCredEnvs},
)})
}

func (suite *ExchangeCMDWithFlagsE2ESuite) SetupSuite() {
t := suite.T()

ctx, flush := tester.NewContext(t)
defer flush()

acct, st, repo, vpr, recorder, cfgFilePath := prepM365Test(t, ctx)

suite.acct = acct
suite.st = st
suite.repo = repo
suite.vpr = vpr
suite.recorder = recorder
suite.cfgFP = cfgFilePath
suite.m365UserID = tester.M365UserID(t)
}

func (suite *ExchangeCMDWithFlagsE2ESuite) TestBackupCreateExchange_badAzureClientID() {
t := suite.T()
ctx, flush := tester.NewContext(t)

defer flush()

suite.recorder.Reset()

cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID,
"--azure-client-id", "invalid-value",
)
cli.BuildCommandTree(cmd)

cmd.SetErr(&suite.recorder)

ctx = print.SetRootCmd(ctx, cmd)

// run the command
err := cmd.ExecuteContext(ctx)
require.Error(t, err, clues.ToCore(err))
}

func (suite *ExchangeCMDWithFlagsE2ESuite) TestBackupCreateExchange_azureIDFromConfigFile() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.vpr)

defer flush()

suite.recorder.Reset()

cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID,
"--config-file", suite.cfgFP)
cli.BuildCommandTree(cmd)

cmd.SetErr(&suite.recorder)

ctx = print.SetRootCmd(ctx, cmd)

// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))

result := suite.recorder.String()
t.Log("backup results", result)

// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.m365UserID)
}

func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupValueFromEnvCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.vpr)

defer flush()

suite.recorder.Reset()

cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID)
cli.BuildCommandTree(cmd)

cmd.SetErr(&suite.recorder)

ctx = print.SetRootCmd(ctx, cmd)

// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))

result := suite.recorder.String()
t.Log("backup results", result)

// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.m365UserID)
}

// AWS flags
func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupInvalidAWSClientIDCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)

defer flush()

suite.recorder.Reset()

cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID,
"--aws-access-key", "invalid-value",
"--aws-secret-access-key", "some-invalid-value",
)
cli.BuildCommandTree(cmd)

cmd.SetErr(&suite.recorder)

ctx = print.SetRootCmd(ctx, cmd)

// run the command
err := cmd.ExecuteContext(ctx)
// since invalid aws creds are explicitly set, should see a failure
require.Error(t, err, clues.ToCore(err))
}

func (suite *ExchangeCMDWithFlagsE2ESuite) TestExchangeBackupAWSValueFromEnvCmd_empty() {
t := suite.T()
ctx, flush := tester.NewContext(t)
ctx = config.SetViper(ctx, suite.vpr)

defer flush()

suite.recorder.Reset()

cmd := tester.StubRootCmd(
"backup", "create", "exchange",
"--user", suite.m365UserID)
cli.BuildCommandTree(cmd)

cmd.SetErr(&suite.recorder)

ctx = print.SetRootCmd(ctx, cmd)

// run the command
err := cmd.ExecuteContext(ctx)
require.NoError(t, err, clues.ToCore(err))

result := suite.recorder.String()
t.Log("backup results", result)

// as an offhand check: the result should contain the m365 user id
assert.Contains(t, result, suite.m365UserID)
}

// ---------------------------------------------------------------------------
// tests with no backups
// ---------------------------------------------------------------------------
Expand Down
18 changes: 16 additions & 2 deletions src/cli/backup/onedrive.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/alcionai/corso/src/cli/flags"
. "github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/utils"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/pkg/backup/details"
Expand Down Expand Up @@ -71,6 +72,10 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
c.Example = oneDriveServiceCommandCreateExamples

flags.AddUserFlag(c)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)

flags.AddFailFastFlag(c)
flags.AddDisableIncrementalsFlag(c)

Expand All @@ -79,6 +84,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
fs.SortFlags = false

flags.AddBackupIDFlag(c, false)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
addFailedItemsFN(c)
addSkippedItemsFN(c)
addRecoveredErrorsFN(c)
Expand All @@ -92,6 +100,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {

flags.AddSkipReduceFlag(c)
flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
flags.AddOneDriveDetailsAndRestoreFlags(c)

case deleteCommand:
Expand All @@ -102,6 +113,9 @@ func addOneDriveCommands(cmd *cobra.Command) *cobra.Command {
c.Example = oneDriveServiceCommandDeleteExamples

flags.AddBackupIDFlag(c, true)
flags.AddCorsoPassphaseFlags(c)
flags.AddAWSCredsFlags(c)
flags.AddAzureCredsFlags(c)
}

return c
Expand Down Expand Up @@ -134,7 +148,7 @@ func createOneDriveCmd(cmd *cobra.Command, args []string) error {
return err
}

r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx)
r, acct, err := utils.AccountConnectAndWriteRepoConfig(ctx, repo.S3Overrides())
if err != nil {
return Only(ctx, err)
}
Expand Down Expand Up @@ -220,7 +234,7 @@ func detailsOneDriveCmd(cmd *cobra.Command, args []string) error {
ctx := cmd.Context()
opts := utils.MakeOneDriveOpts(cmd)

r, _, _, err := utils.GetAccountAndConnect(ctx)
r, _, _, err := utils.GetAccountAndConnect(ctx, repo.S3Overrides())
if err != nil {
return Only(ctx, err)
}
Expand Down
Loading

0 comments on commit 8c66116

Please sign in to comment.