Skip to content

Commit

Permalink
move stage diffing logic to workspace
Browse files Browse the repository at this point in the history
  • Loading branch information
@djelusic
djelusic committed Oct 21, 2021
1 parent 2ab365b commit 2c39a8e
Show file tree
Hide file tree
Showing 5 changed files with 236 additions and 184 deletions.
82 changes: 22 additions & 60 deletions cli/cmd/deploy/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,26 +25,17 @@ type Args struct {
}

type Cmd struct {
awsClient *aws.AWS
functionsDiff resourceDiff
publicDiff resourceDiff
configChanged bool
awsClient *aws.AWS
diff *workspace.StageDiff

store *workspace.FileStore
stage *workspace.Stage
path string

functionsForUpload []uploadData
buildDuration time.Duration
uploadDuration time.Duration
uploadBytes int64
updateDuration time.Duration
}

type uploadData struct {
name string
binaryPath string
s3Key string
buildDuration time.Duration
uploadDuration time.Duration
uploadBytes int64
updateDuration time.Duration
}

func New(a Args) (*Cmd, error) {
Expand Down Expand Up @@ -90,21 +81,20 @@ func (d *Cmd) deploy() error {
return log.Wrap(err)
}
ui.Info("")
d.applyConfiguration()
if !d.HasUpdates() {
ui.Info("No changes - nothing to deploy")
return nil
}
if len(d.functionsForUpload) > 0 {
if len(d.diff.UpdatedFunctions()) > 0 {
ui.Info("==> Uploading...")
if err := d.uploadTimer(func() error { return d.upload() }); err != nil {
if err := d.uploadTimer(func() error { return d.uploadFunctions() }); err != nil {
return log.Wrap(err)
}
ui.Info("")
}

if d.hasFunctionUpdates() {
if d.infrastructureChanged() {
if d.diff.HasFunctionUpdates() {
if d.diff.InfrastructureChanged() {
ui.Info("==> Setting up AWS infrastructure...")
} else {
ui.Info("==> Updating...")
Expand All @@ -116,7 +106,7 @@ func (d *Cmd) deploy() error {
ui.Info("")
}

if d.publicDiff.hasUpdates() {
if d.diff.HasPublicUpdates() {
if err := d.setAWSclient(); err != nil {
return log.Wrap(err)
}
Expand All @@ -139,38 +129,24 @@ func (d *Cmd) deploy() error {
return nil
}

func (d *Cmd) applyConfiguration() {
d.configChanged = d.stage.ApplyConfiguration()
}

func (d *Cmd) HasUpdates() bool {
return d.functionsDiff.hasUpdates() ||
d.publicDiff.hasUpdates() ||
d.configChanged
}

func (d *Cmd) hasFunctionUpdates() bool {
return d.functionsDiff.hasUpdates() ||
d.configChanged
}

func (d *Cmd) infrastructureChanged() bool {
return d.functionsDiff.infrastructureChanged() ||
d.publicDiff.infrastructureChanged() ||
d.configChanged
return d.diff.HasUpdates()
}

func (d *Cmd) buildAndFindDiffs() error {
fd, err := d.functionUpdates()
lf, err := d.localFunctions()
if err != nil {
return log.Wrap(err)
}
d.functionsDiff = fd
pd, err := d.publicSiteUpdates()
lp, err := d.localPublicSites()
if err != nil {
return log.Wrap(err)
}
d.publicDiff = pd
diff, err := d.stage.ApplyLocalChanges(lf, lp)
if err != nil {
return log.Wrap(err)
}
d.diff = diff
return nil
}

Expand All @@ -183,7 +159,7 @@ func (d *Cmd) callBackend() error {
if err := backend.Call(DeployHTTPMethod, d.backendRequest(), &rsp); err != nil {
return log.Wrap(err)
}
if d.infrastructureChanged() {
if d.diff.InfrastructureChanged() {
d.updateStage(rsp)
}
return nil
Expand All @@ -200,14 +176,14 @@ func (d *Cmd) backendRequest() dto.DeployRequest {
for _, f := range d.stage.Functions {
df := d.workspaceFunction2dto(*f)
fns = append(fns, df)
for _, fn := range d.functionsDiff.updated {
for _, fn := range d.diff.UpdatedFunctions() {
if fn == f.Name {
fnsu = append(fnsu, df)
}
}
}
req.FunctionsForUpdate = fnsu
if d.infrastructureChanged() {
if d.diff.InfrastructureChanged() {
req.StageTemplate = &dto.StageTemplate{
Project: d.stage.Project().Name,
Bucket: d.stage.Account().Bucket,
Expand Down Expand Up @@ -242,20 +218,6 @@ func (d *Cmd) updateStage(rsp dto.DeployResponse) {
d.stage.SetPublicBucket(rsp.PublicBucket)
}

type resourceDiff struct {
added []string
removed []string
updated []string
}

func (d *resourceDiff) infrastructureChanged() bool {
return len(d.added) > 0 || len(d.removed) > 0
}

func (d *resourceDiff) hasUpdates() bool {
return d.infrastructureChanged() || len(d.updated) > 0
}

func (d *Cmd) buildTimer(cb func() error) error {
return timer(&d.buildDuration, cb)
}
Expand Down
113 changes: 27 additions & 86 deletions cli/cmd/deploy/functions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
Expand All @@ -20,31 +19,29 @@ import (
"github.com/mantil-io/mantil/workspace"
)

func (d *Cmd) functionUpdates() (resourceDiff, error) {
var diff resourceDiff
localFuncs, err := d.localDirs(FunctionsDir)
func (d *Cmd) localFunctions() ([]workspace.Resource, error) {
localFuncNames, err := d.localDirs(FunctionsDir)
if err != nil {
return diff, err
return nil, log.Wrap(err)
}
var stageFuncs []string
for _, f := range d.stage.Functions {
stageFuncs = append(stageFuncs, f.Name)
}
diff.added = diffArrays(localFuncs, stageFuncs)
if err := d.stage.AddFunctions(diff.added); err != nil {
var rerr *workspace.ErrReservedName
if errors.As(err, &rerr) {
return diff, log.WithUserMessage(rerr, "\"%s\" is reserved name", rerr.Name)
var localFuncs []workspace.Resource
for _, n := range localFuncNames {
ui.Info(n)
funcDir := path.Join(d.path, FunctionsDir, n)
if err := d.buildTimer(func() error { return d.buildFunction(BinaryName, funcDir) }); err != nil {
return nil, log.Wrap(err)
}
var verr workspace.ValidationError
if errors.As(err, &verr) {
return diff, log.WithUserMessage(err, verr.UserMessage())
binaryPath := path.Join(funcDir, BinaryName)
hash, err := fileHash(binaryPath)
if err != nil {
return nil, log.WithUserMessage(err, fmt.Sprintf("Hashing %s failed", binaryPath))
}
localFuncs = append(localFuncs, workspace.Resource{
Name: n,
Hash: hash,
})
}
diff.removed = diffArrays(stageFuncs, localFuncs)
d.stage.RemoveFunctions(diff.removed)
diff.updated, err = d.prepareFunctionsForDeploy()
return diff, err
return localFuncs, nil
}

func (d *Cmd) localDirs(path string) ([]string, error) {
Expand All @@ -65,36 +62,6 @@ func (d *Cmd) localDirs(path string) ([]string, error) {
return dirs, nil
}

// prepareFunctionsForDeploy goes through stage functions, checks which ones have changed
// and uploads new version to s3 if necessary
func (d *Cmd) prepareFunctionsForDeploy() ([]string, error) {
var updatedFunctions []string
d.functionsForUpload = make([]uploadData, 0)
for _, f := range d.stage.Functions {
ui.Info("%s", f.Name)
funcDir := path.Join(d.path, FunctionsDir, f.Name)
if err := d.buildTimer(func() error { return d.buildFunction(BinaryName, funcDir) }); err != nil {
return nil, log.Wrap(err)
}

binaryPath := path.Join(funcDir, BinaryName)
hash, err := fileHash(binaryPath)
if err != nil {
return nil, log.WithUserMessage(err, fmt.Sprintf("Hashing %s failed", binaryPath))
}
if hash != f.Hash {
updatedFunctions = append(updatedFunctions, f.Name)
f.SetHash(hash)
d.functionsForUpload = append(d.functionsForUpload, uploadData{
name: f.Name,
s3Key: f.S3Key,
binaryPath: binaryPath,
})
}
}
return updatedFunctions, nil
}

func (d *Cmd) buildFunction(name, funcDir string) error {
bl := shell.NewBufferedLogger()
err := shell.Exec(shell.ExecOptions{
Expand All @@ -110,10 +77,15 @@ func (d *Cmd) buildFunction(name, funcDir string) error {
return err
}

func (d *Cmd) upload() error {
for _, f := range d.functionsForUpload {
ui.Info(f.name)
if err := d.uploadBinaryToS3(f.s3Key, f.binaryPath); err != nil {
func (d *Cmd) uploadFunctions() error {
for _, n := range d.diff.UpdatedFunctions() {
f := d.stage.FindFunction(n)
if f == nil {
continue
}
path := filepath.Join(d.path, FunctionsDir, n, BinaryName)
ui.Info(n)
if err := d.uploadBinaryToS3(f.S3Key, path); err != nil {
return log.WithUserMessage(err, "Failed to upload file to s3")
}
}
Expand Down Expand Up @@ -185,34 +157,3 @@ func createZipForFile(path, name string) ([]byte, error) {

return buf.Bytes(), nil
}

// returns a1 - a2
func diffArrays(a1 []string, a2 []string) []string {
m := make(map[string]bool)
for _, e := range a2 {
m[e] = true
}
var diff []string
for _, e := range a1 {
if m[e] {
continue
}
diff = append(diff, e)
}
return diff
}

// returns a1 n a2
func intersectArrays(a1 []string, a2 []string) []string {
m := make(map[string]bool)
for _, e := range a1 {
m[e] = true
}
var intersection []string
for _, e := range a2 {
if m[e] {
intersection = append(intersection, e)
}
}
return intersection
}
49 changes: 11 additions & 38 deletions cli/cmd/deploy/public.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,54 +11,27 @@ import (
"golang.org/x/mod/sumdb/dirhash"
)

func (d *Cmd) publicSiteUpdates() (resourceDiff, error) {
var diff resourceDiff
localSites, err := d.localDirs(PublicDir)
func (d *Cmd) localPublicSites() ([]workspace.Resource, error) {
localPublicNames, err := d.localDirs(PublicDir)
if err != nil {
return diff, err
return nil, log.Wrap(err)
}
var stageSites []string
for _, s := range d.stage.Public.Sites {
stageSites = append(stageSites, s.Name)
}
diff.added = diffArrays(localSites, stageSites)
for _, a := range diff.added {
hash, err := d.publicSiteHash(a)
var localPublic []workspace.Resource
for _, n := range localPublicNames {
hash, err := d.publicSiteHash(n)
if err != nil {
return diff, err
return nil, log.Wrap(err)
}
d.stage.Public.Sites = append(d.stage.Public.Sites, &workspace.PublicSite{
Name: a,
localPublic = append(localPublic, workspace.Resource{
Name: n,
Hash: hash,
})
diff.updated = append(diff.updated, a)
}
diff.removed = diffArrays(stageSites, localSites)
for _, r := range diff.removed {
for idx, s := range d.stage.Public.Sites {
if s.Name == r {
d.stage.Public.Sites = append(d.stage.Public.Sites[:idx], d.stage.Public.Sites[idx+1:]...)
}
}
}
intersection := intersectArrays(localSites, stageSites)
for _, i := range intersection {
hash, err := d.publicSiteHash(i)
if err != nil {
return diff, err
}
for _, s := range d.stage.Public.Sites {
if s.Name == i && hash != s.Hash {
s.Hash = hash
diff.updated = append(diff.updated, i)
}
}
}
return diff, nil
return localPublic, err
}

func (d *Cmd) updatePublicSiteContent() error {
for _, u := range d.publicDiff.updated {
for _, u := range d.diff.UpdatedPublicSites() {
var site *workspace.PublicSite
for _, s := range d.stage.Public.Sites {
if s.Name == u {
Expand Down
Loading

0 comments on commit 2c39a8e

Please sign in to comment.