Merge pull request #5 from d2r2/feat/release-v0.3.3

Changes to the release 0.3.3

.gitignore

@@ -1,4 +1,5 @@
 *.log
+*.pprof
 gorsync
 go-rsync
 packaging/fpm_packages/packages/*

backup/abstract.go

@@ -6,11 +6,18 @@ import (
 	"github.com/d2r2/go-rsync/core"
 )
 
+// Notifier interface is used as a contract to provide
+// event-driven mechanism, to map backup process steps with,
+// for instance, user interface.
 type Notifier interface {
+
+	// Pair of calls to report about 1st pass start and completion.
 	NotifyPlanStage_NodeStructureStartInquiry(sourceID int,
 		sourceRsync string) error
 	NotifyPlanStage_NodeStructureDoneInquiry(sourceID int,
 		sourceRsync string, dir *core.Dir) error
+
+	// Pair of calls to report about 2nd pass start and completion.
 	NotifyBackupStage_FolderStartBackup(rootDest string,
 		paths core.SrcDstPath, backupType core.FolderBackupType,
 		leftToBackup core.FolderSize,

backup/common.go

@@ -6,6 +6,9 @@ import (
 	"github.com/d2r2/go-logger"
 )
 
+// You can manage verbosity of log output
+// in the package by changing last parameter value
+// (comment/uncomment corresponding lines).
 var LocalLog = logger.NewPackageLogger("backup",
 	// logger.DebugLevel,
 	logger.InfoLevel,

backup/config.go

@@ -3,16 +3,17 @@ package backup
 import (
 	"fmt"
 
-	"github.com/BurntSushi/toml"
 	"github.com/d2r2/go-rsync/core"
 	"github.com/d2r2/go-rsync/rsync"
 )
 
+/*
 type IRsyncConfigurable interface {
 	GetRsyncParams(addExtraParams []string) []string
 }
+*/
 
-// Node contain information about single rsync source backup.
+// Node contain information about single RSYNC source backup.
 type Node struct {
 	Module  Module
 	RootDir *core.Dir
@@ -26,6 +27,8 @@ type Plan struct {
 	BackupSize core.FolderSize
 }
 
+// GetModules returns all RSYNC source/destination blocks
+// defined in single (specific) backup profile.
 func (v *Plan) GetModules() []Module {
 	modules := []Module{}
 	for _, item := range v.Nodes {
@@ -46,25 +49,20 @@ type Config struct {
 	NumberOfPreviousBackupToUse        *int   `toml:"number_of_previous_backup_to_use"`
 	EnableLowLevelLogForRsync          *bool  `toml:"enable_low_level_log_rsync"`
 	EnableIntensiveLowLevelLogForRsync *bool  `toml:"enable_intensive_low_level_log_rsync"`
-	// rsync --compress
-	RsyncCompressFileTransfer *bool `toml:"rsync_compress_file_transfer"`
-	// rsync --links
-	RsyncRecreateSymlinks *bool `toml:"rsync_recreate_symlinks"`
-	// rsync --perms
-	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"`
-	// rsync --group
-	RsyncTransferSourceGroup *bool `toml:"rsync_transfer_source_group"`
-	// rsync --owner
-	RsyncTransferSourceOwner *bool `toml:"rsync_transfer_source_owner"`
-	// rsync --devices
-	RsyncTransferDeviceFiles *bool `toml:"rsync_transfer_device_files"`
-	// rsync --specials
-	RsyncTransferSpecialFiles *bool `toml:"rsync_transfer_special_files"`
+
+	RsyncTransferSourceOwner       *bool `toml:"rsync_transfer_source_owner"`       // rsync --owner
+	RsyncTransferSourceGroup       *bool `toml:"rsync_transfer_source_group"`       // rsync --group
+	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"` // rsync --perms
+	RsyncRecreateSymlinks          *bool `toml:"rsync_recreate_symlinks"`           // rsync --links
+	RsyncTransferDeviceFiles       *bool `toml:"rsync_transfer_device_files"`       // rsync --devices
+	RsyncTransferSpecialFiles      *bool `toml:"rsync_transfer_special_files"`      // rsync --specials
+	RsyncCompressFileTransfer      *bool `toml:"rsync_compress_file_transfer"`      // rsync --compress
 
 	// BackupNode list contain all RSYNC sources to backup in one session.
 	//Modules []Module `toml:"backup_module"`
 }
 
+/*
 func NewConfig(filePath string) (*Config, error) {
 	var config Config
 	if _, err := toml.DecodeFile(filePath, &config); err != nil {
@@ -73,52 +71,25 @@ func NewConfig(filePath string) (*Config, error) {
 	LocalLog.Debug(f("%+v", config))
 	return &config, nil
 }
-
-// Prepare RSYNC CLI parameters to run console RSYNC process.
-func (conf *Config) GetRsyncParams(addExtraParams []string) []string {
-	var params []string
-	if conf.RsyncCompressFileTransfer != nil && *conf.RsyncCompressFileTransfer {
-		params = append(params, "--compress")
-	}
-	if conf.RsyncTransferSourceOwner != nil && *conf.RsyncTransferSourceOwner {
-		params = append(params, "--owner")
-	}
-	if conf.RsyncTransferSourceGroup != nil && *conf.RsyncTransferSourceGroup {
-		params = append(params, "--group")
-	}
-	if conf.RsyncTransferSourcePermissions != nil && *conf.RsyncTransferSourcePermissions {
-		params = append(params, "--perms")
-	}
-	if conf.RsyncRecreateSymlinks != nil && *conf.RsyncRecreateSymlinks {
-		params = append(params, "--links")
-	}
-	if conf.RsyncTransferDeviceFiles != nil && *conf.RsyncTransferDeviceFiles {
-		params = append(params, "--devices")
-	}
-	if conf.RsyncTransferSpecialFiles != nil && *conf.RsyncTransferSpecialFiles {
-		params = append(params, "--specials")
-	}
-	params = append(params, addExtraParams...)
-	return params
-}
+*/
 
 func (conf *Config) usePreviousBackupEnabled() bool {
-	var usePreviousBackup bool = true
+	var usePreviousBackup = true
 	if conf.UsePreviousBackup != nil {
 		usePreviousBackup = *conf.UsePreviousBackup
 	}
 	return usePreviousBackup
 }
 
 func (conf *Config) numberOfPreviousBackupToUse() int {
-	var numberOfPreviousBackupToUse int = 1
+	var numberOfPreviousBackupToUse = 1
 	if conf.NumberOfPreviousBackupToUse != nil {
 		numberOfPreviousBackupToUse = *conf.NumberOfPreviousBackupToUse
 	}
 	return numberOfPreviousBackupToUse
 }
 
-func (conf *Config) getRsyncSettings() *rsync.Logging {
+func (conf *Config) getRsyncLoggingSettings() *rsync.Logging {
 	logging := &rsync.Logging{}
 	if conf.EnableLowLevelLogForRsync != nil {
 		logging.EnableLog = *conf.EnableLowLevelLogForRsync
@@ -140,19 +111,64 @@ func (conf *Config) getBackupBlockSizeSettings() *backupBlockSizeSettings {
 	return blockSize
 }
 
+// Module signify RSYNC source/destination block, with
+// source/destination URLs and other auxiliary options.
+// Used as configuration data in the backup session code.
 type Module struct {
-	SourceRsync          string  `toml:"src_rsync"`
-	DestSubPath          string  `toml:"dst_subpath"`
+	SourceRsync string `toml:"src_rsync"`
+	DestSubPath string `toml:"dst_subpath"`
+
 	ChangeFilePermission string  `toml:"rsync_change_file_permission"`
 	AuthPassword         *string `toml:"module_auth_password"`
+
+	RsyncTransferSourceOwner       *bool `toml:"rsync_transfer_source_owner"`       // rsync --owner
+	RsyncTransferSourceGroup       *bool `toml:"rsync_transfer_source_group"`       // rsync --group
+	RsyncTransferSourcePermissions *bool `toml:"rsync_transfer_source_permissions"` // rsync --perms
+	RsyncRecreateSymlinks          *bool `toml:"rsync_recreate_symlinks"`           // rsync --links
+	RsyncTransferDeviceFiles       *bool `toml:"rsync_transfer_device_files"`       // rsync --devices
+	RsyncTransferSpecialFiles      *bool `toml:"rsync_transfer_special_files"`      // rsync --specials
 }
 
-// Prepare RSYNC CLI parameters to run console RSYNC process.
-func (module *Module) GetRsyncParams(addExtraParams []string) []string {
+// GetRsyncParams prepare RSYNC CLI parameters to run console RSYNC process.
+func GetRsyncParams(conf *Config, module *Module, addExtraParams []string) []string {
 	var params []string
+	if module.RsyncTransferSourceOwner != nil && *module.RsyncTransferSourceOwner ||
+		module.RsyncTransferSourceOwner == nil && conf.RsyncTransferSourceOwner != nil &&
+			*conf.RsyncTransferSourceOwner {
+		params = append(params, "--owner")
+	}
+	if module.RsyncTransferSourceGroup != nil && *module.RsyncTransferSourceGroup ||
+		module.RsyncTransferSourceGroup == nil && conf.RsyncTransferSourceGroup != nil &&
+			*conf.RsyncTransferSourceGroup {
+		params = append(params, "--group")
+	}
+	if module.RsyncTransferSourcePermissions != nil && *module.RsyncTransferSourcePermissions ||
+		module.RsyncTransferSourcePermissions == nil && conf.RsyncTransferSourcePermissions != nil &&
+			*conf.RsyncTransferSourcePermissions {
+		params = append(params, "--perms")
+	}
+	if module.RsyncRecreateSymlinks != nil && *module.RsyncRecreateSymlinks ||
+		module.RsyncRecreateSymlinks == nil && conf.RsyncRecreateSymlinks != nil &&
+			*conf.RsyncRecreateSymlinks {
+		params = append(params, "--links")
+	}
+	if module.RsyncTransferDeviceFiles != nil && *module.RsyncTransferDeviceFiles ||
+		module.RsyncTransferDeviceFiles == nil && conf.RsyncTransferDeviceFiles != nil &&
+			*conf.RsyncTransferDeviceFiles {
+		params = append(params, "--devices")
+	}
+	if module.RsyncTransferSpecialFiles != nil && *module.RsyncTransferSpecialFiles ||
+		module.RsyncTransferSpecialFiles == nil && conf.RsyncTransferSpecialFiles != nil &&
+			*conf.RsyncTransferSpecialFiles {
+		params = append(params, "--specials")
+	}
+	if conf.RsyncCompressFileTransfer != nil && *conf.RsyncCompressFileTransfer {
+		params = append(params, "--compress")
+	}
 	if module.ChangeFilePermission != "" {
 		params = append(params, fmt.Sprintf("--chmod=%s", module.ChangeFilePermission))
 	}
+
 	params = append(params, addExtraParams...)
 	return params
 }

backup/deduplication.go

@@ -266,7 +266,7 @@ func CreateMetadataSignatureFile(modules []Module, destPath string) error {
 
 // EncodeSignatures encode NodeSignatures object to self-describing binary format.
 func EncodeSignatures(signs NodeSignatures) (string, error) {
-	b := bytes.Buffer{}
+	var b bytes.Buffer
 	e := gob.NewEncoder(&b)
 	err := e.Encode(signs)
 	if err != nil {

backup/heuristic.go

@@ -117,6 +117,7 @@ func findDownNonMeasuredDirByWeight(dir *core.Dir, weight int) *core.Dir {
 	}
 }
 
+/*
 func findUpNonMeasuredDirByWeight(dir *core.Dir, weight int) *core.Dir {
 	item := dir
 	parent := item.Parent
@@ -133,6 +134,7 @@ func findUpNonMeasuredDirByWeight(dir *core.Dir, weight int) *core.Dir {
 		parent = item.Parent
 	}
 }
+*/
 
 func MeasureDir(ctx context.Context, password *string, dir *core.Dir, retryCount *int, log *rsync.Logging,
 	blockSize *backupBlockSizeSettings) (int, error) {

backup/process.go

@@ -21,8 +21,8 @@ var (
 	SingleSplitLogLine string = strings.Repeat("-", 100)
 )
 
-// Perform 1st stage (plan stage) to measure RSYNC source volume to backup
-// and find optimal traverse path of source directory tree.
+// BuildBackupPlan perform 1st stage (plan stage) to measure RSYNC source volume
+// to backup and find optimal traverse path of source directory tree.
 // Use plan built in 1st stage later in 2nd stage.
 func BuildBackupPlan(ctx context.Context, lg logger.PackageLog, config *Config,
 	modules []Module, notifier Notifier) (*Plan, *Progress, error) {
@@ -46,7 +46,7 @@ func BuildBackupPlan(ctx context.Context, lg logger.PackageLog, config *Config,
 
 	// create specific RSYNC log file (might be activated in
 	// backup session preference for debug purpose)
-	rsyncLog := config.getRsyncSettings()
+	rsyncLog := config.getRsyncLoggingSettings()
 	if rsyncLog.EnableLog {
 		log = core.NewProxyLog(nil, "rsync", 5, "2006-01-02T15:04:05",
 			func(line string) error {
@@ -171,11 +171,13 @@ func estimateNode(ctx context.Context, password *string, module Module, progress
 	return dir, &backupSize2, nil
 }
 
-// Perform whole 2nd stage (backup stage) here, than save and report completion to session logs.
-func (this *Plan) RunBackup(progress *Progress, destPath string, errorHookCall rsync.ErrorHookCall) error {
+// RunBackup perform whole 2nd stage (backup stage) here, then save and
+// report about completion to session logs.
+func (plan *Plan) RunBackup(progress *Progress, destPath string,
+	errorHookCall rsync.ErrorHookCall) error {
 
 	// Execute backup stage
-	err := runBackup(this, progress, destPath, errorHookCall)
+	err := runBackup(plan, progress, destPath, errorHookCall)
 	if err != nil {
 		progress.Log.Error(locale.T(MsgLogBackupStageCriticalError,
 			struct{ Error error }{Error: err}))
@@ -277,7 +279,7 @@ func runBackup(plan *Plan, progress *Progress, destPath string, errorHookCall rs
 		sourceID := GenerateSourceID(node.Module.SourceRsync)
 		prevBackups2 := prevBackups.FilterBySourceID(sourceID)
 		// run specific RSYNC source to backup
-		err := runBackupNode(plan, node, plan.Config,
+		err := runBackupNode(plan, node, /*plan.Config,*/
 			progress, destPath2, errorHookCall, prevBackups2)
 		if err != nil {
 			return err
@@ -324,7 +326,7 @@ func runBackup(plan *Plan, progress *Progress, destPath string, errorHookCall rs
 }
 
 // Perform backup of one source defined in backup session preferences.
-func runBackupNode(plan *Plan, node Node, config *Config, progress *Progress, destRootPath string,
+func runBackupNode(plan *Plan, node Node, progress *Progress, destRootPath string,
 	errorHookCall rsync.ErrorHookCall, prevBackups *PrevBackups) error {
 
 	paths := core.SrcDstPath{
@@ -333,7 +335,7 @@ func runBackupNode(plan *Plan, node Node, config *Config, progress *Progress, de
 	}
 
 	progress.Progress = &core.SizeProgress{}
-	err := backupDir(node.RootDir, &node.Module, config,
+	err := backupDir(node.RootDir, &node.Module,
 		plan, progress, paths, errorHookCall, prevBackups.GetDirPaths())
 	return err
 }
@@ -410,10 +412,8 @@ func reportProgress(sessionErr, retryErr error, size core.FolderSize,
 }
 
 // Major function to make all necessary RSYNC calls to execute backup process step by step.
-func backupDir(dir *core.Dir, module *Module, config *Config,
-	plan *Plan, progress *Progress,
-	paths core.SrcDstPath, errorHookCall rsync.ErrorHookCall,
-	prevBackupPaths []string) error {
+func backupDir(dir *core.Dir, module *Module, plan *Plan, progress *Progress,
+	paths core.SrcDstPath, errorHookCall rsync.ErrorHookCall, prevBackupPaths []string) error {
 
 	var err error
 	var backupType core.FolderBackupType
@@ -432,12 +432,11 @@ func backupDir(dir *core.Dir, module *Module, config *Config,
 		}
 		// run backup in "skip mode"
 		options := rsync.NewOptions(rsync.WithDefaultParams(
-			module.GetRsyncParams(plan.Config.GetRsyncParams(defParams)))).
-			AddParams("--delete", "--dirs").
+			GetRsyncParams(plan.Config, module, defParams))).AddParams("--delete", "--dirs").
 			// AddParams("--super").
 			// AddParams("--fake-super").
-			AddParams(f("--include=%s", config.SigFileIgnoreBackup), "--exclude=*").
-			SetRetryCount(config.RsyncRetryCount).
+			AddParams(f("--include=%s", plan.Config.SigFileIgnoreBackup), "--exclude=*").
+			SetRetryCount(plan.Config.RsyncRetryCount).
 			SetAuthPassword(module.AuthPassword).
 			// minimum size for empty signature file
 			SetErrorHook(rsync.NewErrorHook(errorHookCall, core.NewFolderSize(1*core.KB)))
@@ -461,11 +460,10 @@ func backupDir(dir *core.Dir, module *Module, config *Config,
 		}
 		// run full backup including content with recursion
 		options := rsync.NewOptions(rsync.WithDefaultParams(
-			module.GetRsyncParams(plan.Config.GetRsyncParams(defParams)))).
-			AddParams("--delete", "--recursive").
+			GetRsyncParams(plan.Config, module, defParams))).AddParams("--delete", "--recursive").
 			// AddParams("--super").
 			// AddParams("--fake-super").
-			SetRetryCount(config.RsyncRetryCount).
+			SetRetryCount(plan.Config.RsyncRetryCount).
 			SetAuthPassword(module.AuthPassword).
 			SetErrorHook(rsync.NewErrorHook(errorHookCall, *dir.Metrics.FullSize))
 
@@ -495,11 +493,10 @@ func backupDir(dir *core.Dir, module *Module, config *Config,
 		}
 		// run backup only folder content without nested folders (flat mode)
 		options := rsync.NewOptions(rsync.WithDefaultParams(
-			module.GetRsyncParams(plan.Config.GetRsyncParams(defParams)))).
-			AddParams("--delete", "--dirs").
+			GetRsyncParams(plan.Config, module, defParams))).AddParams("--delete", "--dirs").
 			// AddParams("--super").
 			// AddParams("--fake-super").
-			SetRetryCount(config.RsyncRetryCount).
+			SetRetryCount(plan.Config.RsyncRetryCount).
 			SetAuthPassword(module.AuthPassword).
 			SetErrorHook(rsync.NewErrorHook(errorHookCall, *dir.Metrics.Size))
 
@@ -527,7 +524,7 @@ func backupDir(dir *core.Dir, module *Module, config *Config,
 			for i, path := range prevBackupPaths2 {
 				prevBackupPaths2[i] = filepath.Join(path, item.Name)
 			}
-			err = backupDir(item, module, config,
+			err = backupDir(item, module,
 				plan, progress, paths.Join(item.Name), errorHookCall, prevBackupPaths2)
 			if err != nil {
 				return err