Skip to content

Commit

Permalink
add docs, cleanup and lint
Browse files Browse the repository at this point in the history
  • Loading branch information
@kivi
kivi authored and Vikas committed Feb 17, 2025
1 parent 2a6fa95 commit 1ec4022
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 160 deletions.
7 changes: 7 additions & 0 deletions docs/content/docs.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2466,6 +2466,13 @@ Using this flag on a sync operation without also using `--update` would cause
all files modified at any time other than the last upload time to be uploaded
again, which is probably not what you want.

### --use-ssh-config ###

When set, then settings from ~/.ssh/config will also be considered for the
sftp backend. Any setting in ~/.ssh/config will have precedence over the
rclone config file.


### -v, -vv, --verbose ###

With `-v` rclone will tell you about each file that is transferred and
Expand Down
8 changes: 4 additions & 4 deletions fs/config/configflags/configflags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ var (
downloadHeaders []string
headers []string
metadataSet []string
useSshConfig bool
useSSHConfig bool
)

// AddFlags adds the non filing system specific flags to the command
Expand All @@ -61,7 +61,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions", "Networking")
flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading", "Metadata")
flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21", "Networking")
flags.BoolVarP(flagSet, &useSshConfig, "use-ssh-config", "", false, "Use ~/.ssh/config file for sftp/ssh connections", "Config")
flags.BoolVarP(flagSet, &useSSHConfig, "use-ssh-config", "", false, "Use ~/.ssh/config file for sftp/ssh connections", "Config")
}

// ParseHeaders converts the strings passed in via the header flags into HTTPOptions
Expand Down Expand Up @@ -203,8 +203,8 @@ func SetFlags(ci *fs.ConfigInfo) {
}

// Process --use-ssh-config
if useSshConfig {
err := sshconfig.LoadSshConfigIntoEnv()
if useSSHConfig {
err := sshconfig.LoadSSHConfigIntoEnv()
if err != nil {
fs.Fatalf(nil, "--use-ssh-config: Failed loading ssh config: %v", err)
}
Expand Down
113 changes: 52 additions & 61 deletions lib/sshconfig/use_sshconfig.go
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
// Package sshconfig functions to convert ssh config file to rclone config and to add to temp env vars
package sshconfig

import (
"fmt"
"github.com/kevinburke/ssh_config"
"io"
"os"
"path/filepath"
"strings"

"github.com/kevinburke/ssh_config"
)

// keyMapping defines the mapping between SSH configuration keys and Rclone configuration keys.
var keyMapping = map[string]string{
"identityfile": "key_file",
"pubkeyfile": "pubkey_file",
Expand All @@ -18,98 +21,86 @@ var keyMapping = map[string]string{
"password": "pass",
}

func ConvertToIniKey(customKey string) string {
if iniKey, found := keyMapping[strings.ToLower(customKey)]; found {
return iniKey
// sshConfig represents the SSH configuration structure
type sshConfig map[string]map[string]string

// LoadSSHConfigIntoEnv loads SSH configuration into environment variables by mapping SSH settings to
// rclone configuration and setting the environment accordingly.
// Returns an error if any step fails during the process.
// Note: type=sftp is added for each host (section), also key_use_agent=true is set, when if key_file was given.
func LoadSSHConfigIntoEnv() error {
path := filepath.Join(os.Getenv("HOME"), ".ssh", "config")
f, err := os.Open(path) // returning file handler, so caller needs close

if err != nil {
return fmt.Errorf("error opening ssh config file: %w", err)
}

return customKey
c, err := mapSSHToRcloneConfig(f)
if err != nil {
return fmt.Errorf("error mapping ssh config to rclone config: %w", err)
}

if err := EnvLoadSSHConfig(c); err != nil {
return fmt.Errorf("error setting Env with ssh config: %v", err)
}

return nil
}

type SshConfig map[string]map[string]string
// EnvLoadSSHConfig sets the environment variables based on the Ssh configuration.
func EnvLoadSSHConfig(sshCfg sshConfig) error {
for sectionName, section := range sshCfg {

func LoadSshConfigIntoEnv() error {
if c, err := LoadSshConfig(); err != nil {
return fmt.Errorf("error loading ssh config: %v", err)
} else {
if err := EnvLoadSshConfig(*c); err != nil {
return fmt.Errorf("error setting Env with ssh config: %v", err)
for key, value := range section {
s := fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(sectionName), strings.ToUpper(convertToIniKey(key)))
if err := os.Setenv(s, value); err != nil {
return err
}
}
}
return nil
}

func LoadSshConfig() (*SshConfig, error) {
path := filepath.Join(os.Getenv("HOME"), ".ssh", "config")
f, err := os.Open(path) // returning file handler, so caller needs close

if err != nil {
return nil, fmt.Errorf("error opening ssh config file: %w", err)
// convertToIniKey converts custom SSH configuration keys to Rclone configuration keys using keyMapping.
func convertToIniKey(customKey string) string {
if iniKey, found := keyMapping[strings.ToLower(customKey)]; found {
return iniKey
}
return MapSshToRcloneConfig(f)

return customKey
}

func MapSshToRcloneConfig(r io.Reader) (*SshConfig, error) {
// mapSSHToRcloneConfig maps Ssh configuration to rclone configuration.
func mapSSHToRcloneConfig(r io.Reader) (sshConfig, error) {
cfg, err := ssh_config.Decode(r)
if err != nil {
return nil, fmt.Errorf("error deocing ssh config file: %w", err)
}
sections := SshConfig(map[string]map[string]string{})
sections := sshConfig(map[string]map[string]string{})

for _, host := range cfg.Hosts {
pattern := host.Patterns[0].String()
sections[pattern] = make(map[string]string)

// ssh configs are always type sftp
sections[pattern]["type"] = "sftp"

for _, node := range host.Nodes {
keyval := strings.Fields(strings.TrimSpace(node.String()))
if len(keyval) > 1 {
key := strings.ToLower(strings.TrimSpace(keyval[0]))
value := strings.TrimSpace(strings.Join(keyval[1:], " "))

//oldValue, containsKey := sections[pattern][key]
//if containsKey {
// sections[pattern][key] = oldValue + "," + value
//}

sections[pattern][key] = value
sections[pattern][convertToIniKey(key)] = value
}
}

//keyFile, err := cfg.GetAll(pattern, "identityfile")
//fmt.Println("Identi", pattern)
//if err != nil {
// fmt.Println("Error", err)
//} else {
// if len(keyFile) == 2 {
// //keyFile = []string{keyFile[1], keyFile[0]}
// }
// sections[pattern]["identityfile"] = strings.Join(keyFile, ",")
//}
}
return &sections, nil
}

func EnvLoadSshConfig(sshCfg SshConfig) error {
for sectionName, section := range sshCfg {
//fmt.Println(sectionName, section)
// check if config already exist
//typeValue, _ := s.gc.GetValue(sectionName, "type")

_, ok := section["identityfile"]
// add missing key_use_agent if there is identityfile or here mapped key_file
_, ok := sections[pattern]["key_file"]
if ok {
//if _, ok := section["pubkey_file"]; !ok {
// section["pubkey_file"] = keyFile + ".pub"
//}
section["key_use_agent"] = "true"
}

section["type"] = "sftp"
for key, value := range section {
s := fmt.Sprintf("RCLONE_CONFIG_%s_%s", strings.ToUpper(sectionName), strings.ToUpper(ConvertToIniKey(key)))
if err := os.Setenv(s, value); err != nil {
return err
}
sections[pattern]["key_use_agent"] = "true"
}
}
return nil
return sections, nil
}
106 changes: 11 additions & 95 deletions lib/sshconfig/use_sshconfig_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package sshconfig

import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

var sshConfigData = `Host one
Expand Down Expand Up @@ -32,7 +33,7 @@ func TestConvertToIniKey(t *testing.T) {
}

for customKey, expectedIniKey := range tests {
result := ConvertToIniKey(customKey)
result := convertToIniKey(customKey)
if result != expectedIniKey {
t.Errorf("expected: %s, got: %s", expectedIniKey, result)
}
Expand All @@ -41,100 +42,15 @@ func TestConvertToIniKey(t *testing.T) {

func TestMapSshToRcloneConfig(t *testing.T) {
r := strings.NewReader(sshConfigData)
result, err := MapSshToRcloneConfig(r)
c, err := mapSSHToRcloneConfig(r)

require.NoError(t, err)
c := *result

assert.Equal(t, "127.1.1.20", c["one"]["hostname"])
assert.Equal(t, "~/.ssh/id_ed123", c["one"]["identityfile"])
assert.Equal(t, "sftp", c["one"]["type"])
assert.Equal(t, "sftp", c["tworclone"]["type"])
assert.Equal(t, "127.1.1.20", c["one"]["host"])
assert.Equal(t, "~/.ssh/id_ed123", c["one"]["key_file"])
assert.Equal(t, "true", c["one"]["key_use_agent"])
assert.Equal(t, "localhost:8090 127.0.0.1:8190", c["tworclone"]["localforward"])
assert.Empty(t, c["towrclone"]["key_use_agent"])
}

//func TestSshConfigFile(t *testing.T) {
// defer setConfigFile(t, configData)()
// data := &Storage{}
//
// require.NoError(t, data.Load())
//
// t.Run("Read", func(t *testing.T) {
// t.Run("HasSection", func(t *testing.T) {
// assert.True(t, data.HasSection("one"))
// assert.False(t, data.HasSection("missing"))
// })
// t.Run("GetSectionList", func(t *testing.T) {
// assert.Equal(t, []string{
// "one",
// "two",
// "three",
// }, data.GetSectionList())
// })
// t.Run("GetKeyList", func(t *testing.T) {
// assert.Equal(t, []string{
// "type",
// "fruit",
// "topping",
// }, data.GetKeyList("two"))
// assert.Equal(t, []string(nil), data.GetKeyList("unicorn"))
// })
// t.Run("GetValue", func(t *testing.T) {
// value, ok := data.GetValue("one", "type")
// assert.True(t, ok)
// assert.Equal(t, "number1", value)
// value, ok = data.GetValue("three", "fruit")
// assert.True(t, ok)
// assert.Equal(t, "banana", value)
// value, ok = data.GetValue("one", "typeX")
// assert.False(t, ok)
// assert.Equal(t, "", value)
// value, ok = data.GetValue("threeX", "fruit")
// assert.False(t, ok)
// assert.Equal(t, "", value)
// })
// })
//}

//func TestSshConfig(t *testing.T) {
// defer setConfigFile(t, configData)()
// s := &Storage{}
//
// require.NoError(t, s.Load())
//
// host1 := "host1"
//
// sshConfig := useessshconfig.SshConfig{}
// sshConfig[host1] = map[string]string{
// "identityfile": "~/.ssh/id_rsa",
// "hostname": "example.com",
// "user": "user-1",
// "port": "22",
// }
// sshConfig["host2"] = map[string]string{
// "hostname": "anotherexample.com",
// "user": "root",
// "port": "2222",
// }
//
// err := s.MergeSshConfig(sshConfig)
// require.NoError(t, err)
// assert.True(t, s.HasSection("one"))
//
// assert.True(t, s.HasSection(host1))
//
// value, ok := s.GetValue(host1, "type")
// assert.True(t, ok)
// assert.Equal(t, "sftp", value)
// value, ok = s.GetValue(host1, "user")
// assert.True(t, ok)
// assert.Equal(t, "user-1", value)
// value, ok = s.GetValue(host1, "host")
// assert.True(t, ok)
// assert.Equal(t, "example.com", value)
// value, ok = s.GetValue(host1, "key_file")
// assert.Equal(t, "~/.ssh/id_rsa", value)
// //value, ok = s.GetValue(host1, "pubkey_file")
// //assert.Equal(t, "~/.ssh/id_rsa.pub", value)
// value, ok = s.GetValue(host1, "key_use_agent")
// assert.Equal(t, "true", value)
//
//}

0 comments on commit 1ec4022

Please sign in to comment.