diff --git a/avalanche/network.go b/avalanche/network.go index f227383..ed36373 100644 --- a/avalanche/network.go +++ b/avalanche/network.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/ava-labs/avalanche-tooling-sdk-go/utils" + "github.com/ava-labs/avalanchego/genesis" "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/constants" diff --git a/constants/constants.go b/constants/constants.go index adba854..836923c 100644 --- a/constants/constants.go +++ b/constants/constants.go @@ -74,6 +74,7 @@ const ( AvalancheGoGitRepo = "https://github.com/ava-labs/avalanchego" SubnetEVMRepoName = "subnet-evm" + CloudNodeAWMRelayerPath = "/home/ubuntu/.awm-relayer" AWMRelayerInstallDir = "awm-relayer" AWMRelayerConfigFilename = "awm-relayer-config.json" diff --git a/examples/interchain_aws.go b/examples/interchain_aws.go new file mode 100644 index 0000000..393dc96 --- /dev/null +++ b/examples/interchain_aws.go @@ -0,0 +1,243 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package examples + +import ( + "context" + "fmt" + "math/big" + "os" + "time" + + "github.com/ava-labs/avalanche-tooling-sdk-go/avalanche" + awsAPI "github.com/ava-labs/avalanche-tooling-sdk-go/cloud/aws" + "github.com/ava-labs/avalanche-tooling-sdk-go/interchain/relayer" + "github.com/ava-labs/avalanche-tooling-sdk-go/key" + "github.com/ava-labs/avalanche-tooling-sdk-go/node" + "github.com/ava-labs/avalanche-tooling-sdk-go/utils" + "github.com/ava-labs/avalanchego/ids" +) + +func InterchainAWSExample( + network avalanche.Network, + chain1RPC string, + chain1PK string, + chain1SubnetID ids.ID, + chain1BlockchainID ids.ID, + chain2RPC string, + chain2PK string, + chain2SubnetID ids.ID, + chain2BlockchainID ids.ID, +) error { + ctx := context.Background() + + // Get the default cloud parameters for AWS + cp, err := node.GetDefaultCloudParams(ctx, node.AWSCloud) + if err != nil { + panic(err) + } + + securityGroupName := "SECURITY_GROUP_NAME" + // Create a new security group in AWS if you do not currently have one in the selected + // AWS region. + sgID, err := awsAPI.CreateSecurityGroup(ctx, securityGroupName, cp.AWSConfig.AWSProfile, cp.Region) + if err != nil { + panic(err) + } + // Set the security group we are using when creating our Avalanche Nodes + cp.AWSConfig.AWSSecurityGroupName = securityGroupName + cp.AWSConfig.AWSSecurityGroupID = sgID + + keyPairName := "KEY_PAIR_NAME" + sshPrivateKeyPath := utils.ExpandHome("PRIVATE_KEY_FILEPATH") + // Create a new AWS SSH key pair if you do not currently have one in your selected AWS region. + // Note that the created key pair can only be used in the region that it was created in. + // The private key to the created key pair will be stored in the filepath provided in + // sshPrivateKeyPath. + if err := awsAPI.CreateSSHKeyPair(ctx, cp.AWSConfig.AWSProfile, cp.Region, keyPairName, sshPrivateKeyPath); err != nil { + panic(err) + } + // Set the key pair we are using when creating our Avalanche Nodes + cp.AWSConfig.AWSKeyPair = keyPairName + + const ( + awmVersion = "v1.4.0" + ) + + // Create two new Avalanche Validator nodes on Fuji Network on AWS without Elastic IPs + // attached. Once CreateNodes is completed, the validators will begin bootstrapping process + // to Primary Network in Fuji Network. Nodes need to finish bootstrapping process + // before they can validate Avalanche Primary Network / Subnet. + // + // SDK function for nodes to start validating Primary Network / Subnet will be available + // in the next Avalanche Tooling SDK release. + hosts, err := node.CreateNodes(ctx, + &node.NodeParams{ + CloudParams: cp, + Count: 1, + Roles: []node.SupportedRole{node.AWMRelayer}, + Network: avalanche.FujiNetwork(), + AWMRelayerVersion: awmVersion, + UseStaticIP: false, + SSHPrivateKeyPath: sshPrivateKeyPath, + }) + if err != nil { + panic(err) + } + if len(hosts) != 1 { + panic("expected 1 host") + } + awmRelayerHost := hosts[0] // single awm-relayer host + + const ( + sshTimeout = 120 * time.Second + sshCommandTimeout = 10 * time.Second + ) + + // Wait for the host to be ready (only needs to be done once for newly created nodes) + fmt.Println("Waiting for SSH shell") + if err := awmRelayerHost.WaitForSSHShell(sshTimeout); err != nil { + panic(err) + } + + fmt.Println("Deploying Interchain Messenger to AWS") + chain1RelayerKey, err := key.NewSoft() + if err != nil { + return err + } + chain2RelayerKey, err := key.NewSoft() + if err != nil { + return err + } + chain1RegistryAddress, chain1MessengerAddress, chain2RegistryAddress, chain2MessengerAddress, err := SetupICM( + chain1RPC, + chain1PK, + chain2RPC, + chain2PK, + ) + if err != nil { + return err + } + // Get default awm-relayer configuration + relayerConfig, err := awmRelayerHost.GetAMWRelayerConfig() + if err != nil { + panic(err) + } + // Add blockchain chain1 to the relayer config, + // setting it both as source and as destination. + // So the relayer will both listed for new messages in it, + // and send to it new messages from other blockchains. + relayer.AddBlockchainToRelayerConfig( + relayerConfig, + chain1RPC, + "", + chain1SubnetID, + chain1BlockchainID, + chain1RegistryAddress, + chain1MessengerAddress, + chain1RelayerKey.C(), + chain1RelayerKey.PrivKeyHex(), + ) + // Add blockchain chain2 to the relayer config, + // setting it both as source and as destination. + // So the relayer will both listed for new messages in it, + // and send to it new messages from other blockchains. + relayer.AddBlockchainToRelayerConfig( + relayerConfig, + chain2RPC, + "", + chain2SubnetID, + chain2BlockchainID, + chain2RegistryAddress, + chain2MessengerAddress, + chain2RelayerKey.C(), + chain2RelayerKey.PrivKeyHex(), + ) + // Set awm-relayer configuration for the host + if err := awmRelayerHost.SetAMWRelayerConfig(relayerConfig); err != nil { + panic(err) + } + + // Fund each relayer key with 10 TOKENs + // Where TOKEN is the native gas token of each blockchain + // Assumes that the TOKEN decimals are 18, so, this equals + // to 1e18 of the smallest gas amount in each chain + fmt.Printf("Funding relayer keys %s, %s\n", chain1RelayerKey.C(), chain2RelayerKey.C()) + desiredRelayerBalance := big.NewInt(0).Mul(big.NewInt(1e18), big.NewInt(10)) + + // chain1PK will have a balance 10 native gas tokens on chain. + if err := relayer.FundRelayer( + relayerConfig, + chain1BlockchainID, + chain1PK, + nil, + desiredRelayerBalance, + ); err != nil { + return err + } + // chain2PK will have a balance 10 native gas tokens on chain2 + if err := relayer.FundRelayer( + relayerConfig, + chain2BlockchainID, + chain2PK, + nil, + desiredRelayerBalance, + ); err != nil { + return err + } + + // send a message from chain1 to chain2 + fmt.Println("Verifying message delivery") + if err := TestMessageDelivery( + chain1RPC, + chain1PK, + chain1MessengerAddress, + chain2BlockchainID, + chain2RPC, + chain2MessengerAddress, + []byte("hello world"), + ); err != nil { + return err + } + + fmt.Println("Message successfully delivered") + + return nil +} + +func main() { + chain1RPC := os.Getenv("CHAIN1_RPC") + chain1PK := os.Getenv("CHAIN1_PK") + chain1SubnetID, err := ids.FromString(os.Getenv("CHAIN1_SUBNET_ID")) + if err != nil { + panic(err) + } + chain1BlockchainID, err := ids.FromString(os.Getenv("CHAIN1_BLOCKCHAIN_ID")) + if err != nil { + panic(err) + } + chain2RPC := os.Getenv("CHAIN2_RPC") + chain2PK := os.Getenv("CHAIN2_PK") + chain2SubnetID, err := ids.FromString(os.Getenv("CHAIN2_SUBNET_ID")) + if err != nil { + panic(err) + } + chain2BlockchainID, err := ids.FromString(os.Getenv("CHAIN2_BLOCKCHAIN_ID")) + if err != nil { + panic(err) + } + if err := InterchainAWSExample( + avalanche.FujiNetwork(), + chain1RPC, + chain1PK, + chain1SubnetID, + chain1BlockchainID, + chain2RPC, + chain2PK, + chain2SubnetID, + chain2BlockchainID, + ); err != nil { + panic(err) + } +} diff --git a/node/config/awmRelayer.go b/node/config/awmRelayer.go new file mode 100644 index 0000000..949d07e --- /dev/null +++ b/node/config/awmRelayer.go @@ -0,0 +1,26 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package services + +import ( + "path/filepath" + + "github.com/ava-labs/avalanche-tooling-sdk-go/constants" +) + +const DockerAWMRelayerPath = "/.awm-relayer" + +func GetRemoteAMWRelayerConfig() string { + return filepath.Join(constants.CloudNodeAWMRelayerPath, constants.AWMRelayerConfigFilename) +} + +func GetDockerAWMRelayerFolder() string { + return filepath.Join(DockerAWMRelayerPath, "storage") +} + +func AWMRelayerFoldersToCreate() []string { + return []string{ + filepath.Join(constants.CloudNodeAWMRelayerPath, "storage"), + } +} diff --git a/node/create.go b/node/create.go index 218e377..fd11d72 100644 --- a/node/create.go +++ b/node/create.go @@ -43,6 +43,9 @@ type NodeParams struct { // AvalancheGoVersion is the version of Avalanche Go to install in the created node AvalancheGoVersion string + // AWMRelayerVersion is the version of AWM Relayer to install in the created node + AWMRelayerVersion string + // UseStaticIP is whether the created node should have static IP attached to it. Note that // assigning Static IP to a node may incur additional charges on AWS / GCP. There could also be // a limit to how many Static IPs you can have in a region in AWS & GCP. @@ -272,7 +275,7 @@ func provisionHost(node Node, nodeParams *NodeParams) error { return err } case AWMRelayer: - if err := provisionAWMRelayerHost(node); err != nil { + if err := provisionAWMRelayerHost(node, nodeParams); err != nil { return err } default: @@ -329,9 +332,13 @@ func provisionMonitoringHost(node Node) error { return nil } -func provisionAWMRelayerHost(node Node) error { // stub - if err := node.ComposeSSHSetupAWMRelayer(); err != nil { +func provisionAWMRelayerHost(node Node, nodeParams *NodeParams) error { + if err := node.RunSSHSetupDockerService(); err != nil { return err } - return node.StartDockerComposeService(utils.GetRemoteComposeFile(), constants.ServiceAWMRelayer, constants.SSHLongRunningScriptTimeout) + if err := node.ComposeSSHSetupAWMRelayer(nodeParams.Network, nodeParams.AWMRelayerVersion); err != nil { + return err + } + + return node.StartDockerCompose(constants.SSHScriptTimeout) } diff --git a/node/docker_compose.go b/node/docker_compose.go index 7a2ad34..947c2b6 100644 --- a/node/docker_compose.go +++ b/node/docker_compose.go @@ -19,12 +19,12 @@ import ( ) type dockerComposeInputs struct { - WithMonitoring bool - WithAvalanchego bool - AvalanchegoVersion string - E2E bool - E2EIP string - E2ESuffix string + WithMonitoring bool + WithAvalanchego bool + Version string + E2E bool + E2EIP string + E2ESuffix string } //go:embed templates/*.docker-compose.yml diff --git a/node/docker_config.go b/node/docker_config.go index d327820..51a5416 100644 --- a/node/docker_config.go +++ b/node/docker_config.go @@ -4,11 +4,17 @@ package node import ( + "encoding/json" + "errors" + "fmt" "os" + "slices" "github.com/ava-labs/avalanche-tooling-sdk-go/constants" + "github.com/ava-labs/avalanche-tooling-sdk-go/interchain/relayer" remoteconfig "github.com/ava-labs/avalanche-tooling-sdk-go/node/config" "github.com/ava-labs/avalanche-tooling-sdk-go/utils" + "github.com/ava-labs/awm-relayer/config" ) // PrepareAvalanchegoConfig creates the config files for the AvalancheGo @@ -102,3 +108,49 @@ func prepareGrafanaConfig() (string, string, string, string, error) { } return grafanaConfigFile.Name(), grafanaDashboardsFile.Name(), grafanaDataSourceFile.Name(), grafanaPromDataSourceFile.Name(), nil } + +func (h *Node) GetAMWRelayerConfig() (*config.Config, error) { + remoteAWMConf := remoteconfig.GetRemoteAMWRelayerConfig() + if !slices.Contains(h.Roles, AWMRelayer) { + return nil, errors.New("node is not an AWM Relayer") + } + + if configExists, err := h.FileExists(remoteAWMConf); err != nil || !configExists { + return nil, fmt.Errorf("%s: config file %s does not exist or not available", h.NodeID, remoteAWMConf) + } + + c, err := h.ReadFileBytes(remoteAWMConf, constants.SSHFileOpsTimeout) + if err != nil { + return nil, fmt.Errorf("%s: failed to read config file %s: %w", h.NodeID, remoteAWMConf, err) + } + + awmConfig := &config.Config{} + if err := json.Unmarshal(c, &awmConfig); err != nil { + return nil, fmt.Errorf("%s: failed to parse config file %s", h.NodeID, remoteAWMConf) + } + + return awmConfig, nil +} + +// AddBlockchainToRelayerConfig adds a blockchain to the AWM relayer config +func (h *Node) SetAMWRelayerConfig(awmConfig *config.Config) error { + if !slices.Contains(h.Roles, AWMRelayer) { + return errors.New("node is not a AWM Relayer") + } + tmpRelayerConfig, err := os.CreateTemp("", "avalancecli-awm-relayer-config-*.yml") + if err != nil { + return err + } + defer os.Remove(tmpRelayerConfig.Name()) + configData, err := relayer.SerializeRelayerConfig(awmConfig) + if err != nil { + return err + } + if _, err := tmpRelayerConfig.Write(configData); err != nil { + return err + } + if err := h.Upload(tmpRelayerConfig.Name(), remoteconfig.GetRemoteAMWRelayerConfig(), constants.SSHFileOpsTimeout); err != nil { + return err + } + return h.RestartDockerComposeService(utils.GetRemoteComposeFile(), constants.ServiceAWMRelayer, constants.SSHLongRunningScriptTimeout) +} diff --git a/node/docker_logs.go b/node/docker_logs.go new file mode 100644 index 0000000..1b8ee4e --- /dev/null +++ b/node/docker_logs.go @@ -0,0 +1,35 @@ +// Copyright (C) 2024, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package node + +import ( + "fmt" + "strings" + "time" + + "github.com/ava-labs/avalanche-tooling-sdk-go/constants" +) + +func (h *Node) GetContainerLogs(containerName string, tailLines uint, timeout time.Duration) ([]string, error) { + if containerName == "" { + return nil, fmt.Errorf("container name cannot be empty") + } + tailLinesString := "all" + if tailLines > 0 { + tailLinesString = fmt.Sprintf("%d", tailLines) + } + output, err := h.Commandf(nil, timeout, "docker logs --tail %s %s", tailLinesString, containerName) + if err != nil { + return nil, err + } + return strings.Split(string(output), "\n"), nil +} + +func (h *Node) GetAvalanchegoLogs(tailLines uint, timeout time.Duration) ([]string, error) { + return h.GetContainerLogs(constants.ServiceAvalanchego, tailLines, timeout) +} + +func (h *Node) GetAWMRelayerLogs(tailLines uint, timeout time.Duration) ([]string, error) { + return h.GetContainerLogs(constants.ServiceAWMRelayer, tailLines, timeout) +} diff --git a/node/docker_ssh.go b/node/docker_ssh.go index f219657..d48e313 100644 --- a/node/docker_ssh.go +++ b/node/docker_ssh.go @@ -9,9 +9,12 @@ import ( "path/filepath" "time" + "github.com/ava-labs/avalanche-tooling-sdk-go/avalanche" "github.com/ava-labs/avalanche-tooling-sdk-go/constants" + "github.com/ava-labs/avalanche-tooling-sdk-go/interchain/relayer" remoteconfig "github.com/ava-labs/avalanche-tooling-sdk-go/node/config" "github.com/ava-labs/avalanche-tooling-sdk-go/utils" + "github.com/ava-labs/avalanchego/utils/logging" ) // ValidateComposeFile validates a docker-compose file on a remote node. @@ -46,12 +49,12 @@ func (h *Node) ComposeSSHSetupNode(networkID string, subnetsToTrack []string, av constants.SSHScriptTimeout, "templates/avalanchego.docker-compose.yml", dockerComposeInputs{ - AvalanchegoVersion: avalancheGoVersion, - WithMonitoring: withMonitoring, - WithAvalanchego: true, - E2E: utils.IsE2E(), - E2EIP: utils.E2EConvertIP(h.IP), - E2ESuffix: utils.E2ESuffix(h.IP), + Version: avalancheGoVersion, + WithMonitoring: withMonitoring, + WithAvalanchego: true, + E2E: utils.IsE2E(), + E2EIP: utils.E2EConvertIP(h.IP), + E2ESuffix: utils.E2ESuffix(h.IP), }) } @@ -114,9 +117,35 @@ func (h *Node) ComposeSSHSetupMonitoring() error { dockerComposeInputs{}) } -func (h *Node) ComposeSSHSetupAWMRelayer() error { +func (h *Node) ComposeSSHSetupAWMRelayer(network avalanche.Network, awmRelayerVersion string) error { + for _, folder := range remoteconfig.AWMRelayerFoldersToCreate() { + if h.MkdirAll(folder, constants.SSHFileOpsTimeout) != nil { + return fmt.Errorf("error creating folder %s on node %s", folder, h.NodeID) + } + } + + // provide basic configuration for AWM Relayer + tmpRelayerConfig, err := os.CreateTemp("", "avalancecli-awm-relayer-config-*.yml") + if err != nil { + return err + } + defer os.Remove(tmpRelayerConfig.Name()) + awmConfig := relayer.CreateBaseRelayerConfig(logging.Info.LowerString(), remoteconfig.GetDockerAWMRelayerFolder(), 0, network) + configData, err := relayer.SerializeRelayerConfig(awmConfig) + if err != nil { + return err + } + if _, err := tmpRelayerConfig.Write(configData); err != nil { + return err + } + // upload the configuration file to the remote node + if err := h.Upload(tmpRelayerConfig.Name(), remoteconfig.GetRemoteAMWRelayerConfig(), constants.SSHFileOpsTimeout); err != nil { + return err + } return h.ComposeOverSSH("Setup AWM Relayer", constants.SSHScriptTimeout, "templates/awmrelayer.docker-compose.yml", - dockerComposeInputs{}) + dockerComposeInputs{ + Version: awmRelayerVersion, + }) } diff --git a/node/ssh.go b/node/ssh.go index fca99ea..41f18b1 100644 --- a/node/ssh.go +++ b/node/ssh.go @@ -135,12 +135,12 @@ func (h *Node) RunSSHUpgradeAvalanchego(avalancheGoVersion string) error { constants.SSHScriptTimeout, "templates/avalanchego.docker-compose.yml", dockerComposeInputs{ - AvalanchegoVersion: avalancheGoVersion, - WithMonitoring: withMonitoring, - WithAvalanchego: true, - E2E: utils.IsE2E(), - E2EIP: utils.E2EConvertIP(h.IP), - E2ESuffix: utils.E2ESuffix(h.IP), + Version: avalancheGoVersion, + WithMonitoring: withMonitoring, + WithAvalanchego: true, + E2E: utils.IsE2E(), + E2EIP: utils.E2EConvertIP(h.IP), + E2ESuffix: utils.E2ESuffix(h.IP), }); err != nil { return err } diff --git a/node/supported.go b/node/supported.go index b60d8bc..3c36cc9 100644 --- a/node/supported.go +++ b/node/supported.go @@ -101,5 +101,8 @@ func CheckRoles(roles []SupportedRole) error { if slices.Contains(roles, Monitor) && len(roles) > 1 { return fmt.Errorf("%v role cannot be combined with other roles", Monitor) } + if slices.Contains(roles, AWMRelayer) && len(roles) > 1 { + return fmt.Errorf("%v role cannot be combined with other roles", AWMRelayer) + } return nil } diff --git a/node/templates/awmrelayer.docker-compose.yml b/node/templates/awmrelayer.docker-compose.yml index 3861c4a..57bc653 100644 --- a/node/templates/awmrelayer.docker-compose.yml +++ b/node/templates/awmrelayer.docker-compose.yml @@ -1,11 +1,10 @@ -name: avalanche-cli +name: awm-relayer services: awm-relayer: - image: avaplatform/awm-relayer + image: avaplatform/awm-relayer:{{ .Version }} container_name: awm-relayer restart: unless-stopped user: "1000:1000" # ubuntu user - network_mode: "host" volumes: - - /home/ubuntu/.avalanche-cli/services/awm-relayer:/.awm-relayer:rw + - /home/ubuntu/.awm-relayer:/.awm-relayer:rw command: 'awm-relayer --config-file /.awm-relayer/awm-relayer-config.json'