Skip to content

Commit

Permalink
Merge pull request onflow#5939 from onflow/bastian/allow-export-of-gl…
Browse files Browse the repository at this point in the history
…obal-registers

Allow export of global registers (owner is empty string)
  • Loading branch information
turbolent authored May 20, 2024
2 parents 55cf969 + 0491083 commit 0436edf
Show file tree
Hide file tree
Showing 8 changed files with 191 additions and 101 deletions.
33 changes: 11 additions & 22 deletions cmd/util/cmd/execution-state-extract/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import (
"path"
"strings"

runtimeCommon "github.com/onflow/cadence/runtime/common"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd/util/cmd/common"
common2 "github.com/onflow/flow-go/cmd/util/common"
"github.com/onflow/flow-go/cmd/util/ledger/migrations"
"github.com/onflow/flow-go/cmd/util/ledger/util"
"github.com/onflow/flow-go/model/bootstrap"
Expand Down Expand Up @@ -259,24 +259,13 @@ func run(*cobra.Command, []string) {
}
}

var exportedAddresses []runtimeCommon.Address
var exportPayloadsForOwners map[string]struct{}

if len(flagOutputPayloadByAddresses) > 0 {

addresses := strings.Split(flagOutputPayloadByAddresses, ",")

for _, hexAddr := range addresses {
b, err := hex.DecodeString(strings.TrimSpace(hexAddr))
if err != nil {
log.Fatal().Err(err).Msgf("cannot hex decode address %s for payload export", strings.TrimSpace(hexAddr))
}

addr, err := runtimeCommon.BytesToAddress(b)
if err != nil {
log.Fatal().Err(err).Msgf("cannot decode address %x for payload export", b)
}

exportedAddresses = append(exportedAddresses, addr)
var err error
exportPayloadsForOwners, err = common2.ParseOwners(strings.Split(flagOutputPayloadByAddresses, ","))
if err != nil {
log.Fatal().Err(err).Msgf("failed to parse addresses")
}
}

Expand Down Expand Up @@ -334,12 +323,12 @@ func run(*cobra.Command, []string) {
var outputMsg string
if len(flagOutputPayloadFileName) > 0 {
// Output is payload file
if len(exportedAddresses) == 0 {
if len(exportPayloadsForOwners) == 0 {
outputMsg = fmt.Sprintf("exporting all payloads to %s", flagOutputPayloadFileName)
} else {
outputMsg = fmt.Sprintf(
"exporting payloads by addresses %v to %s",
flagOutputPayloadByAddresses,
"exporting payloads for owners %v to %s",
common2.OwnersToString(exportPayloadsForOwners),
flagOutputPayloadFileName,
)
}
Expand Down Expand Up @@ -397,7 +386,7 @@ func run(*cobra.Command, []string) {
!flagNoMigration,
flagInputPayloadFileName,
flagOutputPayloadFileName,
exportedAddresses,
exportPayloadsForOwners,
flagSortPayloads,
opts,
)
Expand All @@ -410,7 +399,7 @@ func run(*cobra.Command, []string) {
flagNWorker,
!flagNoMigration,
flagOutputPayloadFileName,
exportedAddresses,
exportPayloadsForOwners,
flagSortPayloads,
opts,
)
Expand Down
13 changes: 6 additions & 7 deletions cmd/util/cmd/execution-state-extract/execution_state_extract.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
syncAtomic "sync/atomic"
"time"

"github.com/onflow/cadence/runtime/common"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"go.uber.org/atomic"
Expand Down Expand Up @@ -39,7 +38,7 @@ func extractExecutionState(
nWorker int, // number of concurrent worker to migration payloads
runMigrations bool,
outputPayloadFile string,
exportPayloadsByAddresses []common.Address,
exportPayloadsForOwners map[string]struct{},
sortPayloads bool,
opts migrators.Options,
) error {
Expand Down Expand Up @@ -145,7 +144,7 @@ func extractExecutionState(
payloads,
nWorker,
outputPayloadFile,
exportPayloadsByAddresses,
exportPayloadsForOwners,
false, // payloads represents entire state.
sortPayloads,
)
Expand Down Expand Up @@ -217,7 +216,7 @@ func extractExecutionStateFromPayloads(
runMigrations bool,
inputPayloadFile string,
outputPayloadFile string,
exportPayloadsByAddresses []common.Address,
exportPayloadsForOwners map[string]struct{},
sortPayloads bool,
opts migrators.Options,
) error {
Expand Down Expand Up @@ -253,7 +252,7 @@ func extractExecutionStateFromPayloads(
payloads,
nWorker,
outputPayloadFile,
exportPayloadsByAddresses,
exportPayloadsForOwners,
inputPayloadsFromPartialState,
sortPayloads,
)
Expand Down Expand Up @@ -288,7 +287,7 @@ func exportPayloads(
payloads []*ledger.Payload,
nWorker int,
outputPayloadFile string,
exportPayloadsByAddresses []common.Address,
exportPayloadsForOwners map[string]struct{},
inputPayloadsFromPartialState bool,
sortPayloads bool,
) error {
Expand All @@ -308,7 +307,7 @@ func exportPayloads(
log,
outputPayloadFile,
payloads,
exportPayloadsByAddresses,
exportPayloadsForOwners,
inputPayloadsFromPartialState,
)
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -406,10 +406,22 @@ func TestExtractPayloadsFromExecutionState(t *testing.T) {
// Verify exported payloads.
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputPayloadFileName)
require.NoError(t, err)
require.Equal(t, len(selectedKeysValues), len(payloadsFromFile))
require.True(t, partialState)

nonGlobalPayloads := make([]*ledger.Payload, 0, len(selectedKeysValues))
for _, payloadFromFile := range payloadsFromFile {
key, err := payloadFromFile.Key()
require.NoError(t, err)

owner := key.KeyParts[0].Value
if len(owner) > 0 {
nonGlobalPayloads = append(nonGlobalPayloads, payloadFromFile)
}
}

require.Equal(t, len(selectedKeysValues), len(nonGlobalPayloads))

for _, payloadFromFile := range nonGlobalPayloads {
k, err := payloadFromFile.Key()
require.NoError(t, err)

Expand Down
46 changes: 13 additions & 33 deletions cmd/util/cmd/extract-payloads-by-address/cmd.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
package extractpayloads

import (
"encoding/hex"
"fmt"
"os"
"strings"

"github.com/rs/zerolog/log"
"github.com/spf13/cobra"

"github.com/onflow/cadence/runtime/common"

"github.com/onflow/flow-go/cmd/util/common"
"github.com/onflow/flow-go/cmd/util/ledger/util"
)

Expand Down Expand Up @@ -60,14 +57,14 @@ func run(*cobra.Command, []string) {
log.Fatal().Msgf("Output file %s exists", flagOutputPayloadFileName)
}

addresses, err := parseAddresses(strings.Split(flagAddresses, ","))
owners, err := common.ParseOwners(strings.Split(flagAddresses, ","))
if err != nil {
log.Fatal().Err(err)
}

log.Info().Msgf(
"extracting payloads with address %v from %s to %s",
addresses,
"extracting payloads with owners %s from %s to %s",
common.OwnersToString(owners),
flagInputPayloadFileName,
flagOutputPayloadFileName,
)
Expand All @@ -77,39 +74,22 @@ func run(*cobra.Command, []string) {
log.Fatal().Err(err)
}

numOfPayloadWritten, err := util.CreatePayloadFile(log.Logger, flagOutputPayloadFileName, payloads, addresses, inputPayloadsFromPartialState)
numOfPayloadWritten, err := util.CreatePayloadFile(
log.Logger,
flagOutputPayloadFileName,
payloads,
owners,
inputPayloadsFromPartialState,
)
if err != nil {
log.Fatal().Err(err)
}

log.Info().Msgf(
"extracted %d payloads with address %v from %s to %s",
"extracted %d payloads with owners %s from %s to %s",
numOfPayloadWritten,
addresses,
common.OwnersToString(owners),
flagInputPayloadFileName,
flagOutputPayloadFileName,
)
}

func parseAddresses(hexAddresses []string) ([]common.Address, error) {
if len(hexAddresses) == 0 {
return nil, fmt.Errorf("at least one address must be provided")
}

addresses := make([]common.Address, len(hexAddresses))
for i, hexAddr := range hexAddresses {
b, err := hex.DecodeString(strings.TrimSpace(hexAddr))
if err != nil {
return nil, fmt.Errorf("address is not hex encoded %s: %w", strings.TrimSpace(hexAddr), err)
}

addr, err := common.BytesToAddress(b)
if err != nil {
return nil, fmt.Errorf("cannot decode address %x", b)
}

addresses[i] = addr
}

return addresses, nil
}
40 changes: 29 additions & 11 deletions cmd/util/cmd/extract-payloads-by-address/extract_payloads_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
package extractpayloads

import (
"bytes"
"crypto/rand"
"encoding/hex"
"path/filepath"
"strings"
"testing"
Expand Down Expand Up @@ -94,10 +92,22 @@ func TestExtractPayloads(t *testing.T) {
// Verify exported payloads.
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputFile)
require.NoError(t, err)
require.Equal(t, len(selectedKeysValues), len(payloadsFromFile))
require.True(t, partialState)

nonGlobalPayloads := make([]*ledger.Payload, 0, len(selectedKeysValues))
for _, payloadFromFile := range payloadsFromFile {
key, err := payloadFromFile.Key()
require.NoError(t, err)

owner := key.KeyParts[0].Value
if len(owner) > 0 {
nonGlobalPayloads = append(nonGlobalPayloads, payloadFromFile)
}
}

require.Equal(t, len(selectedKeysValues), len(nonGlobalPayloads))

for _, payloadFromFile := range nonGlobalPayloads {
k, err := payloadFromFile.Key()
require.NoError(t, err)

Expand All @@ -108,9 +118,7 @@ func TestExtractPayloads(t *testing.T) {
})
})

t.Run("no payloads", func(t *testing.T) {

emptyAddress := common.Address{}
t.Run("empty address", func(t *testing.T) {

unittest.RunWithTempDir(t, func(datadir string) {

Expand All @@ -127,9 +135,6 @@ func TestExtractPayloads(t *testing.T) {
keys, values := getSampleKeyValues(i)

for j, key := range keys {
if bytes.Equal(key.KeyParts[0].Value, emptyAddress[:]) {
continue
}
keysValues[key.String()] = keyPair{
key: key,
value: values[j],
Expand All @@ -147,7 +152,7 @@ func TestExtractPayloads(t *testing.T) {
Cmd.SetArgs([]string{
"--input-filename", inputFile,
"--output-filename", outputFile,
"--addresses", hex.EncodeToString(emptyAddress[:]),
"--addresses", ",",
})

err = Cmd.Execute()
Expand All @@ -156,8 +161,21 @@ func TestExtractPayloads(t *testing.T) {
// Verify exported payloads.
partialState, payloadsFromFile, err := util.ReadPayloadFile(zerolog.Nop(), outputFile)
require.NoError(t, err)
require.Equal(t, 0, len(payloadsFromFile))
require.True(t, partialState)

var nonGlobalPayloads []*ledger.Payload
for _, payloadFromFile := range payloadsFromFile {
key, err := payloadFromFile.Key()
require.NoError(t, err)

owner := key.KeyParts[0].Value
if len(owner) > 0 {
nonGlobalPayloads = append(nonGlobalPayloads, payloadFromFile)
}
}

require.Equal(t, 0, len(nonGlobalPayloads))

})
})
}
Expand Down
60 changes: 60 additions & 0 deletions cmd/util/common/address.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package common

import (
"encoding/hex"
"fmt"
"strings"

"github.com/onflow/flow-go/model/flow"
)

func ParseOwners(hexAddresses []string) (map[string]struct{}, error) {
if len(hexAddresses) == 0 {
return nil, fmt.Errorf("at least one address must be provided")
}

addresses := make(map[string]struct{}, len(hexAddresses))
for _, hexAddr := range hexAddresses {
hexAddr = strings.TrimSpace(hexAddr)

if len(hexAddr) > 0 {
addr, err := ParseAddress(hexAddr)
if err != nil {
return nil, err
}

addresses[string(addr[:])] = struct{}{}
} else {
// global registers has empty address
addresses[""] = struct{}{}
}
}

return addresses, nil
}

func ParseAddress(hexAddr string) (flow.Address, error) {
b, err := hex.DecodeString(hexAddr)
if err != nil {
return flow.Address{}, fmt.Errorf(
"address is not hex encoded %s: %w",
strings.TrimSpace(hexAddr),
err,
)
}

return flow.BytesToAddress(b), nil
}

func OwnersToString(owners map[string]struct{}) string {
var sb strings.Builder
index := 0
for owner := range owners {
if index > 0 {
sb.WriteRune(',')
}
_, _ = fmt.Fprintf(&sb, "%x", owner)
index++
}
return sb.String()
}
Loading

0 comments on commit 0436edf

Please sign in to comment.