file-exporter: Use strict encoding when writing files.

conduit/plugins/exporters/filewriter/file_exporter.go

@@ -94,7 +94,7 @@ func (exp *fileExporter) Receive(exportData data.BlockData) error {
 		}
 
 		blockFile := path.Join(exp.cfg.BlocksDir, fmt.Sprintf(exp.cfg.FilenamePattern, exportData.Round()))
-		err := util.EncodeToFile(blockFile, exportData, true)
+		err := util.EncodeJSONToFile(blockFile, exportData, true)
 		if err != nil {
 			return fmt.Errorf("Receive(): failed to write file %s: %w", blockFile, err)
 		}

conduit/plugins/exporters/filewriter/file_exporter_test.go

@@ -3,6 +3,7 @@ package filewriter
 import (
 	"context"
 	"fmt"
+	"os"
 	"path"
 	"testing"
 
@@ -111,6 +112,9 @@ func sendData(t *testing.T, fileExp exporters.Exporter, config string, numRounds
 	block := data.BlockData{
 		BlockHeader: sdk.BlockHeader{
 			Round: 3,
+			StateProofTracking: map[sdk.StateProofType]sdk.StateProofTrackingData{
+				0: {StateProofNextRound: 2},
+			},
 		},
 		Payset:      nil,
 		Delta:       nil,
@@ -130,17 +134,20 @@ func sendData(t *testing.T, fileExp exporters.Exporter, config string, numRounds
 	require.Contains(t, err.Error(), "received round 3, expected round 0")
 
 	// write block to file
-	for i := 0; i < numRounds; i++ {
+	for i := sdk.Round(0); int(i) < numRounds; i++ {
 		block = data.BlockData{
 			BlockHeader: sdk.BlockHeader{
-				Round: sdk.Round(i),
+				Round: i,
+				StateProofTracking: map[sdk.StateProofType]sdk.StateProofTrackingData{
+					0: {StateProofNextRound: i},
+				},
 			},
 			Payset: nil,
 			Delta: &sdk.LedgerStateDelta{
 				PrevTimestamp: 1234,
 			},
 			Certificate: &map[string]interface{}{
-				"Round":  sdk.Round(i),
+				"Round":  i,
 				"Period": 2,
 				"Step":   2,
 			},
@@ -162,10 +169,15 @@ func TestExporterReceive(t *testing.T) {
 	for i := 0; i < 5; i++ {
 		filename := fmt.Sprintf(FilePattern, i)
 		path := fmt.Sprintf("%s/blocks/%s", tempdir, filename)
-		assert.FileExists(t, path)
+		require.FileExists(t, path)
+
+		blockBytes, err := os.ReadFile(path)
+		fmt.Println(string(blockBytes))
+		require.NoError(t, err)
+		assert.NotContains(t, string(blockBytes), " 0: ")
 
 		var blockData data.BlockData
-		err := util.DecodeFromFile(path, &blockData, true)
+		err = util.DecodeJSONFromFile(path, &blockData, true)
 		require.Equal(t, sdk.Round(i), blockData.BlockHeader.Round)
 		require.NoError(t, err)
 		require.NotNil(t, blockData.Certificate)
@@ -197,7 +209,7 @@ func TestPatternOverride(t *testing.T) {
 		assert.FileExists(t, path)
 
 		var blockData data.BlockData
-		err := util.DecodeFromFile(path, &blockData, true)
+		err := util.DecodeJSONFromFile(path, &blockData, true)
 		require.Equal(t, sdk.Round(i), blockData.BlockHeader.Round)
 		require.NoError(t, err)
 		require.NotNil(t, blockData.Certificate)
@@ -223,7 +235,7 @@ func TestDropCertificate(t *testing.T) {
 		path := fmt.Sprintf("%s/%s", tempdir, filename)
 		assert.FileExists(t, path)
 		var blockData data.BlockData
-		err := util.DecodeFromFile(path, &blockData, true)
+		err := util.DecodeJSONFromFile(path, &blockData, true)
 		assert.NoError(t, err)
 		assert.Nil(t, blockData.Certificate)
 	}

conduit/plugins/importers/filereader/filereader.go

@@ -72,7 +72,7 @@ func (r *fileReader) Init(ctx context.Context, cfg plugins.PluginConfig, logger
 
 	genesisFile := path.Join(r.cfg.BlocksDir, "genesis.json")
 	var genesis sdk.Genesis
-	err = util.DecodeFromFile(genesisFile, &genesis, false)
+	err = util.DecodeJSONFromFile(genesisFile, &genesis, false)
 	if err != nil {
 		return nil, fmt.Errorf("Init(): failed to process genesis file: %w", err)
 	}
@@ -98,7 +98,7 @@ func (r *fileReader) GetBlock(rnd uint64) (data.BlockData, error) {
 		filename := path.Join(r.cfg.BlocksDir, fmt.Sprintf(r.cfg.FilenamePattern, rnd))
 		var blockData data.BlockData
 		start := time.Now()
-		err := util.DecodeFromFile(filename, &blockData, false)
+		err := util.DecodeJSONFromFile(filename, &blockData, false)
 		if err != nil && errors.Is(err, fs.ErrNotExist) {
 			// If the file read failed because the file didn't exist, wait before trying again
 			if attempts == 0 {

conduit/plugins/importers/filereader/filereader_test.go

@@ -54,7 +54,7 @@ func initializeTestData(t *testing.T, dir string, numRounds int) sdk.Genesis {
 		Timestamp:   1234,
 	}
 
-	err := util.EncodeToFile(path.Join(dir, "genesis.json"), genesisA, true)
+	err := util.EncodeJSONToFile(path.Join(dir, "genesis.json"), genesisA, true)
 	require.NoError(t, err)
 
 	for i := 0; i < numRounds; i++ {
@@ -67,7 +67,7 @@ func initializeTestData(t *testing.T, dir string, numRounds int) sdk.Genesis {
 			Certificate: nil,
 		}
 		blockFile := path.Join(dir, fmt.Sprintf(filewriter.FilePattern, i))
-		err = util.EncodeToFile(blockFile, block, true)
+		err = util.EncodeJSONToFile(blockFile, block, true)
 		require.NoError(t, err)
 	}
 

util/util.go

@@ -25,15 +25,29 @@ const (
 	checksumLength = 4
 )
 
+var prettyHandle *codec.JsonHandle
+
 var base32Encoder = base32.StdEncoding.WithPadding(base32.NoPadding)
 
-// EncodeToFile is used to encode an object to a file. If the file ends in .gz it will be gzipped.
-func EncodeToFile(filename string, v interface{}, pretty bool) error {
+func init() {
+	prettyHandle = new(codec.JsonHandle)
+	prettyHandle.ErrorIfNoField = json.CodecHandle.ErrorIfNoField
+	prettyHandle.ErrorIfNoArrayExpand = json.CodecHandle.ErrorIfNoArrayExpand
+	prettyHandle.Canonical = json.CodecHandle.Canonical
+	prettyHandle.RecursiveEmptyCheck = json.CodecHandle.RecursiveEmptyCheck
+	prettyHandle.Indent = json.CodecHandle.Indent
+	prettyHandle.HTMLCharsAsIs = json.CodecHandle.HTMLCharsAsIs
+	prettyHandle.MapKeyAsString = true
+	prettyHandle.Indent = 2
+}
+
+// EncodeJSONToFile is used to encode an object to a file. If the file ends in .gz it will be gzipped.
+func EncodeJSONToFile(filename string, v interface{}, pretty bool) error {
 	var writer io.Writer
 
 	file, err := os.Create(filename)
 	if err != nil {
-		return fmt.Errorf("EncodeToFile(): failed to create %s: %w", filename, err)
+		return fmt.Errorf("EncodeJSONToFile(): failed to create %s: %w", filename, err)
 	}
 	defer file.Close()
 
@@ -46,22 +60,22 @@ func EncodeToFile(filename string, v interface{}, pretty bool) error {
 		writer = file
 	}
 
-	handle := json.CodecHandle
+	var handle *codec.JsonHandle
 	if pretty {
-		handle.Indent = 2
+		handle = prettyHandle
 	} else {
-		handle.Indent = 0
+		handle = protocol.JSONStrictHandle
 	}
 	enc := codec.NewEncoder(writer, handle)
 	return enc.Encode(v)
 }
 
-// DecodeFromFile is used to decode a file to an object.
-func DecodeFromFile(filename string, v interface{}, strict bool) error {
+// DecodeJSONFromFile is used to decode a file to an object.
+func DecodeJSONFromFile(filename string, v interface{}, strict bool) error {
 	// Streaming into the decoder was slow.
 	fileBytes, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return fmt.Errorf("DecodeFromFile(): failed to read %s: %w", filename, err)
+		return fmt.Errorf("DecodeJSONFromFile(): failed to read %s: %w", filename, err)
 	}
 
 	var reader io.Reader = bytes.NewReader(fileBytes)
@@ -70,13 +84,15 @@ func DecodeFromFile(filename string, v interface{}, strict bool) error {
 		gz, err := gzip.NewReader(reader)
 		defer gz.Close()
 		if err != nil {
-			return fmt.Errorf("DecodeFromFile(): failed to make gzip reader: %w", err)
+			return fmt.Errorf("DecodeJSONFromFile(): failed to make gzip reader: %w", err)
 		}
 		reader = gz
 	}
-	var handle *codec.JsonHandle = json.LenientCodecHandle
+	var handle *codec.JsonHandle
 	if strict {
 		handle = json.CodecHandle
+	} else {
+		handle = json.LenientCodecHandle
 	}
 
 	enc := codec.NewDecoder(reader, handle)

util/util_test.go

@@ -16,47 +16,52 @@ func TestEncodeToAndFromFile(t *testing.T) {
 	tempdir := t.TempDir()
 
 	type test struct {
-		One string `json:"one"`
-		Two int    `json:"two"`
+		One    string         `json:"one"`
+		Two    int            `json:"two"`
+		Strict map[int]string `json:"strict"`
 	}
 	data := test{
 		One: "one",
 		Two: 2,
+		Strict: map[int]string{
+			0: "int-key",
+		},
 	}
 
 	{
 		pretty := path.Join(tempdir, "pretty.json")
-		err := EncodeToFile(pretty, data, true)
+		err := EncodeJSONToFile(pretty, data, true)
 		require.NoError(t, err)
 		require.FileExists(t, pretty)
 		var testDecode test
-		err = DecodeFromFile(pretty, &testDecode, false)
+		err = DecodeJSONFromFile(pretty, &testDecode, false)
 		require.Equal(t, data, testDecode)
 
 		// Check the pretty printing
 		bytes, err := ioutil.ReadFile(pretty)
 		require.NoError(t, err)
 		require.Contains(t, string(bytes), "  \"one\": \"one\",\n")
+		require.Contains(t, string(bytes), "\"0\": \"int-key\"")
 	}
 
 	{
 		small := path.Join(tempdir, "small.json")
-		err := EncodeToFile(small, data, false)
+		err := EncodeJSONToFile(small, data, false)
 		require.NoError(t, err)
 		require.FileExists(t, small)
 		var testDecode test
-		err = DecodeFromFile(small, &testDecode, false)
+		err = DecodeJSONFromFile(small, &testDecode, false)
 		require.Equal(t, data, testDecode)
 	}
 
 	// gzip test
 	{
 		small := path.Join(tempdir, "small.json.gz")
-		err := EncodeToFile(small, data, false)
+		err := EncodeJSONToFile(small, data, false)
 		require.NoError(t, err)
 		require.FileExists(t, small)
 		var testDecode test
-		err = DecodeFromFile(small, &testDecode, false)
+		err = DecodeJSONFromFile(small, &testDecode, false)
 		require.Equal(t, data, testDecode)
 	}
 }