diff --git a/simulators/ethereum/rpc-compat/Dockerfile b/simulators/ethereum/rpc-compat/Dockerfile index 886123f65d..bcbbc396fa 100644 --- a/simulators/ethereum/rpc-compat/Dockerfile +++ b/simulators/ethereum/rpc-compat/Dockerfile @@ -5,7 +5,7 @@ ENV GOPROXY=${GOPROXY} ARG branch=main ENV GIT_REF=${branch} -RUN apk add --update git ca-certificates gcc musl-dev linux-headers +RUN apk add --update git ca-certificates gcc musl-dev linux-headers make # Clone the tests repo. # Allow the user to specify a branch or commit to checkout @@ -15,6 +15,9 @@ RUN git init /execution-apis && \ git fetch --depth 1 origin $GIT_REF && \ git checkout FETCH_HEAD; +# Build the spec using the execution-apis Makefile. +RUN cd /execution-apis && make build + # To run local tests, copy the directory into the same as the simulator and # uncomment the line below # ADD tests /execution-apis/tests @@ -30,5 +33,6 @@ ADD . /source WORKDIR /source COPY --from=builder /source/rpc-compat . COPY --from=builder /execution-apis/tests ./tests +COPY --from=builder /execution-apis/openrpc.json ./openrpc.json ENTRYPOINT ["./rpc-compat"] diff --git a/simulators/ethereum/rpc-compat/go.mod b/simulators/ethereum/rpc-compat/go.mod index c00bc73408..53f57304c3 100644 --- a/simulators/ethereum/rpc-compat/go.mod +++ b/simulators/ethereum/rpc-compat/go.mod @@ -8,6 +8,8 @@ require ( github.com/ethereum/go-ethereum v1.14.5 github.com/ethereum/hive v0.0.0-20240715150147-c87a99dccfce github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 + github.com/open-rpc/meta-schema v0.0.0-20210416041958-626a15d0a618 + github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 ) diff --git a/simulators/ethereum/rpc-compat/go.sum b/simulators/ethereum/rpc-compat/go.sum index ffa04cf3d3..b0e86264be 100644 --- a/simulators/ethereum/rpc-compat/go.sum +++ b/simulators/ethereum/rpc-compat/go.sum @@ -93,8 +93,12 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/open-rpc/meta-schema v0.0.0-20210416041958-626a15d0a618 h1:EoH8oqYGi6BElF3PnUr65GoPVTtaDlnYkrVZct1Q/Sg= +github.com/open-rpc/meta-schema v0.0.0-20210416041958-626a15d0a618/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= +github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/simulators/ethereum/rpc-compat/main.go b/simulators/ethereum/rpc-compat/main.go index d40f4956a9..f81ad27d95 100644 --- a/simulators/ethereum/rpc-compat/main.go +++ b/simulators/ethereum/rpc-compat/main.go @@ -8,6 +8,7 @@ import ( "math" "net" "net/http" + "os" "regexp" "strings" "time" @@ -15,6 +16,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/hive/hivesim" "github.com/nsf/jsondiff" + openrpc "github.com/open-rpc/meta-schema" + jsonschema "github.com/santhosh-tekuri/jsonschema/v5" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) @@ -34,6 +37,12 @@ func main() { panic(err) } + // Load method result schemas from the OpenRPC spec. + schemas, err := loadMethodSchemas("openrpc.json") + if err != nil { + panic(fmt.Sprintf("failed to load OpenRPC spec: %v", err)) + } + // Run the test suite. suite := hivesim.Suite{ Name: "rpc-compat", @@ -50,7 +59,7 @@ conformance with the execution API specification.`[1:], Files: files, Run: func(t *hivesim.T, c *hivesim.Client) { sendForkchoiceUpdated(t, c) - runAllTests(t, c, c.Type) + runAllTests(t, c, c.Type, schemas) }, AlwaysRun: true, }) @@ -58,7 +67,7 @@ conformance with the execution API specification.`[1:], hivesim.MustRunSuite(sim, suite) } -func runAllTests(t *hivesim.T, c *hivesim.Client, clientName string) { +func runAllTests(t *hivesim.T, c *hivesim.Client, clientName string, schemas map[string]openrpc.JSONSchemaObject) { _, testPattern := t.Sim.TestPattern() re := regexp.MustCompile(testPattern) tests := loadTests(t, "tests", re) @@ -68,7 +77,7 @@ func runAllTests(t *hivesim.T, c *hivesim.Client, clientName string) { Name: fmt.Sprintf("%s (%s)", test.name, clientName), Description: test.comment, Run: func(t *hivesim.T) { - if err := runTest(t, c, &test); err != nil { + if err := runTest(t, c, &test, schemas); err != nil { t.Fatal(err) } }, @@ -76,17 +85,19 @@ func runAllTests(t *hivesim.T, c *hivesim.Client, clientName string) { } } -func runTest(t *hivesim.T, c *hivesim.Client, test *rpcTest) error { +func runTest(t *hivesim.T, c *hivesim.Client, test *rpcTest, schemas map[string]openrpc.JSONSchemaObject) error { var ( - client = &http.Client{Timeout: 5 * time.Second} - url = fmt.Sprintf("http://%s", net.JoinHostPort(c.IP.String(), "8545")) - err error - respBytes []byte + client = &http.Client{Timeout: 5 * time.Second} + url = fmt.Sprintf("http://%s", net.JoinHostPort(c.IP.String(), "8545")) + err error + respBytes []byte + lastMethod string ) for _, msg := range test.messages { if msg.send { - // Send request. + // Send request, track the method name for schema lookup. + lastMethod = gjson.Get(msg.data, "method").String() t.Log(">> ", msg.data) respBytes, err = postHttp(client, url, strings.NewReader(msg.data)) if err != nil { @@ -104,15 +115,17 @@ func runTest(t *hivesim.T, c *hivesim.Client, test *rpcTest) error { return fmt.Errorf("invalid JSON response") } - // For speconly tests, ensure the response type matches the expected type. + // For speconly tests, validate the response result against the OpenRPC schema. hasError := gjson.Get(resp, "error").Exists() if !hasError && test.speconly { - errors := checkJSONStructure(gjson.Parse(msg.data), gjson.Parse(resp), ".") - if len(errors) > 0 { - for _, err := range errors { - t.Log(err) - } - return fmt.Errorf("response type does not match expected") + schema, ok := schemas[lastMethod] + if !ok { + return fmt.Errorf("no schema found for method %s", lastMethod) + } + result := json.RawMessage(gjson.Get(resp, "result").Raw) + if err := validateResult(schema, result, lastMethod); err != nil { + t.Log(err) + return fmt.Errorf("response does not match schema for %s: %v", lastMethod, err) } respBytes = nil continue @@ -155,62 +168,52 @@ func runTest(t *hivesim.T, c *hivesim.Client, test *rpcTest) error { return nil } -// checkJSONStructure checks whether the `actual` value matches the type structure -// of the `expected` value. -func checkJSONStructure(expected, actual gjson.Result, path string) []string { - var errors []string - - buildPath := func(key string) string { - if path != "." { - return path + "." + key - } - return "." + key +// loadMethodSchemas reads the dereferenced OpenRPC spec and returns a map of +// method name to its result JSON schema. +func loadMethodSchemas(path string) (map[string]openrpc.JSONSchemaObject, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err } - - if expected.Type != gjson.JSON { - return errors + var doc openrpc.OpenrpcDocument + if err := json.Unmarshal(data, &doc); err != nil { + return nil, err } - - if expected.IsArray() { - if !actual.IsArray() { - errors = append(errors, fmt.Sprintf("%s: expected array but got %s", path, actual.Type)) + schemas := make(map[string]openrpc.JSONSchemaObject) + for _, method := range *doc.Methods { + if method.MethodObject == nil { + continue } - return errors - } - - // Check all expected keys exist with correct types - expected.ForEach(func(key, value gjson.Result) bool { - keyPath := buildPath(key.String()) - actualValue := actual.Get(key.String()) - - if !actualValue.Exists() { - errors = append(errors, fmt.Sprintf("%s: missing key", keyPath)) - return true + m := method.MethodObject + if m.Result == nil || m.Result.ContentDescriptorObject == nil { + continue } - - if value.Type != actualValue.Type && !(value.Type == gjson.JSON && actualValue.Type == gjson.JSON) { - errors = append(errors, fmt.Sprintf("%s: type mismatch (expected %s, got %s)", - keyPath, value.Type, actualValue.Type)) - return true + cd := m.Result.ContentDescriptorObject + if cd.Schema == nil || cd.Schema.JSONSchemaObject == nil { + continue } - - if value.IsObject() || value.IsArray() { - errors = append(errors, checkJSONStructure(value, actualValue, keyPath)...) - } - return true - }) - - // Check for unexpected keys - if actual.IsObject() { - actual.ForEach(func(key, value gjson.Result) bool { - if !expected.Get(key.String()).Exists() { - errors = append(errors, fmt.Sprintf("%s: unexpected key in response", buildPath(key.String()))) - } - return true - }) + schemas[string(*m.Name)] = *cd.Schema.JSONSchemaObject } + return schemas, nil +} - return errors +// validateResult validates the result value against the method's result schema. +func validateResult(schema openrpc.JSONSchemaObject, result json.RawMessage, method string) error { + draft := openrpc.Schema("https://json-schema.org/draft/2019-09/schema") + schema.Schema = &draft + b, err := json.Marshal(schema) + if err != nil { + return fmt.Errorf("unable to marshal schema: %v", err) + } + s, err := jsonschema.CompileString(method+".result", string(b)) + if err != nil { + return err + } + var x interface{} + if err := json.Unmarshal(result, &x); err != nil { + return err + } + return s.Validate(x) } func numbersEqual(a, b json.Number) bool { diff --git a/simulators/ethereum/rpc-compat/testload_speconly_test.go b/simulators/ethereum/rpc-compat/testload_speconly_test.go index 429a2d792e..b9735ecd3c 100644 --- a/simulators/ethereum/rpc-compat/testload_speconly_test.go +++ b/simulators/ethereum/rpc-compat/testload_speconly_test.go @@ -1,114 +1,10 @@ package main import ( - "slices" "strings" "testing" - - "github.com/tidwall/gjson" ) -func TestCompareKeysOnly(t *testing.T) { - tests := []struct { - name string - actual string - expected string - shouldError bool - errors []string - }{ - { - name: "matching keys with different values", - actual: `{"id":1,"result":"0xabc123","jsonrpc":"2.0"}`, - expected: `{"id":2,"result":"0xdef456","jsonrpc":"1.0"}`, - shouldError: false, - }, - { - name: "missing key in actual", - actual: `{"id":1,"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":"0x123","jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".result: missing key"}, - }, - { - name: "extra key in actual is not allowed", - actual: `{"id":1,"result":"0x123","jsonrpc":"2.0","extra":"field"}`, - expected: `{"id":1,"result":"0x456","jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".extra: unexpected key in response"}, - }, - { - name: "nested objects - matching structure", - actual: `{"id":1,"result":{"block":"0x1","hash":"0xabc"},"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":{"block":"0x2","hash":"0xdef"},"jsonrpc":"2.0"}`, - shouldError: false, - }, - { - name: "nested objects - missing nested key", - actual: `{"id":1,"result":{"block":"0x1"},"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":{"block":"0x2","hash":"0xdef"},"jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".result.hash: missing key"}, - }, - { - name: "nested objects - extra nested key", - actual: `{"id":1,"result":{"block":"0x1","hash":"0xabc","extra":"key"},"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":{"block":"0x2","hash":"0xdef"},"jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".result.extra: unexpected key in response"}, - }, - { - name: "arrays - only check structure exists", - actual: `{"id":1,"result":[1,2,3],"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":[4,5,6,7,8],"jsonrpc":"2.0"}`, - shouldError: false, - }, - { - name: "null when string expected - type mismatch", - actual: `{"id":1,"result":null,"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":"0x123","jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".result: type mismatch (expected String, got Null)"}, - }, - { - name: "null when null expected - ok", - actual: `{"id":1,"result":null,"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":null,"jsonrpc":"2.0"}`, - shouldError: false, - }, - { - name: "string when number expected - type mismatch", - actual: `{"id":"1","result":"0x123","jsonrpc":"2.0"}`, - expected: `{"id":1,"result":"0x456","jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".id: type mismatch (expected Number, got String)"}, - }, - { - name: "object when array expected - type mismatch", - actual: `{"id":1,"result":{"key":"value"},"jsonrpc":"2.0"}`, - expected: `{"id":1,"result":[1,2,3],"jsonrpc":"2.0"}`, - shouldError: true, - errors: []string{".result: expected array but got JSON"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := checkJSONStructure(gjson.Parse(tt.expected), gjson.Parse(tt.actual), ".") - if tt.shouldError { - if len(err) == 0 { - t.Errorf("expected error but got none") - } else if !slices.Equal(err, tt.errors) { - t.Errorf("errors mismatch\n got: %v\n want: %v", err, tt.errors) - } - } else { - if len(err) > 0 { - t.Errorf("unexpected errors: %v", err) - } - } - }) - } -} - func TestSpecOnlyParsing(t *testing.T) { testContent := `// This is a test // speconly: true