diff --git a/.changeset/twenty-items-sneeze.md b/.changeset/twenty-items-sneeze.md new file mode 100644 index 0000000000000..44ec82f57dc19 --- /dev/null +++ b/.changeset/twenty-items-sneeze.md @@ -0,0 +1,5 @@ +--- +'@eth-optimism/proxyd': minor +--- + +Put special errors in a dedicated metric, pass along the content-type header diff --git a/go/proxyd/backend.go b/go/proxyd/backend.go index 5a402281110c2..7fef165ef1677 100644 --- a/go/proxyd/backend.go +++ b/go/proxyd/backend.go @@ -152,12 +152,17 @@ func NewBackend( func (b *Backend) Forward(ctx context.Context, req *RPCReq) (*RPCRes, error) { if !b.allowedRPCMethods.Has(req.Method) { + // use unknown below to prevent DOS vector that fills up memory + // with arbitrary method names. + RecordRPCError(ctx, b.Name, MethodUnknown, ErrMethodNotWhitelisted) return nil, ErrMethodNotWhitelisted } if !b.Online() { + RecordRPCError(ctx, b.Name, req.Method, ErrBackendOffline) return nil, ErrBackendOffline } if b.IsRateLimited() { + RecordRPCError(ctx, b.Name, req.Method, ErrBackendOverCapacity) return nil, ErrBackendOverCapacity } @@ -167,22 +172,19 @@ func (b *Backend) Forward(ctx context.Context, req *RPCReq) (*RPCRes, error) { for i := 0; i <= b.maxRetries; i++ { RecordRPCForward(ctx, b.Name, req.Method, RPCRequestSourceHTTP) respTimer := prometheus.NewTimer(rpcBackendRequestDurationSumm.WithLabelValues(b.Name, req.Method)) - resB, err := b.doForward(req) + res, err := b.doForward(req) if err != nil { lastError = err log.Warn("backend request failed, trying again", "err", err, "name", b.Name) respTimer.ObserveDuration() + RecordRPCError(ctx, b.Name, req.Method, err) time.Sleep(calcBackoff(i)) continue } respTimer.ObserveDuration() - - res := new(RPCRes) - // don't mark the backend down if they give us a bad response body - if err := json.Unmarshal(resB, res); err != nil { - return nil, ErrBackendBadResponse + if res.IsError() { + RecordRPCError(ctx, b.Name, req.Method, res.Error) } - return res, nil } @@ -271,7 +273,7 @@ func (b *Backend) setOffline() { } } -func (b *Backend) doForward(rpcReq *RPCReq) ([]byte, error) { +func (b *Backend) doForward(rpcReq *RPCReq) (*RPCRes, error) { body := mustMarshalJSON(rpcReq) httpReq, err := http.NewRequest("POST", b.rpcURL, bytes.NewReader(body)) @@ -283,22 +285,30 @@ func (b *Backend) doForward(rpcReq *RPCReq) ([]byte, error) { httpReq.SetBasicAuth(b.authUsername, b.authPassword) } - res, err := b.client.Do(httpReq) + httpReq.Header.Set("content-type", "application/json") + + httpRes, err := b.client.Do(httpReq) if err != nil { return nil, wrapErr(err, "error in backend request") } - if res.StatusCode != 200 { - return nil, fmt.Errorf("response code %d", res.StatusCode) + // Alchemy returns a 400 on bad JSONs, so handle that case + if httpRes.StatusCode != 200 && httpRes.StatusCode != 400 { + return nil, fmt.Errorf("response code %d", httpRes.StatusCode) } - defer res.Body.Close() - resB, err := ioutil.ReadAll(io.LimitReader(res.Body, b.maxResponseSize)) + defer httpRes.Body.Close() + resB, err := ioutil.ReadAll(io.LimitReader(httpRes.Body, b.maxResponseSize)) if err != nil { return nil, wrapErr(err, "error reading response body") } - return resB, nil + res := new(RPCRes) + if err := json.Unmarshal(resB, res); err != nil { + return nil, ErrBackendBadResponse + } + + return res, nil } type BackendGroup struct { @@ -329,6 +339,7 @@ func (b *BackendGroup) Forward(ctx context.Context, rpcReq *RPCReq) (*RPCRes, er return res, nil } + RecordUnserviceableRequest(ctx, RPCRequestSourceHTTP) return nil, ErrNoBackends } @@ -413,12 +424,14 @@ func (w *WSProxier) clientPump(ctx context.Context, errC chan error) { req, err := w.parseClientMsg(msg) if err != nil { var id *int + method := MethodUnknown if req != nil { id = req.ID + method = req.Method } outConn = w.clientConn msg = mustMarshalJSON(NewRPCErrorRes(id, err)) - RecordRPCError(ctx, SourceClient, err) + RecordRPCError(ctx, BackendProxyd, method, err) } else { RecordRPCForward(ctx, w.backend.Name, req.Method, RPCRequestSourceWS) } @@ -462,7 +475,7 @@ func (w *WSProxier) backendPump(ctx context.Context, errC chan error) { msg = mustMarshalJSON(NewRPCErrorRes(id, err)) } if res.IsError() { - RecordRPCError(ctx, SourceBackend, res.Error) + RecordRPCError(ctx, w.backend.Name, MethodUnknown, res.Error) } err = w.clientConn.WriteMessage(msgType, msg) diff --git a/go/proxyd/metrics.go b/go/proxyd/metrics.go index fcebef780abeb..4296676e38f05 100644 --- a/go/proxyd/metrics.go +++ b/go/proxyd/metrics.go @@ -5,6 +5,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "strconv" + "strings" ) const ( @@ -13,9 +14,10 @@ const ( RPCRequestSourceHTTP = "http" RPCRequestSourceWS = "ws" + BackendProxyd = "proxyd" SourceClient = "client" SourceBackend = "backend" - SourceProxyd = "proxyd" + MethodUnknown = "unknown" ) var ( @@ -42,10 +44,22 @@ var ( Help: "Count of total RPC errors.", }, []string{ "auth", - "source", + "backend_name", + "method_name", "error_code", }) + rpcSpecialErrorsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: MetricsNamespace, + Name: "rpc_special_errors_total", + Help: "Count of total special RPC errors.", + }, []string{ + "auth", + "backend_name", + "method_name", + "error_type", + }) + rpcBackendRequestDurationSumm = promauto.NewSummaryVec(prometheus.SummaryOpts{ Namespace: MetricsNamespace, Name: "rpc_backend_request_duration_seconds", @@ -78,7 +92,7 @@ var ( Help: "Count of total requests that were rejected due to no backends being available.", }, []string{ "auth", - "source", + "request_source", }) httpRequestsTotal = promauto.NewCounter(prometheus.CounterOpts{ @@ -111,22 +125,30 @@ var ( }, []string{ "source", }) + + rpcSpecialErrors = []string{ + "nonce too low", + "gas price too high", + "gas price too low", + "invalid parameters", + } ) func RecordRedisError(source string) { redisErrorsTotal.WithLabelValues(source).Inc() } -func RecordRPCError(ctx context.Context, source string, err error) { +func RecordRPCError(ctx context.Context, backendName, method string, err error) { rpcErr, ok := err.(*RPCErr) var code int if ok { + MaybeRecordSpecialRPCError(ctx, backendName, method, rpcErr) code = rpcErr.Code } else { code = -1 } - rpcErrorsTotal.WithLabelValues(GetAuthCtx(ctx), source, strconv.Itoa(code)).Inc() + rpcErrorsTotal.WithLabelValues(GetAuthCtx(ctx), backendName, method, strconv.Itoa(code)).Inc() } func RecordWSMessage(ctx context.Context, backendName, source string) { @@ -140,3 +162,13 @@ func RecordUnserviceableRequest(ctx context.Context, source string) { func RecordRPCForward(ctx context.Context, backendName, method, source string) { rpcForwardsTotal.WithLabelValues(GetAuthCtx(ctx), backendName, method, source).Inc() } + +func MaybeRecordSpecialRPCError(ctx context.Context, backendName, method string, rpcErr *RPCErr) { + errMsg := strings.ToLower(rpcErr.Message) + for _, errStr := range rpcSpecialErrors { + if strings.Contains(errMsg, errStr) { + rpcSpecialErrorsTotal.WithLabelValues(GetAuthCtx(ctx), backendName, method, errStr).Inc() + return + } + } +} diff --git a/go/proxyd/server.go b/go/proxyd/server.go index 7c0124f539f35..454d5eec56d0c 100644 --- a/go/proxyd/server.go +++ b/go/proxyd/server.go @@ -78,33 +78,21 @@ func (s *Server) HandleRPC(w http.ResponseWriter, r *http.Request) { req, err := ParseRPCReq(io.LimitReader(r.Body, s.maxBodySize)) if err != nil { log.Info("rejected request with bad rpc request", "source", "rpc", "err", err) - RecordRPCError(ctx, SourceClient, err) + RecordRPCError(ctx, BackendProxyd, MethodUnknown, err) writeRPCError(w, nil, err) return } backendRes, err := s.backends.Forward(ctx, req) if err != nil { - if errors.Is(err, ErrNoBackends) { - RecordUnserviceableRequest(ctx, RPCRequestSourceHTTP) - RecordRPCError(ctx, SourceProxyd, err) - } else if errors.Is(err, ErrMethodNotWhitelisted) { - RecordRPCError(ctx, SourceClient, err) - } else { - RecordRPCError(ctx, SourceBackend, err) - } log.Error("error forwarding RPC request", "method", req.Method, "err", err) writeRPCError(w, req.ID, err) return } - if backendRes.IsError() { - RecordRPCError(ctx, SourceBackend, backendRes.Error) - } - enc := json.NewEncoder(w) if err := enc.Encode(backendRes); err != nil { log.Error("error encoding response", "err", err) - RecordRPCError(ctx, SourceProxyd, err) + RecordRPCError(ctx, BackendProxyd, req.Method, err) writeRPCError(w, req.ID, err) return }