proxyd: Handle unexpected JSON-RPC responses (#2628)

This fixes a bug where the infura backend would be labeled offline because it
returns an unexpected JSON-RPC response. Unexpected, but well-formed,
JSON-RPC response are handled specially. Such errors are surfaced up to
the backend proxier so failover still occurs.

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Murphy Law 2022-06-08 09:56:24 -04:00 committed by GitHub
parent 02e05187a9
commit 69f189c0ea
2 changed files with 78 additions and 4 deletions

@ -73,6 +73,8 @@ var (
Message: "gateway timeout",
HTTPErrorCode: 504,
}
ErrBackendUnexpectedJSONRPC = errors.New("backend returned an unexpected JSON-RPC response")
)
func ErrInvalidRequest(msg string) *RPCErr {
@ -228,7 +230,20 @@ func (b *Backend) Forward(ctx context.Context, reqs []*RPCReq, isBatch bool) ([]
)
res, err := b.doForward(ctx, reqs, isBatch)
if err != nil {
switch err {
case nil: // do nothing
// ErrBackendUnexpectedJSONRPC occurs because infura responds with a single JSON-RPC object
// to a batch request whenever any Request Object in the batch would induce a partial error.
// We don't label the the backend offline in this case. But the error is still returned to
// callers so failover can occur if needed.
case ErrBackendUnexpectedJSONRPC:
log.Debug(
"Reecived unexpected JSON-RPC response",
"name", b.Name,
"req_id", GetReqID(ctx),
"err", err,
)
default:
lastError = err
log.Warn(
"backend request failed, trying again",
@ -244,7 +259,7 @@ func (b *Backend) Forward(ctx context.Context, reqs []*RPCReq, isBatch bool) ([]
timer.ObserveDuration()
MaybeRecordErrorsInRPCRes(ctx, b.Name, reqs, res)
return res, nil
return res, err
}
b.setOffline()
@ -387,12 +402,15 @@ func (b *Backend) doForward(ctx context.Context, rpcReqs []*RPCReq, isBatch bool
var res []*RPCRes
if err := json.Unmarshal(resB, &res); err != nil {
// Infura may return a single JSON-RPC response if, for example, the batch contains a request for an unsupported method
if responseIsNotBatched(resB) {
return nil, ErrBackendUnexpectedJSONRPC
}
return nil, ErrBackendBadResponse
}
// Alas! Certain node providers (Infura) always return a single JSON object for some types of errors
if len(rpcReqs) != len(res) {
return nil, ErrBackendBadResponse
return nil, ErrBackendUnexpectedJSONRPC
}
// capture the HTTP status code in the response. this will only
@ -407,6 +425,11 @@ func (b *Backend) doForward(ctx context.Context, rpcReqs []*RPCReq, isBatch bool
return res, nil
}
func responseIsNotBatched(b []byte) bool {
var r RPCRes
return json.Unmarshal(b, &r) == nil
}
// sortBatchRPCResponse sorts the RPCRes slice according to the position of its corresponding ID in the RPCReq slice
func sortBatchRPCResponse(req []*RPCReq, res []*RPCRes) {
pos := make(map[string]int, len(req))

@ -8,6 +8,7 @@ import (
"testing"
"time"
"github.com/alicebob/miniredis"
"github.com/ethereum-optimism/optimism/proxyd"
"github.com/stretchr/testify/require"
)
@ -15,6 +16,7 @@ import (
const (
goodResponse = `{"jsonrpc": "2.0", "result": "hello", "id": 999}`
noBackendsResponse = `{"error":{"code":-32011,"message":"no backends available for method"},"id":999,"jsonrpc":"2.0"}`
unexpectedResponse = `{"error":{"code":-32011,"message":"some error"},"id":999,"jsonrpc":"2.0"}`
)
func TestFailover(t *testing.T) {
@ -240,3 +242,52 @@ func TestBatchWithPartialFailover(t *testing.T) {
require.Equal(t, 2, len(badBackend.Requests()))
require.Equal(t, 2, len(goodBackend.Requests()))
}
func TestInfuraFailoverOnUnexpectedResponse(t *testing.T) {
InitLogger()
// Scenario:
// 1. Send batch to BAD_BACKEND (Infura)
// 2. Infura fails completely due to a partially errorneous batch request (one of N+1 request object is invalid)
// 3. Assert that the request batch is re-routed to the failover provider
// 4. Assert that BAD_BACKEND is NOT labeled offline
// 5. Assert that BAD_BACKEND is NOT retried
redis, err := miniredis.Run()
require.NoError(t, err)
defer redis.Close()
config := ReadConfig("failover")
config.Server.MaxUpstreamBatchSize = 2
config.BackendOptions.MaxRetries = 2
// Setup redis to detect offline backends
config.Redis.URL = fmt.Sprintf("redis://127.0.0.1:%s", redis.Port())
goodBackend := NewMockBackend(BatchedResponseHandler(200, goodResponse, goodResponse))
defer goodBackend.Close()
badBackend := NewMockBackend(SingleResponseHandler(200, unexpectedResponse))
defer badBackend.Close()
require.NoError(t, os.Setenv("GOOD_BACKEND_RPC_URL", goodBackend.URL()))
require.NoError(t, os.Setenv("BAD_BACKEND_RPC_URL", badBackend.URL()))
client := NewProxydClient("http://127.0.0.1:8545")
shutdown, err := proxyd.Start(config)
require.NoError(t, err)
defer shutdown()
res, statusCode, err := client.SendBatchRPC(
NewRPCReq("1", "eth_chainId", nil),
NewRPCReq("2", "eth_chainId", nil),
)
require.NoError(t, err)
require.Equal(t, 200, statusCode)
RequireEqualJSON(t, []byte(asArray(goodResponse, goodResponse)), res)
require.Equal(t, 1, len(badBackend.Requests()))
require.Equal(t, 1, len(goodBackend.Requests()))
rr, err := proxyd.NewRedisRateLimiter(config.Redis.URL)
require.NoError(t, err)
online, err := rr.IsBackendOnline("bad")
require.NoError(t, err)
require.Equal(t, true, online)
}