diff --git a/proxyd/backend.go b/proxyd/backend.go index db19ff6..95653eb 100644 --- a/proxyd/backend.go +++ b/proxyd/backend.go @@ -989,13 +989,21 @@ func weightedShuffle(backends []*Backend) { func (bg *BackendGroup) orderedBackendsForRequest() []*Backend { if bg.Consensus != nil { return bg.loadBalancedConsensusGroup() - } else if bg.WeightedRouting { - result := make([]*Backend, len(bg.Backends)) - copy(result, bg.Backends) - weightedShuffle(result) - return result } else { - return bg.Backends + healthy := make([]*Backend, 0, len(bg.Backends)) + unhealthy := make([]*Backend, 0, len(bg.Backends)) + for _, be := range bg.Backends { + if be.IsHealthy() { + healthy = append(healthy, be) + } else { + unhealthy = append(unhealthy, be) + } + } + if bg.WeightedRouting { + weightedShuffle(healthy) + weightedShuffle(unhealthy) + } + return append(healthy, unhealthy...) } } diff --git a/proxyd/integration_tests/failover_test.go b/proxyd/integration_tests/failover_test.go index 501542a..1d371a6 100644 --- a/proxyd/integration_tests/failover_test.go +++ b/proxyd/integration_tests/failover_test.go @@ -87,6 +87,52 @@ func TestFailover(t *testing.T) { }) } + // the bad endpoint has had 10 requests with 8 error (3xx/4xx/5xx) responses, it should be marked as unhealthy and deprioritized + t.Run("bad endpoint marked as unhealthy", func(t *testing.T) { + res, statusCode, err := client.SendRPC("eth_chainId", nil) + require.NoError(t, err) + require.Equal(t, 200, statusCode) + RequireEqualJSON(t, []byte(goodResponse), res) + require.Equal(t, 1, len(goodBackend.Requests())) + require.Equal(t, 0, len(badBackend.Requests())) // bad backend is not called anymore + goodBackend.Reset() + badBackend.Reset() + }) + + t.Run("bad endpoint is still called if good endpoint also went bad", func(t *testing.T) { + goodBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + _, _ = w.Write([]byte("[{}]")) + })) + badBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(503) + _, _ = w.Write([]byte(unexpectedResponse)) + })) + res, statusCode, _ := client.SendRPC("eth_chainId", nil) + require.Equal(t, 503, statusCode) + RequireEqualJSON(t, []byte(noBackendsResponse), res) // return no backend available since both failed + require.Equal(t, 1, len(goodBackend.Requests())) + require.Equal(t, 1, len(badBackend.Requests())) // bad backend is still called + goodBackend.Reset() + badBackend.Reset() + }) +} + +func TestFailoverMore(t *testing.T) { + goodBackend := NewMockBackend(BatchedResponseHandler(200, goodResponse)) + defer goodBackend.Close() + badBackend := NewMockBackend(nil) + defer badBackend.Close() + + require.NoError(t, os.Setenv("GOOD_BACKEND_RPC_URL", goodBackend.URL())) + require.NoError(t, os.Setenv("BAD_BACKEND_RPC_URL", badBackend.URL())) + + config := ReadConfig("failover") + client := NewProxydClient("http://127.0.0.1:8545") + _, shutdown, err := proxyd.Start(config) + require.NoError(t, err) + defer shutdown() + t.Run("backend times out and falls back to another", func(t *testing.T) { badBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(2 * time.Second)