tornadocontrib/infra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add healthcheck ordering to simple failover mode (#38)

Browse Source 
* add healthcheck to simple failover

* update comment

* update comment

* add another test
This commit is contained in:
cody-wang-cb 2024-08-01 19:17:25 -04:00 committed by GitHub
parent 0fb094feb4
commit ba221ab80f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 60 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
proxyd/backend.go
View File

@ -989,13 +989,21 @@ func weightedShuffle(backends []*Backend) {
func (bg *BackendGroup) orderedBackendsForRequest() []*Backend {
if bg.Consensus != nil {
return bg.loadBalancedConsensusGroup()
} else if bg.WeightedRouting {
result := make([]*Backend, len(bg.Backends))
copy(result, bg.Backends)
weightedShuffle(result)
return result
} else {
return bg.Backends
healthy := make([]*Backend, 0, len(bg.Backends))
unhealthy := make([]*Backend, 0, len(bg.Backends))
for _, be := range bg.Backends {
if be.IsHealthy() {
healthy = append(healthy, be)
} else {
unhealthy = append(unhealthy, be)
}
}
if bg.WeightedRouting {
weightedShuffle(healthy)
weightedShuffle(unhealthy)
}
return append(healthy, unhealthy...)
}
}

46
proxyd/integration_tests/failover_test.go
View File

@ -87,6 +87,52 @@ func TestFailover(t *testing.T) {
})
}
// the bad endpoint has had 10 requests with 8 error (3xx/4xx/5xx) responses, it should be marked as unhealthy and deprioritized
t.Run("bad endpoint marked as unhealthy", func(t *testing.T) {
res, statusCode, err := client.SendRPC("eth_chainId", nil)
require.NoError(t, err)
require.Equal(t, 200, statusCode)
RequireEqualJSON(t, []byte(goodResponse), res)
require.Equal(t, 1, len(goodBackend.Requests()))
require.Equal(t, 0, len(badBackend.Requests())) // bad backend is not called anymore
goodBackend.Reset()
badBackend.Reset()
})
t.Run("bad endpoint is still called if good endpoint also went bad", func(t *testing.T) {
goodBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)
_, _ = w.Write([]byte("[{}]"))
}))
badBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(503)
_, _ = w.Write([]byte(unexpectedResponse))
}))
res, statusCode, _ := client.SendRPC("eth_chainId", nil)
require.Equal(t, 503, statusCode)
RequireEqualJSON(t, []byte(noBackendsResponse), res) // return no backend available since both failed
require.Equal(t, 1, len(goodBackend.Requests()))
require.Equal(t, 1, len(badBackend.Requests())) // bad backend is still called
goodBackend.Reset()
badBackend.Reset()
})
}
func TestFailoverMore(t *testing.T) {
goodBackend := NewMockBackend(BatchedResponseHandler(200, goodResponse))
defer goodBackend.Close()
badBackend := NewMockBackend(nil)
defer badBackend.Close()
require.NoError(t, os.Setenv("GOOD_BACKEND_RPC_URL", goodBackend.URL()))
require.NoError(t, os.Setenv("BAD_BACKEND_RPC_URL", badBackend.URL()))
config := ReadConfig("failover")
client := NewProxydClient("http://127.0.0.1:8545")
_, shutdown, err := proxyd.Start(config)
require.NoError(t, err)
defer shutdown()
t.Run("backend times out and falls back to another", func(t *testing.T) {
badBackend.SetHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)