package integration_tests import ( "context" "fmt" "net/http" "os" "path" "testing" "time" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum-optimism/optimism/proxyd" ms "github.com/ethereum-optimism/optimism/proxyd/tools/mockserver/handler" "github.com/stretchr/testify/require" ) func setup_failover(t *testing.T) (map[string]nodeContext, *proxyd.BackendGroup, *ProxydHTTPClient, func(), []time.Time, []time.Time) { // setup mock servers node1 := NewMockBackend(nil) node2 := NewMockBackend(nil) dir, err := os.Getwd() require.NoError(t, err) responses := path.Join(dir, "testdata/consensus_responses.yml") h1 := ms.MockedHandler{ Overrides: []*ms.MethodTemplate{}, Autoload: true, AutoloadFile: responses, } h2 := ms.MockedHandler{ Overrides: []*ms.MethodTemplate{}, Autoload: true, AutoloadFile: responses, } require.NoError(t, os.Setenv("NODE1_URL", node1.URL())) require.NoError(t, os.Setenv("NODE2_URL", node2.URL())) node1.SetHandler(http.HandlerFunc(h1.Handler)) node2.SetHandler(http.HandlerFunc(h2.Handler)) // setup proxyd config := ReadConfig("fallback") svr, shutdown, err := proxyd.Start(config) require.NoError(t, err) // expose the proxyd client client := NewProxydClient("http://127.0.0.1:8545") // expose the backend group bg := svr.BackendGroups["node"] require.NotNil(t, bg) require.NotNil(t, bg.Consensus) require.Equal(t, 2, len(bg.Backends)) // should match config // convenient mapping to access the nodes by name nodes := map[string]nodeContext{ "normal": { mockBackend: node1, backend: bg.Backends[0], handler: &h1, }, "fallback": { mockBackend: node2, backend: bg.Backends[1], handler: &h2, }, } normalTimestamps := []time.Time{} fallbackTimestamps := []time.Time{} return nodes, bg, client, shutdown, normalTimestamps, fallbackTimestamps } func TestFallback(t *testing.T) { nodes, bg, client, shutdown, normalTimestamps, fallbackTimestamps := setup_failover(t) defer nodes["normal"].mockBackend.Close() defer nodes["fallback"].mockBackend.Close() defer shutdown() ctx := context.Background() // Use Update to Advance the Candidate iteration update := func() { for _, be := range bg.Primaries() { bg.Consensus.UpdateBackend(ctx, be) } for _, be := range bg.Fallbacks() { healthyCandidates := bg.Consensus.FilterCandidates(bg.Primaries()) if len(healthyCandidates) == 0 { bg.Consensus.UpdateBackend(ctx, be) } } bg.Consensus.UpdateBackendGroupConsensus(ctx) } override := func(node string, method string, block string, response string) { if _, ok := nodes[node]; !ok { t.Fatalf("node %s does not exist in the nodes map", node) } nodes[node].handler.AddOverride(&ms.MethodTemplate{ Method: method, Block: block, Response: response, }) } overrideBlock := func(node string, blockRequest string, blockResponse string) { override(node, "eth_getBlockByNumber", blockRequest, buildResponse(map[string]string{ "number": blockResponse, "hash": "hash_" + blockResponse, })) } overrideBlockHash := func(node string, blockRequest string, number string, hash string) { override(node, "eth_getBlockByNumber", blockRequest, buildResponse(map[string]string{ "number": number, "hash": hash, })) } overridePeerCount := func(node string, count int) { override(node, "net_peerCount", "", buildResponse(hexutil.Uint64(count).String())) } overrideNotInSync := func(node string) { override(node, "eth_syncing", "", buildResponse(map[string]string{ "startingblock": "0x0", "currentblock": "0x0", "highestblock": "0x100", })) } containsNode := func(backends []*proxyd.Backend, name string) bool { for _, be := range backends { // Note: Currently checks for name but would like to expose fallback better if be.Name == name { return true } } return false } // TODO: Improvement instead of simple array, // ensure normal and backend are returned in strict order recordLastUpdates := func(backends []*proxyd.Backend) []time.Time { lastUpdated := []time.Time{} for _, be := range backends { lastUpdated = append(lastUpdated, bg.Consensus.GetLastUpdate(be)) } return lastUpdated } // convenient methods to manipulate state and mock responses reset := func() { for _, node := range nodes { node.handler.ResetOverrides() node.mockBackend.Reset() } bg.Consensus.ClearListeners() bg.Consensus.Reset() normalTimestamps = []time.Time{} fallbackTimestamps = []time.Time{} } /* triggerFirstNormalFailure: will trigger consensus group into fallback mode old consensus group should be returned one time, and fallback group should be enabled Fallback will be returned subsequent update */ triggerFirstNormalFailure := func() { overridePeerCount("normal", 0) update() require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) nodes["fallback"].mockBackend.Reset() } t.Run("Test fallback Mode will not be exited, unless state changes", func(t *testing.T) { reset() triggerFirstNormalFailure() for i := 0; i < 10; i++ { update() require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) } }) t.Run("Test Healthy mode will not be exited unless state changes", func(t *testing.T) { reset() for i := 0; i < 10; i++ { update() require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) _, statusCode, err := client.SendRPC("eth_getBlockByNumber", []interface{}{"0x101", false}) require.Equal(t, 200, statusCode) require.Nil(t, err, "error not nil") require.Equal(t, "0x101", bg.Consensus.GetLatestBlockNumber().String()) require.Equal(t, "0xe1", bg.Consensus.GetSafeBlockNumber().String()) require.Equal(t, "0xc1", bg.Consensus.GetFinalizedBlockNumber().String()) } // TODO: Remove these, just here so compiler doesn't complain overrideNotInSync("normal") overrideBlock("normal", "safe", "0xb1") overrideBlockHash("fallback", "0x102", "0x102", "wrong_hash") }) t.Run("trigger normal failure, subsequent update return failover in consensus group, and fallback mode enabled", func(t *testing.T) { reset() triggerFirstNormalFailure() update() require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) }) t.Run("trigger healthy -> fallback, update -> healthy", func(t *testing.T) { reset() update() require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) triggerFirstNormalFailure() update() require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) overridePeerCount("normal", 5) update() require.Equal(t, 1, len(bg.Consensus.GetConsensusGroup())) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) }) t.Run("Ensure fallback is not updated when in normal mode", func(t *testing.T) { reset() for i := 0; i < 10; i++ { update() ts := recordLastUpdates(bg.Backends) normalTimestamps = append(normalTimestamps, ts[0]) fallbackTimestamps = append(fallbackTimestamps, ts[1]) require.False(t, normalTimestamps[i].IsZero()) require.True(t, fallbackTimestamps[i].IsZero()) require.True(t, containsNode(bg.Consensus.GetConsensusGroup(), "normal")) require.False(t, containsNode(bg.Consensus.GetConsensusGroup(), "fallback")) // consensus at block 0x101 require.Equal(t, "0x101", bg.Consensus.GetLatestBlockNumber().String()) require.Equal(t, "0xe1", bg.Consensus.GetSafeBlockNumber().String()) require.Equal(t, "0xc1", bg.Consensus.GetFinalizedBlockNumber().String()) } }) /* Set Normal backend to Fail -> both backends should be updated */ t.Run("Ensure both nodes are quieried in fallback mode", func(t *testing.T) { reset() triggerFirstNormalFailure() for i := 0; i < 10; i++ { update() ts := recordLastUpdates(bg.Backends) normalTimestamps = append(normalTimestamps, ts[0]) fallbackTimestamps = append(fallbackTimestamps, ts[1]) // Both Nodes should be updated again require.False(t, normalTimestamps[i].IsZero()) require.False(t, fallbackTimestamps[i].IsZero(), fmt.Sprintf("Error: Fallback timestamp: %v was not queried on iteratio %d", fallbackTimestamps[i], i), ) if i > 0 { require.Greater(t, normalTimestamps[i], normalTimestamps[i-1]) require.Greater(t, fallbackTimestamps[i], fallbackTimestamps[i-1]) } } }) t.Run("Ensure both nodes are quieried in fallback mode", func(t *testing.T) { reset() triggerFirstNormalFailure() for i := 0; i < 10; i++ { update() ts := recordLastUpdates(bg.Backends) normalTimestamps = append(normalTimestamps, ts[0]) fallbackTimestamps = append(fallbackTimestamps, ts[1]) // Both Nodes should be updated again require.False(t, normalTimestamps[i].IsZero()) require.False(t, fallbackTimestamps[i].IsZero(), fmt.Sprintf("Error: Fallback timestamp: %v was not queried on iteratio %d", fallbackTimestamps[i], i), ) if i > 0 { require.Greater(t, normalTimestamps[i], normalTimestamps[i-1]) require.Greater(t, fallbackTimestamps[i], fallbackTimestamps[i-1]) } } }) t.Run("Healthy -> Fallback -> Healthy with timestamps", func(t *testing.T) { reset() for i := 0; i < 10; i++ { update() ts := recordLastUpdates(bg.Backends) normalTimestamps = append(normalTimestamps, ts[0]) fallbackTimestamps = append(fallbackTimestamps, ts[1]) // Normal is queried, fallback is not require.False(t, normalTimestamps[i].IsZero()) require.True(t, fallbackTimestamps[i].IsZero(), fmt.Sprintf("Error: Fallback timestamp: %v was not queried on iteratio %d", fallbackTimestamps[i], i), ) if i > 0 { require.Greater(t, normalTimestamps[i], normalTimestamps[i-1]) // Fallbacks should be zeros require.Equal(t, fallbackTimestamps[i], fallbackTimestamps[i-1]) } } offset := 10 triggerFirstNormalFailure() for i := 0; i < 10; i++ { update() ts := recordLastUpdates(bg.Backends) normalTimestamps = append(normalTimestamps, ts[0]) fallbackTimestamps = append(fallbackTimestamps, ts[1]) // Both Nodes should be updated again require.False(t, normalTimestamps[i+offset].IsZero()) require.False(t, fallbackTimestamps[i+offset].IsZero()) require.Greater(t, normalTimestamps[i+offset], normalTimestamps[i+offset-1]) require.Greater(t, fallbackTimestamps[i+offset], fallbackTimestamps[i+offset-1]) } overridePeerCount("normal", 5) offset = 20 for i := 0; i < 10; i++ { update() ts := recordLastUpdates(bg.Backends) normalTimestamps = append(normalTimestamps, ts[0]) fallbackTimestamps = append(fallbackTimestamps, ts[1]) // Normal Node will be updated require.False(t, normalTimestamps[i+offset].IsZero()) require.Greater(t, normalTimestamps[i+offset], normalTimestamps[i+offset-1]) // fallback should not be updating if offset+i > 21 { require.Equal(t, fallbackTimestamps[i+offset], fallbackTimestamps[i+offset-1]) } } }) }