From 8a624bb2ca2ef2ce56c3b57f3253fc9b5f6da6ac Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Thu, 13 Oct 2022 14:54:30 -0500 Subject: [PATCH 1/4] proxyd: Allow disabling backend rate limiting The backend rate limiter is in place to protect upstreams like the sequencer. However, in many cases it isn't needed and it causes unnecessary requests to Redis. This PR allows this to be disabled, and disables this by default. --- proxyd/proxyd/backend_rate_limiter.go | 28 ++++++++++++++++++- proxyd/proxyd/config.go | 15 +++++----- .../testdata/backend_rate_limit.toml | 5 +++- proxyd/proxyd/proxyd.go | 12 +++++--- 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/proxyd/proxyd/backend_rate_limiter.go b/proxyd/proxyd/backend_rate_limiter.go index 03c6436..3cc6fae 100644 --- a/proxyd/proxyd/backend_rate_limiter.go +++ b/proxyd/proxyd/backend_rate_limiter.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/hex" "fmt" + "math" "sync" "time" @@ -256,5 +257,30 @@ func randStr(l int) string { return hex.EncodeToString(b) } -type ServerRateLimiter struct { +type NoopBackendRateLimiter struct{} + +var noopBackendRateLimiter = &NoopBackendRateLimiter{} + +func (n *NoopBackendRateLimiter) IsBackendOnline(name string) (bool, error) { + return true, nil +} + +func (n *NoopBackendRateLimiter) SetBackendOffline(name string, duration time.Duration) error { + return nil +} + +func (n *NoopBackendRateLimiter) IncBackendRPS(name string) (int, error) { + return math.MaxInt, nil +} + +func (n *NoopBackendRateLimiter) IncBackendWSConns(name string, max int) (bool, error) { + return true, nil +} + +func (n *NoopBackendRateLimiter) DecBackendWSConns(name string) error { + return nil +} + +func (n *NoopBackendRateLimiter) FlushBackendWSConns(names []string) error { + return nil } diff --git a/proxyd/proxyd/config.go b/proxyd/proxyd/config.go index d0a32d6..de221d9 100644 --- a/proxyd/proxyd/config.go +++ b/proxyd/proxyd/config.go @@ -41,13 +41,14 @@ type MetricsConfig struct { } type RateLimitConfig struct { - UseRedis bool `toml:"use_redis"` - BaseRate int `toml:"base_rate"` - BaseInterval TOMLDuration `toml:"base_interval"` - ExemptOrigins []string `toml:"exempt_origins"` - ExemptUserAgents []string `toml:"exempt_user_agents"` - ErrorMessage string `toml:"error_message"` - MethodOverrides map[string]*RateLimitMethodOverride `toml:"method_overrides"` + UseRedis bool `toml:"use_redis"` + EnableBackendRateLimiter bool `toml:"enable_backend_rate_limiter"` + BaseRate int `toml:"base_rate"` + BaseInterval TOMLDuration `toml:"base_interval"` + ExemptOrigins []string `toml:"exempt_origins"` + ExemptUserAgents []string `toml:"exempt_user_agents"` + ErrorMessage string `toml:"error_message"` + MethodOverrides map[string]*RateLimitMethodOverride `toml:"method_overrides"` } type RateLimitMethodOverride struct { diff --git a/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml b/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml index eca6580..e29bb19 100644 --- a/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml +++ b/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml @@ -15,4 +15,7 @@ max_rps = 2 backends = ["good"] [rpc_method_mappings] -eth_chainId = "main" \ No newline at end of file +eth_chainId = "main" + +[rate_limit] +enable_backend_limiter = true \ No newline at end of file diff --git a/proxyd/proxyd/proxyd.go b/proxyd/proxyd/proxyd.go index 5685633..3e21d71 100644 --- a/proxyd/proxyd/proxyd.go +++ b/proxyd/proxyd/proxyd.go @@ -53,11 +53,15 @@ func Start(config *Config) (func(), error) { var lim BackendRateLimiter var err error - if redisClient == nil { - log.Warn("redis is not configured, using local rate limiter") - lim = NewLocalBackendRateLimiter() + if config.RateLimit.EnableBackendRateLimiter { + if redisClient != nil { + lim = NewRedisRateLimiter(redisClient) + } else { + log.Warn("redis is not configured, using local rate limiter") + lim = NewLocalBackendRateLimiter() + } } else { - lim = NewRedisRateLimiter(redisClient) + lim = noopBackendRateLimiter } // While modifying shared globals is a bad practice, the alternative From 24a3668bc8d3657ed239f6005b553d7cd183e7d4 Mon Sep 17 00:00:00 2001 From: Zach Howard Date: Fri, 14 Oct 2022 14:56:27 -0400 Subject: [PATCH 2/4] adds log level conf to proxyd (#3704) --- proxyd/proxyd/cmd/proxyd/main.go | 15 +++++++++++++++ proxyd/proxyd/config.go | 1 + proxyd/proxyd/example.config.toml | 2 ++ 3 files changed, 18 insertions(+) diff --git a/proxyd/proxyd/cmd/proxyd/main.go b/proxyd/proxyd/cmd/proxyd/main.go index db3828b..c184a1d 100644 --- a/proxyd/proxyd/cmd/proxyd/main.go +++ b/proxyd/proxyd/cmd/proxyd/main.go @@ -37,6 +37,21 @@ func main() { log.Crit("error reading config file", "err", err) } + // update log level from config + logLevel, err := log.LvlFromString(config.Server.LogLevel) + if err != nil { + logLevel = log.LvlInfo + if config.Server.LogLevel != "" { + log.Warn("invalid server.log_level set: " + config.Server.LogLevel) + } + } + log.Root().SetHandler( + log.LvlFilterHandler( + logLevel, + log.StreamHandler(os.Stdout, log.JSONFormat()), + ), + ) + shutdown, err := proxyd.Start(config) if err != nil { log.Crit("error starting proxyd", "err", err) diff --git a/proxyd/proxyd/config.go b/proxyd/proxyd/config.go index d0a32d6..9890121 100644 --- a/proxyd/proxyd/config.go +++ b/proxyd/proxyd/config.go @@ -14,6 +14,7 @@ type ServerConfig struct { WSPort int `toml:"ws_port"` MaxBodySizeBytes int64 `toml:"max_body_size_bytes"` MaxConcurrentRPCs int64 `toml:"max_concurrent_rpcs"` + LogLevel string `toml:"log_level"` // TimeoutSeconds specifies the maximum time spent serving an HTTP request. Note that isn't used for websocket connections TimeoutSeconds int `toml:"timeout_seconds"` diff --git a/proxyd/proxyd/example.config.toml b/proxyd/proxyd/example.config.toml index 2573910..fb8fea9 100644 --- a/proxyd/proxyd/example.config.toml +++ b/proxyd/proxyd/example.config.toml @@ -19,6 +19,8 @@ ws_port = 8085 # Maximum client body size, in bytes, that the server will accept. max_body_size_bytes = 10485760 max_concurrent_rpcs = 1000 +# Server log level +log_level = "info" [redis] # URL to a Redis instance. From fd5e974bd562fe34a96109c7d15b8c7be3f4c99c Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Sat, 15 Oct 2022 09:59:11 -0500 Subject: [PATCH 3/4] fix tests --- .../integration_tests/testdata/backend_rate_limit.toml | 2 +- .../integration_tests/testdata/out_of_service_interval.toml | 5 ++++- proxyd/proxyd/integration_tests/testdata/ws.toml | 3 +++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml b/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml index e29bb19..17500f3 100644 --- a/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml +++ b/proxyd/proxyd/integration_tests/testdata/backend_rate_limit.toml @@ -18,4 +18,4 @@ backends = ["good"] eth_chainId = "main" [rate_limit] -enable_backend_limiter = true \ No newline at end of file +enable_backend_rate_limiter = true \ No newline at end of file diff --git a/proxyd/proxyd/integration_tests/testdata/out_of_service_interval.toml b/proxyd/proxyd/integration_tests/testdata/out_of_service_interval.toml index 6663721..4611251 100644 --- a/proxyd/proxyd/integration_tests/testdata/out_of_service_interval.toml +++ b/proxyd/proxyd/integration_tests/testdata/out_of_service_interval.toml @@ -19,4 +19,7 @@ ws_url = "$BAD_BACKEND_RPC_URL" backends = ["bad", "good"] [rpc_method_mappings] -eth_chainId = "main" \ No newline at end of file +eth_chainId = "main" + +[rate_limit] +enable_backend_rate_limiter = true \ No newline at end of file diff --git a/proxyd/proxyd/integration_tests/testdata/ws.toml b/proxyd/proxyd/integration_tests/testdata/ws.toml index 4642e6b..7340717 100644 --- a/proxyd/proxyd/integration_tests/testdata/ws.toml +++ b/proxyd/proxyd/integration_tests/testdata/ws.toml @@ -26,3 +26,6 @@ backends = ["good"] [rpc_method_mappings] eth_chainId = "main" + +[rate_limit] +enable_backend_rate_limiter = true \ No newline at end of file From fe8a89e61dcdfc05b7b84b135a45d232f30dad48 Mon Sep 17 00:00:00 2001 From: Matthew Slipper Date: Mon, 17 Oct 2022 07:08:33 -0600 Subject: [PATCH 4/4] proxyd: Support pattern matching in origin and user agent Adds support for pattern matchin in exempt origins. This should help with some of the issues Synthetix and others are seeing. --- proxyd/proxyd/server.go | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/proxyd/proxyd/server.go b/proxyd/proxyd/server.go index e86ba4c..1ee10fa 100644 --- a/proxyd/proxyd/server.go +++ b/proxyd/proxyd/server.go @@ -8,6 +8,7 @@ import ( "io" "math" "net/http" + "regexp" "strconv" "strings" "sync" @@ -49,8 +50,8 @@ type Server struct { upgrader *websocket.Upgrader mainLim FrontendRateLimiter overrideLims map[string]FrontendRateLimiter - limExemptOrigins map[string]bool - limExemptUserAgents map[string]bool + limExemptOrigins []*regexp.Regexp + limExemptUserAgents []*regexp.Regexp rpcServer *http.Server wsServer *http.Server cache RPCCache @@ -104,15 +105,23 @@ func NewServer( } var mainLim FrontendRateLimiter - limExemptOrigins := make(map[string]bool) - limExemptUserAgents := make(map[string]bool) + limExemptOrigins := make([]*regexp.Regexp, 0) + limExemptUserAgents := make([]*regexp.Regexp, 0) if rateLimitConfig.BaseRate > 0 { mainLim = limiterFactory(time.Duration(rateLimitConfig.BaseInterval), rateLimitConfig.BaseRate, "main") for _, origin := range rateLimitConfig.ExemptOrigins { - limExemptOrigins[strings.ToLower(origin)] = true + pattern, err := regexp.Compile(origin) + if err != nil { + return nil, err + } + limExemptOrigins = append(limExemptOrigins, pattern) } for _, agent := range rateLimitConfig.ExemptUserAgents { - limExemptUserAgents[strings.ToLower(agent)] = true + pattern, err := regexp.Compile(agent) + if err != nil { + return nil, err + } + limExemptUserAgents = append(limExemptUserAgents, pattern) } } else { mainLim = NoopFrontendRateLimiter @@ -548,11 +557,22 @@ func (s *Server) populateContext(w http.ResponseWriter, r *http.Request) context } func (s *Server) isUnlimitedOrigin(origin string) bool { - return s.limExemptOrigins[strings.ToLower(origin)] + for _, pat := range s.limExemptOrigins { + if pat.MatchString(origin) { + return true + } + } + + return false } func (s *Server) isUnlimitedUserAgent(origin string) bool { - return s.limExemptUserAgents[strings.ToLower(origin)] + for _, pat := range s.limExemptUserAgents { + if pat.MatchString(origin) { + return true + } + } + return false } func setCacheHeader(w http.ResponseWriter, cached bool) {