From 4a288db1645ba04c5755e93b565cebb4f2edc652 Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:07:17 -0700 Subject: [PATCH 01/11] add cooldown --- op-ufm/op-ufm/pkg/provider/roundtrip.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/op-ufm/op-ufm/pkg/provider/roundtrip.go b/op-ufm/op-ufm/pkg/provider/roundtrip.go index f404f33..aa443f1 100644 --- a/op-ufm/op-ufm/pkg/provider/roundtrip.go +++ b/op-ufm/op-ufm/pkg/provider/roundtrip.go @@ -58,6 +58,16 @@ func (p *Provider) RoundTrip(ctx context.Context) { // used for actual round trip time (disregard retry time) roundTripStartedAt := time.Now() for { + // sleep until we get a clear to send + for { + coolDown := time.Duration(p.config.SendTransactionCoolDown) - time.Since(p.txPool.LastSend) + if coolDown > 0 { + time.Sleep(coolDown) + } else { + break + } + } + tx := p.createTx(nonce) txHash = tx.Hash() From a2f166a9a62130666905f492174335c786bc6154 Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:08:09 -0700 Subject: [PATCH 02/11] config update --- op-ufm/op-ufm/pkg/config/config.go | 1 + op-ufm/op-ufm/pkg/provider/tx_pool.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/op-ufm/op-ufm/pkg/config/config.go b/op-ufm/op-ufm/pkg/config/config.go index 39c96ce..3f0abda 100644 --- a/op-ufm/op-ufm/pkg/config/config.go +++ b/op-ufm/op-ufm/pkg/config/config.go @@ -64,6 +64,7 @@ type ProviderConfig struct { SendInterval TOMLDuration `toml:"send_interval"` SendTransactionRetryInterval TOMLDuration `toml:"send_transaction_retry_interval"` SendTransactionRetryTimeout TOMLDuration `toml:"send_transaction_retry_timeout"` + SendTransactionCoolDown TOMLDuration `toml:"send_transaction_cool_down"` ReceiptRetrievalInterval TOMLDuration `toml:"receipt_retrieval_interval"` ReceiptRetrievalTimeout TOMLDuration `toml:"receipt_retrieval_timeout"` diff --git a/op-ufm/op-ufm/pkg/provider/tx_pool.go b/op-ufm/op-ufm/pkg/provider/tx_pool.go index af0c9b1..f6de257 100644 --- a/op-ufm/op-ufm/pkg/provider/tx_pool.go +++ b/op-ufm/op-ufm/pkg/provider/tx_pool.go @@ -16,6 +16,9 @@ type NetworkTransactionPool struct { Transactions map[string]*TransactionState Expected int Nonce uint64 + + // Last time a transaction was sent + LastSend time.Time } type TransactionState struct { From b5cda060b06c47bae607a6a8713f4a6afa160c13 Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:09:03 -0700 Subject: [PATCH 03/11] improve error metrics --- op-ufm/op-ufm/pkg/metrics/clients/eth.go | 10 +++++----- op-ufm/op-ufm/pkg/metrics/clients/signer.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/op-ufm/op-ufm/pkg/metrics/clients/eth.go b/op-ufm/op-ufm/pkg/metrics/clients/eth.go index 6986762..5c01d9a 100644 --- a/op-ufm/op-ufm/pkg/metrics/clients/eth.go +++ b/op-ufm/op-ufm/pkg/metrics/clients/eth.go @@ -22,7 +22,7 @@ func Dial(providerName string, url string) (*InstrumentedEthClient, error) { start := time.Now() c, err := ethclient.Dial(url) if err != nil { - metrics.RecordError(providerName, "ethclient.Dial") + metrics.RecordErrorDetails(providerName, "ethclient.Dial", err) return nil, err } metrics.RecordRPCLatency(providerName, "ethclient", "Dial", time.Since(start)) @@ -34,7 +34,7 @@ func (i *InstrumentedEthClient) TransactionByHash(ctx context.Context, hash comm tx, isPending, err := i.c.TransactionByHash(ctx, hash) if err != nil { if !i.ignorableErrors(err) { - metrics.RecordError(i.providerName, "ethclient.TransactionByHash") + metrics.RecordErrorDetails(i.providerName, "ethclient.TransactionByHash", err) } return nil, false, err } @@ -46,7 +46,7 @@ func (i *InstrumentedEthClient) PendingNonceAt(ctx context.Context, address stri start := time.Now() nonce, err := i.c.PendingNonceAt(ctx, common.HexToAddress(address)) if err != nil { - metrics.RecordError(i.providerName, "ethclient.PendingNonceAt") + metrics.RecordErrorDetails(i.providerName, "ethclient.PendingNonceAt", err) return 0, err } metrics.RecordRPCLatency(i.providerName, "ethclient", "PendingNonceAt", time.Since(start)) @@ -58,7 +58,7 @@ func (i *InstrumentedEthClient) TransactionReceipt(ctx context.Context, txHash c receipt, err := i.c.TransactionReceipt(ctx, txHash) if err != nil { if !i.ignorableErrors(err) { - metrics.RecordError(i.providerName, "ethclient.TransactionReceipt") + metrics.RecordErrorDetails(i.providerName, "ethclient.TransactionReceipt", err) } return nil, err } @@ -71,7 +71,7 @@ func (i *InstrumentedEthClient) SendTransaction(ctx context.Context, tx *types.T err := i.c.SendTransaction(ctx, tx) if err != nil { if !i.ignorableErrors(err) { - metrics.RecordError(i.providerName, "ethclient.SendTransaction") + metrics.RecordErrorDetails(i.providerName, "ethclient.SendTransaction", err) } return err } diff --git a/op-ufm/op-ufm/pkg/metrics/clients/signer.go b/op-ufm/op-ufm/pkg/metrics/clients/signer.go index 17a6cd2..54f1163 100644 --- a/op-ufm/op-ufm/pkg/metrics/clients/signer.go +++ b/op-ufm/op-ufm/pkg/metrics/clients/signer.go @@ -22,7 +22,7 @@ func NewSignerClient(providerName string, logger log.Logger, endpoint string, tl start := time.Now() c, err := signer.NewSignerClient(logger, endpoint, tlsConfig) if err != nil { - metrics.RecordError(providerName, "signer.NewSignerClient") + metrics.RecordErrorDetails(providerName, "signer.NewSignerClient", err) return nil, err } metrics.RecordRPCLatency(providerName, "signer", "NewSignerClient", time.Since(start)) @@ -33,7 +33,7 @@ func (i *InstrumentedSignerClient) SignTransaction(ctx context.Context, chainId start := time.Now() tx, err := i.c.SignTransaction(ctx, chainId, tx) if err != nil { - metrics.RecordError(i.providerName, "signer.SignTransaction") + metrics.RecordErrorDetails(i.providerName, "signer.SignTransaction", err) return nil, err } metrics.RecordRPCLatency(i.providerName, "signer", "SignTransaction", time.Since(start)) From 9c56cde7838d755819749937a4849b67fc8506ed Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:12:13 -0700 Subject: [PATCH 04/11] use network nonce --- op-ufm/op-ufm/pkg/provider/roundtrip.go | 43 ++++++++++++------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/op-ufm/op-ufm/pkg/provider/roundtrip.go b/op-ufm/op-ufm/pkg/provider/roundtrip.go index aa443f1..3868248 100644 --- a/op-ufm/op-ufm/pkg/provider/roundtrip.go +++ b/op-ufm/op-ufm/pkg/provider/roundtrip.go @@ -33,26 +33,10 @@ func (p *Provider) RoundTrip(ctx context.Context) { return } - var nonce uint64 - p.txPool.M.Lock() - if p.txPool.Nonce == uint64(0) { - nonce, err = client.PendingNonceAt(ctx, p.walletConfig.Address) - if err != nil { - log.Error("cant get nounce", - "provider", p.name, - "err", err) - p.txPool.M.Unlock() - return - } - p.txPool.Nonce = nonce - } else { - p.txPool.Nonce++ - nonce = p.txPool.Nonce - } - p.txPool.M.Unlock() - txHash := common.Hash{} attempt := 0 + nonce := uint64(0) + // used for timeout firstAttemptAt := time.Now() // used for actual round trip time (disregard retry time) @@ -68,6 +52,14 @@ func (p *Provider) RoundTrip(ctx context.Context) { } } + nonce, err = client.PendingNonceAt(ctx, p.walletConfig.Address) + if err != nil { + log.Error("cant get nounce", + "provider", p.name, + "err", err) + return + } + tx := p.createTx(nonce) txHash = tx.Hash() @@ -79,7 +71,6 @@ func (p *Provider) RoundTrip(ctx context.Context) { "err", err) return } - txHash = signedTx.Hash() roundTripStartedAt = time.Now() @@ -88,14 +79,15 @@ func (p *Provider) RoundTrip(ctx context.Context) { if err.Error() == txpool.ErrAlreadyKnown.Error() || err.Error() == txpool.ErrReplaceUnderpriced.Error() || err.Error() == core.ErrNonceTooLow.Error() { + if time.Since(firstAttemptAt) >= time.Duration(p.config.SendTransactionRetryTimeout) { log.Error("send transaction timed out (known already)", "provider", p.name, "hash", txHash.Hex(), + "nonce", nonce, "elapsed", time.Since(firstAttemptAt), - "attempt", attempt, - "nonce", nonce) - metrics.RecordError(p.name, "ethclient.SendTransaction.nonce") + "attempt", attempt) + metrics.RecordErrorDetails(p.name, "ethclient.SendTransaction", err) return } log.Warn("tx already known, incrementing nonce and trying again", @@ -118,6 +110,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { } else { log.Error("cant send transaction", "provider", p.name, + "nonce", nonce, "err", err) metrics.RecordErrorDetails(p.name, "ethclient.SendTransaction", err) return @@ -141,6 +134,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { SentAt: sentAt, SeenBy: make(map[string]time.Time), } + p.txPool.LastSend = sentAt p.txPool.M.Unlock() var receipt *types.Receipt @@ -150,6 +144,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { log.Error("receipt retrieval timed out", "provider", p.name, "hash", txHash, + "nonce", nonce, "elapsed", time.Since(sentAt)) return } @@ -157,6 +152,8 @@ func (p *Provider) RoundTrip(ctx context.Context) { if attempt%10 == 0 { log.Debug("checking for receipt...", "provider", p.name, + "hash", txHash, + "nonce", nonce, "attempt", attempt, "elapsed", time.Since(sentAt)) } @@ -165,6 +162,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { log.Error("cant get receipt for transaction", "provider", p.name, "hash", txHash.Hex(), + "nonce", nonce, "err", err) return } @@ -178,6 +176,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { log.Info("got transaction receipt", "hash", txHash.Hex(), + "nonce", nonce, "roundTripLatency", roundTripLatency, "provider", p.name, "blockNumber", receipt.BlockNumber, From 51dc1f0e840c5f404094f52cc493d96d4310cab9 Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:14:26 -0700 Subject: [PATCH 05/11] use network gas --- op-ufm/op-ufm/pkg/metrics/clients/eth.go | 34 +++++++++ op-ufm/op-ufm/pkg/provider/roundtrip.go | 97 +++++++++++++++++++----- op-ufm/op-ufm/pkg/provider/tx_pool.go | 1 - 3 files changed, 112 insertions(+), 20 deletions(-) diff --git a/op-ufm/op-ufm/pkg/metrics/clients/eth.go b/op-ufm/op-ufm/pkg/metrics/clients/eth.go index 5c01d9a..92d7c25 100644 --- a/op-ufm/op-ufm/pkg/metrics/clients/eth.go +++ b/op-ufm/op-ufm/pkg/metrics/clients/eth.go @@ -2,6 +2,7 @@ package clients import ( "context" + "math/big" "time" "github.com/ethereum-optimism/optimism/op-ufm/pkg/metrics" @@ -79,6 +80,39 @@ func (i *InstrumentedEthClient) SendTransaction(ctx context.Context, tx *types.T return err } +func (i *InstrumentedEthClient) EstimateGas(ctx context.Context, msg ethereum.CallMsg) (uint64, error) { + start := time.Now() + gas, err := i.c.EstimateGas(ctx, msg) + if err != nil { + metrics.RecordErrorDetails(i.providerName, "ethclient.EstimateGas", err) + return 0, err + } + metrics.RecordRPCLatency(i.providerName, "ethclient", "EstimateGas", time.Since(start)) + return gas, err +} + +func (i *InstrumentedEthClient) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + start := time.Now() + gasTipCap, err := i.c.SuggestGasTipCap(ctx) + if err != nil { + metrics.RecordErrorDetails(i.providerName, "ethclient.SuggestGasTipCap", err) + return nil, err + } + metrics.RecordRPCLatency(i.providerName, "ethclient", "SuggestGasTipCap", time.Since(start)) + return gasTipCap, err +} + +func (i *InstrumentedEthClient) HeaderByNumber(ctx context.Context, number *big.Int) (*types.Header, error) { + start := time.Now() + header, err := i.c.HeaderByNumber(ctx, number) + if err != nil { + metrics.RecordErrorDetails(i.providerName, "ethclient.HeaderByNumber", err) + return nil, err + } + metrics.RecordRPCLatency(i.providerName, "ethclient", "HeaderByNumber", time.Since(start)) + return header, err +} + func (i *InstrumentedEthClient) ignorableErrors(err error) bool { msg := err.Error() // we dont use errors.Is because eth client actually uses errors.New, diff --git a/op-ufm/op-ufm/pkg/provider/roundtrip.go b/op-ufm/op-ufm/pkg/provider/roundtrip.go index 3868248..7cd12aa 100644 --- a/op-ufm/op-ufm/pkg/provider/roundtrip.go +++ b/op-ufm/op-ufm/pkg/provider/roundtrip.go @@ -2,6 +2,7 @@ package provider import ( "context" + "math/big" "time" "github.com/ethereum-optimism/optimism/op-ufm/pkg/metrics" @@ -21,7 +22,7 @@ import ( // RoundTrip send a new transaction to measure round trip latency func (p *Provider) RoundTrip(ctx context.Context) { - log.Debug("roundTripLatency", + log.Debug("RoundTrip", "provider", p.name) client, err := iclients.Dial(p.name, p.config.URL) @@ -40,8 +41,9 @@ func (p *Provider) RoundTrip(ctx context.Context) { // used for timeout firstAttemptAt := time.Now() // used for actual round trip time (disregard retry time) - roundTripStartedAt := time.Now() + var roundTripStartedAt time.Time for { + // sleep until we get a clear to send for { coolDown := time.Duration(p.config.SendTransactionCoolDown) - time.Since(p.txPool.LastSend) @@ -52,17 +54,16 @@ func (p *Provider) RoundTrip(ctx context.Context) { } } - nonce, err = client.PendingNonceAt(ctx, p.walletConfig.Address) + tx, err := p.createTx(ctx, client, nonce) + nonce = tx.Nonce() if err != nil { - log.Error("cant get nounce", + log.Error("cant create tx", "provider", p.name, + "nonce", nonce, "err", err) return } - tx := p.createTx(nonce) - txHash = tx.Hash() - signedTx, err := p.sign(ctx, tx) if err != nil { log.Error("cant sign tx", @@ -90,15 +91,13 @@ func (p *Provider) RoundTrip(ctx context.Context) { metrics.RecordErrorDetails(p.name, "ethclient.SendTransaction", err) return } + log.Warn("tx already known, incrementing nonce and trying again", "provider", p.name, "nonce", nonce) time.Sleep(time.Duration(p.config.SendTransactionRetryInterval)) - p.txPool.M.Lock() - p.txPool.Nonce++ - nonce = p.txPool.Nonce - p.txPool.M.Unlock() + nonce++ attempt++ if attempt%10 == 0 { log.Debug("retrying send transaction...", @@ -184,20 +183,80 @@ func (p *Provider) RoundTrip(ctx context.Context) { "gasUsed", receipt.GasUsed) } -func (p *Provider) createTx(nonce uint64) *types.Transaction { - toAddress := common.HexToAddress(p.walletConfig.Address) +func (p *Provider) createTx(ctx context.Context, client *iclients.InstrumentedEthClient, nonce uint64) (*types.Transaction, error) { + var err error + if nonce == 0 { + nonce, err = client.PendingNonceAt(ctx, p.walletConfig.Address) + if err != nil { + log.Error("cant get nounce", + "provider", p.name, + "nonce", nonce, + "err", err) + return nil, err + } + } + + gasTipCap, err := client.SuggestGasTipCap(ctx) + if err != nil { + log.Error("cant get gas tip cap", + "provider", p.name, + "err", err) + return nil, err + } + gasTipCap = new(big.Int).Mul(gasTipCap, big.NewInt(110)) + gasTipCap = new(big.Int).Div(gasTipCap, big.NewInt(100)) + + head, err := client.HeaderByNumber(ctx, nil) + if err != nil { + log.Error("cant get base fee from head", + "provider", p.name, + "err", err) + return nil, err + } + baseFee := head.BaseFee + + gasFeeCap := new(big.Int).Add( + gasTipCap, + new(big.Int).Mul(baseFee, big.NewInt(2))) + + addr := common.HexToAddress(p.walletConfig.Address) var data []byte - tx := types.NewTx(&types.DynamicFeeTx{ + dynamicTx := &types.DynamicFeeTx{ ChainID: &p.walletConfig.ChainID, Nonce: nonce, - GasFeeCap: &p.walletConfig.GasFeeCap, - GasTipCap: &p.walletConfig.GasTipCap, - Gas: p.walletConfig.GasLimit, - To: &toAddress, + GasFeeCap: gasFeeCap, + GasTipCap: gasTipCap, + To: &addr, Value: &p.walletConfig.TxValue, Data: data, + } + + gas, err := client.EstimateGas(ctx, ethereum.CallMsg{ + From: addr, + To: &addr, + GasFeeCap: gasFeeCap, + GasTipCap: gasTipCap, + Data: dynamicTx.Data, + Value: dynamicTx.Value, }) - return tx + if err != nil { + log.Error("cant estimate gas", + "provider", p.name, + "err", err) + return nil, err + } + dynamicTx.Gas = gas + tx := types.NewTx(dynamicTx) + + log.Debug("tx created", + "provider", p.name, + "nonce", nonce, + "gas", dynamicTx.Gas, + "gasTipCap", dynamicTx.GasTipCap, + "gasFeeCap", dynamicTx.GasFeeCap, + ) + + return tx, nil } func (p *Provider) sign(ctx context.Context, tx *types.Transaction) (*types.Transaction, error) { diff --git a/op-ufm/op-ufm/pkg/provider/tx_pool.go b/op-ufm/op-ufm/pkg/provider/tx_pool.go index f6de257..ffdd214 100644 --- a/op-ufm/op-ufm/pkg/provider/tx_pool.go +++ b/op-ufm/op-ufm/pkg/provider/tx_pool.go @@ -15,7 +15,6 @@ type NetworkTransactionPool struct { M sync.Mutex Transactions map[string]*TransactionState Expected int - Nonce uint64 // Last time a transaction was sent LastSend time.Time From c7c9a5285089154377706672df148aebc339b2ba Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:14:41 -0700 Subject: [PATCH 06/11] remove gas configs --- op-ufm/op-ufm/pkg/config/config.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/op-ufm/op-ufm/pkg/config/config.go b/op-ufm/op-ufm/pkg/config/config.go index 3f0abda..e0c80d6 100644 --- a/op-ufm/op-ufm/pkg/config/config.go +++ b/op-ufm/op-ufm/pkg/config/config.go @@ -48,10 +48,7 @@ type WalletConfig struct { PrivateKey string `toml:"private_key"` // transaction parameters - TxValue big.Int `toml:"tx_value"` - GasLimit uint64 `toml:"gas_limit"` - GasTipCap big.Int `toml:"gas_tip_cap"` - GasFeeCap big.Int `toml:"gas_fee_cap"` + TxValue big.Int `toml:"tx_value"` } type ProviderConfig struct { @@ -131,12 +128,6 @@ func (c *Config) Validate() error { if wallet.TxValue.BitLen() == 0 { return errors.Errorf("wallet [%s] tx_value is missing", name) } - if wallet.GasLimit == 0 { - return errors.Errorf("wallet [%s] gas_limit is missing", name) - } - if wallet.GasFeeCap.BitLen() == 0 { - return errors.Errorf("wallet [%s] gas_fee_cap is missing", name) - } } for name, provider := range c.Providers { @@ -155,6 +146,9 @@ func (c *Config) Validate() error { if provider.SendTransactionRetryTimeout == 0 { return errors.Errorf("provider [%s] send_transaction_retry_timeout is missing", name) } + if provider.SendTransactionCoolDown == 0 { + return errors.Errorf("provider [%s] send_transaction_cool_down is missing", name) + } if provider.ReceiptRetrievalInterval == 0 { return errors.Errorf("provider [%s] receipt_retrieval_interval is missing", name) } From 5d8d892185e55a29f4ddd350864a0a090db9f32a Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:15:19 -0700 Subject: [PATCH 07/11] send mutex --- op-ufm/op-ufm/pkg/provider/roundtrip.go | 3 +++ op-ufm/op-ufm/pkg/provider/tx_pool.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/op-ufm/op-ufm/pkg/provider/roundtrip.go b/op-ufm/op-ufm/pkg/provider/roundtrip.go index 7cd12aa..42b5659 100644 --- a/op-ufm/op-ufm/pkg/provider/roundtrip.go +++ b/op-ufm/op-ufm/pkg/provider/roundtrip.go @@ -34,6 +34,9 @@ func (p *Provider) RoundTrip(ctx context.Context) { return } + p.txPool.ExclusiveSend.Lock() + defer p.txPool.ExclusiveSend.Unlock() + txHash := common.Hash{} attempt := 0 nonce := uint64(0) diff --git a/op-ufm/op-ufm/pkg/provider/tx_pool.go b/op-ufm/op-ufm/pkg/provider/tx_pool.go index ffdd214..2ea5f6a 100644 --- a/op-ufm/op-ufm/pkg/provider/tx_pool.go +++ b/op-ufm/op-ufm/pkg/provider/tx_pool.go @@ -18,6 +18,8 @@ type NetworkTransactionPool struct { // Last time a transaction was sent LastSend time.Time + // Prevents concurrent transaction send + ExclusiveSend sync.Mutex } type TransactionState struct { From c6abfd09c904eb6278ff0083cdc473d07d15817d Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:15:32 -0700 Subject: [PATCH 08/11] add debug tools to docker file --- op-ufm/op-ufm/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/op-ufm/op-ufm/Dockerfile b/op-ufm/op-ufm/Dockerfile index ad87f21..db9b84a 100644 --- a/op-ufm/op-ufm/Dockerfile +++ b/op-ufm/op-ufm/Dockerfile @@ -18,9 +18,10 @@ COPY --from=builder /app/entrypoint.sh /bin/entrypoint.sh COPY --from=builder /app/bin/ufm /bin/ufm RUN apk update && \ - apk add ca-certificates && \ chmod +x /bin/entrypoint.sh +RUN apk add ca-certificates jq curl bind-tools + VOLUME /etc/ufm EXPOSE 8080 From 09a9abd3f71a3731568d4b86c9945811ca8afaa8 Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Wed, 6 Sep 2023 15:16:01 -0700 Subject: [PATCH 09/11] update example.config.toml --- op-ufm/op-ufm/example.config.toml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/op-ufm/op-ufm/example.config.toml b/op-ufm/op-ufm/example.config.toml index 13e4cd5..4022057 100644 --- a/op-ufm/op-ufm/example.config.toml +++ b/op-ufm/op-ufm/example.config.toml @@ -52,13 +52,15 @@ url = "http://localhost:8551" # Read only providers are only used to check for transactions read_only = true # Interval to poll the provider for expected transactions -read_interval = "1s" +read_interval = "10s" # Interval to submit new transactions to the provider -send_interval = "5s" -# Wallet to be used for sending transactions -wallet = "default" -# Network to pool transactions, i.e. providers in the same network will check transactions from each other -network = "op-goerli" +send_interval = "30s" +# Interval between send transaction when we get "already known" txpool err +send_transaction_retry_interval = "100ms" +# Max time to retry +send_transaction_retry_timeout = "5s" +# Interval between each send transaction to the same network +send_transaction_cool_down = "30s" # Interval between receipt retrieval receipt_retrieval_interval = "500ms" # Max time to check for receipt @@ -72,13 +74,15 @@ url = "http://localhost:8552" # Read only providers are only used to check for transactions read_only = false # Interval to poll the provider for expected transactions -read_interval = "2s" +read_interval = "10s" # Interval to submit new transactions to the provider -send_interval = "3s" -# Wallet to be used for sending transactions -wallet = "default" -# Network to pool transactions, i.e. providers in the same network will check transactions from each other -network = "op-goerli" +send_interval = "30s" +# Interval between send transaction when we get "already known" txpool err +send_transaction_retry_interval = "100ms" +# Max time to retry +send_transaction_retry_timeout = "5s" +# Interval between each send transaction to the same network +send_transaction_cool_down = "30s" # Interval between receipt retrieval receipt_retrieval_interval = "500ms" # Max time to check for receipt From cfbaf252b28f6b512c3f642eb741abf7f5bfc295 Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Thu, 7 Sep 2023 13:25:35 -0700 Subject: [PATCH 10/11] remove gas configs --- op-ufm/op-ufm/example.config.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/op-ufm/op-ufm/example.config.toml b/op-ufm/op-ufm/example.config.toml index 4022057..13b62a3 100644 --- a/op-ufm/op-ufm/example.config.toml +++ b/op-ufm/op-ufm/example.config.toml @@ -39,12 +39,6 @@ address = "0x0000000000000000000000000000000000000000" private_key = "0000000000000000000000000000000000000000000000000000000000000000" # Transaction value in wei tx_value = 100000000000000 -# Gas limit -gas_limit = 21000 -# Gas tip cap -gas_tip_cap = 2000000000 -# Fee cap -gas_fee_cap = 20000000000 [providers.p1] # URL to the RPC provider From a231f5a16cb9f72f1ededbc541b2454e43a3622d Mon Sep 17 00:00:00 2001 From: Felipe Andrade Date: Thu, 7 Sep 2023 13:25:50 -0700 Subject: [PATCH 11/11] timeout metrics --- op-ufm/op-ufm/pkg/provider/roundtrip.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/op-ufm/op-ufm/pkg/provider/roundtrip.go b/op-ufm/op-ufm/pkg/provider/roundtrip.go index 42b5659..3145abc 100644 --- a/op-ufm/op-ufm/pkg/provider/roundtrip.go +++ b/op-ufm/op-ufm/pkg/provider/roundtrip.go @@ -84,6 +84,11 @@ func (p *Provider) RoundTrip(ctx context.Context) { err.Error() == txpool.ErrReplaceUnderpriced.Error() || err.Error() == core.ErrNonceTooLow.Error() { + log.Warn("cant send transaction (retryable)", + "provider", p.name, + "err", err, + "nonce", nonce) + if time.Since(firstAttemptAt) >= time.Duration(p.config.SendTransactionRetryTimeout) { log.Error("send transaction timed out (known already)", "provider", p.name, @@ -91,7 +96,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { "nonce", nonce, "elapsed", time.Since(firstAttemptAt), "attempt", attempt) - metrics.RecordErrorDetails(p.name, "ethclient.SendTransaction", err) + metrics.RecordErrorDetails(p.name, "send.timeout", err) return } @@ -148,6 +153,7 @@ func (p *Provider) RoundTrip(ctx context.Context) { "hash", txHash, "nonce", nonce, "elapsed", time.Since(sentAt)) + metrics.RecordErrorDetails(p.name, "receipt.timeout", err) return } time.Sleep(time.Duration(p.config.ReceiptRetrievalInterval)) @@ -251,9 +257,12 @@ func (p *Provider) createTx(ctx context.Context, client *iclients.InstrumentedEt dynamicTx.Gas = gas tx := types.NewTx(dynamicTx) - log.Debug("tx created", + log.Info("tx created", "provider", p.name, - "nonce", nonce, + "from", addr, + "to", dynamicTx.To, + "nonce", dynamicTx.Nonce, + "value", dynamicTx.Value, "gas", dynamicTx.Gas, "gasTipCap", dynamicTx.GasTipCap, "gasFeeCap", dynamicTx.GasFeeCap,