From 795548580a94b9c96a6b1ea5493ee87d56070c13 Mon Sep 17 00:00:00 2001 From: yenicelik Date: Sun, 21 May 2023 16:40:42 -0400 Subject: [PATCH 1/6] now should also group by archive_needed (turns out this was wanted) --- scripts/generate-requests-and-stats.sh | 2 +- scripts/manual-tests/42-simple-balance.sh | 14 +++++++------- web3_proxy/src/stats/influxdb_queries.rs | 23 ++++------------------- 3 files changed, 12 insertions(+), 27 deletions(-) diff --git a/scripts/generate-requests-and-stats.sh b/scripts/generate-requests-and-stats.sh index ecae4466..a05e55e7 100644 --- a/scripts/generate-requests-and-stats.sh +++ b/scripts/generate-requests-and-stats.sh @@ -5,4 +5,4 @@ # https://github.com/INFURA/versus # ./ethspam | ./versus --stop-after 100 "http://localhost:8544/" # Pipe into the endpoint ..., add a bearer token and all that -./ethspam http://127.0.0.1:8544 | ./versus --stop-after 100 http://localhost:8544 +./ethspam http://127.0.0.1:8544/rpc/01H0ZZJDNNEW49FRFS4D9SPR8B | ./versus --concurrency=4 --stop-after 100 http://localhost:8544/rpc/01H0ZZJDNNEW49FRFS4D9SPR8B diff --git a/scripts/manual-tests/42-simple-balance.sh b/scripts/manual-tests/42-simple-balance.sh index ce6e32da..724d4809 100644 --- a/scripts/manual-tests/42-simple-balance.sh +++ b/scripts/manual-tests/42-simple-balance.sh @@ -24,14 +24,14 @@ curl -X POST http://127.0.0.1:8544/user/login \ -H 'Content-Type: application/json' \ -d '{ "address": "0xeb3e928a2e54be013ef8241d4c9eaf4dfae94d5a", - "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078654233453932384132453534424530313345463832343164344339456146344466414539344435610a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a203031475a4b384b4847305259474737514e5132475037464444470a4973737565642041743a20323032332d30352d30345431313a33333a32312e3533363734355a0a45787069726174696f6e2054696d653a20323032332d30352d30345431313a35333a32312e3533363734355a", - "sig": "cebd9effff15f4517e53522dbe91798d59dc0df0299faaec25d3f6443fa121f847e4311d5ca7386e75b87d6d45df92b8ced58c822117519c666ab1a6b2fc7bd21b", + "msg": "0x6c6c616d616e6f6465732e636f6d2077616e747320796f7520746f207369676e20696e207769746820796f757220457468657265756d206163636f756e743a0a3078654233453932384132453534424530313345463832343164344339456146344466414539344435610a0af09fa699f09fa699f09fa699f09fa699f09fa6990a0a5552493a2068747470733a2f2f6c6c616d616e6f6465732e636f6d2f0a56657273696f6e3a20310a436861696e2049443a20310a4e6f6e63653a20303148305a5a48434356324b32324738544850535758485131480a4973737565642041743a20323032332d30352d32315432303a32303a34332e3033353539315a0a45787069726174696f6e2054696d653a20323032332d30352d32315432303a34303a34332e3033353539315a", + "sig": "7591251840bf75d2ab7c895bc566a49d2f4c3ad6bb14d7256258a59e52055fc94c11f8f3836f5311b52fc18aca40867cd85802636645e1d757494800631cad381c", "version": "3", "signer": "MEW" }' -# bearer token is: 01GZK8MHHGQWK4VPGF97HS91MB -# scret key is: 01GZK65YNV0P0WN2SCXYTW3R9S +# bearer token is: 01H0ZZJDQ2F02MAXZR5K1X5NCP +# scret key is: 01H0ZZJDNNEW49FRFS4D9SPR8B # 01GZH2PS89EJJY6V8JFCVTQ4BX # 01GZH2PS7CTHA3TAZ4HXCTX6KQ @@ -42,7 +42,7 @@ curl -X POST http://127.0.0.1:8544/user/login \ # Check the balance of the user # Balance seems to be returning properly (0, in this test case) curl \ --H "Authorization: Bearer 01GZK8MHHGQWK4VPGF97HS91MB" \ +-H "Authorization: Bearer 01H0ZZJDQ2F02MAXZR5K1X5NCP" \ -X GET "127.0.0.1:8544/user/balance" @@ -73,10 +73,10 @@ curl \ ## Check if calling an RPC endpoint logs the stats ## This one does already even it seems -for i in {1..100} +for i in {1..300} do curl \ - -X POST "127.0.0.1:8544/rpc/01GZK65YNV0P0WN2SCXYTW3R9S" \ + -X POST "127.0.0.1:8544/rpc/01H0ZZJDNNEW49FRFS4D9SPR8B" \ -H "Content-Type: application/json" \ --data '{"method":"eth_blockNumber","params":[],"id":1,"jsonrpc":"2.0"}' done diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs index 368df1f1..0d75881c 100644 --- a/web3_proxy/src/stats/influxdb_queries.rs +++ b/web3_proxy/src/stats/influxdb_queries.rs @@ -73,6 +73,7 @@ pub async fn query_user_stats<'a>( let mut join_candidates: Vec = vec![ "_time".to_string(), "_measurement".to_string(), + "archive_needed".to_string(), "chain_id".to_string(), ]; @@ -184,35 +185,19 @@ pub async fn query_user_stats<'a>( {filter_chain_id} {drop_method} - // cumsum = base base |> aggregateWindow(every: {query_window_seconds}s, fn: sum, createEmpty: false) |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") |> drop(columns: ["balance"]) - |> map(fn: (r) => ({{ r with "archive_needed": if r.archive_needed == "true" then r.frontend_requests else 0}})) |> map(fn: (r) => ({{ r with "error_response": if r.error_response == "true" then r.frontend_requests else 0}})) - |> group(columns: ["_time", "_measurement", "chain_id", "method", "rpc_secret_key_id"]) + |> group(columns: ["_time", "_measurement", "archive_needed", "chain_id", "method", "rpc_secret_key_id"]) |> sort(columns: ["frontend_requests"]) |> map(fn:(r) => ({{ r with "sum_credits_used": float(v: r["sum_credits_used"]) }})) - |> cumulativeSum(columns: ["archive_needed", "error_response", "backend_requests", "cache_hits", "cache_misses", "frontend_requests", "sum_credits_used", "sum_request_bytes", "sum_response_bytes", "sum_response_millis"]) + |> cumulativeSum(columns: ["error_response", "backend_requests", "cache_hits", "cache_misses", "frontend_requests", "sum_credits_used", "sum_request_bytes", "sum_response_bytes", "sum_response_millis"]) |> sort(columns: ["frontend_requests"], desc: true) |> limit(n: 1) |> group() - |> sort(columns: ["_time", "_measurement", "chain_id", "method", "rpc_secret_key_id"], desc: true) - - // balance = base - // |> toFloat() - // |> aggregateWindow(every: {query_window_seconds}s, fn: mean, createEmpty: false) - // |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") - // |> group(columns: ["_time", "_measurement", "chain_id", "method", "rpc_secret_key_id"]) - // |> mean(column: "balance") - // |> group() - // |> sort(columns: ["_time", "_measurement", "chain_id", "method", "rpc_secret_key_id"], desc: true) - - // join( - // tables: {{cumsum, balance}}, - // on: {join_candidates} - // ) + |> sort(columns: ["_time", "_measurement", "archive_needed", "chain_id", "method", "rpc_secret_key_id"], desc: true) "#); info!("Raw query to db is: {:?}", query); From 46afe8b1e4f61292cbee48136c91c9a903e3ee46 Mon Sep 17 00:00:00 2001 From: yenicelik Date: Sun, 21 May 2023 17:13:15 -0400 Subject: [PATCH 2/6] influxdb query also groups by archive_needed now --- web3_proxy/src/stats/influxdb_queries.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs index 0d75881c..eb4a56b8 100644 --- a/web3_proxy/src/stats/influxdb_queries.rs +++ b/web3_proxy/src/stats/influxdb_queries.rs @@ -423,14 +423,20 @@ pub async fn query_user_stats<'a>( } } else if key == "archive_needed" { match value { - influxdb2_structmap::value::Value::Long(inner) => { + influxdb2_structmap::value::Value::String(inner) => { out.insert( "archive_needed".to_owned(), - serde_json::Value::Number(inner.into()), + if inner == "true" { + serde_json::Value::Bool(true) + } else if inner == "false" { + serde_json::Value::Bool(false) + } else { + serde_json::Value::String("error".to_owned()) + }, ); } _ => { - error!("archive_needed should always be a Long!"); + error!("archive_needed should always be a String!"); } } } else if key == "error_response" { From 29678b025ebab5bdaa51374a6b087df99a254774 Mon Sep 17 00:00:00 2001 From: yenicelik Date: Sun, 21 May 2023 20:27:46 -0400 Subject: [PATCH 3/6] updated error_response --- web3_proxy/src/stats/influxdb_queries.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs index eb4a56b8..f31bf2e9 100644 --- a/web3_proxy/src/stats/influxdb_queries.rs +++ b/web3_proxy/src/stats/influxdb_queries.rs @@ -75,6 +75,7 @@ pub async fn query_user_stats<'a>( "_measurement".to_string(), "archive_needed".to_string(), "chain_id".to_string(), + "error_response".to_string(), ]; // Include a hashmap to go from rpc_secret_key_id to the rpc_secret_key @@ -190,14 +191,14 @@ pub async fn query_user_stats<'a>( |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") |> drop(columns: ["balance"]) |> map(fn: (r) => ({{ r with "error_response": if r.error_response == "true" then r.frontend_requests else 0}})) - |> group(columns: ["_time", "_measurement", "archive_needed", "chain_id", "method", "rpc_secret_key_id"]) + |> group(columns: ["_time", "_measurement", "archive_needed", "chain_id", "error_response", "method", "rpc_secret_key_id"]) |> sort(columns: ["frontend_requests"]) |> map(fn:(r) => ({{ r with "sum_credits_used": float(v: r["sum_credits_used"]) }})) - |> cumulativeSum(columns: ["error_response", "backend_requests", "cache_hits", "cache_misses", "frontend_requests", "sum_credits_used", "sum_request_bytes", "sum_response_bytes", "sum_response_millis"]) + |> cumulativeSum(columns: ["backend_requests", "cache_hits", "cache_misses", "frontend_requests", "sum_credits_used", "sum_request_bytes", "sum_response_bytes", "sum_response_millis"]) |> sort(columns: ["frontend_requests"], desc: true) |> limit(n: 1) |> group() - |> sort(columns: ["_time", "_measurement", "archive_needed", "chain_id", "method", "rpc_secret_key_id"], desc: true) + |> sort(columns: ["_time", "_measurement", "archive_needed", "chain_id", "error_response", "method", "rpc_secret_key_id"], desc: true) "#); info!("Raw query to db is: {:?}", query); @@ -441,10 +442,16 @@ pub async fn query_user_stats<'a>( } } else if key == "error_response" { match value { - influxdb2_structmap::value::Value::Long(inner) => { + influxdb2_structmap::value::Value::String(inner) => { out.insert( "error_response".to_owned(), - serde_json::Value::Number(inner.into()), + if inner == "true" { + serde_json::Value::Bool(true) + } else if inner == "false" { + serde_json::Value::Bool(false) + } else { + serde_json::Value::String("error".to_owned()) + }, ); } _ => { From 59634139c902bde1c8ba6541be14ee73ab28b420 Mon Sep 17 00:00:00 2001 From: yenicelik Date: Sun, 21 May 2023 20:27:52 -0400 Subject: [PATCH 4/6] updated error_response --- web3_proxy/src/stats/influxdb_queries.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs index f31bf2e9..69d1110c 100644 --- a/web3_proxy/src/stats/influxdb_queries.rs +++ b/web3_proxy/src/stats/influxdb_queries.rs @@ -190,7 +190,6 @@ pub async fn query_user_stats<'a>( |> aggregateWindow(every: {query_window_seconds}s, fn: sum, createEmpty: false) |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") |> drop(columns: ["balance"]) - |> map(fn: (r) => ({{ r with "error_response": if r.error_response == "true" then r.frontend_requests else 0}})) |> group(columns: ["_time", "_measurement", "archive_needed", "chain_id", "error_response", "method", "rpc_secret_key_id"]) |> sort(columns: ["frontend_requests"]) |> map(fn:(r) => ({{ r with "sum_credits_used": float(v: r["sum_credits_used"]) }})) From 47f07261edec7f911c4a0e2885d141ccfb908ded Mon Sep 17 00:00:00 2001 From: yenicelik Date: Wed, 24 May 2023 13:34:50 +0200 Subject: [PATCH 5/6] detailed request must come with a bearer token --- web3_proxy/src/stats/influxdb_queries.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs index 69d1110c..6246ac54 100644 --- a/web3_proxy/src/stats/influxdb_queries.rs +++ b/web3_proxy/src/stats/influxdb_queries.rs @@ -41,6 +41,13 @@ pub async fn query_user_stats<'a>( None => 0, }; + // Return an error if the bearer is set, but the StatType is Detailed + if stat_response_type == StatType::Detailed && user_id == 0 { + return Err(Web3ProxyError::BadRequest( + "Detailed Stats Response requires you to authorize with a bearer token".to_owned(), + )); + } + let db_replica = app .db_replica() .context("query_user_stats needs a db replica")?; From b6cab88c4eb5678f08af3422085273e769e93b3f Mon Sep 17 00:00:00 2001 From: yenicelik Date: Wed, 24 May 2023 13:35:10 +0200 Subject: [PATCH 6/6] removed joined_on variable that was not used --- web3_proxy/src/stats/influxdb_queries.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/web3_proxy/src/stats/influxdb_queries.rs b/web3_proxy/src/stats/influxdb_queries.rs index 6246ac54..4b597118 100644 --- a/web3_proxy/src/stats/influxdb_queries.rs +++ b/web3_proxy/src/stats/influxdb_queries.rs @@ -77,14 +77,6 @@ pub async fn query_user_stats<'a>( "opt_in_proxy" }; - let mut join_candidates: Vec = vec![ - "_time".to_string(), - "_measurement".to_string(), - "archive_needed".to_string(), - "chain_id".to_string(), - "error_response".to_string(), - ]; - // Include a hashmap to go from rpc_secret_key_id to the rpc_secret_key let mut rpc_key_id_to_key = HashMap::new(); @@ -140,9 +132,6 @@ pub async fn query_user_stats<'a>( )); } - // Make the tables join on the rpc_key_id as well: - join_candidates.push("rpc_secret_key_id".to_string()); - // Iterate, pop and add to string f!( r#"|> filter(fn: (r) => contains(value: r["rpc_secret_key_id"], set: {:?}))"#, @@ -177,13 +166,8 @@ pub async fn query_user_stats<'a>( let drop_method = match stat_response_type { StatType::Aggregated => f!(r#"|> drop(columns: ["method"])"#), - StatType::Detailed => { - // Make the tables join on the method column as well - join_candidates.push("method".to_string()); - "".to_string() - } + StatType::Detailed => "".to_string(), }; - let join_candidates = f!(r#"{:?}"#, join_candidates); let query = f!(r#" base = from(bucket: "{bucket}")