From 094519d0582bac363018b1f2a1ba06624a2f95a0 Mon Sep 17 00:00:00 2001 From: "Dike.w" Date: Wed, 4 Sep 2024 09:39:01 +0800 Subject: [PATCH] beaconserver: simulated beacon api server for op-stack (#2678) only some necessary apis are implemented. --- beacon/fakebeacon/api_func.go | 87 ++++++++++++++++++++++++++++ beacon/fakebeacon/handlers.go | 88 +++++++++++++++++++++++++++++ beacon/fakebeacon/server.go | 97 ++++++++++++++++++++++++++++++++ beacon/fakebeacon/server_test.go | 90 +++++++++++++++++++++++++++++ beacon/fakebeacon/utils.go | 65 +++++++++++++++++++++ cmd/geth/config.go | 20 +++++-- cmd/geth/main.go | 7 +++ cmd/utils/flags.go | 20 +++++++ 8 files changed, 470 insertions(+), 4 deletions(-) create mode 100644 beacon/fakebeacon/api_func.go create mode 100644 beacon/fakebeacon/handlers.go create mode 100644 beacon/fakebeacon/server.go create mode 100644 beacon/fakebeacon/server_test.go create mode 100644 beacon/fakebeacon/utils.go diff --git a/beacon/fakebeacon/api_func.go b/beacon/fakebeacon/api_func.go new file mode 100644 index 000000000..674bf7fb3 --- /dev/null +++ b/beacon/fakebeacon/api_func.go @@ -0,0 +1,87 @@ +package fakebeacon + +import ( + "context" + "sort" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" +) + +type BlobSidecar struct { + Blob kzg4844.Blob `json:"blob"` + Index int `json:"index"` + KZGCommitment kzg4844.Commitment `json:"kzg_commitment"` + KZGProof kzg4844.Proof `json:"kzg_proof"` +} + +type APIGetBlobSidecarsResponse struct { + Data []*BlobSidecar `json:"data"` +} + +type ReducedGenesisData struct { + GenesisTime string `json:"genesis_time"` +} + +type APIGenesisResponse struct { + Data ReducedGenesisData `json:"data"` +} + +type ReducedConfigData struct { + SecondsPerSlot string `json:"SECONDS_PER_SLOT"` +} + +type IndexedBlobHash struct { + Index int // absolute index in the block, a.k.a. position in sidecar blobs array + Hash common.Hash // hash of the blob, used for consistency checks +} + +func configSpec() ReducedConfigData { + return ReducedConfigData{SecondsPerSlot: "1"} +} + +func beaconGenesis() APIGenesisResponse { + return APIGenesisResponse{Data: ReducedGenesisData{GenesisTime: "0"}} +} + +func beaconBlobSidecars(ctx context.Context, backend ethapi.Backend, slot uint64, indices []int) (APIGetBlobSidecarsResponse, error) { + var blockNrOrHash rpc.BlockNumberOrHash + header, err := fetchBlockNumberByTime(ctx, int64(slot), backend) + if err != nil { + log.Error("Error fetching block number", "slot", slot, "indices", indices) + return APIGetBlobSidecarsResponse{}, err + } + sideCars, err := backend.GetBlobSidecars(ctx, header.Hash()) + if err != nil { + log.Error("Error fetching Sidecars", "blockNrOrHash", blockNrOrHash, "err", err) + return APIGetBlobSidecarsResponse{}, err + } + sort.Ints(indices) + fullBlob := len(indices) == 0 + res := APIGetBlobSidecarsResponse{} + idx := 0 + curIdx := 0 + for _, sideCar := range sideCars { + for i := 0; i < len(sideCar.Blobs); i++ { + //hash := kZGToVersionedHash(sideCar.Commitments[i]) + if !fullBlob && curIdx >= len(indices) { + break + } + if fullBlob || idx == indices[curIdx] { + res.Data = append(res.Data, &BlobSidecar{ + Index: idx, + Blob: sideCar.Blobs[i], + KZGCommitment: sideCar.Commitments[i], + KZGProof: sideCar.Proofs[i], + }) + curIdx++ + } + idx++ + } + } + + return res, nil +} diff --git a/beacon/fakebeacon/handlers.go b/beacon/fakebeacon/handlers.go new file mode 100644 index 000000000..3d3768aa4 --- /dev/null +++ b/beacon/fakebeacon/handlers.go @@ -0,0 +1,88 @@ +package fakebeacon + +import ( + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/prysmaticlabs/prysm/v5/api/server/structs" + field_params "github.com/prysmaticlabs/prysm/v5/config/fieldparams" + "github.com/prysmaticlabs/prysm/v5/network/httputil" +) + +var ( + versionMethod = "/eth/v1/node/version" + specMethod = "/eth/v1/config/spec" + genesisMethod = "/eth/v1/beacon/genesis" + sidecarsMethodPrefix = "/eth/v1/beacon/blob_sidecars/{slot}" +) + +func VersionMethod(w http.ResponseWriter, r *http.Request) { + resp := &structs.GetVersionResponse{ + Data: &structs.Version{ + Version: "", + }, + } + httputil.WriteJson(w, resp) +} + +func SpecMethod(w http.ResponseWriter, r *http.Request) { + httputil.WriteJson(w, &structs.GetSpecResponse{Data: configSpec()}) +} + +func GenesisMethod(w http.ResponseWriter, r *http.Request) { + httputil.WriteJson(w, beaconGenesis()) +} + +func (s *Service) SidecarsMethod(w http.ResponseWriter, r *http.Request) { + indices, err := parseIndices(r.URL) + if err != nil { + httputil.HandleError(w, err.Error(), http.StatusBadRequest) + return + } + segments := strings.Split(r.URL.Path, "/") + slot, err := strconv.ParseUint(segments[len(segments)-1], 10, 64) + if err != nil { + httputil.HandleError(w, "not a valid slot(timestamp)", http.StatusBadRequest) + return + } + + resp, err := beaconBlobSidecars(r.Context(), s.backend, slot, indices) + if err != nil { + httputil.HandleError(w, err.Error(), http.StatusBadRequest) + return + } + httputil.WriteJson(w, resp) +} + +// parseIndices filters out invalid and duplicate blob indices +func parseIndices(url *url.URL) ([]int, error) { + rawIndices := url.Query()["indices"] + indices := make([]int, 0, field_params.MaxBlobsPerBlock) + invalidIndices := make([]string, 0) +loop: + for _, raw := range rawIndices { + ix, err := strconv.Atoi(raw) + if err != nil { + invalidIndices = append(invalidIndices, raw) + continue + } + if ix >= field_params.MaxBlobsPerBlock { + invalidIndices = append(invalidIndices, raw) + continue + } + for i := range indices { + if ix == indices[i] { + continue loop + } + } + indices = append(indices, ix) + } + + if len(invalidIndices) > 0 { + return nil, fmt.Errorf("requested blob indices %v are invalid", invalidIndices) + } + return indices, nil +} diff --git a/beacon/fakebeacon/server.go b/beacon/fakebeacon/server.go new file mode 100644 index 000000000..91f48a2fb --- /dev/null +++ b/beacon/fakebeacon/server.go @@ -0,0 +1,97 @@ +package fakebeacon + +import ( + "net/http" + "strconv" + + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/gorilla/mux" + "github.com/prysmaticlabs/prysm/v5/api/server" +) + +const ( + DefaultAddr = "localhost" + DefaultPort = 8686 +) + +type Config struct { + Enable bool + Addr string + Port int +} + +func defaultConfig() *Config { + return &Config{ + Enable: false, + Addr: DefaultAddr, + Port: DefaultPort, + } +} + +type Service struct { + cfg *Config + router *mux.Router + backend ethapi.Backend +} + +func NewService(cfg *Config, backend ethapi.Backend) *Service { + cfgs := defaultConfig() + if cfg.Addr != "" { + cfgs.Addr = cfg.Addr + } + if cfg.Port > 0 { + cfgs.Port = cfg.Port + } + + s := &Service{ + cfg: cfgs, + backend: backend, + } + router := s.newRouter() + s.router = router + return s +} + +func (s *Service) Run() { + _ = http.ListenAndServe(s.cfg.Addr+":"+strconv.Itoa(s.cfg.Port), s.router) +} + +func (s *Service) newRouter() *mux.Router { + r := mux.NewRouter() + r.Use(server.NormalizeQueryValuesHandler) + for _, e := range s.endpoints() { + r.HandleFunc(e.path, e.handler).Methods(e.methods...) + } + return r +} + +type endpoint struct { + path string + handler http.HandlerFunc + methods []string +} + +func (s *Service) endpoints() []endpoint { + return []endpoint{ + { + path: versionMethod, + handler: VersionMethod, + methods: []string{http.MethodGet}, + }, + { + path: specMethod, + handler: SpecMethod, + methods: []string{http.MethodGet}, + }, + { + path: genesisMethod, + handler: GenesisMethod, + methods: []string{http.MethodGet}, + }, + { + path: sidecarsMethodPrefix, + handler: s.SidecarsMethod, + methods: []string{http.MethodGet}, + }, + } +} diff --git a/beacon/fakebeacon/server_test.go b/beacon/fakebeacon/server_test.go new file mode 100644 index 000000000..0b74f565b --- /dev/null +++ b/beacon/fakebeacon/server_test.go @@ -0,0 +1,90 @@ +package fakebeacon + +import ( + "context" + "fmt" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +// +//func TestFetchBlockNumberByTime(t *testing.T) { +// blockNum, err := fetchBlockNumberByTime(context.Background(), 1724052941, client) +// assert.Nil(t, err) +// assert.Equal(t, uint64(41493946), blockNum) +// +// blockNum, err = fetchBlockNumberByTime(context.Background(), 1734052941, client) +// assert.Equal(t, err, errors.New("time too large")) +// +// blockNum, err = fetchBlockNumberByTime(context.Background(), 1600153618, client) +// assert.Nil(t, err) +// assert.Equal(t, uint64(493946), blockNum) +//} +// +//func TestBeaconBlobSidecars(t *testing.T) { +// indexBlobHash := []IndexedBlobHash{ +// {Hash: common.HexToHash("0x01231952ecbaede62f8d0398b656072c072db36982c9ef106fbbc39ce14f983c"), Index: 0}, +// {Hash: common.HexToHash("0x012c21a8284d2d707bb5318e874d2e1b97a53d028e96abb702b284a2cbb0f79c"), Index: 1}, +// {Hash: common.HexToHash("0x011196c8d02536ede0382aa6e9fdba6c460169c0711b5f97fcd701bd8997aee3"), Index: 2}, +// {Hash: common.HexToHash("0x019c86b46b27401fb978fd175d1eb7dadf4976d6919501b0c5280d13a5bab57b"), Index: 3}, +// {Hash: common.HexToHash("0x01e00db7ee99176b3fd50aab45b4fae953292334bbf013707aac58c455d98596"), Index: 4}, +// {Hash: common.HexToHash("0x0117d23b68123d578a98b3e1aa029661e0abda821a98444c21992eb1e5b7208f"), Index: 5}, +// //{Hash: common.HexToHash("0x01e00db7ee99176b3fd50aab45b4fae953292334bbf013707aac58c455d98596"), Index: 1}, +// } +// +// resp, err := beaconBlobSidecars(context.Background(), 1724055046, []int{0, 1, 2, 3, 4, 5}) // block: 41494647 +// assert.Nil(t, err) +// assert.NotNil(t, resp) +// assert.NotEmpty(t, resp.Data) +// for i, sideCar := range resp.Data { +// assert.Equal(t, indexBlobHash[i].Index, sideCar.Index) +// assert.Equal(t, indexBlobHash[i].Hash, kZGToVersionedHash(sideCar.KZGCommitment)) +// } +// +// apiscs := make([]*BlobSidecar, 0, len(indexBlobHash)) +// // filter and order by hashes +// for _, h := range indexBlobHash { +// for _, apisc := range resp.Data { +// if h.Index == int(apisc.Index) { +// apiscs = append(apiscs, apisc) +// break +// } +// } +// } +// +// assert.Equal(t, len(apiscs), len(resp.Data)) +// assert.Equal(t, len(apiscs), len(indexBlobHash)) +//} + +type TimeToSlotFn func(timestamp uint64) (uint64, error) + +// GetTimeToSlotFn returns a function that converts a timestamp to a slot number. +func GetTimeToSlotFn(ctx context.Context) (TimeToSlotFn, error) { + genesis := beaconGenesis() + config := configSpec() + + genesisTime, _ := strconv.ParseUint(genesis.Data.GenesisTime, 10, 64) + secondsPerSlot, _ := strconv.ParseUint(config.SecondsPerSlot, 10, 64) + if secondsPerSlot == 0 { + return nil, fmt.Errorf("got bad value for seconds per slot: %v", config.SecondsPerSlot) + } + timeToSlotFn := func(timestamp uint64) (uint64, error) { + if timestamp < genesisTime { + return 0, fmt.Errorf("provided timestamp (%v) precedes genesis time (%v)", timestamp, genesisTime) + } + return (timestamp - genesisTime) / secondsPerSlot, nil + } + return timeToSlotFn, nil +} + +func TestAPI(t *testing.T) { + slotFn, err := GetTimeToSlotFn(context.Background()) + assert.Nil(t, err) + + expTx := uint64(123151345) + gotTx, err := slotFn(expTx) + assert.Nil(t, err) + assert.Equal(t, expTx, gotTx) +} diff --git a/beacon/fakebeacon/utils.go b/beacon/fakebeacon/utils.go new file mode 100644 index 000000000..cc6fe889b --- /dev/null +++ b/beacon/fakebeacon/utils.go @@ -0,0 +1,65 @@ +package fakebeacon + +import ( + "context" + "errors" + "fmt" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/rpc" +) + +func fetchBlockNumberByTime(ctx context.Context, ts int64, backend ethapi.Backend) (*types.Header, error) { + // calc the block number of the ts. + currentHeader := backend.CurrentHeader() + blockTime := int64(currentHeader.Time) + if ts > blockTime { + return nil, errors.New("time too large") + } + blockNum := currentHeader.Number.Uint64() + estimateEndNumber := int64(blockNum) - (blockTime-ts)/3 + // find the end number + for { + header, err := backend.HeaderByNumber(ctx, rpc.BlockNumber(estimateEndNumber)) + if err != nil { + time.Sleep(time.Duration(rand.Int()%180) * time.Millisecond) + continue + } + if header == nil { + estimateEndNumber -= 1 + time.Sleep(time.Duration(rand.Int()%180) * time.Millisecond) + continue + } + headerTime := int64(header.Time) + if headerTime == ts { + return header, nil + } + + // let the estimateEndNumber a little bigger than real value + if headerTime > ts+8 { + estimateEndNumber -= (headerTime - ts) / 3 + } else if headerTime < ts { + estimateEndNumber += (ts-headerTime)/3 + 1 + } else { + // search one by one + for headerTime >= ts { + header, err = backend.HeaderByNumber(ctx, rpc.BlockNumber(estimateEndNumber-1)) + if err != nil { + time.Sleep(time.Duration(rand.Int()%180) * time.Millisecond) + continue + } + headerTime = int64(header.Time) + if headerTime == ts { + return header, nil + } + estimateEndNumber -= 1 + if headerTime < ts { //found the real endNumber + return nil, fmt.Errorf("block not found by time %d", ts) + } + } + } + } +} diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 5c829a2f7..3e0da27ed 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" + "github.com/ethereum/go-ethereum/beacon/fakebeacon" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" @@ -92,10 +93,11 @@ type ethstatsConfig struct { } type gethConfig struct { - Eth ethconfig.Config - Node node.Config - Ethstats ethstatsConfig - Metrics metrics.Config + Eth ethconfig.Config + Node node.Config + Ethstats ethstatsConfig + Metrics metrics.Config + FakeBeacon fakebeacon.Config } func loadConfig(file string, cfg *gethConfig) error { @@ -242,6 +244,16 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL) } + if ctx.IsSet(utils.FakeBeaconAddrFlag.Name) { + cfg.FakeBeacon.Addr = ctx.String(utils.FakeBeaconAddrFlag.Name) + } + if ctx.IsSet(utils.FakeBeaconPortFlag.Name) { + cfg.FakeBeacon.Port = ctx.Int(utils.FakeBeaconPortFlag.Name) + } + if cfg.FakeBeacon.Enable || ctx.IsSet(utils.FakeBeaconEnabledFlag.Name) { + go fakebeacon.NewService(&cfg.FakeBeacon, backend).Run() + } + git, _ := version.VCS() utils.SetupMetrics(ctx, utils.EnableBuildInfo(git.Commit, git.Date), diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d4ac750d5..264f7cb64 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -232,6 +232,12 @@ var ( utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, } + + fakeBeaconFlags = []cli.Flag{ + utils.FakeBeaconEnabledFlag, + utils.FakeBeaconAddrFlag, + utils.FakeBeaconPortFlag, + } ) var app = flags.NewApp("the go-ethereum command line interface") @@ -286,6 +292,7 @@ func init() { consoleFlags, debug.Flags, metricsFlags, + fakeBeaconFlags, ) flags.AutoEnvVars(app.Flags, "GETH") diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 049908857..1a0f0a040 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/beacon/fakebeacon" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/core" @@ -1146,6 +1147,25 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Value: params.DefaultExtraReserveForBlobRequests, Category: flags.MiscCategory, } + + // Fake beacon + FakeBeaconEnabledFlag = &cli.BoolFlag{ + Name: "fake-beacon", + Usage: "Enable the HTTP-RPC server of fake-beacon", + Category: flags.APICategory, + } + FakeBeaconAddrFlag = &cli.StringFlag{ + Name: "fake-beacon.addr", + Usage: "HTTP-RPC server listening addr of fake-beacon", + Value: fakebeacon.DefaultAddr, + Category: flags.APICategory, + } + FakeBeaconPortFlag = &cli.IntFlag{ + Name: "fake-beacon.port", + Usage: "HTTP-RPC server listening port of fake-beacon", + Value: fakebeacon.DefaultPort, + Category: flags.APICategory, + } ) var (