improve build process
This commit is contained in:
parent
f15d30a98d
commit
5b60a43466
@ -1,13 +1,20 @@
|
|||||||
|
.DS_Store
|
||||||
flamegraph.svg
|
flamegraph.svg
|
||||||
perf.data
|
perf.data
|
||||||
perf.data.old
|
perf.data.old
|
||||||
|
|
||||||
/bin/
|
/*.md
|
||||||
/config/devel*
|
/.env
|
||||||
/config/production*
|
/.git
|
||||||
/data/
|
/.gitignore
|
||||||
/docker-compose*
|
/.vscode
|
||||||
/Dockerfile
|
/Dockerfile
|
||||||
/Jenkinsfile
|
/Jenkinsfile
|
||||||
|
/bin
|
||||||
|
/config/devel*
|
||||||
|
/config/production*
|
||||||
|
/data
|
||||||
|
/docker-compose*
|
||||||
|
/docs
|
||||||
/redis-cell-server/
|
/redis-cell-server/
|
||||||
/target
|
/target
|
||||||
|
5
.gitignore
vendored
5
.gitignore
vendored
@ -1,7 +1,8 @@
|
|||||||
.DS_Store
|
.DS_Store
|
||||||
/config/*.toml
|
|
||||||
/data
|
|
||||||
flamegraph.svg
|
flamegraph.svg
|
||||||
perf.data
|
perf.data
|
||||||
perf.data.old
|
perf.data.old
|
||||||
|
|
||||||
|
/config/*.toml
|
||||||
|
/data
|
||||||
/target
|
/target
|
||||||
|
50
Dockerfile
50
Dockerfile
@ -2,12 +2,17 @@ FROM debian:bullseye-slim as builder
|
|||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
ENV CARGO_TERM_COLOR always
|
ENV CARGO_TERM_COLOR always
|
||||||
|
ENV PATH "/root/.foundry/bin:/root/.cargo/bin:${PATH}"
|
||||||
|
|
||||||
# install rustup dependencies
|
# install rustup dependencies
|
||||||
# also install web3-proxy system dependencies. most things are rust-only, but not everything
|
# also install web3-proxy system dependencies. most things are rust-only, but not everything
|
||||||
RUN apt-get update && \
|
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||||
apt-get install --yes \
|
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||||
|
\
|
||||||
|
apt-get update && \
|
||||||
|
apt-get install --no-install-recommends --yes \
|
||||||
build-essential \
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
cmake \
|
cmake \
|
||||||
curl \
|
curl \
|
||||||
git \
|
git \
|
||||||
@ -17,51 +22,54 @@ RUN apt-get update && \
|
|||||||
libssl-dev \
|
libssl-dev \
|
||||||
libzstd-dev \
|
libzstd-dev \
|
||||||
make \
|
make \
|
||||||
pkg-config \
|
pkg-config
|
||||||
&& \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# install rustup
|
# install rustup
|
||||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
|
||||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none
|
|
||||||
|
|
||||||
# install the correct version of rust
|
# install our desired version of rust
|
||||||
# we need nightly for a few features
|
COPY rust-toolchain.toml ./
|
||||||
COPY rust-toolchain.toml .
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
RUN /root/.cargo/bin/rustup update
|
\
|
||||||
|
rustup update
|
||||||
|
|
||||||
# a next-generation test runner for Rust projects.
|
# a next-generation test runner for Rust projects.
|
||||||
# We only pay the installation cost once,
|
# We only pay the installation cost once,
|
||||||
# it will be cached from the second build onwards
|
# it will be cached from the second build onwards
|
||||||
# TODO: more mount type cache?
|
# TODO: more mount type cache?
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
|
\
|
||||||
cargo install cargo-nextest
|
cargo install cargo-nextest
|
||||||
|
|
||||||
# foundry is needed to run tests
|
# foundry is needed to run tests
|
||||||
# TODO: do this in a seperate FROM and COPY it in
|
# TODO: do this in a seperate FROM and COPY it in
|
||||||
ENV PATH /root/.foundry/bin:$PATH
|
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
RUN curl -L https://foundry.paradigm.xyz | bash && foundryup
|
\
|
||||||
|
curl -L https://foundry.paradigm.xyz | bash && foundryup
|
||||||
# copy the application
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
ENV WEB3_PROXY_FEATURES "rdkafka-src,connectinfo"
|
ENV WEB3_PROXY_FEATURES "rdkafka-src,connectinfo"
|
||||||
|
|
||||||
FROM builder as build_tests
|
FROM builder as build_tests
|
||||||
|
|
||||||
# test the application with cargo-nextest
|
# test the application with cargo-nextest
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=bind,target=. \
|
||||||
|
--mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
--mount=type=cache,target=/app/target \
|
--mount=type=cache,target=/app/target \
|
||||||
RUST_LOG=web3_proxy=trace,info cargo nextest run --features "$WEB3_PROXY_FEATURES" --no-default-features
|
\
|
||||||
|
RUST_LOG=web3_proxy=trace,info cargo nextest run --features "$WEB3_PROXY_FEATURES" --no-default-features && \
|
||||||
|
touch /test_success
|
||||||
|
|
||||||
|
# TODO: does this split of build_tests and build_app actually do anything if they both mount the same cache?
|
||||||
FROM builder as build_app
|
FROM builder as build_app
|
||||||
|
|
||||||
# build the application
|
# build the application
|
||||||
# using a "release" profile (which install does by default) is **very** important
|
# using a "release" profile (which install does by default) is **very** important
|
||||||
# TODO: use the "faster_release" profile which builds with `codegen-units = 1`
|
# TODO: use the "faster_release" profile which builds with `codegen-units = 1`
|
||||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
RUN --mount=type=bind,target=.,rw \
|
||||||
|
--mount=type=cache,target=/usr/local/cargo/registry \
|
||||||
--mount=type=cache,target=/app/target \
|
--mount=type=cache,target=/app/target \
|
||||||
|
\
|
||||||
cargo install \
|
cargo install \
|
||||||
--features "$WEB3_PROXY_FEATURES" \
|
--features "$WEB3_PROXY_FEATURES" \
|
||||||
--locked \
|
--locked \
|
||||||
@ -88,6 +96,8 @@ CMD [ "--config", "/web3-proxy.toml", "proxyd" ]
|
|||||||
# TODO: lower log level when done with prototyping
|
# TODO: lower log level when done with prototyping
|
||||||
ENV RUST_LOG "warn,ethers_providers::rpc=off,web3_proxy=debug,web3_proxy::rpcs::consensus=info,web3_proxy_cli=debug"
|
ENV RUST_LOG "warn,ethers_providers::rpc=off,web3_proxy=debug,web3_proxy::rpcs::consensus=info,web3_proxy_cli=debug"
|
||||||
|
|
||||||
|
# we copy something from build_tests just so that docker actually builds it
|
||||||
|
COPY --from=build_tests /test_success /
|
||||||
COPY --from=build_app /usr/local/bin/* /usr/local/bin/
|
COPY --from=build_app /usr/local/bin/* /usr/local/bin/
|
||||||
|
|
||||||
# make sure the app works
|
# make sure the app works
|
||||||
|
36
bugs.md
36
bugs.md
@ -1,36 +0,0 @@
|
|||||||
# deadlock
|
|
||||||
|
|
||||||
goerli_1 | 1 deadlocks detected
|
|
||||||
goerli_1 | Deadlock #0
|
|
||||||
goerli_1 | Thread Id 139881298757376
|
|
||||||
goerli_1 | 0: 0x5608f7f762d9 - backtrace::backtrace::trace::hbe74611947a262af
|
|
||||||
goerli_1 | 1: 0x5608f7f7a967 - backtrace::capture::Backtrace::new::h667fe9ee7ec04c33
|
|
||||||
goerli_1 | 2: 0x5608f7f6ed33 - parking_lot_core::parking_lot::deadlock_impl::on_unpark::h78879313dd6461e5
|
|
||||||
goerli_1 | 3: 0x5608f730dcd4 - parking_lot::raw_mutex::RawMutex::lock_slow::h9c58bf1ec322b8f6
|
|
||||||
goerli_1 | 4: 0x5608f78f2e87 - <moka::common::concurrent::housekeeper::ThreadPoolHousekeeper<T> as core::ops::drop::Drop>::drop::h4887dbe8ef7d7472
|
|
||||||
goerli_1 | 5: 0x5608f7909362 - alloc::sync::Arc<T>::drop_slow::h3de3d854b76812ea
|
|
||||||
goerli_1 | 6: 0x5608f7919596 - core::ptr::drop_in_place<moka::future::cache::Cache<web3_proxy::app_stats::UserProxyResponseKey,alloc::sync::Arc<web3_proxy::app_stats::ProxyResponseAggregate>,ahash::random_state::RandomState>>::h1bf4d8ebf87406ed
|
|
||||||
goerli_1 | 7: 0x5608f791ac00 - triomphe::arc::Arc<T>::drop_slow::h246e78aee1f2a265
|
|
||||||
goerli_1 | 8: 0x5608f78e38bd - crossbeam_epoch::deferred::Deferred::new::call::h395b93588d5e21a9
|
|
||||||
goerli_1 | 9: 0x5608f72fbaa2 - crossbeam_epoch::internal::Global::collect::h77479fc8b8898340
|
|
||||||
goerli_1 | 10: 0x5608f73ef22c - <moka::sync_base::base_cache::Inner<K,V,S> as moka::common::concurrent::housekeeper::InnerSync>::sync::h07f3f4f6db1c2598
|
|
||||||
goerli_1 | 11: 0x5608f75e4ee3 - moka::common::concurrent::housekeeper::ThreadPoolHousekeeper<T>::call_sync::h11b70044870c94f4
|
|
||||||
goerli_1 | 12: 0x5608f75e4b03 - moka::common::concurrent::housekeeper::ThreadPoolHousekeeper<T>::start_periodical_sync_job::{{closure}}::hdc1c253b1b156548
|
|
||||||
goerli_1 | 13: 0x5608f7cc8d15 - scheduled_thread_pool::Worker::run_job::hb3ae60b61103071b
|
|
||||||
goerli_1 | 14: 0x5608f7cc8b8b - scheduled_thread_pool::Worker::run::h760e10fe3281c379
|
|
||||||
goerli_1 | 15: 0x5608f7ccb294 - std::sys_common::backtrace::__rust_begin_short_backtrace::hc3b55a28c2ef3a5f
|
|
||||||
goerli_1 | 16: 0x5608f7cc9cb5 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hf330c4157d74cf0e
|
|
||||||
goerli_1 | 17: 0x5608f7fc8dd3 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h56d5fc072706762b
|
|
||||||
goerli_1 | at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/alloc/src/boxed.rs:1935:9
|
|
||||||
goerli_1 | <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h41deef8e33b824bb
|
|
||||||
goerli_1 | at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/alloc/src/boxed.rs:1935:9
|
|
||||||
goerli_1 | std::sys::unix::thread::Thread::new::thread_start::ha6436304a1170bba
|
|
||||||
goerli_1 | at /rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/std/src/sys/unix/thread.rs:108:17
|
|
||||||
goerli_1 | 18: 0x7f3b309e9ea7 - start_thread
|
|
||||||
goerli_1 | 19: 0x7f3b307bfaef - clone
|
|
||||||
goerli_1 | 20: 0x0 - <unknown>
|
|
||||||
goerli_1 |
|
|
||||||
|
|
||||||
also saw deadlocks on other chains (arbitrum, goerli, gnosis, optimism, polygon, fantom). though luckily not on eth. and it seems like it kept going.
|
|
||||||
i'm going to guess that the problem is nested caches.
|
|
||||||
refactor to maybe use a dashmap at one level? or flatten into one level and use channels more
|
|
Loading…
Reference in New Issue
Block a user