From 800374e681738409ba3cd5bbe43d1dc96f726880 Mon Sep 17 00:00:00 2001 From: Bryan Stitt Date: Fri, 28 Jul 2023 10:43:02 -0700 Subject: [PATCH] sccache for builds --- Cargo.toml | 5 +-- Dockerfile | 99 +++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dc357ffc..63ef4c73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,11 @@ resolver = "2" [profile.release] # `debug = true` so that sentry can give us line numbers debug = true +# `link time optimization = true = fat` +lto = true [profile.faster_release] inherits = "release" -# spend longer compiling for a slightly faster binary +# spend longer compiling for a faster binary codegen-units = 1 - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 16e38950..a2ce7a59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,19 @@ FROM debian:bullseye-slim as rust WORKDIR /app +# Incrementally compiled crates cannot be cached with sccache +ENV CARGO_INCREMENTAL false ENV CARGO_UNSTABLE_SPARSE_REGISTRY true ENV CARGO_TERM_COLOR always +SHELL [ "/bin/bash", "-c" ] +ENV SHELL /bin/bash ENV PATH "/root/.foundry/bin:/root/.cargo/bin:${PATH}" # install rustup dependencies # also install web3-proxy system dependencies. most things are rust-only, but not everything RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ - set -eux; \ + set -eux -o pipefail; \ \ apt-get update; \ apt-get install --no-install-recommends --yes \ @@ -29,7 +33,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ # install rustup RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ - set -eux; \ + set -eux -o pipefail; \ \ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none --profile=minimal @@ -38,7 +42,7 @@ RUN --mount=type=cache,target=/root/.cargo/git \ COPY rust-toolchain.toml ./ RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ - set -eux; \ + set -eux -o pipefail; \ \ cargo check || [ "$?" -eq 101 ] @@ -46,17 +50,25 @@ RUN --mount=type=cache,target=/root/.cargo/git \ RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ - set -eux; \ + set -eux -o pipefail; \ \ curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh >/tmp/install-binstall.sh; \ bash /tmp/install-binstall.sh; \ rm -rf /tmp/* # sccache +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_SESSION_TOKEN +ARG BUILD_JOBS=4 +ARG SCCACHE_BUCKET=llamanodes-sccache +ARG SCCACHE_REGION=us-east-2 +ARG SCCACHE_S3_KEY_PREFIX=web3-proxy +ARG SCCACHE_S3_USE_SSL=true RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ - set -eux; \ + set -eux -o pipefail; \ \ cargo binstall -y sccache; \ sccache -s @@ -66,25 +78,62 @@ ENV RUSTC_WRAPPER "/root/.cargo/bin/sccache" # nextest runs tests in parallel (done its in own FROM so that it can run in parallel) # TODO: i'd like to use binaries for these, but i had trouble with arm and binstall FROM rust as rust_nextest + +# keep the ARGs from the previous step +# we probably won't need sccache since we install with binstall. But compiling is a fallback for binstall, so it is possible. +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_SESSION_TOKEN +ARG BUILD_JOBS +ARG SCCACHE_BUCKET +ARG SCCACHE_REGION +ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_S3_USE_SSL + RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ - set -eux; \ + set -eux -o pipefail; \ \ - cargo binstall -y cargo-nextest + sccache -s; \ + cargo binstall -y cargo-nextest; \ + sccache -s # foundry/anvil are needed to run tests (done its in own FROM so that it can run in parallel) FROM rust as rust_foundry + +# keep the ARGs from the previous step +# we probably won't need sccache since foundry tries to use binaries. But compiling is a fallback, so it is possible. +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_SESSION_TOKEN +ARG BUILD_JOBS +ARG SCCACHE_BUCKET +ARG SCCACHE_REGION +ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_S3_USE_SSL + RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ - set -eux; \ + set -eux -o pipefail; \ \ + sccache -s; \ curl -L https://foundry.paradigm.xyz | bash && foundryup; \ sccache -s FROM rust as rust_with_env +# keep the ARGs from the previous step +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_SESSION_TOKEN +ARG BUILD_JOBS +ARG SCCACHE_BUCKET +ARG SCCACHE_REGION +ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_S3_USE_SSL + # changing our features doesn't change any of the steps above ENV WEB3_PROXY_FEATURES "deadlock_detection,rdkafka-src" @@ -96,7 +145,7 @@ RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ --mount=type=cache,target=/app/target \ - set -eux; \ + set -eux -o pipefail; \ \ [ -e "$(pwd)/payment-contracts/src/contracts/mod.rs" ] || touch "$(pwd)/payment-contracts/build.rs"; \ cargo --locked --verbose fetch @@ -107,20 +156,32 @@ FROM rust_with_env as build_tests COPY --from=rust_foundry /root/.foundry/bin/anvil /root/.foundry/bin/ COPY --from=rust_nextest /root/.cargo/bin/cargo-nextest* /root/.cargo/bin/ +# keep the ARGs from the previous step +# sccache should be a huge savings here +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_SESSION_TOKEN +ARG BUILD_JOBS +ARG SCCACHE_BUCKET +ARG SCCACHE_REGION +ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_S3_USE_SSL + # test the application with cargo-nextest RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ --mount=type=cache,target=/app/target \ - set -eux; \ + set -eux -o pipefail; \ \ + sccache -s; \ [ -e "$(pwd)/payment-contracts/src/contracts/mod.rs" ] || touch "$(pwd)/payment-contracts/build.rs"; \ RUST_LOG=web3_proxy=trace,info \ cargo \ --frozen \ --offline \ - --verbose \ nextest run \ + --build-jobs "$BUILD_JOBS" \ --features "$WEB3_PROXY_FEATURES" --no-default-features \ ; \ sccache -s; \ @@ -128,6 +189,17 @@ RUN --mount=type=cache,target=/root/.cargo/git \ FROM rust_with_env as build_app +# keep the ARGs from the previous step +# sccache should be a huge savings here +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG AWS_SESSION_TOKEN +ARG BUILD_JOBS +ARG SCCACHE_BUCKET +ARG SCCACHE_REGION +ARG SCCACHE_S3_KEY_PREFIX +ARG SCCACHE_S3_USE_SSL + # # build the release application # # using a "release" profile (which install does by default) is **very** important # # TODO: use the "faster_release" profile which builds with `codegen-units = 1` (but compile is SLOW) @@ -135,17 +207,18 @@ RUN --mount=type=cache,target=/root/.cargo/git \ --mount=type=cache,target=/root/.cargo/registry \ --mount=type=cache,target=/root/.cache/sccache \ --mount=type=cache,target=/app/target \ - set -eux; \ + set -eux -o pipefail; \ \ + sccache -s; \ [ -e "$(pwd)/payment-contracts/src/contracts/mod.rs" ] || touch "$(pwd)/payment-contracts/build.rs"; \ cargo install \ + --jobs "$BUILD_JOBS" \ --features "$WEB3_PROXY_FEATURES" \ --frozen \ --no-default-features \ --offline \ --path ./web3_proxy \ --root /usr/local \ - --verbose \ ; \ sccache -s; \ /usr/local/bin/web3_proxy_cli --help | grep 'Usage: web3_proxy_cli'