sccache for builds

2023-07-28 10:43:02 -07:00 · 2023-07-28 10:43:02 -07:00 · 800374e681
commit 800374e681
parent ca92b3646c
2 changed files with 89 additions and 15 deletions
--- a/Cargo.toml
+++ b/Cargo.toml
@ -14,10 +14,11 @@ resolver = "2"
 [profile.release]
 # `debug = true` so that sentry can give us line numbers
 debug = true
+# `link time optimization = true = fat`
+lto = true

 [profile.faster_release]
 inherits = "release"

-# spend longer compiling for a slightly faster binary
+# spend longer compiling for a faster binary
 codegen-units = 1
- 
--- a/99
+++ b/99
@ -1,15 +1,19 @@
 FROM debian:bullseye-slim as rust

 WORKDIR /app
+# Incrementally compiled crates cannot be cached with sccache
+ENV CARGO_INCREMENTAL false
 ENV CARGO_UNSTABLE_SPARSE_REGISTRY true
 ENV CARGO_TERM_COLOR always
+SHELL [ "/bin/bash", "-c" ]
+ENV SHELL /bin/bash
 ENV PATH "/root/.foundry/bin:/root/.cargo/bin:${PATH}"

 # install rustup dependencies
 # also install web3-proxy system dependencies. most things are rust-only, but not everything
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    set -eux; \
+    set -eux -o pipefail; \
    \
    apt-get update; \
    apt-get install --no-install-recommends --yes \
@ -29,7 +33,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 # install rustup
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
-    set -eux; \
+    set -eux -o pipefail; \
    \
    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain none --profile=minimal

@ -38,7 +42,7 @@ RUN --mount=type=cache,target=/root/.cargo/git \
 COPY rust-toolchain.toml ./
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
-    set -eux; \
+    set -eux -o pipefail; \
    \
    cargo check || [ "$?" -eq 101 ]

@ -46,17 +50,25 @@ RUN --mount=type=cache,target=/root/.cargo/git \
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
-    set -eux; \
+    set -eux -o pipefail; \
    \
    curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh >/tmp/install-binstall.sh; \
    bash /tmp/install-binstall.sh; \
    rm -rf /tmp/*

 # sccache
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG BUILD_JOBS=4
+ARG SCCACHE_BUCKET=llamanodes-sccache
+ARG SCCACHE_REGION=us-east-2
+ARG SCCACHE_S3_KEY_PREFIX=web3-proxy
+ARG SCCACHE_S3_USE_SSL=true
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
-    set -eux; \
+    set -eux -o pipefail; \
    \
    cargo binstall -y sccache; \
    sccache -s
@ -66,25 +78,62 @@ ENV RUSTC_WRAPPER "/root/.cargo/bin/sccache"
 # nextest runs tests in parallel (done its in own FROM so that it can run in parallel)
 # TODO: i'd like to use binaries for these, but i had trouble with arm and binstall
 FROM rust as rust_nextest
+
+# keep the ARGs from the previous step
+# we probably won't need sccache since we install with binstall. But compiling is a fallback for binstall, so it is possible.
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG BUILD_JOBS
+ARG SCCACHE_BUCKET
+ARG SCCACHE_REGION
+ARG SCCACHE_S3_KEY_PREFIX
+ARG SCCACHE_S3_USE_SSL
+
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
-    set -eux; \
+    set -eux -o pipefail; \
    \
-    cargo binstall -y cargo-nextest
+    sccache -s; \
+    cargo binstall -y cargo-nextest; \
+    sccache -s

 # foundry/anvil are needed to run tests (done its in own FROM so that it can run in parallel)
 FROM rust as rust_foundry
+
+# keep the ARGs from the previous step
+# we probably won't need sccache since foundry tries to use binaries. But compiling is a fallback, so it is possible.
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG BUILD_JOBS
+ARG SCCACHE_BUCKET
+ARG SCCACHE_REGION
+ARG SCCACHE_S3_KEY_PREFIX
+ARG SCCACHE_S3_USE_SSL
+
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
-    set -eux; \
+    set -eux -o pipefail; \
    \
+    sccache -s; \
    curl -L https://foundry.paradigm.xyz | bash && foundryup; \
    sccache -s

 FROM rust as rust_with_env

+# keep the ARGs from the previous step
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG BUILD_JOBS
+ARG SCCACHE_BUCKET
+ARG SCCACHE_REGION
+ARG SCCACHE_S3_KEY_PREFIX
+ARG SCCACHE_S3_USE_SSL
+
 # changing our features doesn't change any of the steps above
 ENV WEB3_PROXY_FEATURES "deadlock_detection,rdkafka-src"

@ -96,7 +145,7 @@ RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
    --mount=type=cache,target=/app/target \
-    set -eux; \
+    set -eux -o pipefail; \
    \
    [ -e "$(pwd)/payment-contracts/src/contracts/mod.rs" ] || touch "$(pwd)/payment-contracts/build.rs"; \
    cargo --locked --verbose fetch
@ -107,20 +156,32 @@ FROM rust_with_env as build_tests
 COPY --from=rust_foundry /root/.foundry/bin/anvil /root/.foundry/bin/
 COPY --from=rust_nextest /root/.cargo/bin/cargo-nextest* /root/.cargo/bin/

+# keep the ARGs from the previous step
+# sccache should be a huge savings here
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG BUILD_JOBS
+ARG SCCACHE_BUCKET
+ARG SCCACHE_REGION
+ARG SCCACHE_S3_KEY_PREFIX
+ARG SCCACHE_S3_USE_SSL
+
 # test the application with cargo-nextest
 RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
    --mount=type=cache,target=/app/target \
-    set -eux; \
+    set -eux -o pipefail; \
    \
+    sccache -s; \
    [ -e "$(pwd)/payment-contracts/src/contracts/mod.rs" ] || touch "$(pwd)/payment-contracts/build.rs"; \
    RUST_LOG=web3_proxy=trace,info \
    cargo \
    --frozen \
    --offline \
-    --verbose \
    nextest run \
+    --build-jobs "$BUILD_JOBS" \
    --features "$WEB3_PROXY_FEATURES" --no-default-features \
    ; \
    sccache -s; \
@ -128,6 +189,17 @@ RUN --mount=type=cache,target=/root/.cargo/git \

 FROM rust_with_env as build_app

+# keep the ARGs from the previous step
+# sccache should be a huge savings here
+ARG AWS_ACCESS_KEY_ID
+ARG AWS_SECRET_ACCESS_KEY
+ARG AWS_SESSION_TOKEN
+ARG BUILD_JOBS
+ARG SCCACHE_BUCKET
+ARG SCCACHE_REGION
+ARG SCCACHE_S3_KEY_PREFIX
+ARG SCCACHE_S3_USE_SSL
+
 # # build the release application
 # # using a "release" profile (which install does by default) is **very** important
 # # TODO: use the "faster_release" profile which builds with `codegen-units = 1` (but compile is SLOW)
@ -135,17 +207,18 @@ RUN --mount=type=cache,target=/root/.cargo/git \
    --mount=type=cache,target=/root/.cargo/registry \
    --mount=type=cache,target=/root/.cache/sccache \
    --mount=type=cache,target=/app/target \
-    set -eux; \
+    set -eux -o pipefail; \
    \
+    sccache -s; \
    [ -e "$(pwd)/payment-contracts/src/contracts/mod.rs" ] || touch "$(pwd)/payment-contracts/build.rs"; \
    cargo install \
+    --jobs "$BUILD_JOBS" \
    --features "$WEB3_PROXY_FEATURES" \
    --frozen \
    --no-default-features \
    --offline \
    --path ./web3_proxy \
    --root /usr/local \
-    --verbose \
    ; \
    sccache -s; \
    /usr/local/bin/web3_proxy_cli --help | grep 'Usage: web3_proxy_cli'