Jenkinsfile from llamanodes/erigon

This commit is contained in:
Bryan Stitt 2023-02-05 00:13:14 -08:00
parent 0e4aab24a7
commit 37830f1156
2 changed files with 104 additions and 42 deletions

145
Jenkinsfile vendored

@ -1,42 +1,84 @@
def amd_image
def arm_image
def intel_image
def restoreMTime() {
sh '''
def buildAndPush() {
// env.BRANCH_NAME is set to the git branch name by default
// env.REGISTRY is the repository url for this pipeline
// env.GIT_SHORT is the git short hash of the currently checked out repo
// env.LATEST_BRANCH is the branch name that gets tagged latest
// env.ARCH is the system architecture. some apps can be generic (amd64, arm64),
// but apps that compile for specific hardware (like web3-proxy) will need more specific tags (amd64_epyc2, arm64_graviton2, intel_xeon3, etc.)
// TODO: check that this system actually matches the given arch
sh '''#!/bin/bash
set -eux -o pipefail
[ -n "$GIT_SHORT" ]
[ -n "$GIT_SHORT" ]
[ -n "$REGISTRY" ]
[ -n "$ARCH" ]
# deterministic mtime on .git keeps Dockerfiles that do 'ADD . .' or similar
# without this, the build process always thinks the directory has changes
git restore-mtime
touch -t $(git show -s --date=format:'%Y%m%d%H%M.%S' --format=%cd HEAD) .git
touch -t "$(git show -s --date=format:'%Y%m%d%H%M.%S' --format=%cd HEAD)" .git
function buildAndPush {
image=$1
buildcache=$2
buildctl build \
--frontend=dockerfile.v0 \
--local context=. \
--local dockerfile=. \
--output "type=image,name=${image},push=true" \
--export-cache type=s3,region=us-east-2,bucket=llamarpc-buildctl-cache,name=${buildcache} \
--import-cache type=s3,region=us-east-2,bucket=llamarpc-buildctl-cache,name=${buildcache} \
;
}
BUILDCACHE="${REGISTRY}:buildcache_${ARCH}"
# build and push a docker image tagged with the short git commit
buildAndPush "${REGISTRY}:git_${GIT_SHORT}_${ARCH}" "${BUILDCACHE}"
# push an image tagged with the branch
# since buildAndPush just ran above, this should be very quick
# TODO: maybe replace slashes in the name with dashes or underscores
buildAndPush "${REGISTRY}:branch_${BRANCH_NAME}_${ARCH}" "${BUILDCACHE}"
if [ "${BRANCH_NAME}" = "${LATEST_BRANCH}" ]; then
buildAndPush "${REGISTRY}:latest_${ARCH}" "${BUILDCACHE}"
fi
'''
}
pipeline {
agent any
options {
ansiColor('xterm')
}
environment {
DOCKER_GIT_TAG="$AWS_ECR_URL/web3-proxy:${GIT_COMMIT.substring(0,8)}"
// AWS_ECR_URL needs to be set in jenkin's config.
// AWS_ECR_URL could really be any docker registry. we just use ECR so that we don't have to manage it
REGISTRY="${AWS_ECR_URL}/web3-proxy"
// branch that should get tagged with "latest_$arch" (stable, main, master, etc.)
LATEST_BRANCH="stable"
// non-buildkit builds are officially deprecated
// buildkit is much faster and handles caching much better than the default build process.
DOCKER_BUILDKIT=1
GIT_SHORT="${GIT_COMMIT.substring(0,8)}"
}
stages {
stage('build and push') {
parallel {
stage('Build and push amd64_epyc2 image') {
stage('build and push amd64_epyc2 image') {
agent {
label 'amd64_epyc2'
}
environment {
ARCH="amd64_epyc2"
}
steps {
script {
DOCKER_GIT_TAG_AMD="$DOCKER_GIT_TAG" + "_amd64_epyc2"
restoreMTime()
try {
amd_image = docker.build("$DOCKER_GIT_TAG_AMD")
} catch (e) {
def err = "amd64_epyc2 build failed: ${e}"
error(err)
}
amd_image.push()
amd_image.push('latest_amd64_epyc2')
buildAndPush()
}
}
}
@ -44,18 +86,12 @@ pipeline {
agent {
label 'arm64_graviton2'
}
environment {
ARCH="arm64_graviton2"
}
steps {
script {
DOCKER_GIT_TAG_ARM="$DOCKER_GIT_TAG" + "_arm64_graviton2"
restoreMTime()
try {
arm_image = docker.build("$DOCKER_GIT_TAG_ARM")
} catch (e) {
def err = "arm64_graviton2 build failed: ${e}"
error(err)
}
arm_image.push()
arm_image.push('latest_arm64_graviton2')
buildAndPush()
}
}
}
@ -63,22 +99,47 @@ pipeline {
agent {
label 'intel_xeon3'
}
environment {
ARCH="intel_xeon3"
}
steps {
script {
DOCKER_GIT_TAG_INTEL="$DOCKER_GIT_TAG" + "_intel_xeon3"
restoreMTime()
try {
intel_image = docker.build("$DOCKER_GIT_TAG_INTEL")
} catch (e) {
def err = "intel_xeon3 build failed: ${e}"
error(err)
}
intel_image.push()
intel_image.push('latest_intel_xeon3')
buildAndPush()
}
}
}
}
}
stage('create (experimental) manifest') {
agent any
steps {
script {
sh '''#!/bin/bash
set -eux -o pipefail
[ -n "$BRANCH_NAME" ]
[ -n "$GIT_SHORT" ]
[ -n "$LATEST_BRANCH" ]
[ -n "$REGISTRY" ]
function manifest {
repo=$1
docker manifest create "${repo}" --amend "${repo}_arm64_graviton2" --amend "${repo}_amd64_epyc2" --amend "${repo}_intel_xeon3"
docker manifest push --purge "${repo}"
}
manifest "${REGISTRY}:git_${GIT_SHORT}"
manifest "${REGISTRY}:branch_${BRANCH_NAME}"
if [ "${BRANCH_NAME}" = "${LATEST_BRANCH}" ]; then
manifest "${REGISTRY}:latest"
fi
'''
}
}
}
}
}

@ -389,6 +389,7 @@ impl Web3Connections {
// multiple blocks with the same fork!
if consensus_saved_block.hash() == old_head_block.hash() {
// no change in hash. no need to use head_block_sender
// TODO: trace level if rpc is backup
debug!(
"con {}{}/{}/{}/{} con={} rpc={}@{}",
includes_backups_str,