Jenkinsfile from llamanodes/erigon
This commit is contained in:
parent
0e4aab24a7
commit
37830f1156
145
Jenkinsfile
vendored
145
Jenkinsfile
vendored
@ -1,42 +1,84 @@
|
|||||||
def amd_image
|
def buildAndPush() {
|
||||||
def arm_image
|
// env.BRANCH_NAME is set to the git branch name by default
|
||||||
def intel_image
|
// env.REGISTRY is the repository url for this pipeline
|
||||||
def restoreMTime() {
|
// env.GIT_SHORT is the git short hash of the currently checked out repo
|
||||||
sh '''
|
// env.LATEST_BRANCH is the branch name that gets tagged latest
|
||||||
|
// env.ARCH is the system architecture. some apps can be generic (amd64, arm64),
|
||||||
|
// but apps that compile for specific hardware (like web3-proxy) will need more specific tags (amd64_epyc2, arm64_graviton2, intel_xeon3, etc.)
|
||||||
|
|
||||||
|
// TODO: check that this system actually matches the given arch
|
||||||
|
sh '''#!/bin/bash
|
||||||
|
set -eux -o pipefail
|
||||||
|
|
||||||
|
[ -n "$GIT_SHORT" ]
|
||||||
|
[ -n "$GIT_SHORT" ]
|
||||||
|
[ -n "$REGISTRY" ]
|
||||||
|
[ -n "$ARCH" ]
|
||||||
|
|
||||||
|
# deterministic mtime on .git keeps Dockerfiles that do 'ADD . .' or similar
|
||||||
|
# without this, the build process always thinks the directory has changes
|
||||||
git restore-mtime
|
git restore-mtime
|
||||||
touch -t $(git show -s --date=format:'%Y%m%d%H%M.%S' --format=%cd HEAD) .git
|
touch -t "$(git show -s --date=format:'%Y%m%d%H%M.%S' --format=%cd HEAD)" .git
|
||||||
|
|
||||||
|
function buildAndPush {
|
||||||
|
image=$1
|
||||||
|
buildcache=$2
|
||||||
|
|
||||||
|
buildctl build \
|
||||||
|
--frontend=dockerfile.v0 \
|
||||||
|
--local context=. \
|
||||||
|
--local dockerfile=. \
|
||||||
|
--output "type=image,name=${image},push=true" \
|
||||||
|
--export-cache type=s3,region=us-east-2,bucket=llamarpc-buildctl-cache,name=${buildcache} \
|
||||||
|
--import-cache type=s3,region=us-east-2,bucket=llamarpc-buildctl-cache,name=${buildcache} \
|
||||||
|
;
|
||||||
|
}
|
||||||
|
|
||||||
|
BUILDCACHE="${REGISTRY}:buildcache_${ARCH}"
|
||||||
|
|
||||||
|
# build and push a docker image tagged with the short git commit
|
||||||
|
buildAndPush "${REGISTRY}:git_${GIT_SHORT}_${ARCH}" "${BUILDCACHE}"
|
||||||
|
|
||||||
|
# push an image tagged with the branch
|
||||||
|
# since buildAndPush just ran above, this should be very quick
|
||||||
|
# TODO: maybe replace slashes in the name with dashes or underscores
|
||||||
|
buildAndPush "${REGISTRY}:branch_${BRANCH_NAME}_${ARCH}" "${BUILDCACHE}"
|
||||||
|
|
||||||
|
if [ "${BRANCH_NAME}" = "${LATEST_BRANCH}" ]; then
|
||||||
|
buildAndPush "${REGISTRY}:latest_${ARCH}" "${BUILDCACHE}"
|
||||||
|
fi
|
||||||
'''
|
'''
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
pipeline {
|
pipeline {
|
||||||
agent any
|
agent any
|
||||||
options {
|
|
||||||
ansiColor('xterm')
|
|
||||||
}
|
|
||||||
environment {
|
environment {
|
||||||
DOCKER_GIT_TAG="$AWS_ECR_URL/web3-proxy:${GIT_COMMIT.substring(0,8)}"
|
// AWS_ECR_URL needs to be set in jenkin's config.
|
||||||
|
// AWS_ECR_URL could really be any docker registry. we just use ECR so that we don't have to manage it
|
||||||
|
REGISTRY="${AWS_ECR_URL}/web3-proxy"
|
||||||
|
|
||||||
|
// branch that should get tagged with "latest_$arch" (stable, main, master, etc.)
|
||||||
|
LATEST_BRANCH="stable"
|
||||||
|
|
||||||
|
// non-buildkit builds are officially deprecated
|
||||||
|
// buildkit is much faster and handles caching much better than the default build process.
|
||||||
DOCKER_BUILDKIT=1
|
DOCKER_BUILDKIT=1
|
||||||
|
|
||||||
|
GIT_SHORT="${GIT_COMMIT.substring(0,8)}"
|
||||||
}
|
}
|
||||||
stages {
|
stages {
|
||||||
stage('build and push') {
|
stage('build and push') {
|
||||||
parallel {
|
parallel {
|
||||||
stage('Build and push amd64_epyc2 image') {
|
stage('build and push amd64_epyc2 image') {
|
||||||
agent {
|
agent {
|
||||||
label 'amd64_epyc2'
|
label 'amd64_epyc2'
|
||||||
}
|
}
|
||||||
|
environment {
|
||||||
|
ARCH="amd64_epyc2"
|
||||||
|
}
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
DOCKER_GIT_TAG_AMD="$DOCKER_GIT_TAG" + "_amd64_epyc2"
|
buildAndPush()
|
||||||
restoreMTime()
|
|
||||||
try {
|
|
||||||
amd_image = docker.build("$DOCKER_GIT_TAG_AMD")
|
|
||||||
} catch (e) {
|
|
||||||
def err = "amd64_epyc2 build failed: ${e}"
|
|
||||||
error(err)
|
|
||||||
}
|
|
||||||
amd_image.push()
|
|
||||||
amd_image.push('latest_amd64_epyc2')
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -44,18 +86,12 @@ pipeline {
|
|||||||
agent {
|
agent {
|
||||||
label 'arm64_graviton2'
|
label 'arm64_graviton2'
|
||||||
}
|
}
|
||||||
|
environment {
|
||||||
|
ARCH="arm64_graviton2"
|
||||||
|
}
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
DOCKER_GIT_TAG_ARM="$DOCKER_GIT_TAG" + "_arm64_graviton2"
|
buildAndPush()
|
||||||
restoreMTime()
|
|
||||||
try {
|
|
||||||
arm_image = docker.build("$DOCKER_GIT_TAG_ARM")
|
|
||||||
} catch (e) {
|
|
||||||
def err = "arm64_graviton2 build failed: ${e}"
|
|
||||||
error(err)
|
|
||||||
}
|
|
||||||
arm_image.push()
|
|
||||||
arm_image.push('latest_arm64_graviton2')
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -63,22 +99,47 @@ pipeline {
|
|||||||
agent {
|
agent {
|
||||||
label 'intel_xeon3'
|
label 'intel_xeon3'
|
||||||
}
|
}
|
||||||
|
environment {
|
||||||
|
ARCH="intel_xeon3"
|
||||||
|
}
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
DOCKER_GIT_TAG_INTEL="$DOCKER_GIT_TAG" + "_intel_xeon3"
|
buildAndPush()
|
||||||
restoreMTime()
|
|
||||||
try {
|
|
||||||
intel_image = docker.build("$DOCKER_GIT_TAG_INTEL")
|
|
||||||
} catch (e) {
|
|
||||||
def err = "intel_xeon3 build failed: ${e}"
|
|
||||||
error(err)
|
|
||||||
}
|
|
||||||
intel_image.push()
|
|
||||||
intel_image.push('latest_intel_xeon3')
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
stage('create (experimental) manifest') {
|
||||||
|
agent any
|
||||||
|
steps {
|
||||||
|
script {
|
||||||
|
sh '''#!/bin/bash
|
||||||
|
set -eux -o pipefail
|
||||||
|
|
||||||
|
[ -n "$BRANCH_NAME" ]
|
||||||
|
[ -n "$GIT_SHORT" ]
|
||||||
|
[ -n "$LATEST_BRANCH" ]
|
||||||
|
[ -n "$REGISTRY" ]
|
||||||
|
|
||||||
|
function manifest {
|
||||||
|
repo=$1
|
||||||
|
|
||||||
|
docker manifest create "${repo}" --amend "${repo}_arm64_graviton2" --amend "${repo}_amd64_epyc2" --amend "${repo}_intel_xeon3"
|
||||||
|
|
||||||
|
docker manifest push --purge "${repo}"
|
||||||
|
}
|
||||||
|
|
||||||
|
manifest "${REGISTRY}:git_${GIT_SHORT}"
|
||||||
|
manifest "${REGISTRY}:branch_${BRANCH_NAME}"
|
||||||
|
|
||||||
|
if [ "${BRANCH_NAME}" = "${LATEST_BRANCH}" ]; then
|
||||||
|
manifest "${REGISTRY}:latest"
|
||||||
|
fi
|
||||||
|
'''
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -389,6 +389,7 @@ impl Web3Connections {
|
|||||||
// multiple blocks with the same fork!
|
// multiple blocks with the same fork!
|
||||||
if consensus_saved_block.hash() == old_head_block.hash() {
|
if consensus_saved_block.hash() == old_head_block.hash() {
|
||||||
// no change in hash. no need to use head_block_sender
|
// no change in hash. no need to use head_block_sender
|
||||||
|
// TODO: trace level if rpc is backup
|
||||||
debug!(
|
debug!(
|
||||||
"con {}{}/{}/{}/{} con={} rpc={}@{}",
|
"con {}{}/{}/{}/{} con={} rpc={}@{}",
|
||||||
includes_backups_str,
|
includes_backups_str,
|
||||||
|
Loading…
Reference in New Issue
Block a user