infra/.circleci/continue_config.yml

557 lines
19 KiB
YAML

version: 2.1
orbs:
go: circleci/go@1.9.0
gcp-cli: circleci/gcp-cli@2.4.1
shellcheck: circleci/shellcheck@3.1.2
path-filtering: circleci/path-filtering@1.0.0
parameters:
run-build-op-conductor-mon:
type: boolean
default: false
run-build-op-ufm:
type: boolean
default: false
run-build-proxyd:
type: boolean
default: false
run-all:
type: boolean
default: false
commands:
gcp-oidc-authenticate:
description: "Authenticate with GCP using a CircleCI OIDC token."
parameters:
project_id:
type: env_var_name
default: GCP_PROJECT_ID
workload_identity_pool_id:
type: env_var_name
default: GCP_WIP_ID
workload_identity_pool_provider_id:
type: env_var_name
default: GCP_WIP_PROVIDER_ID
service_account_email:
type: env_var_name
default: GCP_SERVICE_ACCOUNT_EMAIL
gcp_cred_config_file_path:
type: string
default: /home/circleci/gcp_cred_config.json
oidc_token_file_path:
type: string
default: /home/circleci/oidc_token.json
steps:
- run:
name: "Create OIDC credential configuration"
command: |
# Store OIDC token in temp file
echo $CIRCLE_OIDC_TOKEN > << parameters.oidc_token_file_path >>
# Create a credential configuration for the generated OIDC ID Token
gcloud iam workload-identity-pools create-cred-config \
"projects/${<< parameters.project_id >>}/locations/global/workloadIdentityPools/${<< parameters.workload_identity_pool_id >>}/providers/${<< parameters.workload_identity_pool_provider_id >>}"\
--output-file="<< parameters.gcp_cred_config_file_path >>" \
--service-account="${<< parameters.service_account_email >>}" \
--credential-source-file=<< parameters.oidc_token_file_path >>
- run:
name: "Authenticate with GCP using OIDC"
command: |
# Configure gcloud to leverage the generated credential configuration
gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>"
# Configure ADC
echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV"
jobs:
log-config-results:
docker:
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov.
environment:
CURRENT_TAG: << pipeline.git.tag >>
steps:
- checkout
- run:
name: Log Configuration Results
command: |
echo "Configuration Results:"
echo "run-build-op-conductor-mon: << pipeline.parameters.run-build-op-conductor-mon >>"
echo "run-build-op-ufm: << pipeline.parameters.run-build-op-ufm >>"
echo "run-build-proxyd: << pipeline.parameters.run-build-proxyd >>"
echo "run-all: << pipeline.parameters.run-all >>"
echo ""
echo "Pipeline Trigger Information:"
echo "pipeline.trigger_source: << pipeline.trigger_source >>"
echo "Is not a scheduled pipeline: $([ "<< pipeline.trigger_source >>" != "scheduled_pipeline" ] && echo "true" || echo "false")"
echo ""
echo "Tag Information:"
echo "Current tag: $CURRENT_TAG"
# Use the same regex patterns as defined in the YAML anchors
if [[ $CURRENT_TAG =~ ^proxyd/v.* ]]; then
echo "proxyd tag regex match: true"
else
echo "proxyd tag regex match: false"
fi
if [[ $CURRENT_TAG =~ ^op-conductor-mon/v.* ]]; then
echo "op-conductor-mon tag regex match: true"
else
echo "op-conductor-mon tag regex match: false"
fi
if [[ $CURRENT_TAG =~ ^op-ufm/v.* ]]; then
echo "op-ufm tag regex match: true"
else
echo "op-ufm tag regex match: false"
fi
docker-build:
environment:
DOCKER_BUILDKIT: 1
parameters:
docker_name:
description: Docker image name
type: string
docker_tags:
description: Docker image tags as csv
type: string
docker_file:
description: Path to Dockerfile
type: string
docker_context:
description: Docker build context
type: string
registry:
description: Docker registry
type: string
default: "us-docker.pkg.dev"
repo:
description: Docker repo
type: string
default: "oplabs-tools-artifacts/images"
machine:
image: default
steps:
- checkout
- run:
command: mkdir -p /tmp/docker_images
- run:
name: Build
command: |
# Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets.
# Building should work without this read only login, but may get rate limited.
if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then
echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin
fi
IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|")
docker build \
$(echo -ne $DOCKER_TAGS | tr '\n' ' ') \
-f <<parameters.docker_file>> \
<<parameters.docker_context>>
- run:
name: Save
command: |
IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
DOCKER_LABELS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g")
echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker save -o /tmp/docker_images/<<parameters.docker_name>>_{}.tar $IMAGE_BASE:{}
- persist_to_workspace:
root: /tmp/docker_images
paths:
- "."
docker-publish:
parameters:
docker_name:
description: Docker image name
type: string
docker_tags:
description: Docker image tags as csv
type: string
registry:
description: Docker registry
type: string
default: "us-docker.pkg.dev"
repo:
description: Docker repo
type: string
default: "oplabs-tools-artifacts/images"
machine:
image: default
steps:
- attach_workspace:
at: /tmp/docker_images
- run:
name: Docker load
command: |
DOCKER_LABELS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g")
echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker load -i /tmp/docker_images/<<parameters.docker_name>>_{}.tar
- gcp-oidc-authenticate
# Below is CircleCI recommended way of specifying nameservers on an Ubuntu box:
# https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan
- run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml
- run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml
- run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml
- run: sudo cat /etc/netplan/50-cloud-init.yaml
- run: sudo netplan apply
- run:
name: Publish
command: |
gcloud auth configure-docker <<parameters.registry>>
IMAGE_BASE="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>"
DOCKER_TAGS=$(echo -ne <<parameters.docker_tags>> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|${IMAGE_BASE}:|")
echo -ne $DOCKER_TAGS | tr ' ' '\n' | xargs -L1 docker push
- when:
condition:
equal: ['main', <<pipeline.git.branch>>]
steps:
- gcp-oidc-authenticate:
service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL
- run:
name: Sign
command: |
git clone --branch v1.0.3 --depth 1 https://github.com/ethereum-optimism/binary_signer
cd binary_signer/signer
IMAGE_PATH="<<parameters.registry>>/<<parameters.repo>>/<<parameters.docker_name>>:<<pipeline.git.revision>>"
echo $IMAGE_PATH
pip3 install -r requirements.txt
python3 ./sign_image.py --command="sign"\
--attestor-project-name="$ATTESTOR_PROJECT_NAME"\
--attestor-name="$ATTESTOR_NAME"\
--image-path="$IMAGE_PATH"\
--signer-logging-level="INFO"\
--attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1"
docker-tag-op-stack-release:
parameters:
registry:
description: Docker registry
type: string
default: "us-docker.pkg.dev"
repo:
description: Docker repo
type: string
default: "oplabs-tools-artifacts/images"
docker:
- image: cimg/python:3.7
resource_class: small
steps:
- gcp-cli/install
- gcp-oidc-authenticate
- checkout
- run:
name: Tag
command: |
gcloud auth configure-docker <<parameters.registry>>
./ops/ci-tag-docker-release/ci-docker-tag-op-stack-release.sh <<parameters.registry>>/<<parameters.repo>> $CIRCLE_TAG $CIRCLE_SHA1
go-lint:
parameters:
module:
description: Go Module Name
type: string
docker:
- image: cimg/go:1.21
steps:
- checkout
- run:
name: run generate
command: |
make generate || go generate ./...
working_directory: <<parameters.module>>
- run:
name: run tidy
command: |
go mod tidy && git diff --exit-code
working_directory: <<parameters.module>>
- run:
name: run lint
command: |
if [ -f .golangci.yml ]; then
golangci-lint run -c .golangci.yml -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
else
golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./...
fi
working_directory: <<parameters.module>>
go-test:
parameters:
module:
description: Go Module Name
type: string
docker:
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov.
- image: cimg/postgres:14.6
environment:
POSTGRES_USER: opc
POSTGRES_HOST_AUTH_METHOD: trust
resource_class: small
steps:
- checkout
- run:
name: go version
command: go version
- run:
name: prep results dir
command: mkdir -p /tmp/test-results
- run:
name: run generate
command: |
make generate || go generate ./...
working_directory: <<parameters.module>>
- run:
name: run tests
command: |
gotestsum --format=standard-verbose --junitfile=/tmp/test-results/<<parameters.module>>.xml \
-- -coverpkg=github.com/ethereum-optimism/infrastructure-services/... -coverprofile=coverage.out ./...
working_directory: <<parameters.module>>
- run:
name: upload coverage
command: codecov --verbose --clean --flags <<parameters.module>>
- store_test_results:
path: /tmp/test-results
py-presubmit:
parameters:
poetry_root:
description: Root of the Poetry project directory.
type: string
docker:
- image: cimg/python:3.11
resource_class: small
steps:
- checkout
- run:
name: prep results dir
command: mkdir -p /tmp/test-results
- run:
name: run presubmit
command: |
poetry install
poetry run presubmit
working_directory: <<parameters.poetry_root>>
build-release:
parameters:
package_name:
description: Package to build
type: string
artifact_path:
description: Path to build artifact
type: string
default: ./bin
release_env:
description: Release environment
type: string
default: prod
docker:
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
steps:
- checkout
- run:
name: Build
command: |
VERSION=$(echo "$CIRCLE_TAG" | grep -Eow 'v.*' || true)
make build-release VERSION=$VERSION RELEASE_ENV=<<parameters.release_env>>
working_directory: <<parameters.package_name>>
- persist_to_workspace:
root: <<parameters.package_name>>/<<parameters.artifact_path>>
paths:
- "."
publish-release:
parameters:
package_name:
description: Package to publish
type: string
artifact_path:
description: Path to build artifact
type: string
default: ./bin
docker:
- image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest
steps:
- attach_workspace:
at: <<parameters.package_name>>/<<parameters.artifact_path>>
- run:
name: "Publish Release on GitHub"
command: |
go install github.com/tcnksm/ghr@v0.16.2
ghr -t "$GITHUB_TOKEN" -u "$CIRCLE_PROJECT_USERNAME" -r "$CIRCLE_PROJECT_REPONAME" -c "$CIRCLE_SHA1" -delete "$CIRCLE_TAG" <<parameters.package_name>>/<<parameters.artifact_path>>
workflows:
logging:
jobs:
- log-config-results:
filters:
tags:
only: /.*/
branches:
ignore: /.*/
op-conductor-mon:
when:
or: [<< pipeline.parameters.run-build-op-conductor-mon >>, << pipeline.parameters.run-all >>]
jobs:
- go-lint:
name: op-conductor-mon-lint
module: op-conductor-mon
- go-test:
name: op-conductor-mon-tests
module: op-conductor-mon
- docker-build:
name: op-conductor-mon-docker-build
docker_file: op-conductor-mon/Dockerfile
docker_name: op-conductor-mon
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
docker_context: .
op-ufm:
when:
or: [<< pipeline.parameters.run-build-op-ufm >>, << pipeline.parameters.run-all >>]
jobs:
- go-lint:
name: op-ufm-lint
module: op-ufm
- go-test:
name: op-ufm-tests
module: op-ufm
- docker-build:
name: op-ufm-docker-build
docker_file: op-ufm/Dockerfile
docker_name: op-ufm
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
docker_context: .
op-proxyd:
when:
or: [<< pipeline.parameters.run-build-proxyd >>, << pipeline.parameters.run-all >>]
jobs:
- go-lint:
name: proxyd-lint
module: proxyd
- go-test:
name: proxyd-tests
module: proxyd
- docker-build:
name: proxyd-docker-build
docker_file: proxyd/Dockerfile
docker_name: proxyd
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
docker_context: .
release:
jobs:
- log-config-results:
filters:
tags:
only: /^(proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
branches:
ignore: /.*/
- hold:
type: approval
filters:
tags:
only: /^(proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
branches:
ignore: /.*/
- docker-build:
name: op-ufm-docker-build
filters:
tags:
only: /^op-ufm\/v.*/
docker_name: op-ufm
docker_tags: <<pipeline.git.revision>>
docker_context: .
docker_file: op-ufm/Dockerfile
context:
- oplabs-gcr-release
requires:
- hold
- docker-publish:
name: op-ufm-docker-publish
docker_name: op-ufm
docker_tags: <<pipeline.git.revision>>
context:
- oplabs-gcr-release
requires:
- op-ufm-docker-build
- docker-tag-op-stack-release:
name: docker-tag-op-ufm-release
filters:
tags:
only: /^op-ufm\/v.*/
branches:
ignore: /.*/
context:
- oplabs-gcr-release
requires:
- op-ufm-docker-publish
- docker-build:
name: proxyd-docker-build
filters:
tags:
only: /^proxyd\/v.*/
docker_name: proxyd
docker_tags: <<pipeline.git.revision>>
docker_context: .
docker_file: proxyd/Dockerfile
context:
- oplabs-gcr-release
requires:
- hold
- docker-publish:
name: proxyd-docker-release
filters:
tags:
only: /^proxyd\/v.*/
docker_name: proxyd
docker_tags: <<pipeline.git.revision>>
context:
- oplabs-gcr-release
requires:
- proxyd-docker-build
- docker-tag-op-stack-release:
name: docker-tag-op-stack-release
filters:
tags:
only: /^proxyd\/v.*/
branches:
ignore: /.*/
context:
- oplabs-gcr-release
requires:
- proxyd-docker-release
- docker-build:
name: op-conductor-mon-docker-build
filters:
tags:
only: /^op-conductor-mon\/v.*/
docker_file: op-conductor-mon/Dockerfile
docker_name: op-conductor-mon
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
docker_context: .
context:
- oplabs-gcr-release
requires:
- hold
- docker-publish:
name: op-conductor-mon-docker-publish
docker_name: op-conductor-mon
docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
context:
- oplabs-gcr-release
requires:
- op-conductor-mon-docker-build
- docker-tag-op-stack-release:
name: docker-tag-op-stack-release
filters:
tags:
only: /^op-conductor-mon\/v.*/
branches:
ignore: /.*/
context:
- oplabs-gcr-release
requires:
- op-conductor-mon-docker-publish