version: 2.1 orbs: go: circleci/go@1.9.0 gcp-cli: circleci/gcp-cli@2.4.1 shellcheck: circleci/shellcheck@3.1.2 path-filtering: circleci/path-filtering@1.0.0 parameters: run-build-op-conductor-mon: type: boolean default: false run-build-op-ufm: type: boolean default: false run-build-proxyd: type: boolean default: false run-all: type: boolean default: false commands: gcp-oidc-authenticate: description: "Authenticate with GCP using a CircleCI OIDC token." parameters: project_id: type: env_var_name default: GCP_PROJECT_ID workload_identity_pool_id: type: env_var_name default: GCP_WIP_ID workload_identity_pool_provider_id: type: env_var_name default: GCP_WIP_PROVIDER_ID service_account_email: type: env_var_name default: GCP_SERVICE_ACCOUNT_EMAIL gcp_cred_config_file_path: type: string default: /home/circleci/gcp_cred_config.json oidc_token_file_path: type: string default: /home/circleci/oidc_token.json steps: - run: name: "Create OIDC credential configuration" command: | # Store OIDC token in temp file echo $CIRCLE_OIDC_TOKEN > << parameters.oidc_token_file_path >> # Create a credential configuration for the generated OIDC ID Token gcloud iam workload-identity-pools create-cred-config \ "projects/${<< parameters.project_id >>}/locations/global/workloadIdentityPools/${<< parameters.workload_identity_pool_id >>}/providers/${<< parameters.workload_identity_pool_provider_id >>}"\ --output-file="<< parameters.gcp_cred_config_file_path >>" \ --service-account="${<< parameters.service_account_email >>}" \ --credential-source-file=<< parameters.oidc_token_file_path >> - run: name: "Authenticate with GCP using OIDC" command: | # Configure gcloud to leverage the generated credential configuration gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>" # Configure ADC echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV" jobs: log-config-results: docker: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov. environment: CURRENT_TAG: << pipeline.git.tag >> steps: - checkout - run: name: Log Configuration Results command: | echo "Configuration Results:" echo "run-build-op-conductor-mon: << pipeline.parameters.run-build-op-conductor-mon >>" echo "run-build-op-ufm: << pipeline.parameters.run-build-op-ufm >>" echo "run-build-proxyd: << pipeline.parameters.run-build-proxyd >>" echo "run-all: << pipeline.parameters.run-all >>" echo "" echo "Pipeline Trigger Information:" echo "pipeline.trigger_source: << pipeline.trigger_source >>" echo "Is not a scheduled pipeline: $([ "<< pipeline.trigger_source >>" != "scheduled_pipeline" ] && echo "true" || echo "false")" echo "" echo "Tag Information:" echo "Current tag: $CURRENT_TAG" # Use the same regex patterns as defined in the YAML anchors if [[ $CURRENT_TAG =~ ^proxyd/v.* ]]; then echo "proxyd tag regex match: true" else echo "proxyd tag regex match: false" fi if [[ $CURRENT_TAG =~ ^op-conductor-mon/v.* ]]; then echo "op-conductor-mon tag regex match: true" else echo "op-conductor-mon tag regex match: false" fi if [[ $CURRENT_TAG =~ ^op-ufm/v.* ]]; then echo "op-ufm tag regex match: true" else echo "op-ufm tag regex match: false" fi docker-build: environment: DOCKER_BUILDKIT: 1 parameters: docker_name: description: Docker image name type: string docker_tags: description: Docker image tags as csv type: string docker_file: description: Path to Dockerfile type: string docker_context: description: Docker build context type: string registry: description: Docker registry type: string default: "us-docker.pkg.dev" repo: description: Docker repo type: string default: "oplabs-tools-artifacts/images" machine: image: default steps: - checkout - run: command: mkdir -p /tmp/docker_images - run: name: Build command: | # Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets. # Building should work without this read only login, but may get rate limited. if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin fi IMAGE_BASE="<>/<>/<>" DOCKER_TAGS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|-t ${IMAGE_BASE}:|") docker build \ $(echo -ne $DOCKER_TAGS | tr '\n' ' ') \ -f <> \ <> - run: name: Save command: | IMAGE_BASE="<>/<>/<>" DOCKER_LABELS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g") echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker save -o /tmp/docker_images/<>_{}.tar $IMAGE_BASE:{} - persist_to_workspace: root: /tmp/docker_images paths: - "." docker-publish: parameters: docker_name: description: Docker image name type: string docker_tags: description: Docker image tags as csv type: string registry: description: Docker registry type: string default: "us-docker.pkg.dev" repo: description: Docker repo type: string default: "oplabs-tools-artifacts/images" machine: image: default steps: - attach_workspace: at: /tmp/docker_images - run: name: Docker load command: | DOCKER_LABELS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g") echo -ne $DOCKER_LABELS | tr ' ' '\n' | xargs -I {} docker load -i /tmp/docker_images/<>_{}.tar - gcp-oidc-authenticate # Below is CircleCI recommended way of specifying nameservers on an Ubuntu box: # https://support.circleci.com/hc/en-us/articles/7323511028251-How-to-set-custom-DNS-on-Ubuntu-based-images-using-netplan - run: sudo sed -i '13 i \ \ \ \ \ \ \ \ \ \ \ \ nameservers:' /etc/netplan/50-cloud-init.yaml - run: sudo sed -i '14 i \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ addresses:' /etc/netplan/50-cloud-init.yaml - run: sudo sed -i "s/addresses:/ addresses":" [8.8.8.8, 8.8.4.4] /g" /etc/netplan/50-cloud-init.yaml - run: sudo cat /etc/netplan/50-cloud-init.yaml - run: sudo netplan apply - run: name: Publish command: | gcloud auth configure-docker <> IMAGE_BASE="<>/<>/<>" DOCKER_TAGS=$(echo -ne <> | sed "s/,/\n/g" | sed "s/[^a-zA-Z0-9\n]/-/g" | sed -e "s|^|${IMAGE_BASE}:|") echo -ne $DOCKER_TAGS | tr ' ' '\n' | xargs -L1 docker push - when: condition: equal: ['main', <>] steps: - gcp-oidc-authenticate: service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL - run: name: Sign command: | git clone --branch v1.0.3 --depth 1 https://github.com/ethereum-optimism/binary_signer cd binary_signer/signer IMAGE_PATH="<>/<>/<>:<>" echo $IMAGE_PATH pip3 install -r requirements.txt python3 ./sign_image.py --command="sign"\ --attestor-project-name="$ATTESTOR_PROJECT_NAME"\ --attestor-name="$ATTESTOR_NAME"\ --image-path="$IMAGE_PATH"\ --signer-logging-level="INFO"\ --attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1" docker-tag-op-stack-release: parameters: registry: description: Docker registry type: string default: "us-docker.pkg.dev" repo: description: Docker repo type: string default: "oplabs-tools-artifacts/images" docker: - image: cimg/python:3.7 resource_class: small steps: - gcp-cli/install - gcp-oidc-authenticate - checkout - run: name: Tag command: | gcloud auth configure-docker <> ./ops/ci-tag-docker-release/ci-docker-tag-op-stack-release.sh <>/<> $CIRCLE_TAG $CIRCLE_SHA1 go-lint: parameters: module: description: Go Module Name type: string docker: - image: cimg/go:1.21 steps: - checkout - run: name: run generate command: | make generate || go generate ./... working_directory: <> - run: name: run tidy command: | go mod tidy && git diff --exit-code working_directory: <> - run: name: run lint command: | if [ -f .golangci.yml ]; then golangci-lint run -c .golangci.yml -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./... else golangci-lint run -E goimports,sqlclosecheck,bodyclose,asciicheck,misspell,errorlint -e "errors.As" -e "errors.Is" --timeout "3m0s" ./... fi working_directory: <> go-test: parameters: module: description: Go Module Name type: string docker: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest # only used to enable codecov. - image: cimg/postgres:14.6 environment: POSTGRES_USER: opc POSTGRES_HOST_AUTH_METHOD: trust resource_class: small steps: - checkout - run: name: go version command: go version - run: name: prep results dir command: mkdir -p /tmp/test-results - run: name: run generate command: | make generate || go generate ./... working_directory: <> - run: name: run tests command: | gotestsum --format=standard-verbose --junitfile=/tmp/test-results/<>.xml \ -- -coverpkg=github.com/ethereum-optimism/infrastructure-services/... -coverprofile=coverage.out ./... working_directory: <> - run: name: upload coverage command: codecov --verbose --clean --flags <> - store_test_results: path: /tmp/test-results py-presubmit: parameters: poetry_root: description: Root of the Poetry project directory. type: string docker: - image: cimg/python:3.11 resource_class: small steps: - checkout - run: name: prep results dir command: mkdir -p /tmp/test-results - run: name: run presubmit command: | poetry install poetry run presubmit working_directory: <> build-release: parameters: package_name: description: Package to build type: string artifact_path: description: Path to build artifact type: string default: ./bin release_env: description: Release environment type: string default: prod docker: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest steps: - checkout - run: name: Build command: | VERSION=$(echo "$CIRCLE_TAG" | grep -Eow 'v.*' || true) make build-release VERSION=$VERSION RELEASE_ENV=<> working_directory: <> - persist_to_workspace: root: <>/<> paths: - "." publish-release: parameters: package_name: description: Package to publish type: string artifact_path: description: Path to build artifact type: string default: ./bin docker: - image: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:latest steps: - attach_workspace: at: <>/<> - run: name: "Publish Release on GitHub" command: | go install github.com/tcnksm/ghr@v0.16.2 ghr -t "$GITHUB_TOKEN" -u "$CIRCLE_PROJECT_USERNAME" -r "$CIRCLE_PROJECT_REPONAME" -c "$CIRCLE_SHA1" -delete "$CIRCLE_TAG" <>/<> workflows: logging: jobs: - log-config-results: filters: tags: only: /.*/ branches: ignore: /.*/ op-conductor-mon: when: or: [<< pipeline.parameters.run-build-op-conductor-mon >>, << pipeline.parameters.run-all >>] jobs: - go-lint: name: op-conductor-mon-lint module: op-conductor-mon - go-test: name: op-conductor-mon-tests module: op-conductor-mon - docker-build: name: op-conductor-mon-docker-build docker_file: op-conductor-mon/Dockerfile docker_name: op-conductor-mon docker_tags: <>,<> docker_context: . op-ufm: when: or: [<< pipeline.parameters.run-build-op-ufm >>, << pipeline.parameters.run-all >>] jobs: - go-lint: name: op-ufm-lint module: op-ufm - go-test: name: op-ufm-tests module: op-ufm - docker-build: name: op-ufm-docker-build docker_file: op-ufm/Dockerfile docker_name: op-ufm docker_tags: <>,<> docker_context: . op-proxyd: when: or: [<< pipeline.parameters.run-build-proxyd >>, << pipeline.parameters.run-all >>] jobs: - go-lint: name: proxyd-lint module: proxyd - go-test: name: proxyd-tests module: proxyd - docker-build: name: proxyd-docker-build docker_file: proxyd/Dockerfile docker_name: proxyd docker_tags: <>,<> docker_context: . release: jobs: - log-config-results: filters: tags: only: /^(proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/ branches: ignore: /.*/ - hold: type: approval filters: tags: only: /^(proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/ branches: ignore: /.*/ - docker-build: name: op-ufm-docker-build filters: tags: only: /^op-ufm\/v.*/ docker_name: op-ufm docker_tags: <> docker_context: . docker_file: op-ufm/Dockerfile context: - oplabs-gcr-release requires: - hold - docker-publish: name: op-ufm-docker-publish docker_name: op-ufm docker_tags: <> context: - oplabs-gcr-release requires: - op-ufm-docker-build - docker-tag-op-stack-release: name: docker-tag-op-ufm-release filters: tags: only: /^op-ufm\/v.*/ branches: ignore: /.*/ context: - oplabs-gcr-release requires: - op-ufm-docker-publish - docker-build: name: proxyd-docker-build filters: tags: only: /^proxyd\/v.*/ docker_name: proxyd docker_tags: <> docker_context: . docker_file: proxyd/Dockerfile context: - oplabs-gcr-release requires: - hold - docker-publish: name: proxyd-docker-release filters: tags: only: /^proxyd\/v.*/ docker_name: proxyd docker_tags: <> context: - oplabs-gcr-release requires: - proxyd-docker-build - docker-tag-op-stack-release: name: docker-tag-op-stack-release filters: tags: only: /^proxyd\/v.*/ branches: ignore: /.*/ context: - oplabs-gcr-release requires: - proxyd-docker-release - docker-build: name: op-conductor-mon-docker-build filters: tags: only: /^op-conductor-mon\/v.*/ docker_file: op-conductor-mon/Dockerfile docker_name: op-conductor-mon docker_tags: <>,<> docker_context: . context: - oplabs-gcr-release requires: - hold - docker-publish: name: op-conductor-mon-docker-publish docker_name: op-conductor-mon docker_tags: <>,<> context: - oplabs-gcr-release requires: - op-conductor-mon-docker-build - docker-tag-op-stack-release: name: docker-tag-op-stack-release filters: tags: only: /^op-conductor-mon\/v.*/ branches: ignore: /.*/ context: - oplabs-gcr-release requires: - op-conductor-mon-docker-publish