Browse Source

Migrate to composite GHA actions (#11718)

This migrates from reusable workflows to composite GHA actions.  This has the following advantages:
1) We can split them into smaller, easier to reason about chunks
2) We can reuse them more freely between each other and workflows
3) They don't complicate the job names
4) In theory they'll be easier to test in isolation.  While composite actions can't be unit-tested, we can easily break them up into nodejs or docker actions that can be.

As a proof of concept, some of our non-Bazel tests are migrated to GHA as well (CMake + Composer)

Closes #11718

COPYBARA_INTEGRATE_REVIEW=https://github.com/protocolbuffers/protobuf/pull/11718 from protocolbuffers:gha-actions 5403307bc0
PiperOrigin-RevId: 506375417
pull/11745/head
Mike Kruskal 1 year ago
committed by Copybara-Service
parent
commit
2b4e4260f0
14 changed files with 350 additions and 162 deletions
  1. +1
    -0
      .github/CODEOWNERS
  2. +76
    -0
      .github/actions/bazel-docker/action.yml
  3. +29
    -0
      .github/actions/internal/docker-auth/action.yml
  4. +58
    -0
      .github/actions/internal/docker-run/action.yml
  5. +32
    -0
      .github/actions/non-bazel-docker/action.yml
  6. +58
    -9
      .github/workflows/test_cpp.yml
  7. +50
    -0
      .github/workflows/test_php.yml
  8. +21
    -19
      .github/workflows/test_ruby_install.yml
  9. +8
    -0
      .github/workflows/test_runner.yml
  10. +0
    -127
      .github/workflows/tool_docker.yml
  11. +3
    -0
      .gitignore
  12. +4
    -4
      cmake/gtest.cmake
  13. +7
    -0
      cmake/install.cmake
  14. +3
    -3
      regenerate_stale_files.sh

+ 1
- 0
.github/CODEOWNERS View File

@ -34,3 +34,4 @@
/kokoro/ @protocolbuffers/protobuf-btr
/third_party/ @protocolbuffers/protobuf-btr
*.bazel @protocolbuffers/protobuf-btr
/.github/ @protocolbuffers/protobuf-btr

+ 76
- 0
.github/actions/bazel-docker/action.yml View File

@ -0,0 +1,76 @@
name: 'Docker Bazel Run'
description: 'Run a Bazel-based docker image for Protobuf CI testing'
inputs:
credentials:
required: true
description: "The GCP credentials to use for reading the docker image"
type: string
image:
required: false
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
description: "The docker image to use"
type: string
bazel-cache:
required: false
description: >
A unique path for the Bazel cache. This will trigger the generation
of a BAZEL_CACHE environment variable inside the container that provides
the appropriate flags for any bazel command.
type: string
bazel:
required: false
description: "The Bazel command to run"
type: string
bash:
required: false
description: "A bash command to run. $BAZEL_FLAGS will be available to use for bazel runs."
type: string
runs:
using: 'composite'
steps:
- name: Authenticate
id: auth
uses: ./.github/actions/internal/docker-auth
with:
credentials: ${{ inputs.credentials }}
- name: Validate inputs
if: ${{ inputs.bash && inputs.bazel}}
shell: bash
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
- name: Initialize Bazel flags
shell: bash
run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV
- name: Configure Bazel caching
shell: bash
# Skip bazel cache for local act runs due to issue with credential files
# and nested docker images
if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
run: >
echo "BAZEL_FLAGS=$BAZEL_FLAGS
--google_credentials=/workspace/$(basename ${{ steps.auth.outputs.credentials-file }})
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
- name: Configure Bazel cache writing
shell: bash
# External PRs should never write to our caches.
if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }}
run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV
- name: Run Bash Docker
uses: ./.github/actions/internal/docker-run
if: ${{ inputs.bash }}
with:
image: ${{ inputs.image }}
run-flags: --entrypoint "/bin/bash"
command: -l -c "${{ inputs.bash }}"
- name: Run Bazel Docker
uses: ./.github/actions/internal/docker-run
if: ${{ !inputs.bash }}
with:
image: ${{ inputs.image }}
command: ${{ inputs.bazel }} $BAZEL_FLAGS

+ 29
- 0
.github/actions/internal/docker-auth/action.yml View File

@ -0,0 +1,29 @@
name: 'Authenticate Docker'
description: 'Authenticate a workflow for Protobuf CI testing'
inputs:
credentials:
required: true
description: "The GCP credentials to use for reading the docker image"
type: string
outputs:
credentials-file:
description: "Credentials file generated for GCP"
value: ${{ steps.auth.outputs.credentials_file_path }}
runs:
using: 'composite'
steps:
- name: Authenticate to Google Cloud
id: auth
uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0
with:
credentials_json: ${{ inputs.credentials }}
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1
- name: Use gcloud CLI
shell: bash
run: gcloud info
- name: Authenticate for GAR use
shell: bash
run: gcloud auth configure-docker -q us-docker.pkg.dev

+ 58
- 0
.github/actions/internal/docker-run/action.yml View File

@ -0,0 +1,58 @@
name: 'Run Docker'
description: 'Run a docker image for Protobuf CI testing'
inputs:
image:
required: true
description: "The docker image to use"
type: string
command:
required: true
description: "A raw docker command to run"
type: string
run-flags:
required: false
description: "Additional flags to pass to docker run"
type: string
# WARNING: loading from cache appears to be slower than pull!
docker-cache:
required: false
description: "Enabled caching of pulled docker images."
runs:
using: 'composite'
steps:
- name: Setup QEMU for possible emulation
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
- name: Check docker cache
if: ${{ inputs.docker-cache }}
id: check-docker-cache
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ci/docker/
key: ${{ inputs.image }}
- name: Pull and store if cache miss
shell: bash
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }}
run: >
time docker pull -q ${{ inputs.image }} &&
mkdir -p ci/docker/$(dirname ${{ inputs.image }}) &&
time docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar
- name: Use the cached image on cache hit
shell: bash
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }}
run: time docker image load --input ./ci/docker/${{ inputs.image }}.tar
- name: Pull fresh docker image
shell: bash
if: ${{ !inputs.docker-cache }}
run: time docker pull -q ${{ inputs.image }}
- name: Run docker
shell: bash
run: >
time docker run ${{ inputs.run-flags}} -v${{ github.workspace }}:/workspace
${{ inputs.image }} ${{ inputs.command }}

+ 32
- 0
.github/actions/non-bazel-docker/action.yml View File

@ -0,0 +1,32 @@
name: 'Docker Non-Bazel Run'
description: 'Run a docker image for Protobuf CI testing with a non-Bazel build system'
inputs:
credentials:
required: true
description: "The GCP credentials to use for reading the docker image"
type: string
command:
required: true
description: A command to run in the docker image
image:
required: false
default: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
description: "The docker image to use"
type: string
runs:
using: 'composite'
steps:
- name: Update stale files using Bazel
uses: ./.github/actions/bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1
credentials: ${{ inputs.credentials }}
bazel-cache: regenerate-stale-files
bash: ./regenerate_stale_files.sh $BAZEL_FLAGS
- name: Run Docker
uses: ./.github/actions/internal/docker-run
with:
image: ${{ inputs.image }}
command: ${{ inputs.command }}

+ 58
- 9
.github/workflows/test_cpp.yml View File

@ -35,12 +35,61 @@ jobs:
- config: { name: "Bazel4" }
targets: "//src/..."
image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:4.2.3-3b71de326b62f67bf754c4dc4016d6a2fa9dd664"
name: mat${{ matrix.none }}
uses: ./.github/workflows/tool_docker.yml
with:
name: Linux ${{ matrix.config.name }}
safe-checkout: ${{ inputs.safe-checkout }}
image: ${{ matrix.image }}
bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} --distinct_host_configuration=false
bazel-cache: cpp_bazel/${{ matrix.config.name }}
secrets: inherit
name: Linux ${{ matrix.config.name }}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
with:
image: ${{ matrix.image }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: cpp_bazel/${{ matrix.config.name }}
bazel: test ${{ matrix.targets }} ${{ matrix.config.flags }} --distinct_host_configuration=false
cmake:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
include:
- command: >
/test.sh
-Dprotobuf_BUILD_CONFORMANCE=ON
-Dprotobuf_BUILD_EXAMPLES=ON
-DCMAKE_CXX_STANDARD=14
- name: Ninja
command: >
/test.sh
-G Ninja
-Dprotobuf_BUILD_CONFORMANCE=ON
-DCMAKE_CXX_STANDARD=14
- name: Shared
command: >
/test.sh
-Dprotobuf_BUILD_CONFORMANCE=ON
-Dprotobuf_BUILD_SHARED_LIBS=ON
-DCMAKE_CXX_STANDARD=14
- name: Install
command: >
/install.sh -DCMAKE_CXX_STANDARD=14 \&\& /test.sh
-Dprotobuf_REMOVE_INSTALLED_HEADERS=ON
-Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
-Dprotobuf_BUILD_CONFORMANCE=ON
-DCMAKE_CXX_STANDARD=14
name: Linux CMake ${{ matrix.name}}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/non-bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: ${{ matrix.command }}

+ 50
- 0
.github/workflows/test_php.yml View File

@ -0,0 +1,50 @@
name: PHP Tests
on:
workflow_call:
inputs:
safe-checkout:
required: true
description: "The SHA key for the commit we want to run over"
type: string
jobs:
linux:
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
include:
- name: 7.3 Debug
version: 7.3.28-dbg
command: composer test \&\& composer test_c
- name: 7.4 Debug
version: 7.4.18-dbg
command: composer test \&\& composer test_c
- name: 8.0 Optimized
version: 8.0.5
command: composer test \&\& composer test_c
- name: 8.0 Debug
version: 8.0.5-dbg
command: composer test \&\& composer test_c
- name: 8.0 Memory Leak
version: 8.0.5-dbg
# Run specialized memory leak & multirequest tests.
command: composer test_c \&\& tests/multirequest.sh \&\& tests/memory_leak_test.sh
- name: 8.0 Valgrind
version: 8.0.5-dbg
command: composer test_valgrind
name: Linux PHP ${{ matrix.name}}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
submodules: recursive
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/non-bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: ${{ matrix.command }}

+ 21
- 19
.github/workflows/test_ruby_install.yml View File

@ -23,22 +23,24 @@ jobs:
- { name: JRuby 9.3, ruby: jruby-9.3.4.0, bazel: 5.1.1}
- { name: Ruby 2.6 (Bazel6), ruby: ruby-2.6.0, bazel: 6.0.0}
- { name: JRuby 9.2 (Bazel6), ruby: jruby-9.2.20.1, bazel: 6.0.0}
name: mat${{ matrix.none }}
uses: ./.github/workflows/tool_docker.yml
with:
name: Linux ${{ matrix.name }}
safe-checkout: ${{ inputs.safe-checkout }}
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }}
run-flags: --entrypoint "/bin/bash"
command: >
-l -c "
bazel --version &&
ruby --version &&
bazel build //ruby:release //:protoc $BAZEL_CACHE &&
gem install bazel-bin/ruby/google-protobuf-* &&
bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto &&
bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto &&
ruby ruby/tests/basic.rb"
secrets: inherit
name: Linux ${{ matrix.name }}
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: ./.github/actions/bazel-docker
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/ruby:${{ matrix.ruby }}-${{ matrix.bazel }}-75e79f791b96e056086f43ace729cf3ebf9a9f5d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: ruby_install/${{ matrix.ruby }}_${{ matrix.bazel }}
bash: >
bazel --version;
ruby --version;
bazel build //ruby:release //:protoc $BAZEL_FLAGS;
gem install bazel-bin/ruby/google-protobuf-*;
bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/test_import_proto2.proto;
bazel-bin/protoc --proto_path=src --proto_path=ruby/tests --proto_path=ruby --ruby_out=ruby tests/basic_test.proto;
ruby ruby/tests/basic.rb

+ 8
- 0
.github/workflows/test_runner.yml View File

@ -105,6 +105,14 @@ jobs:
safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
secrets: inherit
php:
name: PHP
needs: [check-tag]
uses: ./.github/workflows/test_php.yml
with:
safe-checkout: ${{ needs.check-tag.outputs.checkout-sha }}
secrets: inherit
php-ext:
name: PHP Extension
needs: [check-tag]


+ 0
- 127
.github/workflows/tool_docker.yml View File

@ -1,127 +0,0 @@
name: Run a Docker workflow
on:
workflow_call:
inputs:
name:
required: True
description: "The name to display for the test"
type: string
image:
required: false
default: 'us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:5.1.1-aec4d74f2eb6938fc53ef7d9a79a4bf2da24abc1'
description: "The docker image to use"
type: string
safe-checkout:
required: true
description: "The SHA key for the commit we want to run over"
type: string
run-flags:
required: false
description: "Additional flags to pass to docker run"
type: string
bazel-cache:
required: false
description: >
A unique path for the Bazel cache. This will trigger the generation
of a BAZEL_CACHE environment variant that provides the appropriate
flags for any bazel command.
type: string
# WARNING: loading from cache appears to be slower than pull!
docker-cache:
required: false
description: "Enabled caching of pulled docker images."
default: false
type: boolean
# Non-Bazel options
command:
required: false
description: "A raw docker command to run"
type: string
# Bazel options
bazel:
required: false
description: "The Bazel command to run"
type: string
jobs:
run:
name: ${{ inputs.name }}
timeout-minutes: 120
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
with:
ref: ${{ inputs.safe-checkout }}
# Authentication
- name: Setup QEMU for possible emulation
id: qemu-arm64
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1.2.0
- name: Authenticate to Google Cloud
uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # v1.0.0
with:
credentials_json: ${{ secrets.GAR_SERVICE_ACCOUNT }}
export_environment_variables: true
- name: Set up Cloud SDK
uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1
- name: Use gcloud CLI
run: gcloud info
- name: Authenticate for GAR use
run: gcloud auth configure-docker -q us-docker.pkg.dev
# Create the docker command
- name: Validate Docker command
if: ${{ inputs.command && inputs.bazel}}
run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1
- name: Configure Bazel caching
# Skip bazel cache for local act runs due to issue with credential files
# and nested docker images
if: ${{ inputs.bazel-cache && !github.event.act_local_test }}
run: >
echo "BAZEL_CACHE=
--google_credentials=/workspace/$(basename $GOOGLE_APPLICATION_CREDENTIALS)
--remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV
- name: Configure Bazel cache updating
# External runs should never write to our caches.
if: ${{ inputs.bazel-cache && !inputs.safe-checkout && !github.event.act_local_test }}
run: echo "BAZEL_CACHE=$BAZEL_CACHE --remote_upload_local_results" >> $GITHUB_ENV
- name: Configure Bazel command
if: ${{ inputs.bazel }}
run: >
echo "DOCKER_COMMAND=${{ inputs.bazel }}
--keep_going --test_output=errors --test_timeout=600
$BAZEL_CACHE" >> $GITHUB_ENV
# Grab Docker image
- name: Check docker cache
if: ${{ inputs.docker-cache }}
id: check-docker-cache
uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4
with:
path: ci/docker/
key: ${{ inputs.image }}
- name: Pull and store if cache miss
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit != 'true' }}
run: >
docker pull ${{ inputs.image }} &&
mkdir -p ci/docker/$(dirname ${{ inputs.image }}) &&
docker image save ${{ inputs.image }} --output ./ci/docker/${{ inputs.image }}.tar
- name: Use the cached image on cache hit
if: ${{ inputs.docker-cache && steps.check-docker-cache.outputs.cache-hit == 'true' }}
run: docker image load --input ./ci/docker/${{ inputs.image }}.tar
- name: Pull fresh docker image
if: ${{ !inputs.docker-cache }}
run: docker pull ${{ inputs.image }}
- name: Run docker
run: >
docker run ${{ inputs.run-flags}}
-v${{ github.workspace }}:/workspace
${{ inputs.image }}
${{ inputs.command || '$DOCKER_COMMAND' }}

+ 3
- 0
.gitignore View File

@ -208,3 +208,6 @@ BenchmarkDotNet.Artifacts/
# Clangd uses these common ephemeral files
.cache
compile_commands.json
# Ignore GHA NodeJS files
.github/**/node_modules/

+ 4
- 4
cmake/gtest.cmake View File

@ -31,10 +31,10 @@ else()
COMPILE_DEFINITIONS
"GTEST_CREATE_SHARED_LIBRARY=1"
)
install(TARGETS gmock EXPORT protobuf-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
if (protobuf_INSTALL)
set(protobuf_INSTALL_TESTS ON)
endif()
endif()
target_link_libraries(gmock ${CMAKE_THREAD_LIBS_INIT})


+ 7
- 0
cmake/install.cmake View File

@ -113,3 +113,10 @@ if(protobuf_INSTALL_EXAMPLES)
DESTINATION "${CMAKE_INSTALL_EXAMPLEDIR}"
COMPONENT protobuf-examples)
endif()
if (protobuf_INSTALL_TESTS)
install(TARGETS gmock EXPORT protobuf-targets
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
endif()

+ 3
- 3
regenerate_stale_files.sh View File

@ -10,13 +10,13 @@ cd $(dirname -- "$0")
# Run and fix all staleness tests.
# LINT.IfChange(staleness_tests)
bazel test //src:cmake_lists_staleness_test || ./bazel-bin/src/cmake_lists_staleness_test --fix
bazel test //src/google/protobuf:well_known_types_staleness_test || ./bazel-bin/src/google/protobuf/well_known_types_staleness_test --fix
bazel test //src:cmake_lists_staleness_test "$@" || ./bazel-bin/src/cmake_lists_staleness_test --fix
bazel test //src/google/protobuf:well_known_types_staleness_test "$@" || ./bazel-bin/src/google/protobuf/well_known_types_staleness_test --fix
# LINT.ThenChange(//depot/google3/third_party/protobuf/github/kokoro/windows/prepare_build_win64.bat:staleness_tests)
# Generate C# code.
# This doesn't currently have Bazel staleness tests, but there's an existing
# shell script that generates everything required. The output files are stable,
# so just regenerating in place should be harmless.
bazel build src/google/protobuf/compiler:protoc
bazel build src/google/protobuf/compiler:protoc "$@"
(export PROTOC=$PWD/bazel-bin/protoc && cd csharp && ./generate_protos.sh)

Loading…
Cancel
Save