commit 292164eddb666591c7b5a4fd62f7f58277ec0674 from: Yaroslav Lobankov via: Yaroslav Lobankov date: Wed Aug 30 08:52:29 2023 UTC Add testing for static build packaging This patch finally brings desired testing for static build packages. How it works: In a few words, we have two workflow files: calling and callable. The callable workflow (static_build_pack_test_deploy.yml) is parametrized and contains all the logic with the building, testing, and deploying packages. It takes just two inputs: package platform and JSON matrix for testing. The calling workflow (packaging.yml) just runs callable one with specific parameters and contains all the logic related to triggering by events and concurrency. The static_build_pack_test_deploy.yml workflow consists of three jobs: `build`, `test`, and `deploy`. Artifacts between jobs are passed via the `upload-artifact` and `download-artifact` actions. The `test` job is a matrix one and verifies packages on provided Linux distros passed through input. After the testing is done, the `deploy` job is intended to upload packages to repositories on a tag push, which means release or pre-release. Note, for starting Docker containers to test packages we use PackPack images because they have almost all requirements to run tests. Follows up #8771 Follows up #8840 Follows up #8866 Closes tarantool/tarantool-qa#322 NO_DOC=ci NO_TEST=ci NO_CHANGELOG=ci commit - 88333d13eb9930904268b314d195b0e811ef5464 commit + 292164eddb666591c7b5a4fd62f7f58277ec0674 blob - /dev/null blob + 14d8545c6665632d722805a0ea7bf3bf3626102c (mode 644) --- /dev/null +++ .github/workflows/packaging.yml @@ -0,0 +1,74 @@ +name: packaging + +on: + push: + branches: + - 'master' + - 'release/**' + tags: + - '**' + pull_request: + types: [ opened, reopened, synchronize, labeled ] + workflow_dispatch: + +concurrency: + # Update of a developer branch cancels the previously scheduled workflow + # run for this branch. However, the 'master' branch, release branch, and + # tag workflow runs are never canceled. + # + # We use a trick here: define the concurrency group as 'workflow run ID' + + # 'workflow run attempt' because it is a unique combination for any run. + # So it effectively discards grouping. + # + # Important: we cannot use `github.sha` as a unique identifier because + # pushing a tag may cancel a run that works on a branch push event. + group: ${{ ( + github.ref == 'refs/heads/master' || + startsWith(github.ref, 'refs/heads/release/') || + startsWith(github.ref, 'refs/tags/')) && + format('{0}-{1}', github.run_id, github.run_attempt) || + format('{0}-{1}', github.workflow, github.ref) }} + cancel-in-progress: true + +jobs: + packaging-x86_64: + # Run on push to the 'master' and release branches of tarantool/tarantool + # or on pull request if the 'full-ci' or 'static-build-ci' label is set. + if: github.repository == 'tarantool/tarantool' && + ( github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'full-ci') || + contains(github.event.pull_request.labels.*.name, 'static-build-ci') ) + + uses: ./.github/workflows/static_build_pack_test_deploy.yml + with: + arch: x86_64 + test-matrix: '{ + "include": [ + {"os": "debian-buster"}, {"os": "debian-bullseye"}, + {"os": "centos-7"}, {"os": "centos-8"}, + {"os": "fedora-35"}, {"os": "fedora-36"}, + {"os": "ubuntu-focal"}, {"os": "ubuntu-jammy"} + ] + }' + secrets: inherit + + packaging-aarch64: + # Run on push to the 'master' and release branches of tarantool/tarantool + # or on pull request if the 'full-ci' or 'static-build-ci' label is set. + if: github.repository == 'tarantool/tarantool' && + ( github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'full-ci') || + contains(github.event.pull_request.labels.*.name, 'static-build-ci') ) + + uses: ./.github/workflows/static_build_pack_test_deploy.yml + with: + arch: aarch64 + test-matrix: '{ + "include": [ + {"os": "debian-buster"}, {"os": "debian-bullseye"}, + {"os": "centos-7"}, {"os": "centos-8"}, + {"os": "fedora-35"}, {"os": "fedora-36"}, + {"os": "ubuntu-focal"}, {"os": "ubuntu-jammy"} + ] + }' + secrets: inherit blob - fc541665be525afddcb28703159108eb461a7690 (mode 644) blob + /dev/null --- .github/workflows/static_build_packaging.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: static_build_packaging - -on: - push: - branches: - - 'master' - - 'release/**' - tags: - - '**' - pull_request: - types: [ opened, reopened, synchronize, labeled ] - workflow_dispatch: - -concurrency: - # Update of a developer branch cancels the previously scheduled workflow - # run for this branch. However, the 'master' branch, release branch, and - # tag workflow runs are never canceled. - # - # We use a trick here: define the concurrency group as 'workflow run ID' + - # 'workflow run attempt' because it is a unique combination for any run. - # So it effectively discards grouping. - # - # Important: we cannot use `github.sha` as a unique identifier because - # pushing a tag may cancel a run that works on a branch push event. - group: ${{ ( - github.ref == 'refs/heads/master' || - startsWith(github.ref, 'refs/heads/release/') || - startsWith(github.ref, 'refs/tags/')) && - format('{0}-{1}', github.run_id, github.run_attempt) || - format('{0}-{1}', github.workflow, github.ref) }} - cancel-in-progress: true - -jobs: - static_build_packaging: - name: 'static_build_packaging (${{ matrix.arch }})' - - # Run on push to the 'master' and release branches of tarantool/tarantool - # or on pull request if the 'full-ci' or 'static-build-ci' label is set. - if: github.repository == 'tarantool/tarantool' && - ( github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'full-ci') || - contains(github.event.pull_request.labels.*.name, 'static-build-ci') ) - - runs-on: ${{ matrix.runner }} - - strategy: - fail-fast: false - matrix: - include: - - runner: ubuntu-20.04-self-hosted - arch: x86_64 - - runner: graviton - arch: aarch64 - - steps: - - name: Prepare checkout - uses: tarantool/actions/prepare-checkout@master - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - submodules: recursive - - uses: ./.github/actions/environment - - name: Install deps - uses: ./.github/actions/install-deps-debian - - name: Build static packages - uses: ./.github/actions/pack-and-deploy - env: - RWS_AUTH: ${{ secrets.RWS_AUTH }} - - name: Upload build artifacts - if: github.ref == 'refs/heads/master' || - startsWith(github.ref, 'refs/heads/release/') || - startsWith(github.ref, 'refs/tags/') - uses: actions/upload-artifact@v3 - with: - name: tarantool-deb-rpm-x86_64-aarch64 - retention-days: 21 - path: | - build/tarantool*.deb - build/tarantool*.rpm - if-no-files-found: error - - name: Send VK Teams message on failure - if: failure() - uses: ./.github/actions/report-job-status - with: - bot-token: ${{ secrets.VKTEAMS_BOT_TOKEN }} blob - /dev/null blob + 00b8884d83b91ce2f5cbcf93abc043f25c5dd5e6 (mode 644) --- /dev/null +++ .github/workflows/static_build_pack_test_deploy.yml @@ -0,0 +1,211 @@ +name: static_build_pack_test_deploy + +on: + workflow_call: + inputs: + arch: + description: Package platform + default: x86_64 + required: false + type: string + test-matrix: + description: JSON matrix for packages testing + default: '{"include": [{"os": "ubuntu-focal"}, {"os": "ubuntu-jammy"}]}' + required: false + type: string + +jobs: + build: + runs-on: [ self-hosted, Linux, '${{ inputs.arch }}', regular ] + + steps: + - name: Prepare checkout + uses: tarantool/actions/prepare-checkout@master + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: recursive + + - uses: ./.github/actions/environment + + - name: Make static build packages + run: make -f .pack.mk package-static + + - name: Pack libraries for testing + run: > + find test/ -name '*.so' | tar -cvzf + ${{ github.workspace }}/build/test-libs-${{ inputs.arch }}.tgz -T - + working-directory: static-build/tarantool-prefix/src/tarantool-build/ + + - name: Upload deb packages + uses: actions/upload-artifact@v3 + with: + name: tarantool-deb-${{ inputs.arch }} + retention-days: 21 + path: build/tarantool*.deb + if-no-files-found: error + + - name: Upload rpm packages + uses: actions/upload-artifact@v3 + with: + name: tarantool-rpm-${{ inputs.arch }} + retention-days: 21 + path: build/tarantool*.rpm + if-no-files-found: error + + - name: Upload test libraries + uses: actions/upload-artifact@v3 + with: + name: tarantool-test-libs-${{ inputs.arch }} + retention-days: 21 + path: build/test-libs*.tgz + if-no-files-found: error + + - name: Send VK Teams message on failure + if: failure() + uses: ./.github/actions/report-job-status + with: + bot-token: ${{ secrets.VKTEAMS_BOT_TOKEN }} + + test: + runs-on: [ self-hosted, Linux, '${{ inputs.arch }}', regular ] + + needs: build + + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.test-matrix) }} + + container: + image: packpack/packpack:${{ matrix.os }} + # Mount /dev to the container to be able to mount a disk image inside it + # for successful run of the .github/actions/environment action. + volumes: + - /dev:/dev + # Our testing expects that the init process (PID 1) will + # reap orphan processes. At least the following test leans + # on it: app-tap/gh-4983-tnt-e-assert-false-hangs.test.lua. + # Add extra privileges to the container for successful run + # of the .github/actions/environment action. + options: '--init --privileged' + + steps: + - name: Prepare checkout + uses: tarantool/actions/prepare-checkout@master + + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Set package manager + run: | + if ${{ startsWith(matrix.os, 'astra') || + startsWith(matrix.os, 'ubuntu') || + startsWith(matrix.os, 'debian') }}; then + echo "PACKAGE_MANAGER=apt" >> $GITHUB_ENV + elif ${{ startsWith(matrix.os, 'centos') || + startsWith(matrix.os, 'el') || + startsWith(matrix.os, 'fedora') || + startsWith(matrix.os, 'redos') }}; then + echo "PACKAGE_MANAGER=yum" >> $GITHUB_ENV + else + echo "Not supported OS provided: ${{ matrix.os }}" + exit 1 + fi + + # Install the 'e2fsprogs' package that contains the 'mkfs' program needed + # for the 'environment' action bellow. + - name: Install e2fsprogs + if: env.PACKAGE_MANAGER != 'apt' + run: ${{ env.PACKAGE_MANAGER }} -y install e2fsprogs + + - uses: ./.github/actions/environment + + - name: Download packages + uses: actions/download-artifact@v3 + with: + name: tarantool-${{ + env.PACKAGE_MANAGER == 'apt' && 'deb' || 'rpm' }}-${{ inputs.arch }} + path: build + + - name: Download test libraries + uses: actions/download-artifact@v3 + with: + name: tarantool-test-libs-${{ inputs.arch }} + path: build + + - name: Install packages + run: ${{ env.PACKAGE_MANAGER }} -y install ./tarantool* + working-directory: build + + - name: Extract test libraries + run: tar -xvzf test-libs*.tgz + working-directory: build + + - name: Update list of available deb packages + if: env.PACKAGE_MANAGER == 'apt' + run: apt update + + - name: Install test dependencies + run: | + ${{ env.PACKAGE_MANAGER }} -y install \ + python3 \ + python3-gevent \ + python3-${{ env.PACKAGE_MANAGER == 'apt' && 'yaml' || 'pyyaml' }} \ + tzdata + + - name: Run tests + run: | + ./test-run.py \ + --force \ + --builddir ../build \ + --executable /usr/bin/tarantool + working-directory: test + + - name: Send VK Teams message on failure + if: failure() + uses: ./.github/actions/report-job-status + with: + bot-token: ${{ secrets.VKTEAMS_BOT_TOKEN }} + + deploy: + if: startsWith(github.ref, 'refs/tags/') && + !endsWith(github.ref, '-entrypoint') + + runs-on: [ self-hosted, Linux, lightweight ] + + needs: test + + steps: + - name: Prepare checkout + uses: tarantool/actions/prepare-checkout@master + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Download deb packages + uses: actions/download-artifact@v3 + with: + name: tarantool-deb-${{ inputs.arch }} + path: build + + - name: Download rpm packages + uses: actions/download-artifact@v3 + with: + name: tarantool-rpm-${{ inputs.arch }} + path: build + + - name: Deploy packages + run: | + case ${{ github.ref }} in + refs/tags/*-alpha*|refs/tags/*-beta*|refs/tags/*-rc*) + REPO_TYPE=pre-release make -f .pack.mk deploy-static + ;; + refs/tags/*) + REPO_TYPE=release make -f .pack.mk deploy-static + ;; + esac + env: + RWS_AUTH: ${{ secrets.RWS_AUTH }}