From 71b6c82364ed84e999d45195aee67181858dff28 Mon Sep 17 00:00:00 2001 From: Ernesto Puerta Date: Thu, 27 Jul 2023 19:22:28 +0200 Subject: [PATCH] ci: run make-check as GH Action Signed-off-by: Ernesto Puerta --- .github/workflows/build-and-test.yml | 81 ++++++++++++++++++++++++++++ install-deps.sh | 54 ++++++++++--------- run-make-check.sh | 4 +- src/script/lib-build.sh | 6 +-- src/script/run-make.sh | 34 +++++++++++- 5 files changed, 146 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/build-and-test.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000000000..7823b5cfbc308 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,81 @@ +name: Build and Test +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] +jobs: + install-dependencies: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - distro-name: centos + distro-release: stream8 + distro-image: quay.io/centos/centos:stream8 + cache_package_path: | + /var/lib/dnf + /var/cache/dnf + cache_package_hashfiles: | + ceph.spec.in + #- distro-name: centos + # distro-release: stream9 + #- ubuntu:20.04 + #- ubuntu:22.04 + container: + image: ${{ matrix.distro-image }} + options: --user root + env: + BASH_ENV: ./run-make-check.sh + SHELLOPTS: xtrace + FOR_MAKE_CHECK: 1 + DNF_INSTALL_OPTIONS: >- + --setopt=install_weak_deps=False + --setopt=keepcache=True + --setopt=fastestmirror=True + --nodocs + defaults: + run: + shell: bash + env: + CACHE_PACKAGE_HASHFILES: | + install-deps.sh + run-make-check.sh + do_cmake.sh + src/script/run-make.sh + src/script/lib-build.sh + name: "${{ matrix.distro-name }}:${{ matrix.distro-release }}" + timeout-minutes: 120 + steps: + - run: | + dnf install -y $DNF_INSTALL_OPTIONS git-core + - run: export + - run: set + - name: Restore package cache + uses: actions/checkout@v3 + with: + submodules: true + - run: git config --global --add safe.directory $GITHUB_WORKSPACE + - name: Restore ccache + uses: actions/cache@v3 + with: + path: ~/.ccache + key: ccache-${{ matrix.distro-name }}-${{ matrix.distro-release }}-${{ github.ref_name }} + restore-keys: | + packages-${{ matrix.distro-name }}-${{ matrix.distro-release }}- + - uses: actions/cache@v3 + with: + path: ${{ matrix.cache_package_path }} + key: packages-${{ matrix.distro-name }}-${{ matrix.distro-release }}-${{ hashFiles(env.CACHE_PACKAGE_HASHFILES, matrix.cache_package_hashfiles) }} + restore-keys: | + packages-${{ matrix.distro-name }}-${{ matrix.distro-release }}- + - run: prepare + - run: ccache -s + - run: configure + - run: build vstart + - name: Test + run: | + build tests + cd build + run diff --git a/install-deps.sh b/install-deps.sh index 611aaeccf24e7..7bb54a511f75d 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -32,7 +32,6 @@ export LC_ALL=C.UTF-8 ARCH=$(uname -m) - function munge_ceph_spec_in { local with_seastar=$1 shift @@ -139,7 +138,7 @@ function install_pkg_on_ubuntu { fi if test -n "$missing_pkgs"; then local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo" - in_jenkins && echo -n "CI_DEBUG: Downloading $shaman_url ... " + in_ci && echo -n "CI_DEBUG: Downloading $shaman_url ... " $SUDO curl --silent --fail --write-out "%{http_code}" --location $shaman_url --output /etc/apt/sources.list.d/$project.list $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs @@ -479,42 +478,47 @@ else fi ;; rocky|centos|fedora|rhel|ol|virtuozzo) - builddepcmd="dnf -y builddep --allowerasing" + builddepcmd="dnf builddep -y $DNF_INSTALL_OPTIONS --allowerasing" echo "Using dnf to install dependencies" case "$ID" in fedora) - $SUDO dnf install -y dnf-utils + $SUDO dnf install -y $DNF_INSTALL_OPTIONS dnf-utils ;; rocky|centos|rhel|ol|virtuozzo) MAJOR_VERSION="$(echo $VERSION_ID | cut -d. -f1)" - $SUDO dnf install -y dnf-utils selinux-policy-targeted + $SUDO dnf install -y $DNF_INSTALL_OPTIONS dnf-utils selinux-policy-targeted rpm --quiet --query epel-release || \ - $SUDO dnf -y install --nogpgcheck https://dl.fedoraproject.org/pub/epel/epel-release-latest-$MAJOR_VERSION.noarch.rpm + $SUDO dnf install -y $DNF_INSTALL_OPTIONS --nogpgcheck https://dl.fedoraproject.org/pub/epel/epel-release-latest-$MAJOR_VERSION.noarch.rpm $SUDO rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$MAJOR_VERSION $SUDO rm -f /etc/yum.repos.d/dl.fedoraproject.org* - if test $ID = centos -a $MAJOR_VERSION = 8 ; then - # Enable 'powertools' or 'PowerTools' repo - $SUDO dnf config-manager --set-enabled $(dnf repolist --all 2>/dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}') - dts_ver=11 - # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies - $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/ - $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save - $SUDO dnf -y module enable javapackages-tools - elif test $ID = rhel -a $MAJOR_VERSION = 8 ; then - dts_ver=11 - $SUDO dnf config-manager --set-enabled "codeready-builder-for-rhel-8-${ARCH}-rpms" - $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/ - $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save - $SUDO dnf -y module enable javapackages-tools - fi + case "$ID-$MAJOR_VERSION" in + centos-8) + # Enable 'powertools' or 'PowerTools' repo + $SUDO dnf config-manager --set-enabled $(dnf repolist --all 2>/dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}') + ;;& + centos-9) + # Enable 'crb' repo + $SUDO dnf config-manager --set-enabled crb + ;;& + rhel-[89]) + $SUDO dnf config-manager --set-enabled "codeready-builder-for-rhel-${MAJOR_VERSION}-${ARCH}-rpms" + ;;& + centos-8|rhel-8) + # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies + $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/${MAJOR_VERSION}/ + $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_${MAJOR_VERSION}_.gpgcheck=0 --save + $SUDO dnf -y module enable javapackages-tools + dts_ver=11 + ;; + esac ;; esac if [ "$INSTALL_EXTRA_PACKAGES" ]; then - $SUDO dnf install -y $INSTALL_EXTRA_PACKAGES + $SUDO dnf install -y $DNF_INSTALL_OPTIONS $INSTALL_EXTRA_PACKAGES fi munge_ceph_spec_in $with_seastar $with_zbd $for_make_check $DIR/ceph.spec # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel - $SUDO dnf install -y python3-devel + $SUDO dnf install -y $DNF_INSTALL_OPTIONS python3-devel $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out [ ${PIPESTATUS[0]} -ne 0 ] && exit 1 if [ -n "$dts_ver" ]; then @@ -525,7 +529,7 @@ else # for rgw motr backend build checks if ! rpm --quiet -q cortx-motr-devel && { [[ $FOR_MAKE_CHECK ]] || $with_rgw_motr; }; then - $SUDO dnf install -y \ + $SUDO dnf install -y $DNF_INSTALL_OPTIONS --nobest \ "$motr_pkgs_url/isa-l-2.30.0-1.el7.${ARCH}.rpm" \ "$motr_pkgs_url/cortx-motr-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm" \ "$motr_pkgs_url/cortx-motr-devel-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm" @@ -554,7 +558,6 @@ fi if $for_make_check; then mkdir -p install-deps-cache top_srcdir=$(pwd) - export XDG_CACHE_HOME=$top_srcdir/install-deps-cache wip_wheelhouse=wheelhouse-wip # # preload python modules so that tox can run without network access @@ -563,7 +566,6 @@ if $for_make_check; then preload_wheels_for_tox $ini done rm -rf $top_srcdir/install-deps-python3 - rm -rf $XDG_CACHE_HOME type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false) fi diff --git a/run-make-check.sh b/run-make-check.sh index 0ebe2b5de6bdf..7a2fac6258dad 100755 --- a/run-make-check.sh +++ b/run-make-check.sh @@ -43,7 +43,7 @@ function run() { fi CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS} - if in_jenkins; then + if in_ci; then if ! ctest $CHECK_MAKEOPTS --no-compress-output --output-on-failure --test-output-size-failed 1024000 -T Test; then # do not return failure, as the jenkins publisher will take care of this rm -fr ${TMPDIR:-/tmp}/ceph-asok.* @@ -72,7 +72,7 @@ function main() { # uses run-make.sh to install-deps FOR_MAKE_CHECK=1 prepare configure "$@" - in_jenkins && echo "CI_DEBUG: Running 'build tests'" + in_ci && echo "CI_DEBUG: Running 'build tests'" build tests echo "make check: successful build on $(git rev-parse HEAD)" FOR_MAKE_CHECK=1 run diff --git a/src/script/lib-build.sh b/src/script/lib-build.sh index 6c4455344379d..ea4a865ff3ec9 100644 --- a/src/script/lib-build.sh +++ b/src/script/lib-build.sh @@ -22,12 +22,12 @@ # shellcheck disable=SC2034 _SOURCED_LIB_BUILD=1 -function in_jenkins() { - [ -n "$JENKINS_HOME" ] +function in_ci() { + [ -n "$JENKINS_HOME" -o -n "$CI" ] } function ci_debug() { - if in_jenkins || [ "${FORCE_CI_DEBUG}" ]; then + if in_ci || [ "${FORCE_CI_DEBUG}" ]; then echo "CI_DEBUG: $*" fi } diff --git a/src/script/run-make.sh b/src/script/run-make.sh index 42d8a94aee1fc..d017eee592271 100755 --- a/src/script/run-make.sh +++ b/src/script/run-make.sh @@ -48,6 +48,36 @@ function detect_ceph_dev_pkgs() { echo "$cmake_opts" } +function get_os_id() { + local OS=$(uname -s) + local ID + if [[ -e /etc/os-release ]]; then + source /etc/os-release + elif [[ "$OS" == FreeBSD ]]; then + ID=freebsd + else + ID=unknown + fi + + echo "$ID" +} + +function install_packages() { + local packages="$1" + + case "$(get_os_id)" in + fedora|rocky|centos|rhel|ol|virtuozzo) + $SUDO dnf install -y $DNF_INSTALL_OPTIONS $packages + ;; + debian|ubuntu|devuan|elementary|softiron) + $SUDO apt-get install -y $APT_INSTALL_OPTIONS $packages + ;; + opensuse*|suse|sles) + $SUDO zypper install -y $ZYPPER_INSTALL_OPTIONS $packages + ;; + esac +} + function prepare() { local which_pkg="which" if command -v apt-get > /dev/null 2>&1 ; then @@ -56,7 +86,7 @@ function prepare() { if test -f ./install-deps.sh ; then ci_debug "Running install-deps.sh" - INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang" + export INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang curl" $DRY_RUN source ./install-deps.sh || return 1 trap clean_up_after_myself EXIT fi @@ -78,7 +108,7 @@ EOM $DRY_RUN export SOURCE_DATE_EPOCH="946684800" $DRY_RUN ccache -o sloppiness=time_macros $DRY_RUN ccache -o run_second_cpp=true - if in_jenkins; then + if in_ci; then # Build host has plenty of space available, let's use it to keep # various versions of the built objects. This could increase the cache hit # if the same or similar PRs are running several times -- 2.39.5