]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
ci: run make-check as GH Action wip-gh_action_make_check-main 52677/head
authorErnesto Puerta <epuertat@redhat.com>
Thu, 27 Jul 2023 17:22:28 +0000 (19:22 +0200)
committerErnesto Puerta <epuertat@redhat.com>
Fri, 4 Aug 2023 10:48:09 +0000 (12:48 +0200)
Signed-off-by: Ernesto Puerta <epuertat@redhat.com>
.github/workflows/build-and-test.yml [new file with mode: 0644]
install-deps.sh
run-make-check.sh
src/script/lib-build.sh
src/script/run-make.sh

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
new file mode 100644 (file)
index 0000000..7823b5c
--- /dev/null
@@ -0,0 +1,81 @@
+name: Build and Test
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+jobs:
+  install-dependencies:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - distro-name: centos
+            distro-release: stream8
+            distro-image: quay.io/centos/centos:stream8
+            cache_package_path: |
+              /var/lib/dnf
+              /var/cache/dnf
+            cache_package_hashfiles: |
+              ceph.spec.in
+          #- distro-name: centos
+          #  distro-release: stream9
+          #- ubuntu:20.04
+          #- ubuntu:22.04
+    container:
+      image: ${{ matrix.distro-image }}
+      options: --user root
+      env:
+        BASH_ENV: ./run-make-check.sh
+        SHELLOPTS: xtrace
+        FOR_MAKE_CHECK: 1
+        DNF_INSTALL_OPTIONS: >-
+          --setopt=install_weak_deps=False
+          --setopt=keepcache=True
+          --setopt=fastestmirror=True
+          --nodocs
+    defaults:
+      run:
+        shell: bash
+    env:
+      CACHE_PACKAGE_HASHFILES: |
+        install-deps.sh
+        run-make-check.sh
+        do_cmake.sh
+        src/script/run-make.sh
+        src/script/lib-build.sh
+    name: "${{ matrix.distro-name }}:${{ matrix.distro-release }}"
+    timeout-minutes: 120
+    steps:
+      - run: |
+          dnf install -y $DNF_INSTALL_OPTIONS git-core
+      - run: export
+      - run: set
+      - name: Restore package cache
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - run: git config --global --add safe.directory $GITHUB_WORKSPACE
+      - name: Restore ccache
+        uses: actions/cache@v3
+        with:
+          path: ~/.ccache
+          key: ccache-${{ matrix.distro-name }}-${{ matrix.distro-release }}-${{ github.ref_name }}
+          restore-keys: |
+            packages-${{ matrix.distro-name }}-${{ matrix.distro-release }}-
+      - uses: actions/cache@v3
+        with:
+          path: ${{ matrix.cache_package_path }}
+          key: packages-${{ matrix.distro-name }}-${{ matrix.distro-release }}-${{ hashFiles(env.CACHE_PACKAGE_HASHFILES, matrix.cache_package_hashfiles) }}
+          restore-keys: |
+            packages-${{ matrix.distro-name }}-${{ matrix.distro-release }}-
+      - run: prepare
+      - run: ccache -s
+      - run: configure
+      - run: build vstart
+      - name: Test
+        run: |
+          build tests
+          cd build
+          run
index 611aaeccf24e73ba4ce2e5788ec1ead6c00f6a2d..7bb54a511f75d7134888a42018b64d691be429a6 100755 (executable)
@@ -32,7 +32,6 @@ export LC_ALL=C.UTF-8
 
 ARCH=$(uname -m)
 
-
 function munge_ceph_spec_in {
     local with_seastar=$1
     shift
@@ -139,7 +138,7 @@ function install_pkg_on_ubuntu {
     fi
     if test -n "$missing_pkgs"; then
         local shaman_url="https://shaman.ceph.com/api/repos/${project}/master/${sha1}/ubuntu/${codename}/repo"
-        in_jenkins && echo -n "CI_DEBUG: Downloading $shaman_url ... "
+        in_ci && echo -n "CI_DEBUG: Downloading $shaman_url ... "
         $SUDO curl --silent --fail --write-out "%{http_code}" --location $shaman_url --output /etc/apt/sources.list.d/$project.list
         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get update -y -o Acquire::Languages=none -o Acquire::Translation=none || true
         $SUDO env DEBIAN_FRONTEND=noninteractive apt-get install --allow-unauthenticated -y $missing_pkgs
@@ -479,42 +478,47 @@ else
         fi
         ;;
     rocky|centos|fedora|rhel|ol|virtuozzo)
-        builddepcmd="dnf -y builddep --allowerasing"
+        builddepcmd="dnf builddep -y $DNF_INSTALL_OPTIONS --allowerasing"
         echo "Using dnf to install dependencies"
         case "$ID" in
             fedora)
-                $SUDO dnf install -y dnf-utils
+                $SUDO dnf install -y $DNF_INSTALL_OPTIONS dnf-utils
                 ;;
             rocky|centos|rhel|ol|virtuozzo)
                 MAJOR_VERSION="$(echo $VERSION_ID | cut -d. -f1)"
-                $SUDO dnf install -y dnf-utils selinux-policy-targeted
+                $SUDO dnf install -y $DNF_INSTALL_OPTIONS dnf-utils selinux-policy-targeted
                 rpm --quiet --query epel-release || \
-                    $SUDO dnf -y install --nogpgcheck https://dl.fedoraproject.org/pub/epel/epel-release-latest-$MAJOR_VERSION.noarch.rpm
+                    $SUDO dnf install -y $DNF_INSTALL_OPTIONS --nogpgcheck https://dl.fedoraproject.org/pub/epel/epel-release-latest-$MAJOR_VERSION.noarch.rpm
                 $SUDO rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-$MAJOR_VERSION
                 $SUDO rm -f /etc/yum.repos.d/dl.fedoraproject.org*
-                if test $ID = centos -a $MAJOR_VERSION = 8 ; then
-                    # Enable 'powertools' or 'PowerTools' repo
-                    $SUDO dnf config-manager --set-enabled $(dnf repolist --all 2>/dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}')
-                    dts_ver=11
-                    # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies
-                    $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
-                    $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
-                    $SUDO dnf -y module enable javapackages-tools
-                elif test $ID = rhel -a $MAJOR_VERSION = 8 ; then
-                    dts_ver=11
-                    $SUDO dnf config-manager --set-enabled "codeready-builder-for-rhel-8-${ARCH}-rpms"
-                    $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/8/
-                    $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_8_.gpgcheck=0 --save
-                    $SUDO dnf -y module enable javapackages-tools
-                fi
+                case "$ID-$MAJOR_VERSION" in
+                    centos-8)
+                        # Enable 'powertools' or 'PowerTools' repo
+                        $SUDO dnf config-manager --set-enabled $(dnf repolist --all 2>/dev/null|gawk 'tolower($0) ~ /^powertools\s/{print $1}')
+                        ;;&
+                    centos-9)
+                        # Enable 'crb' repo
+                        $SUDO dnf config-manager --set-enabled crb
+                        ;;&
+                    rhel-[89])
+                        $SUDO dnf config-manager --set-enabled "codeready-builder-for-rhel-${MAJOR_VERSION}-${ARCH}-rpms"
+                        ;;&
+                    centos-8|rhel-8)
+                        # before EPEL8 and PowerTools provide all dependencies, we use sepia for the dependencies
+                        $SUDO dnf config-manager --add-repo http://apt-mirror.front.sepia.ceph.com/lab-extras/${MAJOR_VERSION}/
+                        $SUDO dnf config-manager --setopt=apt-mirror.front.sepia.ceph.com_lab-extras_${MAJOR_VERSION}_.gpgcheck=0 --save
+                        $SUDO dnf -y module enable javapackages-tools
+                        dts_ver=11
+                        ;;
+                esac
                 ;;
         esac
         if [ "$INSTALL_EXTRA_PACKAGES" ]; then
-            $SUDO dnf install -y $INSTALL_EXTRA_PACKAGES
+            $SUDO dnf install -y $DNF_INSTALL_OPTIONS $INSTALL_EXTRA_PACKAGES
         fi
         munge_ceph_spec_in $with_seastar $with_zbd $for_make_check $DIR/ceph.spec
         # for python3_pkgversion macro defined by python-srpm-macros, which is required by python3-devel
-        $SUDO dnf install -y python3-devel
+        $SUDO dnf install -y $DNF_INSTALL_OPTIONS python3-devel
         $SUDO $builddepcmd $DIR/ceph.spec 2>&1 | tee $DIR/yum-builddep.out
         [ ${PIPESTATUS[0]} -ne 0 ] && exit 1
         if [ -n "$dts_ver" ]; then
@@ -525,7 +529,7 @@ else
         # for rgw motr backend build checks
         if ! rpm --quiet -q cortx-motr-devel &&
               { [[ $FOR_MAKE_CHECK ]] || $with_rgw_motr; }; then
-            $SUDO dnf install -y \
+            $SUDO dnf install -y $DNF_INSTALL_OPTIONS --nobest \
                   "$motr_pkgs_url/isa-l-2.30.0-1.el7.${ARCH}.rpm" \
                   "$motr_pkgs_url/cortx-motr-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm" \
                   "$motr_pkgs_url/cortx-motr-devel-2.0.0-1_git3252d623_any.el8.${ARCH}.rpm"
@@ -554,7 +558,6 @@ fi
 if $for_make_check; then
     mkdir -p install-deps-cache
     top_srcdir=$(pwd)
-    export XDG_CACHE_HOME=$top_srcdir/install-deps-cache
     wip_wheelhouse=wheelhouse-wip
     #
     # preload python modules so that tox can run without network access
@@ -563,7 +566,6 @@ if $for_make_check; then
         preload_wheels_for_tox $ini
     done
     rm -rf $top_srcdir/install-deps-python3
-    rm -rf $XDG_CACHE_HOME
     type git > /dev/null || (echo "Dashboard uses git to pull dependencies." ; false)
 fi
 
index 0ebe2b5de6bdf70f9c2d1e24cad4bb1db7373f82..7a2fac6258dad6931c52b42047119842cf80f08e 100755 (executable)
@@ -43,7 +43,7 @@ function run() {
     fi
 
     CHECK_MAKEOPTS=${CHECK_MAKEOPTS:-$DEFAULT_MAKEOPTS}
-    if in_jenkins; then
+    if in_ci; then
         if ! ctest $CHECK_MAKEOPTS --no-compress-output --output-on-failure --test-output-size-failed 1024000 -T Test; then
             # do not return failure, as the jenkins publisher will take care of this
             rm -fr ${TMPDIR:-/tmp}/ceph-asok.*
@@ -72,7 +72,7 @@ function main() {
     # uses run-make.sh to install-deps
     FOR_MAKE_CHECK=1 prepare
     configure "$@"
-    in_jenkins && echo "CI_DEBUG: Running 'build tests'"
+    in_ci && echo "CI_DEBUG: Running 'build tests'"
     build tests
     echo "make check: successful build on $(git rev-parse HEAD)"
     FOR_MAKE_CHECK=1 run
index 6c4455344379d373774ddbda1400dd16fb9eb0a2..ea4a865ff3ec93e262f448d7e33bb124d5c6cc13 100644 (file)
 # shellcheck disable=SC2034
 _SOURCED_LIB_BUILD=1
 
-function in_jenkins() {
-    [ -n "$JENKINS_HOME" ]
+function in_ci() {
+    [ -n "$JENKINS_HOME" -o -n "$CI" ]
 }
 
 function ci_debug() {
-    if in_jenkins || [ "${FORCE_CI_DEBUG}" ]; then
+    if in_ci || [ "${FORCE_CI_DEBUG}" ]; then
         echo "CI_DEBUG: $*"
     fi
 }
index 42d8a94aee1fc05c3d9cb6f55f92945fe46e3935..d017eee592271be60f9d72bfedde90bdabd21c08 100755 (executable)
@@ -48,6 +48,36 @@ function detect_ceph_dev_pkgs() {
     echo "$cmake_opts"
 }
 
+function get_os_id() {
+    local OS=$(uname -s)
+    local ID
+    if [[ -e /etc/os-release ]]; then
+        source /etc/os-release
+    elif [[ "$OS" == FreeBSD ]]; then
+        ID=freebsd
+    else
+        ID=unknown
+    fi
+
+    echo "$ID"
+}
+
+function install_packages() {
+    local packages="$1"
+
+    case "$(get_os_id)" in
+        fedora|rocky|centos|rhel|ol|virtuozzo)
+            $SUDO dnf install -y $DNF_INSTALL_OPTIONS $packages
+            ;;
+        debian|ubuntu|devuan|elementary|softiron)
+            $SUDO apt-get install -y $APT_INSTALL_OPTIONS $packages
+            ;;
+        opensuse*|suse|sles)
+            $SUDO zypper install -y $ZYPPER_INSTALL_OPTIONS $packages
+            ;;
+    esac
+}
+
 function prepare() {
     local which_pkg="which"
     if command -v apt-get > /dev/null 2>&1 ; then
@@ -56,7 +86,7 @@ function prepare() {
 
     if test -f ./install-deps.sh ; then
         ci_debug "Running install-deps.sh"
-        INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang"
+        export INSTALL_EXTRA_PACKAGES="ccache git $which_pkg clang curl"
         $DRY_RUN source ./install-deps.sh || return 1
         trap clean_up_after_myself EXIT
     fi
@@ -78,7 +108,7 @@ EOM
     $DRY_RUN export SOURCE_DATE_EPOCH="946684800"
     $DRY_RUN ccache -o sloppiness=time_macros
     $DRY_RUN ccache -o run_second_cpp=true
-    if in_jenkins; then
+    if in_ci; then
         # Build host has plenty of space available, let's use it to keep
         # various versions of the built objects. This could increase the cache hit
         # if the same or similar PRs are running several times