From: Vallari Agrawal <vallari.agrawal@ibm.com>
Date: Thu, 13 Feb 2025 07:15:55 +0000 (+0530)
Subject: qa/suites/nvmeof: add more asserts to scalability_test
X-Git-Tag: v20.0.0~66^2~2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=877c726ce7f821d595e4500c42bd898b2bc91bd4;p=ceph.git

qa/suites/nvmeof: add more asserts to scalability_test

Signed-off-by: Vallari Agrawal <vallari.agrawal@ibm.com>
---

diff --git a/qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml b/qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml
index c24ce631dfa..8a969f01ff2 100644
--- a/qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml
+++ b/qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml
@@ -36,6 +36,6 @@ tasks:
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c,nvmeof.d
         - nvmeof/scalability_test.sh nvmeof.b,nvmeof.c
     env:
-      SCALING_DELAYS: '120'
+      SCALING_DELAYS: '300'
       RBD_POOL: mypool
       NVMEOF_GROUP: mygroup0
diff --git a/qa/workunits/nvmeof/scalability_test.sh b/qa/workunits/nvmeof/scalability_test.sh
index 395d9b5d8d2..f95659cd5a9 100755
--- a/qa/workunits/nvmeof/scalability_test.sh
+++ b/qa/workunits/nvmeof/scalability_test.sh
@@ -16,10 +16,40 @@ status_checks() {
     expected_count=$1
 
     output=$(ceph nvme-gw show $POOL $GROUP) 
-    nvme_show=$(echo $output | grep -o '"AVAILABLE"' | wc -l)
-    if [ "$nvme_show" -ne "$expected_count" ]; then
-        return 1
+    # nvme_show=$(echo $output | grep -o '"AVAILABLE"' | wc -l)
+    # if [ "$nvme_show" -ne "$expected_count" ]; then
+    #    return 1
+    # fi
+    total_ns=$(echo $output | jq '.["num-namespaces"]')
+    total_gws=$(echo $output | jq '.["num gws"]')
+    expected_avg_ns=$((total_ns / total_gws))
+
+    if [ "$total_gws" -ne "$expected_count" ]; then
+       return 1
     fi
+    expected_ns_count=$(( $NVMEOF_NAMESPACES_COUNT * $NVMEOF_SUBSYSTEMS_COUNT )) 
+    if [ "$total_ns" -ne "$expected_ns_count" ]; then
+       return 1
+    fi
+
+    gateways=$(echo "$output" | jq -c '.["Created Gateways:"][]')
+
+    echo "$gateways" | while read -r gw; do
+        gw_id=$(echo "$gw" | jq -r '.["gw-id"]')
+        availability=$(echo "$gw" | jq -r '.["Availability"]')
+        num_namespaces=$(echo "$gw" | jq '.["num-namespaces"]')
+
+        if [[ "$availability" != "AVAILABLE" ]]; then
+            echo "Gateway $gw_id is not AVAILABLE."
+            exit 1
+        fi
+
+        diff=$((num_namespaces - expected_avg_ns))
+        if [[ $diff -lt -1 || $diff -gt 1 ]]; then
+            echo "Gateway $gw_id has num-namespaces ($num_namespaces), expected around $expected_ns. Indicates a problem in ns load-balancing."
+            exit 1
+        fi
+    done
 
     orch_ls=$(ceph orch ls)
     if ! echo "$orch_ls" | grep -q "$expected_count/$expected_count"; then