qa/suites/nvmeof: add mtls test

Add qa/workunits/nvmeof/mtls_test.sh which enables
mtls config and redeploy, then verify and disables
mtls config.

Signed-off-by: Vallari Agrawal <vallari.agrawal@ibm.com>

diff --git a/qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml b/qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml
new file mode 100644 (file)
index 0000000..8eb4f6d
--- /dev/null
+++ b/qa/suites/nvmeof/basic/workloads/nvmeof_mtls.yaml
@@ -0,0 +1,36 @@
+tasks:
+- nvmeof:
+    installer: host.a
+    gw_image: quay.io/ceph/nvmeof:latest # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 3
+      namespaces_count: 20
+      cli_image: quay.io/ceph/nvmeof-cli:latest
+      create_mtls_secrets: true
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
+        - nvmeof/fio_test.sh --rbd_iostat
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
+      IOSTAT_INTERVAL: '10'
+      RUNTIME: '60'
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.0:
+        - nvmeof/mtls_test.sh

diff --git a/qa/tasks/nvmeof.py b/qa/tasks/nvmeof.py
index b89f123c97e032e2c0de958ade63332fa166bce4..42e357294d9655cf02649da2fa95c1432c75318b 100644 (file)
--- a/qa/tasks/nvmeof.py
+++ b/qa/tasks/nvmeof.py
@@ -32,6 +32,7 @@ class Nvmeof(Task):
             gateway_config:
                 namespaces_count: 10
                 cli_version: latest
+                create_mtls_secrets: False 
                     
     """
 
@@ -69,6 +70,7 @@ class Nvmeof(Task):
         self.serial = gateway_config.get('serial', 'SPDK00000000000001')
         self.port = gateway_config.get('port', '4420')
         self.srport = gateway_config.get('srport', '5500')
+        self.create_mtls_secrets = gateway_config.get('create_mtls_secrets', False)
 
     def deploy_nvmeof(self):
         """
@@ -147,7 +149,38 @@ class Nvmeof(Task):
                 started=True,
             )
         log.info("[nvmeof]: executed deploy_nvmeof successfully!")
-        
+
+    def write_mtls_config(self, gateway_ips):
+        log.info("[nvmeof]: writing mtls config...")
+        allowed_ips = ""
+        for ip in gateway_ips:
+            allowed_ips += ("IP:" + ip + ",")
+        self.remote.run(
+            args=[
+                "sudo", "openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes", "-keyout", "/etc/ceph/server.key",
+                "-out", "/etc/ceph/server.crt", "-days", "3650", "-subj", "/CN=my.server", "-addext", f"subjectAltName={allowed_ips[:-1]}" 
+            ]
+        )
+        self.remote.run(
+            args=[
+                "sudo", "openssl", "req", "-x509", "-newkey", "rsa:4096", "-nodes", "-keyout", "/etc/ceph/client.key",
+                "-out", "/etc/ceph/client.crt", "-days", "3650", "-subj", "/CN=client1"
+            ]
+        )
+        secrets_files = {"/etc/ceph/server.key": None, 
+                 "/etc/ceph/server.crt": None, 
+                 "/etc/ceph/client.key": None, 
+                 "/etc/ceph/client.crt": None, 
+                }
+        for file in secrets_files.keys():
+            secrets_files[file] = self.remote.read_file(path=file, sudo=True)
+
+        for remote in self.ctx.cluster.remotes.keys():
+            for remote_file in secrets_files.keys():
+                data = secrets_files[remote_file]
+                remote.sudo_write_file(path=remote_file, data=data, mode='0644')
+        log.info("[nvmeof]: written mtls config!")
+
     def set_gateway_cfg(self):
         log.info('[nvmeof]: running set_gateway_cfg...')
         ip_address = self.remote.ip_address
@@ -174,6 +207,8 @@ class Nvmeof(Task):
                 data=conf_data,
                 sudo=True
             )
+        if self.create_mtls_secrets: 
+            self.write_mtls_config(gateway_ips)
         log.info("[nvmeof]: executed set_gateway_cfg successfully!")
 
 

diff --git a/qa/workunits/nvmeof/mtls_test.sh b/qa/workunits/nvmeof/mtls_test.sh
new file mode 100755 (executable)
index 0000000..e13ca53
--- /dev/null
+++ b/qa/workunits/nvmeof/mtls_test.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -ex
+source /etc/ceph/nvmeof.env
+
+# install yq
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /tmp/yq && chmod +x /tmp/yq
+
+subjectAltName=$(echo "$NVMEOF_GATEWAY_IP_ADDRESSES" | sed 's/,/,IP:/g')
+
+# create mtls spec files
+ceph orch ls nvmeof --export > /tmp/gw-conf-original.yaml
+sudo /tmp/yq ".spec.enable_auth=true | \
+    .spec.root_ca_cert=\"mountcert\" | \
+    .spec.client_cert = load_str(\"/etc/ceph/client.crt\") | \
+    .spec.client_key = load_str(\"/etc/ceph/client.key\") | \
+    .spec.server_cert = load_str(\"/etc/ceph/server.crt\") | \
+    .spec.server_key = load_str(\"/etc/ceph/server.key\")" /tmp/gw-conf-original.yaml > /tmp/gw-conf-with-mtls.yaml
+cp /tmp/gw-conf-original.yaml /tmp/gw-conf-without-mtls.yaml 
+sudo /tmp/yq '.spec.enable_auth=false' -i /tmp/gw-conf-without-mtls.yaml
+
+wait_for_service() {
+    MAX_RETRIES=30
+    for ((RETRY_COUNT=1; RETRY_COUNT<=MAX_RETRIES; RETRY_COUNT++)); do
+
+        if ceph orch ls --refresh | grep -q "nvmeof"; then
+            echo "Found nvmeof in the output!"
+            break
+        fi
+        if [ $RETRY_COUNT -eq $MAX_RETRIES ]; then
+            echo "Reached maximum retries ($MAX_RETRIES). Exiting."
+            break
+        fi
+        sleep 5
+    done
+    ceph orch ps
+    ceph orch ls --refresh
+}
+
+# deploy mtls
+cat /tmp/gw-conf-with-mtls.yaml 
+ceph orch apply -i /tmp/gw-conf-with-mtls.yaml
+ceph orch redeploy nvmeof.mypool.mygroup0 
+sleep 100
+wait_for_service
+
+
+# test
+IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES"
+for i in "${!gateway_ips[@]}"
+do
+    ip="${gateway_ips[i]}"
+    sudo podman run -v /etc/ceph/server.crt:/server.crt:z -v /etc/ceph/client.crt:/client.crt:z \
+        -v /etc/ceph/client.key:/client.key:z  \
+        -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT \
+        --client-key /client.key --client-cert /client.crt --server-cert /server.crt --format json subsystem list
+done
+
+
+# remove mtls
+cat /tmp/gw-conf-without-mtls.yaml 
+ceph orch apply -i /tmp/gw-conf-without-mtls.yaml
+ceph orch redeploy nvmeof.mypool.mygroup0 
+sleep 100
+wait_for_service
+
+
+# test
+IFS=',' read -ra gateway_ips <<< "$NVMEOF_GATEWAY_IP_ADDRESSES"
+for i in "${!gateway_ips[@]}"
+do
+    ip="${gateway_ips[i]}"
+    sudo podman run -it $NVMEOF_CLI_IMAGE --server-address $ip --server-port $NVMEOF_SRPORT \
+        --format json subsystem list
+done
+