]> git.apps.os.sepia.ceph.com Git - ceph-ansible.git/commitdiff
rgw: add support for multiple rgw instances on a single host
authorguihecheng <guihecheng@cmiot.chinamobile.com>
Fri, 9 Nov 2018 00:56:57 +0000 (08:56 +0800)
committerGuillaume Abrioux <gabrioux@redhat.com>
Fri, 18 Jan 2019 10:12:28 +0000 (11:12 +0100)
With this, we could have multiple rgw instances on a single host
with a single run, don't have to use rgw-standalone.yml which does not
seems able to bind ports separately.
If you want to have multiple rgw instances, just change 'radosgw_instances'
to the number you want, which defaults to 1.
Not compatible with Multi-Site yet.

Signed-off-by: guihecheng <guihecheng@cmiot.chinamobile.com>
18 files changed:
group_vars/all.yml.sample
infrastructure-playbooks/purge-cluster.yml
infrastructure-playbooks/purge-docker-cluster.yml
roles/ceph-config/templates/ceph.conf.j2
roles/ceph-defaults/defaults/main.yml
roles/ceph-facts/tasks/facts.yml
roles/ceph-handler/templates/restart_rgw_daemon.sh.j2
roles/ceph-rgw/handlers/main.yml
roles/ceph-rgw/tasks/common.yml
roles/ceph-rgw/tasks/docker/start_docker_rgw.yml
roles/ceph-rgw/tasks/pre_requisite.yml
roles/ceph-rgw/tasks/start_radosgw.yml
roles/ceph-rgw/templates/ceph-radosgw.service.j2
tests/functional/centos/7/cluster/group_vars/all
tests/functional/centos/7/docker-collocation/group_vars/all
tests/functional/centos/7/docker/group_vars/all
tests/functional/tests/rgw/test_rgw.py
tests/functional/tests/rgw/test_rgw_tuning.py

index cc64812160309f96fafecb88a9d8630b1f676b8c..e69f2fda99671bfe1967ba1194a69305b3661f3c 100644 (file)
@@ -405,6 +405,7 @@ dummy:
 #radosgw_address: 0.0.0.0
 #radosgw_address_block: subnet
 #radosgw_keystone_ssl: false # activate this when using keystone PKI keys
+#radosgw_num_instances: 1
 # Rados Gateway options
 #email_address: foo@bar.com
 
index 2e918e3ee33c18c7e7654990526452bd1c5d2590..31b79235eaa73377e7ec4a7bd23edd785bf58c2d 100644 (file)
 
   - name: stop ceph rgws with systemd
     service:
-      name: ceph-radosgw@rgw.{{ ansible_hostname }}
+      name: ceph-radosgw@rgw.*
       state: stopped
       enabled: no
     failed_when: false
index aa7d22af8c0171b87a18f2818200958b27cea592..7ce366802560530bfebc06ed4996bfecfac48d2f 100644 (file)
 
   - name: disable ceph rgw service (new unit name)
     service:
-      name: "ceph-radosgw@rgw.{{ ansible_hostname }}"
+      name: "ceph-radosgw@*"
       state: stopped
       enabled: no
     ignore_errors: true
   - name: remove ceph rgw container
     docker_container:
       image: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}"
-      name: "ceph-rgw-{{ ansible_hostname }}"
+      name: "ceph-rgw-{{ ansible_hostname }}-*"
       state: absent
     ignore_errors: true
 
index 4ce937f073880184465449c03f639e404e79d14f..b73361b7585dc01f07fa77fb36958923f5d90ce7 100644 (file)
@@ -101,11 +101,15 @@ osd memory target = {{ _osd_memory_target | default(osd_memory_target) }}
 {% for host in groups[rgw_group_name] %}
 {% set _rgw_hostname = hostvars[host]['rgw_hostname'] | default(hostvars[host]['ansible_hostname']) %}
 {# {{ hostvars[host]['rgw_hostname'] }} for backward compatibility, fqdn issues. See bz1580408 #}
-[client.rgw.{{ _rgw_hostname }}]
+{% if hostvars[host]['rgw_instances'] is defined %}
+{% for instance in hostvars[host]['rgw_instances'] %}
+[client.rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}]
 host = {{ _rgw_hostname }}
-keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname }}/keyring
-log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] }}.log
-rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ hostvars[host]['_radosgw_address'] }}:{{ radosgw_frontend_port }} {{ radosgw_frontend_options }}
+keyring = /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ _rgw_hostname + '.' + instance['instance_name'] }}/keyring
+log file = /var/log/ceph/{{ cluster }}-rgw-{{ hostvars[host]['ansible_hostname'] + '.' + instance['instance_name'] }}.log
+rgw frontends = {{ radosgw_frontend_type }} {{ 'port' if radosgw_frontend_type == 'civetweb' else 'endpoint' }}={{ instance['radosgw_address'] }}:{{ instance['radosgw_frontend_port'] }} {{ radosgw_frontend_options }}
+{% endfor %}
+{% endif %}
 {% endfor %}
 {% endif %}
 
index 0de35990b0af58b89da7f1f866b73063d7245de1..7ff5994d18682b3e200c77d74b55decf12eef150 100644 (file)
@@ -397,6 +397,7 @@ radosgw_interface: interface
 radosgw_address: 0.0.0.0
 radosgw_address_block: subnet
 radosgw_keystone_ssl: false # activate this when using keystone PKI keys
+radosgw_num_instances: 1
 # Rados Gateway options
 email_address: foo@bar.com
 
index eb703ce5a86a5a6e3f068c5d89bf5899d55b2f35..3973797fc79011f3508216235f175338339e8580 100644 (file)
   import_tasks: set_radosgw_address.yml
   when:
     - inventory_hostname in groups.get(rgw_group_name, [])
+
+- name: set_fact rgw_instances
+  set_fact:
+    rgw_instances: "{{ rgw_instances|default([]) | union([{'instance_name': 'rgw' + item|string, 'radosgw_address': _radosgw_address, 'radosgw_frontend_port': radosgw_frontend_port|int + item|int}]) }}"
+  with_sequence: start=0 end={{ radosgw_num_instances|int - 1 }}
+  when:
+    - inventory_hostname in groups.get(rgw_group_name, [])
index 4d970ffa8cde1ab16bf896fbbe42d43bf15324f9..b6f58b08b24da54efadb1ce4cb575d64fb425b60 100644 (file)
@@ -2,20 +2,48 @@
 
 RETRIES="{{ handler_health_rgw_check_retries }}"
 DELAY="{{ handler_health_rgw_check_delay }}"
-RGW_NAME="{{ ansible_hostname }}"
-RGW_PORT="{{ radosgw_frontend_port }}"
+HOST_NAME="{{ ansible_hostname }}"
+RGW_NUMS={{ radosgw_num_instances }}
+RGW_BASE_PORT={{ radosgw_frontend_port }}
+declare -a DOCKER_EXECS
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  DOCKER_EXECS[i]=""
 {% if containerized_deployment %}
-DOCKER_EXEC="{{ container_binary }} exec ceph-rgw-{{ ansible_hostname }}"
+  CONTAINER_NAME="ceph-rgw-${HOST_NAME}-rgw${i}"
+  DOCKER_EXECS[i]="{{ container_binary }} exec ${CONTAINER_NAME}"
 {% endif %}
+done
+declare -a SOCKETS
 # Backward compatibility
-$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-client.rgw.{{ ansible_fqdn }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-client.rgw.{{ ansible_fqdn }}.asok
-$DOCKER_EXEC test -S /var/run/ceph/{{ cluster }}-client.rgw.{{ ansible_hostname }}.asok && SOCKET=/var/run/ceph/{{ cluster }}-client.rgw.{{ ansible_hostname }}.asok
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  SOCKET[i]="EMPTY_SOCKET"
+  ${DOCKER_EXECS[i]} test -S /var/run/ceph/{{ cluster }}-client.rgw.{{ ansible_fqdn }}.asok && SOCKETS[i]=/var/run/ceph/{{ cluster }}-client.rgw.{{ ansible_fqdn }}.asok
+  ${DOCKER_EXECS[i]} test -S /var/run/ceph/{{ cluster }}-client.rgw.${HOST_NAME}.rgw${i}.asok && SOCKETS[i]=/var/run/ceph/{{ cluster }}-client.rgw.${HOST_NAME}.rgw${i}.asok
+done
 RGW_IP={{ hostvars[inventory_hostname]['_radosgw_address'] }}
 
+check_socket() {
+  local i=$1
+  local succ=0
+  local count=10
+  # Wait and ensure the socket exists after restarting the daemon
+  while [ $count -ne 0 ]; do
+    ${DOCKER_EXECS[i]} test -S ${SOCKETS[i]} && succ=$((succ+1)) && break
+    sleep $DELAY
+    let count=count-1
+  done
+  if [ $succ -ne 1 ]; then
+    echo "Socket file ${SOCKETS[i]} could not be found, which means Rados Gateway is not running. Showing ceph-rgw unit logs now:"
+    journalctl -u ceph-radosgw@rgw.${HOST_NAME}.rgw${i}
+    exit 1
+  fi
+}
+
 check_for_curl_or_wget() {
-  if $DOCKER_EXEC command -v wget &>/dev/null; then
+  local i=$1
+  if ${DOCKER_EXECS[i]} command -v wget &>/dev/null; then
     rgw_test_command="wget --quiet"
-  elif $DOCKER_EXEC command -v curl &>/dev/null; then
+  elif ${DOCKER_EXECS[i]} command -v curl &>/dev/null; then
     rgw_test_command="curl --fail --silent --output /dev/null"
   else
     echo "It seems that neither curl or wget are available on your system."
@@ -25,27 +53,32 @@ check_for_curl_or_wget() {
 }
 
 check_rest() {
-  check_for_curl_or_wget
+  local i=$1
+  check_for_curl_or_wget ${i}
+  local succ=0
   while [ $RETRIES -ne 0 ]; do
-    test "$rgw_test_command http://$RGW_IP:$RGW_PORT" && exit 0
+    test "$rgw_test_command http://$RGW_IP:$((RGW_BASE_PORT+i))" && succ=$((succ+1)) && break
     sleep $DELAY
     let RETRIES=RETRIES-1
   done
-  # If we reach this point, it means there is a problem with the connection to rgw
-  echo "Error connecting locally to Rados Gateway service: http://$rgw_listen"
-  exit 1
+  if [ $succ -ne 1 ]; then
+    # If we reach this point, it means there is a problem with the connection to rgw
+    echo "Error connecting locally to Rados Gateway service: http://$rgw_listen"
+    exit 1
+  fi
 }
 
 # First, restart the daemon
-systemctl restart ceph-radosgw@rgw.${RGW_NAME}
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  systemctl restart ceph-radosgw@rgw.${HOST_NAME}.rgw${i}
+done
+
+# Check socket files
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  check_socket ${i}
+done
 
-COUNT=10
-# Wait and ensure the socket exists after restarting the daemon
-while [ $COUNT -ne 0 ]; do
-  $DOCKER_EXEC test -S $SOCKET && check_rest
-  sleep $DELAY
-  let COUNT=COUNT-1
+# Check rest
+for ((i=0; i<${RGW_NUMS}; i++)); do
+  check_rest ${i}
 done
-echo "Socket file ${SOCKET} could not be found, which means Rados Gateway is not running. Showing ceph-rgw unit logs now:"
-journalctl -u ceph-radosgw@rgw.${RGW_NAME}
-exit 1
index 129078767b31ae36d0a41710486c4b0de394047c..ce906fa95c07d01fc0f261862d7951604fb80310 100644 (file)
@@ -6,5 +6,5 @@
 
 - name: restart rgw
   service:
-    name: ceph-radosgw@rgw.{{ ansible_hostname }}
+    name: ceph-radosgw@*
     state: restarted
index 5c36ef19b09de655881cf4d760feaf67d5db43ed..268a0c0bb18363863fb148937f3404c00226c8d8 100644 (file)
@@ -7,9 +7,19 @@
     group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
     mode: "0755"
   with_items:
-    - /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}
     - "{{ rbd_client_admin_socket_path }}"
 
+- name: create rados gateway instance directories
+  file:
+    path: "/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}"
+    state: directory
+    owner: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    group: "{{ ceph_uid if containerized_deployment else 'ceph' }}"
+    mode: "0755"
+  with_items: "{{ rgw_instances }}"
+  when:
+    - rgw_instances is defined
+
 - name: copy ceph keyring(s) if needed
   copy:
     src: "{{ fetch_directory }}/{{ fsid }}/{{ item.name }}"
index 83c270d7549a18094568d9d20cdc201313d0e8ab..90db90925b000328875dd61741c3fcf886f71d7c 100644 (file)
@@ -1,4 +1,16 @@
 ---
+- name: generate environment file
+  become: true
+  copy:
+    dest: "/var/lib/ceph/radosgw/ceph-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/EnvironmentFile"
+    owner: "root"
+    group: "root"
+    mode: "0644"
+    content: |
+      INST_NAME={{ item.instance_name }}
+      INST_PORT={{ item.radosgw_frontend_port }}
+  with_items: "{{ rgw_instances }}"
+
 - name: generate systemd unit file
   become: true
   template:
@@ -26,7 +38,8 @@
 
 - name: systemd start rgw container
   systemd:
-    name: "ceph-radosgw@rgw.{{ ansible_hostname }}.service"
+    name: ceph-radosgw@rgw.{{ ansible_hostname }}.{{ item.instance_name }}
     state: started
     enabled: yes
-    daemon_reload: yes
\ No newline at end of file
+    daemon_reload: yes
+  with_items: "{{ rgw_instances }}"
index d736c8915cdb832f90400b3af2712f9c8160325c..6219d27990f7b1b2b92d7be9ab53c59a8d47eab3 100644 (file)
@@ -1,15 +1,19 @@
 ---
-- name: create rados gateway keyring
-  command: ceph --cluster {{ cluster }} --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring auth get-or-create client.rgw.{{ ansible_hostname }} osd 'allow rwx' mon 'allow rw' -o /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring
+- name: create rados gateway instance keyring
+  command: ceph --cluster {{ cluster }} --name client.bootstrap-rgw --keyring /var/lib/ceph/bootstrap-rgw/{{ cluster }}.keyring auth get-or-create client.rgw.{{ ansible_hostname }}.{{ item.instance_name }} osd 'allow rwx' mon 'allow rw' -o /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring
   args:
-    creates: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring
+    creates: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring
   changed_when: false
-  when: cephx
+  with_items: "{{ rgw_instances }}"
+  when:
+    - cephx
 
-- name: set rados gateway key permissions
+- name: set rados gateway instance key permissions
   file:
-    path: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}/keyring
+    path: /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_hostname }}.{{ item.instance_name }}/keyring
     owner: "ceph"
     group: "ceph"
     mode: "0600"
-  when: cephx
+  with_items: "{{ rgw_instances }}"
+  when:
+    - cephx
index ca6f2131622fcb5d54478f7f7da0683982d26d09..4bac51cc1cbe031803484d494e294241f9104c39 100644 (file)
   when:
     - ceph_rgw_systemd_overrides is defined
 
-- name: start rgw
+- name: start rgw instance
   service:
-    name: ceph-radosgw@rgw.{{ ansible_hostname }}
+    name: ceph-radosgw@rgw.{{ ansible_hostname }}.{{ item.instance_name }}
     state: started
     enabled: yes
+  with_items: "{{ rgw_instances }}"
 
 - name: enable the ceph-radosgw.target service
   systemd:
index 0d7944c1eeb0a771eb8334bc58f1c206314898c9..1e05a8cec9ed1f86879c4314ce26ce237ac51fe3 100644 (file)
@@ -3,9 +3,9 @@ Description=Ceph RGW
 After=docker.service
 
 [Service]
-EnvironmentFile=-/etc/environment
-ExecStartPre=-/usr/bin/{{ container_binary }} stop ceph-rgw-{{ ansible_hostname }}
-ExecStartPre=-/usr/bin/{{ container_binary }} rm ceph-rgw-{{ ansible_hostname }}
+EnvironmentFile=/var/lib/ceph/radosgw/ceph-%i/EnvironmentFile
+ExecStartPre=-/usr/bin/{{ container_binary }} stop ceph-rgw-{{ ansible_hostname }}-${INST_NAME}
+ExecStartPre=-/usr/bin/{{ container_binary }} rm ceph-rgw-{{ ansible_hostname }}-${INST_NAME}
 ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   --memory={{ ceph_rgw_docker_memory_limit }} \
   {% if (container_binary == 'docker' and ceph_docker_version.split('.')[0] is version_compare('13', '>=')) or container_binary == 'podman' -%}
@@ -19,10 +19,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \
   -v /etc/localtime:/etc/localtime:ro \
   -e CEPH_DAEMON=RGW \
   -e CLUSTER={{ cluster }} \
-  --name=ceph-rgw-{{ ansible_hostname }} \
+  -e RGW_NAME={{ ansible_hostname }}.${INST_NAME} \
+  -e RGW_CIVETWEB_PORT=${INST_PORT} \
+  --name=ceph-rgw-{{ ansible_hostname }}-${INST_NAME} \
   {{ ceph_rgw_docker_extra_env }} \
   {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}
-ExecStopPost=-/usr/bin/{{ container_binary }} stop ceph-rgw-{{ ansible_hostname }}
+ExecStopPost=-/usr/bin/{{ container_binary }} stop ceph-rgw-{{ ansible_hostname }}-${INST_NAME}
 Restart=always
 RestartSec=10s
 TimeoutStartSec=120
index e5342684ab2f7101916f57088748614765758c5f..940f08e523687293f533aaaf8b3b470273f2e872 100644 (file)
@@ -4,6 +4,7 @@ ceph_repository: community
 public_network: "192.168.1.0/24"
 cluster_network: "192.168.2.0/24"
 radosgw_interface: eth1
+radosgw_num_instances: 2
 ceph_conf_overrides:
   global:
     osd_pool_default_size: 1
index 71554cfd3bfe780eb99087f75418e791cd6e689d..fe3e7ab3efd0ce4ef9e53adad1e1192fd9abd567 100644 (file)
@@ -6,6 +6,7 @@ docker: True
 containerized_deployment: True
 monitor_interface: eth1
 radosgw_interface: eth1
+radosgw_num_instances: 2
 ceph_mon_docker_subnet: "{{ public_network }}"
 ceph_docker_on_openstack: False
 public_network: "192.168.15.0/24"
@@ -15,4 +16,4 @@ rgw_bucket_default_quota_max_objects: 1638400
 ceph_conf_overrides:
   global:
     osd_pool_default_pg_num: 8
-    osd_pool_default_size: 1
\ No newline at end of file
+    osd_pool_default_size: 1
index f9582c12743060fdcb4f7fc0a73b3af1c171df40..c9d554b094d9869daf08a57a57116d6cd1941e17 100644 (file)
@@ -6,6 +6,7 @@ docker: True
 containerized_deployment: True
 monitor_interface: eth1
 radosgw_interface: eth1
+radosgw_num_instances: 2
 ceph_mon_docker_subnet: "{{ public_network }}"
 ceph_docker_on_openstack: False
 public_network: "192.168.17.0/24"
index 98f5c51e97b40921f72e2d4740eaf2099ee3a220..d2b1482fc11e261202b7798f7579499cb78aaee7 100644 (file)
@@ -12,16 +12,20 @@ class TestRGWs(object):
         assert result
 
     def test_rgw_service_is_running(self, node, host):
-        service_name = "ceph-radosgw@rgw.{hostname}".format(
-            hostname=node["vars"]["inventory_hostname"]
-        )
-        assert host.service(service_name).is_running
+               for i in range(int(node["vars"]["radosgw_num_instances"])):
+                       service_name = "ceph-radosgw@rgw.{hostname}.rgw{seq}".format(
+                               hostname=node["vars"]["inventory_hostname"],
+                               seq=i
+                       )
+                       assert host.service(service_name).is_running
 
     def test_rgw_service_is_enabled(self, node, host):
-        service_name = "ceph-radosgw@rgw.{hostname}".format(
-            hostname=node["vars"]["inventory_hostname"]
-        )
-        assert host.service(service_name).is_enabled
+               for i in range(int(node["vars"]["radosgw_num_instances"])):
+                       service_name = "ceph-radosgw@rgw.{hostname}.rgw{seq}".format(
+                               hostname=node["vars"]["inventory_hostname"],
+                               seq=i
+                       )
+                       assert host.service(service_name).is_enabled
 
     def test_rgw_is_up(self, node, host):
         hostname = node["vars"]["inventory_hostname"]
@@ -30,7 +34,7 @@ class TestRGWs(object):
             container_binary = 'docker'
             if host.exists('podman') and host.ansible("setup")["ansible_facts"]["ansible_distribution"] == 'Fedora':  # noqa E501
                 container_binary = 'podman'
-            docker_exec_cmd = '{container_binary} exec ceph-rgw-{hostname}'.format(  # noqa E501
+            docker_exec_cmd = '{container_binary} exec ceph-rgw-{hostname}-rgw0'.format(  # noqa E501
                 hostname=hostname, container_binary=container_binary)
         else:
             docker_exec_cmd = ''
@@ -42,11 +46,17 @@ class TestRGWs(object):
         output = host.check_output(cmd)
         daemons = [i for i in json.loads(
             output)["servicemap"]["services"]["rgw"]["daemons"]]
-        assert hostname in daemons
+        for i in range(int(node["vars"]["radosgw_num_instances"])):
+            instance_name = "{hostname}.rgw{seq}".format(
+                hostname=hostname,
+                seq=i
+            )
+            assert instance_name in daemons
 
     @pytest.mark.no_docker
     def test_rgw_http_endpoint(self, node, host):
         # rgw frontends ip_addr is configured on eth1
         ip_addr = host.interface("eth1").addresses[0]
-        assert host.socket(
-            "tcp://{ip_addr}:{port}".format(ip_addr=ip_addr, port=8080)).is_listening  # noqa E501
+        for i in range(int(node["vars"]["radosgw_num_instances"])):
+                       assert host.socket(
+                               "tcp://{ip_addr}:{port}".format(ip_addr=ip_addr, port=(8080+i))).is_listening  # noqa E501
index 3dff9cea64016cba2f38f6fbe6400cb1879e8538..55617808494a4fff0b6de4e6e104d242bc2bc457 100644 (file)
@@ -13,7 +13,7 @@ class TestRGWs(object):
 
     @pytest.mark.no_docker
     def test_rgw_bucket_default_quota_is_applied(self, node, host):
-        radosgw_admin_cmd = "sudo radosgw-admin --cluster={cluster} -n client.rgw.{hostname} --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}/keyring user create --uid=test --display-name Test".format(  # noqa E501
+        radosgw_admin_cmd = "sudo radosgw-admin --cluster={cluster} -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring user create --uid=test --display-name Test".format(  # noqa E501
             hostname=node["vars"]["inventory_hostname"],
             cluster=node['cluster_name']
         )
@@ -24,7 +24,7 @@ class TestRGWs(object):
 
     @pytest.mark.no_docker
     def test_rgw_tuning_pools_are_set(self, node, host):
-        cmd = "sudo ceph --cluster={cluster} --connect-timeout 5 -n client.rgw.{hostname} --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}/keyring osd dump".format(  # noqa E501
+        cmd = "sudo ceph --cluster={cluster} --connect-timeout 5 -n client.rgw.{hostname}.rgw0 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring osd dump".format(  # noqa E501
             hostname=node["vars"]["inventory_hostname"],
             cluster=node['cluster_name']
         )
@@ -42,7 +42,7 @@ class TestRGWs(object):
         container_binary = 'docker'
         if host.exists('podman') and host.ansible("setup")["ansible_facts"]["ansible_distribution"] == 'Fedora':  # noqa E501
             container_binary = 'podman'
-        cmd = "sudo {container_binary} exec ceph-rgw-{hostname} ceph --cluster={cluster} -n client.rgw.{hostname} --connect-timeout 5 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}/keyring  osd dump".format(  # noqa E501
+        cmd = "sudo {container_binary} exec ceph-rgw-{hostname}-rgw0 ceph --cluster={cluster} -n client.rgw.{hostname}.rgw0 --connect-timeout 5 --keyring /var/lib/ceph/radosgw/{cluster}-rgw.{hostname}.rgw0/keyring  osd dump".format(  # noqa E501
             hostname=hostname,
             cluster=cluster,
             container_binary=container_binary