From 71c5e6a816a1eb2c69f70499dc5aadd9bf81fb37 Mon Sep 17 00:00:00 2001 From: Seena Fallah Date: Sun, 18 Feb 2024 03:41:41 +0100 Subject: [PATCH] container: cleanup container systemd units * Make common params of container args in a var to avoid duplication * The /var/lib/ceph/crash mount was missing after 637ca81c9cf801e4d1d125dc8a2492b90fd78eea * Add CEPH_USE_RANDOM_NONCE as it's needed when running inside container (can be removed for squid later) * Add NODE_NAME as some part of ceph code relies on this var * add default logging opts for Signed-off-by: Seena Fallah --- group_vars/all.yml.sample | 19 ++++++++++++- .../tasks/create_ceph_initial_dirs.yml | 1 + .../templates/ceph-crash.service.j2 | 9 ++++--- roles/ceph-defaults/defaults/main.yml | 17 +++++++++++- roles/ceph-facts/tasks/facts.yml | 2 +- roles/ceph-mds/templates/ceph-mds.service.j2 | 15 ++++++----- roles/ceph-mgr/templates/ceph-mgr.service.j2 | 19 ++++++------- roles/ceph-mon/templates/ceph-mon.service.j2 | 27 +++++++++---------- roles/ceph-osd/tasks/crush_rules.yml | 2 +- roles/ceph-osd/templates/systemd-run.j2 | 17 +++++++----- .../templates/ceph-radosgw.service.j2 | 13 ++++++--- 11 files changed, 94 insertions(+), 47 deletions(-) diff --git a/group_vars/all.yml.sample b/group_vars/all.yml.sample index 88002cbbb..166a2618c 100644 --- a/group_vars/all.yml.sample +++ b/group_vars/all.yml.sample @@ -534,7 +534,24 @@ dummy: #containerized_deployment: false #container_binary: #timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}" - +#ceph_common_container_params: +# envs: +# NODE_NAME: "{{ ansible_facts['hostname'] }}" +# CEPH_USE_RANDOM_NONCE: "1" +# CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" +# TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}" +# args: +# - --setuser=ceph +# - --setgroup=ceph +# - --default-log-to-file=false +# - --default-log-to-stderr=true +# - --default-log-stderr-prefix="debug " +# volumes: +# - /var/lib/ceph/crash:/var/lib/ceph/crash:z +# - /var/run/ceph:/var/run/ceph:z +# - /var/log/ceph:/var/log/ceph:z +# - /etc/ceph:/etc/ceph:z +# - /etc/localtime:/etc/localtime:ro # this is only here for usage with the rolling_update.yml playbook # do not ever change this here diff --git a/roles/ceph-config/tasks/create_ceph_initial_dirs.yml b/roles/ceph-config/tasks/create_ceph_initial_dirs.yml index a131ac215..64936e171 100644 --- a/roles/ceph-config/tasks/create_ceph_initial_dirs.yml +++ b/roles/ceph-config/tasks/create_ceph_initial_dirs.yml @@ -13,6 +13,7 @@ - /var/lib/ceph/osd - /var/lib/ceph/mds - /var/lib/ceph/tmp + - /var/lib/ceph/crash - /var/lib/ceph/radosgw - /var/lib/ceph/bootstrap-rgw - /var/lib/ceph/bootstrap-mgr diff --git a/roles/ceph-crash/templates/ceph-crash.service.j2 b/roles/ceph-crash/templates/ceph-crash.service.j2 index 1424eda83..bba5fe7aa 100644 --- a/roles/ceph-crash/templates/ceph-crash.service.j2 +++ b/roles/ceph-crash/templates/ceph-crash.service.j2 @@ -24,9 +24,12 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-crash-%i \ {% if cluster != 'ceph' %} -e CEPH_ARGS="--cluster {{ cluster }}" \ {% endif %} --v /var/lib/ceph/crash:/var/lib/ceph/crash:z \ --v /etc/localtime:/etc/localtime:ro \ --v /etc/ceph:/etc/ceph:z \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} --entrypoint=/usr/bin/ceph-crash {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" diff --git a/roles/ceph-defaults/defaults/main.yml b/roles/ceph-defaults/defaults/main.yml index 6f97e7e99..bd01a91ec 100644 --- a/roles/ceph-defaults/defaults/main.yml +++ b/roles/ceph-defaults/defaults/main.yml @@ -526,7 +526,22 @@ ceph_client_docker_registry: "{{ ceph_docker_registry }}" containerized_deployment: false container_binary: timeout_command: "{{ 'timeout --foreground -s KILL ' ~ docker_pull_timeout if (docker_pull_timeout != '0') and (ceph_docker_dev_image is undefined or not ceph_docker_dev_image) else '' }}" - +ceph_common_container_params: + envs: + NODE_NAME: "{{ ansible_facts['hostname'] }}" + CEPH_USE_RANDOM_NONCE: "1" + CONTAINER_IMAGE: "{{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }}" + TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES: "{{ ceph_tcmalloc_max_total_thread_cache }}" + args: + - --default-log-to-file=false + - --default-log-to-stderr=true + - --default-log-stderr-prefix="debug " + volumes: + - /var/lib/ceph/crash:/var/lib/ceph/crash:z + - /var/run/ceph:/var/run/ceph:z + - /var/log/ceph:/var/log/ceph:z + - /etc/ceph:/etc/ceph:z + - /etc/localtime:/etc/localtime:ro # this is only here for usage with the rolling_update.yml playbook # do not ever change this here diff --git a/roles/ceph-facts/tasks/facts.yml b/roles/ceph-facts/tasks/facts.yml index d65e49315..e176ecc11 100644 --- a/roles/ceph-facts/tasks/facts.yml +++ b/roles/ceph-facts/tasks/facts.yml @@ -197,7 +197,7 @@ block: - &read-osd-pool-default-crush-rule name: Read osd pool default crush rule - ansible.builtin.command: grep 'osd pool default crush rule' /etc/ceph/{{ cluster }}.conf + ansible.builtin.command: grep 'osd_pool_default_crush_rule' /etc/ceph/{{ cluster }}.conf register: crush_rule_variable changed_when: false check_mode: false diff --git a/roles/ceph-mds/templates/ceph-mds.service.j2 b/roles/ceph-mds/templates/ceph-mds.service.j2 index f57b8098f..9f56ec61b 100644 --- a/roles/ceph-mds/templates/ceph-mds.service.j2 +++ b/roles/ceph-mds/templates/ceph-mds.service.j2 @@ -30,16 +30,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ --cpus={{ cpu_limit }} \ -v /var/lib/ceph/bootstrap-mds:/var/lib/ceph/bootstrap-mds:z \ -v /var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:/var/lib/ceph/mds/{{ cluster }}-{{ ansible_facts['hostname'] }}:z \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/log/ceph:/var/log/ceph:z \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {{ ceph_mds_docker_extra_env }} \ --name=ceph-mds-{{ ansible_facts['hostname'] }} \ --entrypoint=/usr/bin/ceph-mds \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} -f -i {{ ansible_facts['hostname'] }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" diff --git a/roles/ceph-mgr/templates/ceph-mgr.service.j2 b/roles/ceph-mgr/templates/ceph-mgr.service.j2 index 419cdacd4..fe614b216 100644 --- a/roles/ceph-mgr/templates/ceph-mgr.service.j2 +++ b/roles/ceph-mgr/templates/ceph-mgr.service.j2 @@ -27,21 +27,22 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ --security-opt label=disable \ --memory={{ ceph_mgr_docker_memory_limit }} \ --cpus={{ ceph_mgr_docker_cpu_limit }} \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} -v /var/lib/ceph/mgr:/var/lib/ceph/mgr:z,rshared \ -v /var/lib/ceph/bootstrap-mgr:/var/lib/ceph/bootstrap-mgr:z \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/log/ceph:/var/log/ceph:z \ - -e CLUSTER={{ cluster }} \ - -e CEPH_DAEMON=MGR \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {{ ceph_mgr_docker_extra_env }} \ --name=ceph-mgr-{{ ansible_facts['hostname'] }} \ --entrypoint=/usr/bin/ceph-mgr \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -f --default-log-to-file=false --default-log-to-stderr=true \ +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} + -f \ -i {{ ansible_facts['hostname'] }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" diff --git a/roles/ceph-mon/templates/ceph-mon.service.j2 b/roles/ceph-mon/templates/ceph-mon.service.j2 index 1b7f850cf..7dc0e5cb7 100644 --- a/roles/ceph-mon/templates/ceph-mon.service.j2 +++ b/roles/ceph-mon/templates/ceph-mon.service.j2 @@ -28,11 +28,10 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \ --memory={{ ceph_mon_docker_memory_limit }} \ --cpus={{ ceph_mon_docker_cpu_limit }} \ --security-opt label=disable \ +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} -v /var/lib/ceph/mon:/var/lib/ceph/mon:z,rshared \ - -v /etc/ceph:/etc/ceph:z \ - -v /var/run/ceph:/var/run/ceph:z \ - -v /etc/localtime:/etc/localtime:ro \ - -v /var/log/ceph:/var/log/ceph:z \ {% if ansible_facts['os_family'] == 'RedHat' -%} -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \ {% endif -%} @@ -42,19 +41,19 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --name ceph-mon-%i \ {% if mon_docker_net_host | bool -%} --net=host \ {% endif -%} - -e IP_VERSION={{ ip_version[-1:] }} \ - -e MON_IP={{ _current_monitor_address }} \ - -e CLUSTER={{ cluster }} \ - -e FSID={{ fsid }} \ - -e MON_PORT={{ ceph_mon_container_listen_port }} \ - -e CEPH_PUBLIC_NETWORK={{ public_network | regex_replace(' ', '') }} \ - -e CONTAINER_IMAGE={{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {{ ceph_mon_docker_extra_env }} \ --entrypoint=/usr/bin/ceph-mon \ {{ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ - -f --default-log-to-file=false --default-log-to-stderr=true \ - -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name][0] }} +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} + -f \ + --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true \ + -i {{ monitor_name }} --mon-data /var/lib/ceph/mon/{{ cluster }}-{{ monitor_name }} \ + --public-addr {{ _current_monitor_address }} --mon-initial-members {{ groups[mon_group_name] | join(',') }} {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" {% else %} diff --git a/roles/ceph-osd/tasks/crush_rules.yml b/roles/ceph-osd/tasks/crush_rules.yml index 5f103f338..303326794 100644 --- a/roles/ceph-osd/tasks/crush_rules.yml +++ b/roles/ceph-osd/tasks/crush_rules.yml @@ -63,7 +63,7 @@ community.general.ini_file: dest: "/etc/ceph/{{ cluster }}.conf" section: "global" - option: "osd pool default crush rule" + option: "osd_pool_default_crush_rule" value: "{{ info_ceph_default_crush_rule_yaml.rule_id }}" mode: "0644" delegate_to: "{{ item }}" diff --git a/roles/ceph-osd/templates/systemd-run.j2 b/roles/ceph-osd/templates/systemd-run.j2 index 9d5ce3fee..73ba1b556 100644 --- a/roles/ceph-osd/templates/systemd-run.j2 +++ b/roles/ceph-osd/templates/systemd-run.j2 @@ -43,23 +43,26 @@ numactl \ {% if ceph_osd_docker_cpuset_mems is defined -%} --cpuset-mems='{{ ceph_osd_docker_cpuset_mems }}' \ {% endif -%} +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} -v /dev:/dev \ --v /etc/localtime:/etc/localtime:ro \ -v /var/lib/ceph/bootstrap-osd/ceph.keyring:/var/lib/ceph/bootstrap-osd/ceph.keyring:z \ -v /var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":/var/lib/ceph/osd/{{ cluster }}-"${OSD_ID}":z \ --v /etc/ceph:/etc/ceph:z \ --v /var/run/ceph:/var/run/ceph:z \ -v /var/run/udev/:/var/run/udev/ \ --v /var/log/ceph:/var/log/ceph:z \ {% if ansible_facts['distribution'] == 'Ubuntu' -%} --security-opt apparmor:unconfined \ {% endif -%} --e CLUSTER={{ cluster }} \ --e TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES={{ ceph_tcmalloc_max_total_thread_cache }} \ +{% for k, v in ceph_common_container_params['envs'].items() %} +-e {{ k }}={{ v }} \ +{% endfor %} -v /run/lvm/:/run/lvm/ \ -e OSD_ID=${OSD_ID} \ --name=ceph-osd-${OSD_ID} \ --entrypoint=/usr/bin/ceph-osd \ {{ ceph_osd_docker_extra_env }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ --f -i ${OSD_ID} \ No newline at end of file +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} +-f -i ${OSD_ID} diff --git a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 index c618437e5..add58124b 100644 --- a/roles/ceph-rgw/templates/ceph-radosgw.service.j2 +++ b/roles/ceph-rgw/templates/ceph-radosgw.service.j2 @@ -34,11 +34,13 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ {% if ceph_rgw_docker_cpuset_mems is defined -%} --cpuset-mems="{{ ceph_rgw_docker_cpuset_mems }}" \ {% endif -%} +{% for v in ceph_common_container_params['volumes'] %} + -v {{ v }} \ +{% endfor %} -v /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:/var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}:z \ - -v /etc/ceph:/etc/ceph \ - -v /var/run/ceph:/var/run/ceph \ - -v /etc/localtime:/etc/localtime \ - -v /var/log/ceph:/var/log/ceph \ +{% for k, v in ceph_common_container_params['envs'].items() %} + -e {{ k }}={{ v }} \ +{% endfor %} {% if ansible_facts['os_family'] == 'RedHat' -%} -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted \ {% endif -%} @@ -50,6 +52,9 @@ ExecStart=/usr/bin/{{ container_binary }} run --rm --net=host \ --entrypoint=/usr/bin/radosgw \ {{ ceph_rgw_docker_extra_env }} \ {{ ceph_docker_registry }}/{{ ceph_docker_image }}:{{ ceph_docker_image_tag }} \ +{% for arg in ceph_common_container_params['args'] %} + {{ arg }} \ +{% endfor %} -f -n client.rgw.{{ ansible_facts['hostname'] }}.${INST_NAME} -k /var/lib/ceph/radosgw/{{ cluster }}-rgw.{{ ansible_facts['hostname'] }}.${INST_NAME}/keyring {% if container_binary == 'podman' %} ExecStop=-/usr/bin/sh -c "/usr/bin/{{ container_binary }} rm -f `cat /%t/%n-cid`" -- 2.39.5