From: Juan Miguel Olmo Martínez Date: Thu, 4 Oct 2018 13:54:42 +0000 (+0200) Subject: mgr/ansible: Ansible orchestrator module X-Git-Tag: v14.1.0~741^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=526cdcdf01e33ece1ea4c36f580b4edbce5bd0f2;p=ceph.git mgr/ansible: Ansible orchestrator module A Ceph Manager Orchestrator that uses a external REST API service to execute Ansible playbooks. get_inventory implementation Signed-off-by: Juan Miguel Olmo Martínez Document how to use CLI through Orchestrator CLI Signed-off-by: Juan Miguel Olmo Martínez --- diff --git a/doc/mgr/ansible.rst b/doc/mgr/ansible.rst new file mode 100644 index 00000000000..e92c22b6365 --- /dev/null +++ b/doc/mgr/ansible.rst @@ -0,0 +1,122 @@ + +.. _ansible-module: + +==================== +Ansible Orchestrator +==================== + +This module is a :ref:`Ceph orchestrator ` module that uses `Ansible Runner Service `_ (a RESTful API server) to execute Ansible playbooks in order to satisfy the different operations supported. + +These operations basically (and for the moment) are: + +- Get an inventory of the Ceph cluster nodes and all the storage devices present in each node +- ... +- ... + + +Usage +===== + +Enable the module: + +:: + + # ceph mgr module enable ansible + +Disable the module + +:: + + # ceph mgr module disable ansible + + +Enable the Ansible orchestrator module and use it with the :ref:`CLI `: + +:: + + ceph mgr module enable orchestrator_cli + ceph mgr module enable ansible + ceph orchestrator set backend ansible + + +Configuration +============= + +Configuration must be set once the module is enabled by first time. + +This can be done in one monitor node via the configuration key facility on a +cluster-wide level (so they apply to all manager instances) as follows:: + + + # ceph config set mgr mgr/ansible/server_addr + # ceph config set mgr mgr/ansible/server_port + # ceph config set mgr mgr/ansible/username + # ceph config set mgr mgr/ansible/password + # ceph config set mgr mgr/ansible/verify_server + +Where: + + * : Is the ip address/hostname of the server where the Ansible Runner Service is available. + * : The port number where the Ansible Runner Service is listening + * : The username of one authorized user in the Ansible Runner Service + * : The password of the authorized user. + * : Either a boolean, in which case it controls whether the server's TLS certificate is verified, or a string, in which case it must be a path to a CA bundle to use in the verification. Defaults to ``True``. + + +Debugging +========= + +Any kind of incident with this orchestrator module can be debugged using the Ceph manager logs: + +Set the right log level in order to debug properly. Remember that the python log levels debug, info, warn, err are mapped into the Ceph severities 20, 4, 1 and 0 respectively. + +And use the "active" manager node: ( "ceph -s" command in one monitor give you this information) + +* Check current debug level:: + + [@mgr0 ~]# ceph daemon mgr.mgr0 config show | grep debug_mgr + "debug_mgr": "1/5", + "debug_mgrc": "1/5", + +* Change the log level to "debug":: + + [mgr0 ~]# ceph daemon mgr.mgr0 config set debug_mgr 20/5 + { + "success": "" + } + +* Restore "info" log level:: + + [mgr0 ~]# ceph daemon mgr.mgr0 config set debug_mgr 1/5 + { + "success": "" + } + + +Operations +========== + +**Inventory:** + +Get the list of storage devices installed in all the cluster nodes. The output format is:: + + [host: + device_name (type_of_device , size_in_bytes)] + +Example:: + + [root@mon0 ~]# ceph orchestrator device ls + 192.168.121.160: + vda (hdd, 44023414784b) + sda (hdd, 53687091200b) + sdb (hdd, 53687091200b) + sdc (hdd, 53687091200b) + 192.168.121.36: + vda (hdd, 44023414784b) + 192.168.121.201: + vda (hdd, 44023414784b) + 192.168.121.70: + vda (hdd, 44023414784b) + sda (hdd, 53687091200b) + sdb (hdd, 53687091200b) + sdc (hdd, 53687091200b) diff --git a/doc/mgr/index.rst b/doc/mgr/index.rst index 0b40cb89809..337127c05fc 100644 --- a/doc/mgr/index.rst +++ b/doc/mgr/index.rst @@ -44,3 +44,4 @@ sensible. Rook plugin DeepSea plugin Insights plugin + Ansible plugin diff --git a/src/pybind/mgr/CMakeLists.txt b/src/pybind/mgr/CMakeLists.txt index 916c672b9ba..f66bba7e4be 100644 --- a/src/pybind/mgr/CMakeLists.txt +++ b/src/pybind/mgr/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(dashboard) add_subdirectory(insights) +add_subdirectory(ansible) diff --git a/src/pybind/mgr/ansible/CMakeLists.txt b/src/pybind/mgr/ansible/CMakeLists.txt new file mode 100644 index 00000000000..c706aa8cbe5 --- /dev/null +++ b/src/pybind/mgr/ansible/CMakeLists.txt @@ -0,0 +1,7 @@ +set(MGR_ANSIBLE_VIRTUALENV ${CEPH_BUILD_VIRTUALENV}/mgr-ansible-virtualenv) + +add_custom_target(mgr-ansible-test-venv + COMMAND ${CMAKE_SOURCE_DIR}/src/tools/setup-virtualenv.sh --python=${MGR_PYTHON_EXECUTABLE} ${MGR_ANSIBLE_VIRTUALENV} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/src/pybind/mgr/ansible + COMMENT "ansible tests virtualenv is being created") +add_dependencies(tests mgr-ansible-test-venv) diff --git a/src/pybind/mgr/ansible/__init__.py b/src/pybind/mgr/ansible/__init__.py new file mode 100644 index 00000000000..ea61a12fd7e --- /dev/null +++ b/src/pybind/mgr/ansible/__init__.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import +import os + +if 'UNITTEST' not in os.environ: + from .module import Module +else: + import sys + import mock + sys.modules['ceph_module'] = mock.Mock() diff --git a/src/pybind/mgr/ansible/ansible_runner_svc.py b/src/pybind/mgr/ansible/ansible_runner_svc.py new file mode 100644 index 00000000000..2989c89ce05 --- /dev/null +++ b/src/pybind/mgr/ansible/ansible_runner_svc.py @@ -0,0 +1,274 @@ +""" +Tool module to interact with the Ansible Runner Service +""" +import requests +import json +import re + +# Ansible Runner events finished + + +# Ansible Runner service API endpoints +API_URL = "api" +LOGIN_URL = "api/v1/login" +PLAYBOOK_EXEC_URL = "api/v1/playbooks" +PLAYBOOK_EVENTS = "api/v1/jobs/%s/events" +EVENT_DATA_URL = "api/v1/jobs/%s/events/%s" + +class ExecutionStatusCode(object): + """Execution status of playbooks ( 'msg' field in playbook status request) + """ + + SUCCESS = 0 # Playbook has been executed succesfully" msg = successful + ERROR = 1 # Playbook has finished with error msg = failed + ON_GOING = 2 # Playbook is being executed msg = running + NOT_LAUNCHED = 3 # Not initialized + + +class PlayBookExecution(object): + """Object to provide all the results of a Playbook execution + """ + + def __init__(self, rest_client, playbook, logger, result_pattern="", the_params={}): + + self.rest_client = rest_client + + # Identifier of the playbook execution + self.play_uuid = "-" + + # Pattern used to extract the result from the events + self.result_task_pattern = result_pattern + + # Playbook name + self.playbook = playbook + + # Params used in the playbook + self.params = the_params + + # Logger + self.log = logger + + def launch(self): + """ Launch the playbook execution + """ + + endpoint = "%s/%s" % (PLAYBOOK_EXEC_URL, self.playbook) + + response = self.rest_client.http_post(endpoint, self.params) + + if response: + self.play_uuid = json.loads(response.text)["data"]["play_uuid"] + self.log.info("Playbook execution launched succesfuly") + else: + # An error launching the execution implies play_uuid empty + self.play_uuid = "" + self.log.error("Playbook launch error. \ + Check request result") + + def get_status(self): + """ Return the status of the execution + + In the msg field of the respons we can find: + "msg": "successful" + "msg": "running" + "msg": "failed" + """ + + status_value = ExecutionStatusCode.NOT_LAUNCHED + + if self.play_uuid == '-': # Initialized + status_value = ExecutionStatusCode.NOT_LAUNCHED + elif self.play_uuid == '': # Error launching playbook + status_value = ExecutionStatusCode.ERROR + else: + endpoint = "%s/%s" % (PLAYBOOK_EXEC_URL, self.play_uuid) + response = self.rest_client.http_get(endpoint) + + if response: + the_status = json.loads(response.text)["msg"] + if the_status == 'successful': + status_value = ExecutionStatusCode.SUCCESS + elif the_status == 'failed': + status_value = ExecutionStatusCode.ERROR + else: + status_value = ExecutionStatusCode.ON_GOING + else: + status_value = ExecutionStatusCode.ERROR + + self.log.info("Requested playbook execution status is: %s", status_value) + return status_value + + def get_result(self, event_filter=""): + """Get the data of the events filtered by a task pattern and + a event filter + + @returns: the events that matches with the patterns provided + """ + + if not self.result_task_pattern or not self.play_uuid: + result_events = {} + + response = self.rest_client.http_get(PLAYBOOK_EVENTS % self.play_uuid) + + if not response: + result_events = {} + else: + events = json.loads(response.text)["data"]["events"] + result_events = {event:data for event,data in events.items() + if "task" in data and + re.match(self.result_task_pattern, data["task"])} + if event_filter: + result_events = {event:data for event,data in result_events.items() + if re.match(event_filter, data['event'])} + + self.log.info("Requested playbook result is: %s", json.dumps(result_events)) + return result_events + +class Client(object): + """An utility object that allows to connect with the Ansible runner service + and execute easily playbooks + """ + + def __init__(self, server_url, user, password, verify_server, logger): + """Provide an https client to make easy interact with the Ansible + Runner Service" + + @param servers_url: The base URL >server>: of the Ansible Runner Service + @param user: User name of the authorized user + @param password: Password of the authotized user + @param verify_server: Either a boolean, in which case it controls whether we verify + the server's TLS certificate, or a string, in which case it must be a path + to a CA bundle to use. Defaults to ``True``. + @param logger: Log file + """ + self.server_url = server_url + self.user = user + self.password = password + self.log = logger + self.auth = (self.user, self.password) + if not verify_server: + self.verify_server = True + elif verify_server.lower().strip() == 'false': + self.verify_server = False + else: + self.verify_server = verify_server + + # Once authenticated this token will be used in all the requests + self.token = "" + + self.server_url = "https://{0}".format(self.server_url) + + # Log in the server and get a token + self.login() + + def login(self): + """ Login with user credentials to obtain a valid token + """ + + response = None + + the_url = "%s/%s" % (self.server_url, LOGIN_URL) + response = requests.get(the_url, + auth = self.auth, + verify = self.verify_server) + + if response.status_code != requests.codes.ok: + self.log.error("login error <<%s>> (%s):%s", + the_url, response.status_code, response.text) + else: + self.log.info("login succesful <<%s>> (%s):%s", + the_url, response.status_code, response.text) + + if response: + self.token = json.loads(response.text)["data"]["token"] + self.log.info("Connection with Ansible Runner Service is operative") + + + def is_operative(self): + """Indicates if the connection with the Ansible runner Server is ok + """ + + # No Token... this means we haven't used yet the service. + if not self.token: + return False + + # Check the service + response = self.http_get(API_URL) + + if response: + return response.status_code == requests.codes.ok + else: + return False + + def http_get(self, endpoint): + """Execute an http get request + + @param endpoint: Ansible Runner service RESTful API endpoint + + @returns: A requests object + """ + + response = None + + try: + the_url = "%s/%s" % (self.server_url, endpoint) + r = requests.get(the_url, + verify = self.verify_server, + headers = {"Authorization": self.token}) + + if r.status_code != requests.codes.ok: + self.log.error("http GET %s <--> (%s - %s)\n%s", + the_url, r.status_code, r.reason, r.text) + else: + self.log.info("http GET %s <--> (%s - %s)", + the_url, r.status_code, r.text) + + response = r + + except Exception: + self.log.exception("Ansible runner service(GET %s)", the_url) + + return response + + def http_post(self, endpoint, payload): + """Execute an http post request + + @param endpoint: Ansible Runner service RESTful API endpoint + @param payload: Dictionary with the data used in the post request + + @returns: A requests object + """ + + response = None + + try: + the_url = "%s/%s" % (self.server_url, endpoint) + r = requests.post(the_url, + verify = self.verify_server, + headers = {"Authorization": self.token, + "Content-type": "application/json"}, + data = payload) + + if r.status_code != requests.codes.ok: + self.log.error("http POST %s [%s] <--> (%s - %s)\n%s", + the_url, payload, r.status_code, r.reason, r.text) + else: + self.log.info("http POST %s <--> (%s - %s)", + the_url, r.status_code, r.text) + response = r + + except Exception: + self.log.exception("Ansible runner service(POST %s)", the_url) + + return response + + def http_put(self, endpoint, payload): + """Execute an http put request + + @param endpoint: Ansible Runner service RESTful API endpoint + @param payload: Dictionary with the data used in the put request + + @returns: A requests object + """ + # TODO + raise NotImplementedError("TODO") diff --git a/src/pybind/mgr/ansible/module.py b/src/pybind/mgr/ansible/module.py new file mode 100644 index 00000000000..862e53aebf2 --- /dev/null +++ b/src/pybind/mgr/ansible/module.py @@ -0,0 +1,382 @@ +""" +ceph-mgr Ansible orchestrator module + +The external Orchestrator is the Ansible runner service (RESTful https service) +""" + +# Python stuff +from threading import Event +import errno +import json + +# Ceph stuff +from mgr_module import MgrModule +import orchestrator + +# Orchestrator stuff +# A Client is used to communicate with the Ansible Runner service +from ansible_runner_svc import Client, PlayBookExecution, ExecutionStatusCode,\ + EVENT_DATA_URL + +# Constants section + +# Time to clean the completions list +WAIT_PERIOD = 10 + + +# List of playbooks names used + +# Name of the playbook used in the "get_inventory" method. +# This playbook is expected to provide a list of storage devices in the host +# where the playbook is executed. +GET_STORAGE_DEVICES_CATALOG_PLAYBOOK = "host-disks.yml" + + + +class AnsibleReadOperation(orchestrator.ReadCompletion): + """ A read operation means to obtain information from the cluster. + """ + + def __init__(self, client, playbook, logger, result_pattern, params): + super(AnsibleReadOperation, self).__init__() + + + # Private attributes + self.playbook = playbook + self._is_complete = False + self._is_errored = False + self._result = [] + + # Error description in operation + self.error = "" + + # Ansible Runner Service client + self.ar_client = client + + # Logger + self.log = logger + + # An aditional filter of result events based in the event + self.event_filter = "" + + # Function assigned dinamically to process the result + self.process_output = None + + # Playbook execution object + self.pb_execution = PlayBookExecution(client, + playbook, + logger, + result_pattern, + params) + + @property + def is_complete(self): + return self._is_complete + + @property + def is_errored(self): + return self._is_errored + + @property + def result(self): + return self._result + + @property + def status(self): + """Return the status code of the operation + updating conceptually 'linked' attributes + """ + current_status = self.pb_execution.get_status() + + self._is_complete = (current_status == ExecutionStatusCode.SUCCESS) or \ + (current_status == ExecutionStatusCode.ERROR) + + self._is_errored = (current_status == ExecutionStatusCode.ERROR) + + return current_status + + def execute_playbook(self): + """Execute the playbook with the provided params. + """ + + self.pb_execution.launch() + + def update_result(self): + """Output of the read operation + + The result of the playbook execution can be customized through the + function provided as 'process_output' attribute + + @return string: Result of the operation formatted if it is possible + """ + + processed_result = [] + + if self._is_complete: + raw_result = self.pb_execution.get_result(self.event_filter) + + if self.process_output: + processed_result = self.process_output( + raw_result, + self.ar_client, + self.pb_execution.play_uuid) + else: + processed_result = raw_result + + self._result = processed_result + + +class AnsibleChangeOperation(orchestrator.WriteCompletion): + """Operations that changes the "cluster" state + + Modifications/Changes (writes) are a two-phase thing, firstly execute + the playbook that is going to change elements in the Ceph Cluster. + When the playbook finishes execution (independently of the result), + the modification/change operation has finished. + """ + def __init__(self): + super(AnsibleChangeOperation, self).__init__() + + self.error = False + @property + def status(self): + """Return the status code of the operation + """ + #TODO + return 0 + + @property + def is_persistent(self): + """ + Has the operation updated the orchestrator's configuration + persistently? Typically this would indicate that an update + had been written to a manifest, but that the update + had not necessarily been pushed out to the cluster. + + In the case of Ansible is always False. + because a initiated playbook execution will need always to be + relaunched if it fails. + """ + + return False + + @property + def is_effective(self): + """Has the operation taken effect on the cluster? + For example, if we were adding a service, has it come up and appeared + in Ceph's cluster maps? + + In the case of Ansible, this will be True if the playbooks has been + executed succesfully. + + @return Boolean: if the playbook has been executed succesfully + """ + + return self.status == ExecutionStatusCode.SUCCESS + + @property + def is_errored(self): + return self.error + + @property + def is_complete(self): + return self.is_errored or (self.is_persistent and self.is_effective) + + +class Module(MgrModule, orchestrator.Orchestrator): + """An Orchestrator that an external Ansible runner service to perform + operations + """ + + OPTIONS = [ + {'name': 'server_url'}, + {'name': 'username'}, + {'name': 'password'}, + {'name': 'verify_server'} # Check server identity (Boolean/path to CA bundle) + + ] + + def __init__(self, *args, **kwargs): + """ + """ + super(Module, self).__init__(*args, **kwargs) + + self.run = False + + self.all_completions = [] + + self.ar_client = None + + def available(self): + """ Check if Ansible Runner service is working + """ + # TODO + return (True, "Everything ready") + + def wait(self, completions): + """Given a list of Completion instances, progress any which are + incomplete. + + @param completions: list of Completion instances + @Returns : List with completions operations pending + """ + + # Check progress and update status in each operation + for operation in completions: + self.log.info("playbook <%s> status:%s", operation.playbook, operation.status) + if operation.is_complete: + operation.update_result() + + completions = filter(lambda x: not x.is_complete, completions) + + self.log.info("Operations pending: %s", len(completions)) + + return completions + + def serve(self): + """ Mandatory for standby modules + """ + self.log.info("Starting Ansible Orchestrator module ...") + + # Verify config options (Just that settings are available) + self.verify_config() + + # Ansible runner service client + try: + self.ar_client = Client(server_url = self.get_config('server_url', ''), + user = self.get_config('username', ''), + password = self.get_config('password', ''), + verify_server = self.get_config('verify_server', True), + logger = self.log) + except Exception: + self.log.exception("Ansible Runner Service not available. " + "Check external server status/TLS identity or " + "connection options. If configuration options changed" + " try to disable/enable the module.") + self.shutdown() + return + + self.run = True + + def shutdown(self): + self.log.info('Stopping Ansible orchestrator module') + self.run = False + + def get_inventory(self, node_filter=None): + """ + + @param : node_filter instance + @Return : A AnsibleReadOperation instance (Completion Object) + """ + + # Create a new read completion object for execute the playbook + ansible_operation = AnsibleReadOperation(client = self.ar_client, + playbook = GET_STORAGE_DEVICES_CATALOG_PLAYBOOK, + logger = self.log, + result_pattern = "RESULTS", + params = "{}") + + # Assign the process_output function + ansible_operation.process_output = process_inventory_json + ansible_operation.event_filter = "runner_on_ok" + + # Execute the playbook to obtain data + ansible_operation.execute_playbook() + + self.all_completions.append(ansible_operation) + + return ansible_operation + + def create_osds(self, osd_spec): + """ + Create one or more OSDs within a single Drive Group. + + The principal argument here is the drive_group member + of OsdSpec: other fields are advisory/extensible for any + finer-grained OSD feature enablement (choice of backing store, + compression/encryption, etc). + + :param osd_spec: OsdCreationSpec + """ + + def verify_config(self): + + if not self.get_config('server_url', ''): + self.log.error( + "No Ansible Runner Service base URL :" + "Try 'ceph config set mgr mgr/%s/server_url :'", + self.module_name) + + if not self.get_config('username', ''): + self.log.error( + "No Ansible Runner Service user. " + "Try 'ceph config set mgr mgr/%s/username '", + self.module_name) + + if not self.get_config('password', ''): + self.log.error( + "No Ansible Runner Service User password. " + "Try 'ceph config set mgr mgr/%s/password '", + self.module_name) + + if not self.get_config('verify_server', ''): + self.log.error( + "TLS server identity verification is enabled by default." + "Use 'ceph config set mgr mgr/{0}/verify_server False' to disable it." + "Use 'ceph config set mgr mgr/{0}/verify_server ' to " + "point the CA bundle path used for verification".format(self.module_name)) + + +# Auxiliary functions +#============================================================================== + +def process_inventory_json(inventory_events, ar_client, playbook_uuid): + """ Adapt the output of the playbook used in 'get_inventory' + to the Orchestrator expected output (list of InventoryNode) + + @param inventory_events: events dict with the results + + Example: + inventory_events = + {'37-100564f1-9fed-48c2-bd62-4ae8636dfcdb': {'host': '192.168.121.254', + 'task': 'RESULTS', + 'event': 'runner_on_ok'}, + '36-2016b900-e38f-7dcd-a2e7-00000000000e': {'host': '192.168.121.252' + 'task': 'RESULTS', + 'event': 'runner_on_ok'}} + @param ar_client: Ansible Runner Service client + @param playbook_uuid: Palybooud identifier + + @return : list of InventoryNode + """ + + #Obtain the needed data for each result event + inventory_nodes = [] + + # Loop over the result events and request the event data + for event_key, data in inventory_events.items(): + event_response = ar_client.http_get(EVENT_DATA_URL % (playbook_uuid, + event_key)) + + # Process the data for each event + if event_response: + event_data = json.loads(event_response.text)["data"]["event_data"] + + free_disks = event_data["res"]["disks_catalog"] + for item, data in free_disks.items(): + if item not in [host.name for host in inventory_nodes]: + + devs = [] + for dev_key, dev_data in data.items(): + if dev_key not in [device.id for device in devs]: + dev = orchestrator.InventoryDevice() + dev.id = dev_key + dev.type = 'hdd' if dev_data["rotational"] else "sdd/nvme" + dev.size = dev_data["sectorsize"] * dev_data["sectors"] + devs.append(dev) + + inventory_nodes.append( + orchestrator.InventoryNode(item, devs)) + + + return inventory_nodes diff --git a/src/pybind/mgr/ansible/run-tox.sh b/src/pybind/mgr/ansible/run-tox.sh new file mode 100644 index 00000000000..951ea23150e --- /dev/null +++ b/src/pybind/mgr/ansible/run-tox.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# run from ./ or from ../ +: ${MGR_ANSIBLE_VIRTUALENV:=/tmp/mgr-ansible-virtualenv} +: ${WITH_PYTHON2:=ON} +: ${WITH_PYTHON3:=ON} +: ${CEPH_BUILD_DIR:=$PWD/.tox} +test -d ansible && cd ansible + +if [ -e tox.ini ]; then + TOX_PATH=$(readlink -f tox.ini) +else + TOX_PATH=$(readlink -f $(dirname $0)/tox.ini) +fi + +# tox.ini will take care of this. +unset PYTHONPATH +export CEPH_BUILD_DIR=$CEPH_BUILD_DIR + +source ${MGR_ANSIBLE_VIRTUALENV}/bin/activate + +if [ "$WITH_PYTHON2" = "ON" ]; then + ENV_LIST+="py27" +fi +if [ "$WITH_PYTHON3" = "ON" ]; then + ENV_LIST+="py3" +fi + +tox -c ${TOX_PATH} -e ${ENV_LIST} diff --git a/src/pybind/mgr/ansible/tests/__init__.py b/src/pybind/mgr/ansible/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/mgr/ansible/tests/pb_execution_events.data b/src/pybind/mgr/ansible/tests/pb_execution_events.data new file mode 100644 index 00000000000..248134a3961 --- /dev/null +++ b/src/pybind/mgr/ansible/tests/pb_execution_events.data @@ -0,0 +1,183 @@ +{ + "status": "OK", + "msg": "", + "data": { + "events": { + "2-6edf768f-2923-44e1-b884-f0227b811cfc": { + "event": "playbook_on_start" + }, + "3-2016b900-e38f-7dcd-a2e7-000000000008": { + "event": "playbook_on_play_start" + }, + "4-2016b900-e38f-7dcd-a2e7-000000000012": { + "event": "playbook_on_task_start", + "task": "Gathering Facts" + }, + "5-19ae1e5e-aa2d-479e-845a-ef0253cc1f99": { + "event": "runner_on_ok", + "host": "192.168.121.245", + "task": "Gathering Facts" + }, + "6-aad3acc4-06a3-4c97-82ff-31e9e484b1f5": { + "event": "runner_on_ok", + "host": "192.168.121.61", + "task": "Gathering Facts" + }, + "7-55298017-3e7d-4734-b316-bbe13ce1da5e": { + "event": "runner_on_ok", + "host": "192.168.121.254", + "task": "Gathering Facts" + }, + "8-2016b900-e38f-7dcd-a2e7-00000000000a": { + "event": "playbook_on_task_start", + "task": "setup" + }, + "9-2085ccb6-e337-4b9f-bc38-1d8bbf9b973f": { + "event": "runner_on_ok", + "host": "192.168.121.254", + "task": "setup" + }, + "10-e14cdbbc-4883-436c-a41c-a8194ec69075": { + "event": "runner_on_ok", + "host": "192.168.121.245", + "task": "setup" + }, + "11-6d815a26-df53-4240-b8b6-2484e88e4f48": { + "event": "runner_on_ok", + "host": "192.168.121.61", + "task": "setup" + }, + "12-2016b900-e38f-7dcd-a2e7-00000000000b": { + "event": "playbook_on_task_start", + "task": "Get a list of block devices (excludes loop and child devices)" + }, + "13-799b0119-ccab-4eca-b30b-a37b0bafa02c": { + "event": "runner_on_ok", + "host": "192.168.121.245", + "task": "Get a list of block devices (excludes loop and child devices)" + }, + "14-6beb6958-4bfd-4a9c-bd2c-d20d00248605": { + "event": "runner_on_ok", + "host": "192.168.121.61", + "task": "Get a list of block devices (excludes loop and child devices)" + }, + "15-3ca99cc8-98ea-4967-8f2d-115426d00b6a": { + "event": "runner_on_ok", + "host": "192.168.121.254", + "task": "Get a list of block devices (excludes loop and child devices)" + }, + "16-2016b900-e38f-7dcd-a2e7-00000000000c": { + "event": "playbook_on_task_start", + "task": "check if disk {{ item }} is free" + }, + "17-8c88141a-08d1-411f-a855-9f7702a49c4e": { + "event": "runner_item_on_failed", + "host": "192.168.121.245", + "task": "check if disk vda is free" + }, + "18-4457db98-6f18-4f63-bfaa-584db5eea05b": { + "event": "runner_on_failed", + "host": "192.168.121.245", + "task": "check if disk {{ item }} is free" + }, + "19-ac3c72cd-1fbb-495a-be69-53fa6029f356": { + "event": "runner_item_on_failed", + "host": "192.168.121.61", + "task": "check if disk vda is free" + }, + "20-d161cb70-ba2e-4571-b029-c6428a566fef": { + "event": "runner_on_failed", + "host": "192.168.121.61", + "task": "check if disk {{ item }} is free" + }, + "21-65f1ce5c-2d86-4cc3-8e10-cff6bf6cbd82": { + "event": "runner_item_on_failed", + "host": "192.168.121.254", + "task": "check if disk sda is free" + }, + "22-7f86dcd4-4ef7-4f5a-9db3-c3780b67cc4b": { + "event": "runner_item_on_failed", + "host": "192.168.121.254", + "task": "check if disk sdb is free" + }, + "23-837bf4f6-a912-46a8-b94b-55aa66a935c4": { + "event": "runner_item_on_ok", + "host": "192.168.121.254", + "task": "check if disk sdc is free" + }, + "24-adf6238d-723f-4783-9226-8475419d466e": { + "event": "runner_item_on_failed", + "host": "192.168.121.254", + "task": "check if disk vda is free" + }, + "25-554661d8-bc34-4885-a589-4960d6b8a487": { + "event": "runner_on_failed", + "host": "192.168.121.254", + "task": "check if disk {{ item }} is free" + }, + "26-2016b900-e38f-7dcd-a2e7-00000000000d": { + "event": "playbook_on_task_start", + "task": "Update hosts freedisk list" + }, + "27-52df484c-30a0-4e3b-9057-02ca345c5790": { + "event": "runner_item_on_skipped", + "host": "192.168.121.254", + "task": "Update hosts freedisk list" + }, + "28-083616ad-3c1f-4fb8-a06c-5d64e670e362": { + "event": "runner_item_on_skipped", + "host": "192.168.121.254", + "task": "Update hosts freedisk list" + }, + "29-bffc68d3-5448-491f-8780-07858285f5cd": { + "event": "runner_item_on_skipped", + "host": "192.168.121.245", + "task": "Update hosts freedisk list" + }, + "30-cca2dfd9-16e9-4fcb-8bf7-c4da7dab5668": { + "event": "runner_on_skipped", + "host": "192.168.121.245", + "task": "Update hosts freedisk list" + }, + "31-158a98ac-7e8d-4ebb-8c53-4467351a2d3a": { + "event": "runner_item_on_ok", + "host": "192.168.121.254", + "task": "Update hosts freedisk list" + }, + "32-06a7e809-8d82-41df-b01d-45d94e519cb7": { + "event": "runner_item_on_skipped", + "host": "192.168.121.254", + "task": "Update hosts freedisk list" + }, + "33-d5cdbb58-728a-4be5-abf1-4a051146e727": { + "event": "runner_item_on_skipped", + "host": "192.168.121.61", + "task": "Update hosts freedisk list" + }, + "34-9b3c570b-22d8-4539-8c94-d0c1cbed8633": { + "event": "runner_on_ok", + "host": "192.168.121.254", + "task": "Update hosts freedisk list" + }, + "35-93336830-03cd-43ff-be87-a7e063ca7547": { + "event": "runner_on_skipped", + "host": "192.168.121.61", + "task": "Update hosts freedisk list" + }, + "36-2016b900-e38f-7dcd-a2e7-00000000000e": { + "event": "playbook_on_task_start", + "task": "RESULTS" + }, + "37-100564f1-9fed-48c2-bd62-4ae8636dfcdb": { + "event": "runner_on_ok", + "host": "192.168.121.254", + "task": "RESULTS" + }, + "38-20a64160-30a1-481f-a3ee-36e491bc7869": { + "event": "playbook_on_stats" + } + }, + "total_events": 37 + } +} + diff --git a/src/pybind/mgr/ansible/tests/test_client_playbooks.py b/src/pybind/mgr/ansible/tests/test_client_playbooks.py new file mode 100644 index 00000000000..4a75b5c7a0f --- /dev/null +++ b/src/pybind/mgr/ansible/tests/test_client_playbooks.py @@ -0,0 +1,266 @@ +import logging +import unittest +import mock +import json + +import requests_mock + +from requests.exceptions import ConnectionError + +from ..ansible_runner_svc import Client, PlayBookExecution, ExecutionStatusCode, \ + LOGIN_URL, API_URL, PLAYBOOK_EXEC_URL, \ + PLAYBOOK_EVENTS + + +SERVER_URL = "ars:5001" +USER = "admin" +PASSWORD = "admin" +CERTIFICATE = "" + +# Playbook attributes +PB_NAME = "test_playbook" +PB_UUID = "1733c3ac" + +# Playbook execution data file +PB_EVENTS_FILE = "./tests/pb_execution_events.data" + +# create console handler and set level to info +logger = logging.getLogger() +handler = logging.StreamHandler() +handler.setLevel(logging.INFO) +formatter = logging.Formatter("%(levelname)s - %(message)s") +handler.setFormatter(formatter) +logger.addHandler(handler) + + +def mock_login(mock_server): + + the_login_url = "https://%s/%s" % (SERVER_URL,LOGIN_URL) + + mock_server.register_uri("GET", + the_login_url, + json={"status": "OK", + "msg": "Token returned", + "data": {"token": "dummy_token"}}, + status_code=200) + + the_api_url = "https://%s/%s" % (SERVER_URL,API_URL) + mock_server.register_uri("GET", + the_api_url, + text="api", + status_code=200) + +def mock_get_pb(mock_server, playbook_name, return_code): + + mock_login(mock_server) + + ars_client = Client(SERVER_URL, USER, PASSWORD, + CERTIFICATE, logger) + + the_pb_url = "https://%s/%s/%s" % (SERVER_URL, PLAYBOOK_EXEC_URL, playbook_name) + + if return_code == 404: + mock_server.register_uri("POST", + the_pb_url, + json={ "status": "NOTFOUND", + "msg": "playbook file not found", + "data": {}}, + status_code=return_code) + elif return_code == 202: + mock_server.register_uri("POST", + the_pb_url, + json={ "status": "STARTED", + "msg": "starting", + "data": { "play_uuid": "1733c3ac" }}, + status_code=return_code) + + return PlayBookExecution(ars_client, playbook_name, logger, + result_pattern = "RESULTS") + +class ARSclientTest(unittest.TestCase): + + def test_server_not_reachable(self): + + with self.assertRaises(ConnectionError): + ars_client = Client(SERVER_URL, USER, PASSWORD, + CERTIFICATE, logger) + + def test_server_wrong_USER(self): + + with requests_mock.Mocker() as mock_server: + the_login_url = "https://%s/%s" % (SERVER_URL,LOGIN_URL) + mock_server.get(the_login_url, + json={"status": "NOAUTH", + "msg": "Access denied invalid login: unknown USER", + "data": {}}, + status_code=401) + + + ars_client = Client(SERVER_URL, USER, PASSWORD, + CERTIFICATE, logger) + + self.assertFalse(ars_client.is_operative(), + "Operative attribute expected to be False") + + def test_server_connection_ok(self): + + with requests_mock.Mocker() as mock_server: + + mock_login(mock_server) + + ars_client = Client(SERVER_URL, USER, PASSWORD, + CERTIFICATE, logger) + + self.assertTrue(ars_client.is_operative(), + "Operative attribute expected to be True") + +class PlayBookExecutionTests(unittest.TestCase): + + + def test_playbook_execution_ok(self): + """Check playbook id is set when the playbook is launched + """ + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, PB_NAME, 202) + + test_pb.launch() + + self.assertEqual(test_pb.play_uuid, PB_UUID, + "Found Unexpected playbook uuid") + + + + def test_playbook_execution_error(self): + """Check playbook id is not set when the playbook is not present + """ + + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, "unknown_playbook", 404) + + test_pb.launch() + + self.assertEqual(test_pb.play_uuid, "", + "Playbook uuid not empty") + + def test_playbook_not_launched(self): + """Check right status code when Playbook execution has not been launched + """ + + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, PB_NAME, 202) + + # Check playbook not launched + self.assertEqual(test_pb.get_status(), + ExecutionStatusCode.NOT_LAUNCHED, + "Wrong status code for playbook not launched") + + def test_playbook_launched(self): + """Check right status code when Playbook execution has been launched + """ + + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, PB_NAME, 202) + + test_pb.launch() + + the_status_url = "https://%s/%s/%s" % (SERVER_URL, + PLAYBOOK_EXEC_URL, + PB_UUID) + mock_server.register_uri("GET", + the_status_url, + json={"status": "OK", + "msg": "running", + "data": {"task": "Step 2", + "last_task_num": 6} + }, + status_code=200) + + self.assertEqual(test_pb.get_status(), + ExecutionStatusCode.ON_GOING, + "Wrong status code for a running playbook") + + self.assertEqual(test_pb.play_uuid, PB_UUID, + "Unexpected playbook uuid") + + def test_playbook_finish_ok(self): + """Check right status code when Playbook execution is succesful + """ + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, PB_NAME, 202) + + test_pb.launch() + + the_status_url = "https://%s/%s/%s" % (SERVER_URL, + PLAYBOOK_EXEC_URL, + PB_UUID) + mock_server.register_uri("GET", + the_status_url, + json={"status": "OK", + "msg": "successful", + "data": {} + }, + status_code=200) + + self.assertEqual(test_pb.get_status(), + ExecutionStatusCode.SUCCESS, + "Wrong status code for a playbook executed succesfully") + + def test_playbook_finish_error(self): + """Check right status code when Playbook execution has failed + """ + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, PB_NAME, 202) + + test_pb.launch() + + the_status_url = "https://%s/%s/%s" % (SERVER_URL, + PLAYBOOK_EXEC_URL, + PB_UUID) + mock_server.register_uri("GET", + the_status_url, + json={"status": "OK", + "msg": "failed", + "data": {} + }, + status_code=200) + + self.assertEqual(test_pb.get_status(), + ExecutionStatusCode.ERROR, + "Wrong status code for a playbook with error") + + def test_playbook_get_result(self): + """ Find the right result event in a set of different events + """ + with requests_mock.Mocker() as mock_server: + + test_pb = mock_get_pb(mock_server, PB_NAME, 202) + + test_pb.launch() + + the_events_url = "https://%s/%s" % (SERVER_URL, + PLAYBOOK_EVENTS % PB_UUID) + + # Get the events stored in a file + pb_events = {} + with open(PB_EVENTS_FILE) as events_file: + pb_events = json.loads(events_file.read()) + + mock_server.register_uri("GET", + the_events_url, + json=pb_events, + status_code=200) + + result = test_pb.get_result("runner_on_ok") + + self.assertEqual(len(result.keys()), 1, + "Unique result event not found") + + self.assertIn("37-100564f1-9fed-48c2-bd62-4ae8636dfcdb", + result.keys(), + "Predefined result event not found") diff --git a/src/pybind/mgr/ansible/tox.ini b/src/pybind/mgr/ansible/tox.ini new file mode 100644 index 00000000000..1df28401be7 --- /dev/null +++ b/src/pybind/mgr/ansible/tox.ini @@ -0,0 +1,18 @@ +[tox] +envlist = py27,py3 +skipsdist = true +toxworkdir = {env:CEPH_BUILD_DIR}/ansible +minversion = 2.8.1 + +[testenv] +deps = + pytest + mock + requests-mock +setenv= + UNITTEST = true + py27: PYTHONPATH = {toxinidir}/../../../../build/lib/cython_modules/lib.2 + py3: PYTHONPATH = {toxinidir}/../../../../build/lib/cython_modules/lib.3 + +commands= + {envbindir}/py.test tests/ diff --git a/src/pybind/mgr/orchestrator_cli/module.py b/src/pybind/mgr/orchestrator_cli/module.py index 85aa7f2c27f..af1bdc73459 100644 --- a/src/pybind/mgr/orchestrator_cli/module.py +++ b/src/pybind/mgr/orchestrator_cli/module.py @@ -78,7 +78,7 @@ class OrchestratorCli(MgrModule): done = False while done is False: - done = self._oremote("wait", completions) + done = self._oremote("wait", completions) == [] if not done: any_nonpersistent = False diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 1e47535e984..ec250fc7214 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -559,6 +559,11 @@ if(WITH_MGR) list(APPEND tox_tests run-tox-mgr-insights) set(MGR_INSIGHTS_VIRTUALENV ${CEPH_BUILD_VIRTUALENV}/mgr-insights-virtualenv) list(APPEND env_vars_for_tox_tests MGR_INSIGHTS_VIRTUALENV=${MGR_INSIGHTS_VIRTUALENV}) + + add_test(NAME run-tox-mgr-ansible COMMAND bash ${CMAKE_SOURCE_DIR}/src/pybind/mgr/ansible/run-tox.sh) + list(APPEND tox_tests run-tox-mgr-ansible) + set(MGR_ANSIBLE_VIRTUALENV ${CEPH_BUILD_VIRTUALENV}/mgr-ansible-virtualenv) + list(APPEND env_vars_for_tox_tests MGR_ANSIBLE_VIRTUALENV=${MGR_ANSIBLE_VIRTUALENV}) endif() set_property(