mgr/rest: Trim request array and limit size

Presently, the requests array in the REST module has the potential to grow
indefinitely, leading to excessive memory consumption, particularly when
dealing with lengthy and intricate request results.

To address this issue, a limit will be imposed on the requests array within
the REST module.
This limitation will be governed by the `mgr/restful/x/max_requests` configuration
parameter specific to the REST module.
when submit_request called we will check request array if exceed max_request option
if it does we will check if the future trimmed request finished and log error
message in case we are trimming un-finished requests.

Fixes: https://tracker.ceph.com/issues/59580
Signed-off-by: Nitzan Mordechai <nmordech@redhat.com>

diff --git a/src/pybind/mgr/restful/module.py b/src/pybind/mgr/restful/module.py
index cb8391ecd08dabc12aa064e2f665443f5a397e9f..c6a69641fa050a1bcba9cc92de356a73725ea4eb 100644 (file)
--- a/src/pybind/mgr/restful/module.py
+++ b/src/pybind/mgr/restful/module.py
@@ -12,6 +12,7 @@ import threading
 import traceback
 import socket
 import fcntl
+from typing import cast
 
 from . import common
 from . import context
@@ -197,6 +198,7 @@ class Module(MgrModule):
         {'name': 'server_port'},
         {'name': 'key_file'},
         {'name': 'enable_auth', 'type': 'bool', 'default': True},
+        {'name': 'max_requests', 'type': 'int', 'default': 500},
     ]
 
     COMMANDS = [
@@ -243,6 +245,7 @@ class Module(MgrModule):
 
         self.stop_server = False
         self.serve_event = threading.Event()
+        self.max_requests = cast(int, self.get_localized_module_option('max_requests', 500))
 
 
     def serve(self):
@@ -599,6 +602,16 @@ class Module(MgrModule):
         with self.requests_lock:
             request = CommandsRequest(_request)
             self.requests.append(request)
+            if len(self.requests) > self.max_requests:
+                req_to_trim = 0
+                for i, req in enumerate(self.requests):
+                    if req.is_finished():
+                        self.log.error("Trimmed one finished request due to exceeded maximum requests limit")
+                        req_to_trim = i
+                        break
+                    else:
+                        self.log.error("Trimmed the oldest unfinished request due to exceeded maximum requests limit")
+                self.requests.pop(req_to_trim)
         if kwargs.get('wait', 0):
             while not request.is_finished():
                 time.sleep(0.001)