pybind/mgr: disable sqlite3/python autocommit

SQLite3 and python's binding were both committing transactions at unintended
points. Turn it off and stop using executescript.

Fixes: https://tracker.ceph.com/issues/65494
Signed-off-by: Patrick Donnelly <pdonnell@redhat.com>

diff --git a/src/pybind/mgr/devicehealth/module.py b/src/pybind/mgr/devicehealth/module.py
index e4356175c6a4298a07c117b9d779e14651a9cb14..e90db88fd1695e10a1763c2c74c2b9f584ce23af 100644 (file)
--- a/src/pybind/mgr/devicehealth/module.py
+++ b/src/pybind/mgr/devicehealth/module.py
@@ -50,31 +50,39 @@ def get_nvme_wear_level(data: Dict[Any, Any]) -> Optional[float]:
 class Module(MgrModule):
 
     # latest (if db does not exist)
-    SCHEMA = """
-CREATE TABLE Device (
-  devid TEXT PRIMARY KEY
-) WITHOUT ROWID;
-CREATE TABLE DeviceHealthMetrics (
-  time DATETIME DEFAULT (strftime('%s', 'now')),
-  devid TEXT NOT NULL REFERENCES Device (devid),
-  raw_smart TEXT NOT NULL,
-  PRIMARY KEY (time, devid)
-);
-"""
+    SCHEMA = [
+        """
+        CREATE TABLE Device (
+            devid TEXT PRIMARY KEY
+        ) WITHOUT ROWID;
+        """,
+        """
+        CREATE TABLE DeviceHealthMetrics (
+            time DATETIME DEFAULT (strftime('%s', 'now')),
+            devid TEXT NOT NULL REFERENCES Device (devid),
+            raw_smart TEXT NOT NULL,
+            PRIMARY KEY (time, devid)
+        );
+        """
+    ]
 
     SCHEMA_VERSIONED = [
         # v1
-        """
-CREATE TABLE Device (
-  devid TEXT PRIMARY KEY
-) WITHOUT ROWID;
-CREATE TABLE DeviceHealthMetrics (
-  time DATETIME DEFAULT (strftime('%s', 'now')),
-  devid TEXT NOT NULL REFERENCES Device (devid),
-  raw_smart TEXT NOT NULL,
-  PRIMARY KEY (time, devid)
-);
-"""
+        [
+            """
+            CREATE TABLE Device (
+            devid TEXT PRIMARY KEY
+            ) WITHOUT ROWID;
+            """,
+            """
+            CREATE TABLE DeviceHealthMetrics (
+                time DATETIME DEFAULT (strftime('%s', 'now')),
+                devid TEXT NOT NULL REFERENCES Device (devid),
+                raw_smart TEXT NOT NULL,
+                PRIMARY KEY (time, devid)
+            );
+            """,
+        ]
     ]
 
     MODULE_OPTIONS = [
@@ -320,6 +328,7 @@ CREATE TABLE DeviceHealthMetrics (
 
         done = False
         with ioctx, self._db_lock, self.db:
+            self.db.execute('BEGIN;')
             count = 0
             for obj in ioctx.list_objects():
                 try:
@@ -512,6 +521,7 @@ CREATE TABLE DeviceHealthMetrics (
         """
 
         with self._db_lock, self.db:
+            self.db.execute('BEGIN;')
             self._create_device(devid)
             self.db.execute(SQL, (devid, json.dumps(data)))
             self._prune_device_metrics()
@@ -566,6 +576,7 @@ CREATE TABLE DeviceHealthMetrics (
         self.log.debug(f"_get_device_metrics: {devid} {sample} {min_sample}")
 
         with self._db_lock, self.db:
+            self.db.execute('BEGIN;')
             if isample:
                 cursor = self.db.execute(SQL_EXACT, (devid, isample))
             else:

diff --git a/src/pybind/mgr/mgr_module.py b/src/pybind/mgr/mgr_module.py
index 5ea2435774c16399eb069af821c7a64d6fe34dce..60c9ce06ca998ccc05a864d4e39c4ca6dd9449c8 100644 (file)
--- a/src/pybind/mgr/mgr_module.py
+++ b/src/pybind/mgr/mgr_module.py
@@ -997,8 +997,8 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
     MODULE_OPTION_DEFAULTS = {}  # type: Dict[str, Any]
 
     # Database Schema
-    SCHEMA = None # type: Optional[str]
-    SCHEMA_VERSIONED = None # type: Optional[List[str]]
+    SCHEMA = None # type: Optional[List[str]]
+    SCHEMA_VERSIONED = None # type: Optional[List[List[str]]]
 
     # Priority definitions for perf counters
     PRIO_CRITICAL = 10
@@ -1183,15 +1183,20 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
             self.appify_pool(self.MGR_POOL_NAME, 'mgr')
 
     def create_skeleton_schema(self, db: sqlite3.Connection) -> None:
-        SQL = """
+        SQL = [
+        """
         CREATE TABLE IF NOT EXISTS MgrModuleKV (
           key TEXT PRIMARY KEY,
           value NOT NULL
         ) WITHOUT ROWID;
-        INSERT OR IGNORE INTO MgrModuleKV (key, value) VALUES ('__version', 0);
+        """,
         """
+        INSERT OR IGNORE INTO MgrModuleKV (key, value) VALUES ('__version', 0);
+        """,
+        ]
 
-        db.executescript(SQL)
+        for sql in SQL:
+            db.execute(sql)
 
     def update_schema_version(self, db: sqlite3.Connection, version: int) -> None:
         SQL = "UPDATE OR ROLLBACK MgrModuleKV SET value = ? WHERE key = '__version';"
@@ -1230,7 +1235,8 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
         if version <= 0:
             self.log.info(f"creating main.db for {self.module_name}")
             assert self.SCHEMA is not None
-            db.executescript(self.SCHEMA)
+            for sql in self.SCHEMA:
+                db.execute(sql)
             self.update_schema_version(db, 1)
         else:
             assert self.SCHEMA_VERSIONED is not None
@@ -1239,8 +1245,8 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
                 raise RuntimeError(f"main.db version is newer ({version}) than module ({latest})")
             for i in range(version, latest):
                 self.log.info(f"upgrading main.db for {self.module_name} from {i-1}:{i}")
-                SQL = self.SCHEMA_VERSIONED[i]
-                db.executescript(SQL)
+                for sql in self.SCHEMA_VERSIONED[i]:
+                    db.execute(sql)
             if version < latest:
                 self.update_schema_version(db, latest)
 
@@ -1251,6 +1257,7 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
 
         kv = self.get_module_option('sqlite3_killpoint')
         with db:
+            db.execute('BEGIN;')
             self.create_skeleton_schema(db)
             if kv == 1:
                 os._exit(120)
@@ -1286,7 +1293,10 @@ class MgrModule(ceph_module.BaseMgrModule, MgrModuleLoggingMixin):
             self.create_mgr_pool()
         uri = f"file:///{self.MGR_POOL_NAME}:{self.module_name}/main.db?vfs=ceph";
         self.log.debug(f"using uri {uri}")
-        db = sqlite3.connect(uri, check_same_thread=False, uri=True)
+        try:
+            db = sqlite3.connect(uri, check_same_thread=False, uri=True, autocommit=False) # type: ignore[call-arg]
+        except TypeError:
+            db = sqlite3.connect(uri, check_same_thread=False, uri=True, isolation_level=None)
         # if libcephsqlite reconnects, update the addrv for blocklist
         with db:
             cur = db.execute('SELECT json_extract(ceph_status(), "$.addr");')