From: Patrick Donnelly Date: Thu, 28 Jan 2021 23:12:19 +0000 (-0800) Subject: test/libcephsqlite,qa: add tests for libcephsqlite X-Git-Tag: v16.2.0~36^2~3 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=a2fec162cf8e401ad6b0d3cd885d025289ca93e1;p=ceph.git test/libcephsqlite,qa: add tests for libcephsqlite Signed-off-by: Patrick Donnelly (cherry picked from commit 54b3c8f7a789068453a25c00730238f08d7fed68) --- diff --git a/qa/suites/rados/basic/ceph.yaml b/qa/suites/rados/basic/ceph.yaml index c0857e14f279..c12a671f013b 100644 --- a/qa/suites/rados/basic/ceph.yaml +++ b/qa/suites/rados/basic/ceph.yaml @@ -10,4 +10,9 @@ overrides: mon osdmap full prune txsize: 2 tasks: - install: + extra_system_packages: + rpm: + - sqlite-devel + deb: + - sqlite3 - ceph: diff --git a/qa/suites/rados/basic/tasks/libcephsqlite.yaml b/qa/suites/rados/basic/tasks/libcephsqlite.yaml new file mode 100644 index 000000000000..12498fb15917 --- /dev/null +++ b/qa/suites/rados/basic/tasks/libcephsqlite.yaml @@ -0,0 +1,24 @@ +overrides: + ceph: + conf: + client: + debug ms: 1 + debug client: 20 + debug cephsqlite: 20 + log-ignorelist: + - POOL_APP_NOT_ENABLED + - do not have an application enabled +tasks: +- exec: + client.0: + - ceph osd pool create cephsqlite + - ceph auth get-or-create client.libcephsqlite mon 'profile simple-rados-client-with-blocklist' osd 'allow rwx pool=cephsqlite' >> /etc/ceph/ceph.keyring +- exec: + client.0: + - ceph_test_libcephsqlite --id libcephsqlite --no-log-to-stderr +- workunit: + clients: + client.0: + - rados/test_libcephsqlite.sh cephsqlite + env: + CEPH_ARGS: --id libcephsqlite --no-log-to-stderr diff --git a/qa/workunits/rados/test_libcephsqlite.sh b/qa/workunits/rados/test_libcephsqlite.sh new file mode 100755 index 000000000000..1810a3f3f72a --- /dev/null +++ b/qa/workunits/rados/test_libcephsqlite.sh @@ -0,0 +1,136 @@ +#!/bin/bash -ex + +# The main point of these tests beyond ceph_test_libcephsqlite is to: +# +# - Ensure you can load the Ceph VFS via the dynamic load extension mechanism +# in SQLite. +# - Check the behavior of a dead application, that it does not hold locks +# indefinitely. + +pool="$1" +ns="$(basename $0)" + +function sqlite { + background="$1" + if [ "$background" = b ]; then + shift + fi + a=$(cat) + printf "%s" "$a" >&2 + # We're doing job control gymnastics here to make sure that sqlite3 is the + # main process (i.e. the process group leader) in the background, not a bash + # function or job pipeline. + sqlite3 -cmd '.output /dev/null' -cmd '.load libcephsqlite.so' -cmd 'pragma journal_mode = PERSIST' -cmd ".open file:///$pool:$ns/baz.db?vfs=ceph" -cmd '.output stdout' <<<"$a" & + if [ "$background" != b ]; then + wait + fi +} + +function striper { + rados --pool=$pool --namespace="$ns" --striper "$@" +} + +function repeat { + n=$1 + shift + for ((i = 0; i < "$n"; ++i)); do + echo "$*" + done +} + +striper rm baz.db || true + +time sqlite < +#include +#include +#include +#include + +#include + +#include +#include +#include "gtest/gtest.h" + +#include "include/uuid.h" +#include "include/rados/librados.hpp" +#include "include/libcephsqlite.h" +#include "SimpleRADOSStriper.h" + +#include "common/ceph_argparse.h" +#include "common/ceph_crypto.h" +#include "common/ceph_time.h" +#include "common/common_init.h" +#include "common/debug.h" + +#define dout_subsys ceph_subsys_client +#undef dout_prefix +#define dout_prefix *_dout << "unittest_libcephsqlite: " + +#define sqlcatchcode(S, code) \ +do {\ + rc = S;\ + if (rc != code) {\ + std::cout << "[" << __FILE__ << ":" << __LINE__ << "]"\ + << " sqlite3 error: " << rc << " `" << sqlite3_errstr(rc)\ + << "': " << sqlite3_errmsg(db) << std::endl;\ + sqlite3_finalize(stmt);\ + stmt = NULL;\ + goto out;\ + }\ +} while (0) + +#define sqlcatch(S) sqlcatchcode(S, SQLITE_OK) + +static boost::intrusive_ptr cct; + +class CephSQLiteTest : public ::testing::Test { +public: + inline static const std::string pool = "cephsqlite"; + + static void SetUpTestSuite() { + librados::Rados cluster; + ASSERT_LE(0, cluster.init_with_context(cct.get())); + ASSERT_LE(0, cluster.connect()); + if (int rc = cluster.pool_create(pool.c_str()); rc < 0 && rc != -EEXIST) { + ASSERT_EQ(0, rc); + } + cluster.shutdown(); + sleep(5); + } + void SetUp() override { + uuid.generate_random(); + ASSERT_LE(0, cluster.init_with_context(cct.get())); + ASSERT_LE(0, cluster.connect()); + ASSERT_LE(0, cluster.wait_for_latest_osdmap()); + ASSERT_EQ(0, db_open()); + } + void TearDown() override { + ASSERT_EQ(SQLITE_OK, sqlite3_close(db)); + db = nullptr; + cluster.shutdown(); + /* Leave database behind for inspection. */ + } + +protected: + int db_open() + { + static const char SQL[] = + "PRAGMA journal_mode = PERSIST;" + "PRAGMA page_size = 65536;" + "PRAGMA cache_size = 32768;" + "PRAGMA temp_store = memory;" + "CREATE TEMPORARY TABLE perf (i INTEGER PRIMARY KEY, v TEXT);" + "CREATE TEMPORARY VIEW p AS" + " SELECT perf.i, J.*" + " FROM perf, json_tree(perf.v) AS J;" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + ; + + sqlite3_stmt *stmt = NULL; + const char *current = SQL; + int rc; + + auto&& name = get_uri(); + sqlcatch(sqlite3_open_v2(name.c_str(), &db, SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_URI, "ceph")); + std::cout << "using database: " << name << std::endl; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_exec(db, current, NULL, NULL, NULL)); + + rc = 0; +out: + sqlite3_finalize(stmt); + return rc; + } + + virtual std::string get_uri() const { + auto uri = fmt::format("file:{}:/{}?vfs=ceph", pool, get_name()); + return uri; + } + virtual std::string get_name() const { + auto name = fmt::format("{}.db", uuid.to_string()); + return name; + } + + sqlite3* db = nullptr; + uuid_d uuid; + librados::Rados cluster; +}; + +TEST_F(CephSQLiteTest, Create) { + static const char SQL[] = + "CREATE TABLE foo (a INT);" + ; + + sqlite3_stmt *stmt = NULL; + const char *current = SQL; + int rc; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; + +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertBulk4096) { + static const char SQL[] = + "PRAGMA page_size = 4096;" + "CREATE TABLE foo (a INT);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT RANDOM()" + " FROM c" + " LIMIT 1000000;" + "PRAGMA page_size;" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 4096); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertBulk) { + static const char SQL[] = + "CREATE TABLE foo (a INT);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT RANDOM()" + " FROM c" + " LIMIT 1000000;" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_exec(db, current, NULL, NULL, NULL)); + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, UpdateBulk) { + static const char SQL[] = + "CREATE TABLE foo (a INT);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT x" + " FROM c" + " LIMIT 1000000;" + "SELECT SUM(a) FROM foo;" + "UPDATE foo" + " SET a = a+a;" + "SELECT SUM(a) FROM foo;" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t sum, sum2; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sum = sqlite3_column_int64(stmt, 0); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sum2 = sqlite3_column_int64(stmt, 0); + ASSERT_EQ(sum*2, sum2); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertRate) { + using clock = ceph::coarse_mono_clock; + using time = ceph::coarse_mono_time; + + static const char SQL[] = + "CREATE TABLE foo (a INT);" + "INSERT INTO foo (a) VALUES (RANDOM());" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + time t1, t2; + int count = 100; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + t1 = clock::now(); + for (int i = 0; i < count; ++i) { + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + } + t2 = clock::now(); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + { + auto diff = std::chrono::duration(t2-t1); + std::cout << "transactions per second: " << count/diff.count() << std::endl; + } + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, DatabaseShrink) { + static const char SQL[] = + "CREATE TABLE foo (a INT);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT x" + " FROM c" + " LIMIT 1000000;" + "DELETE FROM foo" + " WHERE RANDOM()%4 < 3;" + "VACUUM;" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + librados::IoCtx ioctx; + std::unique_ptr rs; + uint64_t size1, size2; + + std::cout << SQL << std::endl; + + ASSERT_EQ(0, cluster.ioctx_create(pool.c_str(), ioctx)); + rs = std::make_unique(ioctx, get_name()); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + ASSERT_EQ(0, rs->lock(1000)); + ASSERT_EQ(0, rs->stat(&size1)); + ASSERT_EQ(0, rs->unlock()); + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + ASSERT_EQ(0, rs->lock(1000)); + ASSERT_EQ(0, rs->stat(&size2)); + ASSERT_EQ(0, rs->unlock()); + ASSERT_LT(size2, size1/2); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertExclusiveRate) { + using clock = ceph::coarse_mono_clock; + using time = ceph::coarse_mono_time; + + static const char SQL[] = + "PRAGMA locking_mode=EXCLUSIVE;" + "CREATE TABLE foo (a INT);" + "INSERT INTO foo (a) VALUES (RANDOM());" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + time t1, t2; + int count = 100; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + t1 = clock::now(); + for (int i = 0; i < count; ++i) { + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + } + t2 = clock::now(); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + { + auto diff = std::chrono::duration(t2-t1); + std::cout << "transactions per second: " << count/diff.count() << std::endl; + } + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertExclusiveWALRate) { + using clock = ceph::coarse_mono_clock; + using time = ceph::coarse_mono_time; + + static const char SQL[] = + "PRAGMA locking_mode=EXCLUSIVE;" + "PRAGMA journal_mode=WAL;" + "CREATE TABLE foo (a INT);" + "INSERT INTO foo (a) VALUES (RANDOM());" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + time t1, t2; + int count = 100; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + t1 = clock::now(); + for (int i = 0; i < count; ++i) { + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + } + t2 = clock::now(); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + { + auto diff = std::chrono::duration(t2-t1); + std::cout << "transactions per second: " << count/diff.count() << std::endl; + } + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, WALTransactionSync) { + static const char SQL[] = + "PRAGMA locking_mode=EXCLUSIVE;" + "PRAGMA journal_mode=WAL;" + "CREATE TABLE foo (a INT);" /* sets up the -wal journal */ + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "BEGIN TRANSACTION;" + "INSERT INTO foo (a) VALUES (RANDOM());" + "END TRANSACTION;" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "SELECT a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_vfs.opf_sync.avgcount' AND" + " b.fullkey = '$.libcephsqlite_vfs.opf_sync.avgcount';" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t id; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + for (int i = 0; i < 10; i++) { + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + } + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + id = sqlite3_last_insert_rowid(db); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, PersistTransactionSync) { + static const char SQL[] = + "BEGIN TRANSACTION;" + "CREATE TABLE foo (a INT);" + "INSERT INTO foo (a) VALUES (RANDOM());" + "END TRANSACTION;" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "SELECT a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_vfs.opf_sync.avgcount' AND" + " b.fullkey = '$.libcephsqlite_vfs.opf_sync.avgcount';" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t id; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + id = sqlite3_last_insert_rowid(db); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 3); /* journal, db, journal header (PERIST) */ + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertExclusiveLock) { + static const char SQL[] = + "PRAGMA locking_mode=EXCLUSIVE;" + "CREATE TABLE foo (a INT);" + "INSERT INTO foo (a) VALUES (RANDOM());" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "SELECT a.atom, b.atom, a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_vfs.opf_lock.avgcount' AND" + " b.fullkey = '$.libcephsqlite_vfs.opf_lock.avgcount';" + "SELECT a.atom, b.atom, a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_striper.lock' AND" + " b.fullkey = '$.libcephsqlite_striper.lock';" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t id; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + id = sqlite3_last_insert_rowid(db); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_GT(sqlite3_column_int64(stmt, 0), 0); + ASSERT_GT(sqlite3_column_int64(stmt, 1), 0); + ASSERT_EQ(sqlite3_column_int64(stmt, 2), 3); /* NONE -> SHARED; SHARED -> RESERVED; RESERVED -> EXCLUSIVE */ + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_GT(sqlite3_column_int64(stmt, 0), 0); + ASSERT_GT(sqlite3_column_int64(stmt, 1), 0); + ASSERT_EQ(sqlite3_column_int64(stmt, 2), 1); /* one actual lock on the striper */ + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, TransactionSizeUpdate) { + static const char SQL[] = + "BEGIN TRANSACTION;" + "CREATE TABLE foo (a INT);" + "INSERT INTO foo (a) VALUES (RANDOM());" + "END TRANSACTION;" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "SELECT a.atom, b.atom, a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_striper.update_size' AND" + " b.fullkey = '$.libcephsqlite_striper.update_size';" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t id; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + id = sqlite3_last_insert_rowid(db); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_GT(sqlite3_column_int64(stmt, 0), 0); + ASSERT_GT(sqlite3_column_int64(stmt, 1), 0); + ASSERT_EQ(sqlite3_column_int64(stmt, 2), 2); /* once for journal write and db write (but not journal header clear!) */ + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, AllocatedGrowth) { + static const char SQL[] = + "CREATE TABLE foo (a BLOB);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT RANDOMBLOB(1<<20)" + " FROM c" + " LIMIT 1024;" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "SELECT a.atom, b.atom, a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_striper.update_allocated' AND" + " b.fullkey = '$.libcephsqlite_striper.update_allocated';" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t id; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + id = sqlite3_last_insert_rowid(db); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_GT(sqlite3_column_int64(stmt, 2), 8); /* max_growth = 128MB, 1024MB of data */ + ASSERT_LT(sqlite3_column_int64(stmt, 2), 12); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + + +TEST_F(CephSQLiteTest, DeleteBulk) { + static const char SQL[] = + "CREATE TABLE foo (a INT);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT x" + " FROM c" + " LIMIT 1000000;" + "DELETE FROM foo" + " WHERE RANDOM()%2 == 0;" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, DropMassive) { + static const char SQL[] = + "CREATE TABLE foo (a BLOB);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO foo (a)" + " SELECT RANDOMBLOB(1<<20)" + " FROM c" + " LIMIT 1024;" + "DROP TABLE foo;" + "VACUUM;" + "INSERT INTO perf (v)" + " VALUES (ceph_perf());" + "SELECT a.atom, b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_striper.shrink' AND" + " b.fullkey = '$.libcephsqlite_striper.shrink';" + "SELECT a.atom-b.atom" + " FROM p AS a, p AS b" + " WHERE a.i = ? AND" + " b.i = ? AND" + " a.fullkey = '$.libcephsqlite_striper.shrink_bytes' AND" + " b.fullkey = '$.libcephsqlite_striper.shrink_bytes';" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + uint64_t id; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + id = sqlite3_last_insert_rowid(db); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_GT(sqlite3_column_int64(stmt, 0), sqlite3_column_int64(stmt, 1)); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatch(sqlite3_bind_int64(stmt, 1, id)); + sqlcatch(sqlite3_bind_int64(stmt, 2, id-1)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_LT(512*(1<<20), sqlite3_column_int64(stmt, 0)); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, InsertMassiveVerify) { + static const char SQL[] = + "CREATE TABLE foo (a BLOB);" + "CREATE TEMPORARY TABLE bar (a BLOB);" + "WITH RECURSIVE c(x) AS" + " (" + " VALUES(1)" + " UNION ALL" + " SELECT x+1" + " FROM c" + " )" + "INSERT INTO bar (a)" + " SELECT RANDOMBLOB(1<<20)" + " FROM c" + " LIMIT 1024;" + "SELECT a FROM bar;" + "INSERT INTO foo (a)" + " SELECT a FROM bar;" + "SELECT a FROM foo;" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + std::vector hashes1, hashes2; + + std::cout << SQL << std::endl; + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const void* blob = sqlite3_column_blob(stmt, 0); + ceph::bufferlist bl; + bl.append(std::string_view((const char*)blob, (size_t)sqlite3_column_bytes(stmt, 0))); + auto digest = ceph::crypto::digest(bl); + hashes1.emplace_back(digest.to_str()); + } + sqlcatchcode(rc, SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + const void* blob = sqlite3_column_blob(stmt, 0); + ceph::bufferlist bl; + bl.append(std::string_view((const char*)blob, (size_t)sqlite3_column_bytes(stmt, 0))); + auto digest = ceph::crypto::digest(bl); + hashes2.emplace_back(digest.to_str()); + } + sqlcatchcode(rc, SQLITE_DONE); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + ASSERT_EQ(hashes1, hashes2); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, PerfValid) { + static const char SQL[] = + "SELECT json_valid(ceph_perf());" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, StatusValid) { + static const char SQL[] = + "SELECT json_valid(ceph_status());" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + ASSERT_EQ(sqlite3_column_int64(stmt, 0), 1); + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + +TEST_F(CephSQLiteTest, StatusFields) { + static const char SQL[] = + "SELECT json_extract(ceph_status(), '$.addr');" + "SELECT json_extract(ceph_status(), '$.id');" + ; + + int rc; + const char *current = SQL; + sqlite3_stmt *stmt = NULL; + + std::cout << SQL << std::endl; + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + { + auto addr = sqlite3_column_text(stmt, 0); + std::cout << addr << std::endl; + } + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); + sqlcatchcode(sqlite3_step(stmt), SQLITE_ROW); + { + auto id = sqlite3_column_int64(stmt, 0); + std::cout << id << std::endl; + ASSERT_GT(id, 0); + } + sqlcatch(sqlite3_finalize(stmt); stmt = NULL); + + rc = 0; +out: + sqlite3_finalize(stmt); + ASSERT_EQ(0, rc); +} + + +int main(int argc, char **argv) { + vector args; + argv_to_vec(argc, (const char **)argv, args); + + std::string conf_file_list; + std::string cluster; + CephInitParameters iparams = ceph_argparse_early_args(args, CEPH_ENTITY_TYPE_CLIENT, &cluster, &conf_file_list); + cct = boost::intrusive_ptr(common_preinit(iparams, CODE_ENVIRONMENT_UTILITY, 0), false); + cct->_conf.parse_config_files(conf_file_list.empty() ? nullptr : conf_file_list.c_str(), &std::cerr, 0); + cct->_conf.parse_env(cct->get_module_type()); // environment variables override + cct->_conf.parse_argv(args); + cct->_conf.apply_changes(nullptr); + common_init_finish(cct.get()); + + ldout(cct, 1) << "sqlite3 version: " << sqlite3_libversion() << dendl; + if (int rc = sqlite3_config(SQLITE_CONFIG_URI, 1); rc) { + lderr(cct) << "sqlite3 config failed: " << rc << dendl; + exit(EXIT_FAILURE); + } + + sqlite3_auto_extension((void (*)())sqlite3_cephsqlite_init); + sqlite3* db = nullptr; + if (int rc = sqlite3_open_v2(":memory:", &db, SQLITE_OPEN_READWRITE, nullptr); rc == SQLITE_OK) { + sqlite3_close(db); + } else { + lderr(cct) << "could not open sqlite3: " << rc << dendl; + exit(EXIT_FAILURE); + } + if (int rc = cephsqlite_setcct(cct.get(), nullptr); rc < 0) { + lderr(cct) << "could not set cct: " << rc << dendl; + exit(EXIT_FAILURE); + } + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/test/test_libcephsqlite.cc b/src/test/test_libcephsqlite.cc deleted file mode 100644 index 248a9682f149..000000000000 --- a/src/test/test_libcephsqlite.cc +++ /dev/null @@ -1,512 +0,0 @@ -/* g++ -I ../src -I ../src/include/ -L ./lib/ -lcephsqlite -lsqlite3 -lrados -lradosstriper -o test_libsqlite ../src/test/test_libcephsqlite.cc - */ -#include - -#include "rados/librados.hpp" -#include "radosstriper/libradosstriper.hpp" -#include "libcephsqlite.h" - -#include -#include -#include - -#include - -#define sqlcatchcode(S, code) \ -do {\ - rc = S;\ - if (rc != code) {\ - if (rc == SQLITE_BUSY) {\ - rc = EAGAIN;\ - } else {\ - std::cerr << "[" << __FILE__ << ":" << __LINE__ << "]"\ - << " sqlite3 error: " << rc << " `" << sqlite3_errstr(rc)\ - << "': " << sqlite3_errmsg(db) << std::endl;\ - if (rc == SQLITE_CONSTRAINT) {\ - rc = EINVAL;\ - } else {\ - rc = EIO;\ - }\ - }\ - sqlite3_finalize(stmt);\ - stmt = NULL;\ - goto out;\ - }\ -} while (0) - -#define sqlcatch(S) sqlcatchcode(S, 0) - -void create_db(sqlite3 *db); -void insert_rand(sqlite3 *db, uint64_t count, uint64_t size); -void insert_db(sqlite3 *db, const char *file_name); -long int count_db(sqlite3 *db); -void list_db(sqlite3 *db); -void delete_db(sqlite3 *db); -const char *sqlite_error_str(int err); -const char *sqlite_exterror_str(int err); - - -int main(int argc, char **argv) -{ - int ret = -1; - librados::Rados cluster; - - ret = cluster.init2("client.admin", "ceph", 0); - if( ret < 0) - { - std::cerr << "Couldn't init cluster "<< ret << std::endl; - } - - // make sure ceph.conf is in /etc/ceph/ and is world readable - ret = cluster.conf_read_file("ceph.conf"); - if( ret < 0) - { - std::cerr << "Couldn't read conf file "<< ret << std::endl; - } - - ret = cluster.connect(); - if(ret < 0) - { - std::cerr << "Couldn't connect to cluster "<< ret << std::endl; - } - else - { - std::cout << "Connected to Cluster"<< std::endl; - } - - std::string cmd = (argv[1] ? argv[1] : ""); - int pool_id = 2; // for cephfs.a.data - - std::cout << "cmd:" << cmd << std::endl; - sqlite3 *db = ceph_sqlite3_open(cluster, "rados-db-test", "DATABASE", pool_id, (cmd == "create")); - - ceph_sqlite3_set_db_params("rados-db-test", 3, (1<<22)); - - if (!db) { - std::cerr << "error: while initializing database" << std::endl; - return 1; - } - - if (cmd == "create") { - create_db(db); - } else if (cmd == "insert_rand") { - insert_rand(db, strtoul(argv[2], NULL, 10), strtoul(argv[3], NULL, 10)); /* argv[2] contains file name of data file */ - } else if (cmd == "insert") { - insert_db(db, argv[2]); /* argv[2] contains file name of data file */ - } else if (cmd == "count") { - std::cout << "total rows: " << count_db(db) << std::endl; - } else if (cmd == "list") { - list_db(db); - } else if (cmd == "delete") { - delete_db(db); - } else { - std::cout << "Usage:" << std::endl; - std::cout << "\t" << argv[0] << " {create|insert |count|list|delete}" << std::endl; - } - - sqlite3_close(db); - cluster.shutdown(); - return 0; -} - -inline -void dump_error(const char *func_name, int line, const char *errmsg, const char *sqlite_errmsg) -{ - std::cerr << func_name << ":" << std::dec << line << ":" << errmsg << ":" << sqlite_errmsg << std::endl; -} - -void create_db(sqlite3 *db) -{ - const char *ddl1 = - "CREATE TABLE IF NOT EXISTS t1(fname TEXT PRIMARY KEY NOT NULL, sha256sum TEXT NOT NULL)"; - const char *ddl2 = - "CREATE UNIQUE INDEX t1fname ON t1(fname);"; - sqlite3_stmt *stmt = NULL; - const char *unused = NULL; - - int ret = sqlite3_prepare_v2(db, ddl1, strlen(ddl1), &stmt, &unused); - std::cerr << __FUNCTION__ << ": prepare:0x" << std::hex << ret << "(" << sqlite_error_str(ret) << ")" << std::endl; - if (ret != SQLITE_OK) { - dump_error(__FUNCTION__, __LINE__, "error: when preparing", sqlite3_errmsg(db)); - goto out; - } - - dump_error(__FUNCTION__, __LINE__, "stepping", ""); - ret = sqlite3_step(stmt); - std::cerr << __FUNCTION__ << ": step:0x" << std::hex << ret << "(" << sqlite_error_str(ret) << ")" << std::endl; - if (ret != SQLITE_DONE) { - dump_error(__FUNCTION__, __LINE__, "error: when stepping", sqlite3_errmsg(db)); - goto out; - } - -#if 0 - sqlite3_finalize(stmt); - - ret = sqlite3_prepare_v2(db, ddl2, strlen(ddl2), &stmt, &unused); - std::cerr << __FUNCTION__ << ": prepare:0x" << std::hex << ret << "(" << sqlite_error_str(ret) << ")" << std::endl; - if (ret != SQLITE_OK) { - dump_error(__FUNCTION__, __LINE__, "error: when preparing", sqlite3_errmsg(db)); - goto out; - } - - ret = sqlite3_step(stmt); - std::cerr << __FUNCTION__ << ": step:0x" << std::hex << ret << "(" << sqlite_error_str(ret) << ")" << std::endl; - if (ret != SQLITE_DONE) { - dump_error(__FUNCTION__, __LINE__, "error: when stepping", sqlite3_errmsg(db)); - } -#endif -out: - sqlite3_finalize(stmt); -} - -void insert_rand(sqlite3 *db, uint64_t count, uint64_t size) -{ - static const char SQL[] = - "CREATE TABLE IF NOT EXISTS rand(text BLOB NOT NULL);" - "INSERT INTO rand VALUES (randomblob(?));" - ; - - sqlite3_stmt *stmt = NULL; - const char *current = SQL; - int rc; - - sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); - sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); - sqlcatch(sqlite3_finalize(stmt)); - - sqlcatch(sqlite3_prepare_v2(db, current, -1, &stmt, ¤t)); - sqlcatch(sqlite3_bind_int64(stmt, 1, (sqlite3_int64)size)); - for (uint64_t i = 0; i < count; i++) { - sqlcatchcode(sqlite3_step(stmt), SQLITE_DONE); - std::cout << "last row inserted: " << sqlite3_last_insert_rowid(db) << std::endl; - } - sqlcatch(sqlite3_finalize(stmt)); - -out: - (void)0; -} - - -void insert_db(sqlite3 *db, const char *file_name) -{ - int row = 0; - std::string col_fname; - std::string col_sha256sum; - - std::ifstream is(file_name); - - const char *dml = "INSERT INTO t1(fname, sha256sum) VALUES(?,?);"; - sqlite3_stmt *stmt = NULL; - const char *unused = NULL; - - while (is >> col_sha256sum >> col_fname) { - if (sqlite3_prepare_v2(db, dml, strlen(dml), &stmt, &unused) != SQLITE_OK) { - dump_error(__FUNCTION__, __LINE__, "error: while preparing", sqlite3_errmsg(db)); - goto out; - } - - if (sqlite3_bind_text(stmt, 1, col_fname.c_str(), strlen(col_fname.c_str()), NULL) != SQLITE_OK) { - std::stringstream ss; - - ss << "error: while attempting to sqlite3_bind_text(col_fname) for row " << row; - dump_error(__FUNCTION__, __LINE__, ss.str().c_str(), sqlite3_errmsg(db)); - goto out; - } - if (sqlite3_bind_text(stmt, 2, col_sha256sum.c_str(), strlen(col_sha256sum.c_str()), NULL) != SQLITE_OK) { - std::stringstream ss; - - ss << "error: while attempting to sqlite3_bind_text(col_sha256sum) for row " << row; - dump_error(__FUNCTION__, __LINE__, ss.str().c_str(), sqlite3_errmsg(db)); - goto out; - } - - int retries = 20; - int ret = -1; - while ((ret = sqlite3_step(stmt)) != SQLITE_DONE) { - usleep(1000); - retries--; - } - if (ret != SQLITE_DONE) { - std::stringstream ss; - - ss << "error:0x" << std::hex << ret << " while attempting to sqlite3_step() for row " << row; - dump_error(__FUNCTION__, __LINE__, ss.str().c_str(), sqlite3_errmsg(db)); - goto out; - } - if (sqlite3_reset(stmt) != SQLITE_OK) { - std::stringstream ss; - - ss << "error: while resetting for row " << row; - dump_error(__FUNCTION__, __LINE__, ss.str().c_str(), sqlite3_errmsg(db)); - goto out; - } - sqlite3_finalize(stmt); - ++row; - std::cerr << "inserted row: " << row << std::endl; - } -out: - return; -} - -long int count_db(sqlite3 *db) -{ - long ret = -1; - const char *dml = "SELECT COUNT(*) FROM t1;"; - sqlite3_stmt *stmt = NULL; - const char *unused = NULL; - - if (sqlite3_prepare_v2(db, dml, strlen(dml), &stmt, &unused) != SQLITE_OK) { - dump_error(__FUNCTION__, __LINE__, "error: when preparing", sqlite3_errmsg(db)); - goto out; - } - - if (sqlite3_step(stmt) != SQLITE_ROW) { - std::stringstream ss; - - ss << "error: while stepping"; - dump_error(__FUNCTION__, __LINE__, ss.str().c_str(), sqlite3_errmsg(db)); - goto out; - } - ret = sqlite3_column_int64(stmt, 0); -out: - sqlite3_finalize(stmt); - return ret; -} - -void list_db(sqlite3 *db) -{ - const char *dml = "SELECT * FROM t1;"; - sqlite3_stmt *stmt = NULL; - const char *unused = NULL; - - if (sqlite3_prepare_v2(db, dml, strlen(dml), &stmt, &unused) != SQLITE_OK) { - dump_error(__FUNCTION__, __LINE__, "error: when preparing", sqlite3_errmsg(db)); - goto out; - } - - while (sqlite3_step(stmt) == SQLITE_ROW) { - std::cout << sqlite3_column_text(stmt, 1) << " " << sqlite3_column_text(stmt, 0) << std::endl; - } -out: - sqlite3_finalize(stmt); -} - -void delete_db(sqlite3 *db) -{ - const char *dml = "DELETE FROM t1; DROP INDEX t1fname; DROP TABLE t1;"; - sqlite3_stmt *stmt = NULL; - const char *unused = NULL; - - if (sqlite3_prepare_v2(db, dml, strlen(dml), &stmt, &unused) != SQLITE_OK) { - dump_error(__FUNCTION__, __LINE__, "error: when preparing", sqlite3_errmsg(db)); - goto out; - } - - if (sqlite3_step(stmt) != SQLITE_DONE) { - dump_error(__FUNCTION__, __LINE__, "error: while deleting table", sqlite3_errmsg(db)); - } -out: - sqlite3_finalize(stmt); -} - -#define CASE(x) case x: return #x - -const char *sqlite_error_str(int err) -{ - switch (err & 0xff) { - CASE(SQLITE_OK); - CASE(SQLITE_ERROR); - CASE(SQLITE_INTERNAL); - CASE(SQLITE_PERM); - CASE(SQLITE_ABORT); - CASE(SQLITE_BUSY); - CASE(SQLITE_LOCKED); - CASE(SQLITE_NOMEM); - CASE(SQLITE_READONLY); - CASE(SQLITE_INTERRUPT); - CASE(SQLITE_IOERR); - CASE(SQLITE_CORRUPT); - CASE(SQLITE_NOTFOUND); - CASE(SQLITE_FULL); - CASE(SQLITE_CANTOPEN); - CASE(SQLITE_PROTOCOL); - CASE(SQLITE_EMPTY); - CASE(SQLITE_SCHEMA); - CASE(SQLITE_TOOBIG); - CASE(SQLITE_CONSTRAINT); - CASE(SQLITE_MISMATCH); - CASE(SQLITE_MISUSE); - CASE(SQLITE_NOLFS); - CASE(SQLITE_AUTH); - CASE(SQLITE_FORMAT); - CASE(SQLITE_RANGE); - CASE(SQLITE_NOTADB); - CASE(SQLITE_NOTICE); - CASE(SQLITE_WARNING); - CASE(SQLITE_ROW); - CASE(SQLITE_DONE); - } - return "NULL"; -} - -const char *sqlite_exterror_str(int err) -{ - switch (err & 0xff) { - case SQLITE_ERROR: - switch (err) { -#ifdef SQLITE_ERROR_MISSING_COLLSEQ - CASE(SQLITE_ERROR_MISSING_COLLSEQ); -#endif -#ifdef SQLITE_ERROR_RETRY - CASE(SQLITE_ERROR_RETRY); -#endif -#ifdef SQLITE_ERROR_SNAPSHOT - CASE(SQLITE_ERROR_SNAPSHOT); -#endif - } - break; - - case SQLITE_IOERR: - switch (err) { - CASE(SQLITE_IOERR_READ); - CASE(SQLITE_IOERR_SHORT_READ); - CASE(SQLITE_IOERR_WRITE); - CASE(SQLITE_IOERR_FSYNC); - CASE(SQLITE_IOERR_DIR_FSYNC); - CASE(SQLITE_IOERR_TRUNCATE); - CASE(SQLITE_IOERR_FSTAT); - CASE(SQLITE_IOERR_UNLOCK); - CASE(SQLITE_IOERR_RDLOCK); - CASE(SQLITE_IOERR_DELETE); - CASE(SQLITE_IOERR_BLOCKED); - CASE(SQLITE_IOERR_NOMEM); - CASE(SQLITE_IOERR_ACCESS); - CASE(SQLITE_IOERR_CHECKRESERVEDLOCK); - CASE(SQLITE_IOERR_LOCK); - CASE(SQLITE_IOERR_CLOSE); - CASE(SQLITE_IOERR_DIR_CLOSE); - CASE(SQLITE_IOERR_SHMOPEN); - CASE(SQLITE_IOERR_SHMSIZE); - CASE(SQLITE_IOERR_SHMLOCK); - CASE(SQLITE_IOERR_SHMMAP); - CASE(SQLITE_IOERR_SEEK); - CASE(SQLITE_IOERR_DELETE_NOENT); - CASE(SQLITE_IOERR_MMAP); - CASE(SQLITE_IOERR_GETTEMPPATH); - CASE(SQLITE_IOERR_CONVPATH); - CASE(SQLITE_IOERR_VNODE); - CASE(SQLITE_IOERR_AUTH); -#ifdef SQLITE_IOERR_BEGIN_ATOMIC - CASE(SQLITE_IOERR_BEGIN_ATOMIC); -#endif -#ifdef SQLITE_IOERR_COMMIT_ATOMIC - CASE(SQLITE_IOERR_COMMIT_ATOMIC); -#endif -#ifdef SQLITE_IOERR_ROLLBACK_ATOMIC - CASE(SQLITE_IOERR_ROLLBACK_ATOMIC); -#endif - } - break; - - case SQLITE_LOCKED: - switch (err) { - CASE(SQLITE_LOCKED_SHAREDCACHE); -#ifdef SQLITE_LOCKED_VTAB - CASE(SQLITE_LOCKED_VTAB); -#endif - } - break; - - case SQLITE_BUSY: - switch (err) { - CASE(SQLITE_BUSY_RECOVERY); - CASE(SQLITE_BUSY_SNAPSHOT); - } - break; - - case SQLITE_CANTOPEN: - switch (err) { - CASE(SQLITE_CANTOPEN_NOTEMPDIR); - CASE(SQLITE_CANTOPEN_ISDIR); - CASE(SQLITE_CANTOPEN_FULLPATH); - CASE(SQLITE_CANTOPEN_CONVPATH); -#ifdef SQLITE_CANTOPEN_DIRTYWAL - CASE(SQLITE_CANTOPEN_DIRTYWAL); -#endif - } - break; - - case SQLITE_CORRUPT: - switch (err) { - CASE(SQLITE_CORRUPT_VTAB); -#ifdef SQLITE_CORRUPT_SEQUENCE - CASE(SQLITE_CORRUPT_SEQUENCE); -#endif - } - break; - - case SQLITE_READONLY: - switch (err) { - CASE(SQLITE_READONLY_RECOVERY); - CASE(SQLITE_READONLY_CANTLOCK); - CASE(SQLITE_READONLY_ROLLBACK); - CASE(SQLITE_READONLY_DBMOVED); -#ifdef SQLITE_READONLY_CANTINIT - CASE(SQLITE_READONLY_CANTINIT); -#endif -#ifdef SQLITE_READONLY_DIRECTORY - CASE(SQLITE_READONLY_DIRECTORY); -#endif - } - break; - - case SQLITE_ABORT: - switch (err) { - CASE(SQLITE_ABORT_ROLLBACK); - } - break; - - case SQLITE_CONSTRAINT: - switch (err) { - CASE(SQLITE_CONSTRAINT_CHECK); - CASE(SQLITE_CONSTRAINT_COMMITHOOK); - CASE(SQLITE_CONSTRAINT_FOREIGNKEY); - CASE(SQLITE_CONSTRAINT_FUNCTION); - CASE(SQLITE_CONSTRAINT_NOTNULL); - CASE(SQLITE_CONSTRAINT_PRIMARYKEY); - CASE(SQLITE_CONSTRAINT_TRIGGER); - CASE(SQLITE_CONSTRAINT_UNIQUE); - CASE(SQLITE_CONSTRAINT_VTAB); - CASE(SQLITE_CONSTRAINT_ROWID); - } - break; - - case SQLITE_NOTICE: - switch (err) { - CASE(SQLITE_NOTICE_RECOVER_WAL); - CASE(SQLITE_NOTICE_RECOVER_ROLLBACK); - } - break; - - case SQLITE_WARNING: - switch (err) { - CASE(SQLITE_WARNING_AUTOINDEX); - } - break; - - case SQLITE_AUTH: - switch (err) { - CASE(SQLITE_AUTH_USER); - } - break; - - case SQLITE_OK: - switch (err) { -#ifdef SQLITE_OK_LOAD_PERMANENTLY - CASE(SQLITE_OK_LOAD_PERMANENTLY); -#endif - } - break; - } - return "EXTNULL"; -}