From d353dbc53a841a0c9a7200b9a198fc380743ba51 Mon Sep 17 00:00:00 2001 From: chenji Date: Fri, 17 Jul 2015 11:17:47 +0800 Subject: [PATCH] rados.py: This module now supports omap operations you can operate omap in python test_rados.py: add unit-test cases Signed-off-by: Ji Chen --- src/pybind/rados.py | 245 ++++++++++++++++++++++++++++++++++ src/test/pybind/test_rados.py | 50 ++++++- 2 files changed, 294 insertions(+), 1 deletion(-) diff --git a/src/pybind/rados.py b/src/pybind/rados.py index de2c1f71de939..271392f526671 100644 --- a/src/pybind/rados.py +++ b/src/pybind/rados.py @@ -832,6 +832,36 @@ Rados object in state %s." % self.state) raise make_ex(ret, "error blacklisting client '%s'" % client_address) +class OmapIterator(object): + """Omap iterator""" + def __init__(self, ioctx, ctx): + self.ioctx = ioctx + self.ctx = ctx + + def __iter__(self): + return self + + def next(self): + """ + Get the next key-value pair in the object + :returns: next rados.OmapItem + """ + key = c_char_p() + value = c_char_p() + lens = c_int() + run_in_thread(self.ioctx.librados.rados_omap_get_next, + (self.ctx, pointer(key), pointer(value), + pointer(lens))) + if key.value is None and value.value is None: + raise StopIteration() + if lens.value: + value.value = value.value[0:lens.value] + return (key.value, value.value) + + def __del__(self): + run_in_thread(self.ioctx.librados.rados_omap_get_end, (self.ctx,)) + + class ObjectIterator(object): """rados.Ioctx Object iterator""" def __init__(self, ioctx): @@ -1070,6 +1100,39 @@ class Completion(object): run_in_thread(self.ioctx.librados.rados_aio_release, (self.rados_comp,)) + +class WriteOpCtx(object): + """write operation context manager""" + def __init__(self, ioctx): + self.ioctx = ioctx + + def __enter__(self): + self.ioctx.librados.rados_create_write_op.restype = c_void_p + ret = run_in_thread(self.ioctx.librados.rados_create_write_op, (None,)) + self.write_op = ret + return ret + + def __exit__(self, type, msg, traceback): + self.ioctx.librados.rados_release_write_op.argtypes = [c_void_p] + run_in_thread(self.ioctx.librados.rados_release_write_op, (c_void_p(self.write_op),)) + + +class ReadOpCtx(object): + """read operation context manager""" + def __init__(self, ioctx): + self.ioctx = ioctx + + def __enter__(self): + self.ioctx.librados.rados_create_read_op.restype = c_void_p + ret = run_in_thread(self.ioctx.librados.rados_create_read_op, (None,)) + self.read_op = ret + return ret + + def __exit__(self, type, msg, traceback): + self.ioctx.librados.rados_release_read_op.argtypes = [c_void_p] + run_in_thread(self.ioctx.librados.rados_release_read_op, (c_void_p(self.read_op),)) + + RADOS_CB = CFUNCTYPE(c_int, c_void_p, c_void_p) @@ -1831,6 +1894,188 @@ returned %d, but should return zero on success." % (self.name, ret)) self.require_ioctx_open() return run_in_thread(self.librados.rados_get_last_version, (self.io,)) + def create_write_op(self): + """ + create write operation object. + need call release_write_op after use + """ + self.librados.rados_create_write_op.restype = c_void_p + return run_in_thread(self.librados.rados_create_write_op, (None,)) + + def create_read_op(self): + """ + create read operation object. + need call release_read_op after use + """ + self.librados.rados_create_read_op.restype = c_void_p + return run_in_thread(self.librados.rados_create_read_op, (None,)) + + def release_write_op(self, write_op): + """ + release memory alloc by create_write_op + """ + self.librados.rados_release_write_op.argtypes = [c_void_p] + run_in_thread(self.librados.rados_release_write_op, (c_void_p(write_op),)) + + def release_read_op(self, read_op): + """ + release memory alloc by create_read_op + :para read_op: read_op object + :type: int + """ + self.librados.rados_release_read_op.argtypes = [c_void_p] + run_in_thread(self.librados.rados_release_read_op, (c_void_p(read_op),)) + + @requires(('write_op', int), ('keys', tuple), ('values', tuple)) + def set_omap(self, write_op, keys, values): + """ + set keys values to write_op + :para write_op: write_operation object + :type write_op: int + :para keys: a tuple of keys + :type keys: tuple + :para values: a tuple of values + :type values: tuple + """ + if len(keys) != len(values): + raise Error("Rados(): keys and values must have the same number of items") + key_num = len(keys) + key_array_type = c_char_p*key_num + key_array = key_array_type() + key_array[:] = keys + + value_array_type = c_char_p*key_num + value_array = value_array_type() + value_array[:] = values + + lens_array_type = c_size_t*key_num + lens_array = lens_array_type() + for index, value in enumerate(values): + lens_array[index] = c_size_t(len(value)) + + run_in_thread(self.librados.rados_write_op_omap_set, + (c_void_p(write_op), byref(key_array), byref(value_array), + byref(lens_array), c_int(key_num),)) + + @requires(('write_op', int), ('oid', str), ('mtime', opt(int)), ('flags', opt(int))) + def operate_write_op(self, write_op, oid, mtime=0, flags=0): + """ + excute the real write operation + :para write_op: write operation object + :type write_op: int + :para oid: object name + :type oid: str + :para mtime: the time to set the mtime to, 0 for the current time + :type mtime: int + :para flags: flags to apply to the entire operation + :type flags: int + """ + run_in_thread(self.librados.rados_write_op_operate, + (c_void_p(write_op), self.io, c_char_p(oid), + c_long(mtime), c_int(flags),)) + + @requires(('read_op', int), ('oid', str), ('flag', opt(int))) + def operate_read_op(self, read_op, oid, flag=0): + """ + excute the real read operation + :para read_op: read operation object + :type read_op: int + :para oid: object name + :type oid: str + :para flag: flags to apply to the entire operation + :type flag: int + """ + run_in_thread(self.librados.rados_read_op_operate, + (c_void_p(read_op), self.io, c_char_p(oid), c_int(flag),)) + + @requires(('read_op', int), ('start_after', str), ('filter_prefix', str), ('max_return', int)) + def get_omap_vals(self, read_op, start_after, filter_prefix, max_return): + """ + get the omap values + :para read_op: read operation object + :type read_op: int + :para start_after: list keys starting after start_after + :type start_after: str + :para filter_prefix: list only keys beginning with filter_prefix + :type filter_prefix: int + :para max_return: list no more than max_return key/value pairs + :type max_return: int + :returns: an iterator over the the requested omap values, return value from this action + """ + prval = c_int() + iter_addr = c_void_p() + run_in_thread(self.librados.rados_read_op_omap_get_vals, + (c_void_p(read_op), c_char_p(start_after), + c_char_p(filter_prefix), c_int(max_return), + byref(iter_addr), pointer(prval))) + return OmapIterator(self, iter_addr), prval.value + + @requires(('read_op', int), ('start_after', str), ('max_return', int)) + def get_omap_keys(self, read_op, start_after, max_return): + """ + get the omap keys + :para read_op: read operation object + :type read_op: int + :para start_after: list keys starting after start_after + :type start_after: str + :para max_return: list no more than max_return key/value pairs + :type max_return: int + :returns: an iterator over the the requested omap values, return value from this action + """ + prval = c_int() + iter_addr = c_void_p() + run_in_thread(self.librados.rados_read_op_omap_get_keys, + (c_void_p(read_op), c_char_p(start_after), + c_int(max_return), byref(iter_addr), pointer(prval))) + return OmapIterator(self, iter_addr), prval.value + + @requires(('read_op', int), ('keys', tuple)) + def get_omap_vals_by_keys(self, read_op, keys): + """ + get the omap values by keys + :para read_op: read operation object + :type read_op: int + :para keys: input key tuple + :type keys: tuple + :returns: an iterator over the the requested omap values, return value from this action + """ + prval = c_int() + iter_addr = c_void_p() + key_num = len(keys) + key_array_type = c_char_p*key_num + key_array = key_array_type() + key_array[:] = keys + run_in_thread(self.librados.rados_read_op_omap_get_vals_by_keys, + (c_void_p(read_op), byref(key_array), c_int(key_num), + byref(iter_addr), pointer(prval))) + return OmapIterator(self, iter_addr), prval.value + + @requires(('write_op', int), ('keys', tuple)) + def remove_omap_keys(self, write_op, keys): + """ + remove omap keys specifiled + :para write_op: write operation object + :type write_op: int + :para keys: input key tuple + :type keys: tuple + """ + key_num = len(keys) + key_array_type = c_char_p*key_num + key_array = key_array_type() + key_array[:] = keys + run_in_thread(self.librados.rados_write_op_omap_rm_keys, + (c_void_p(write_op), byref(key_array), c_int(key_num))) + + @requires(('write_op', int)) + def clear_omap(self, write_op): + """ + Remove all key/value pairs from an object + :para write_op: write operation object + :type write_op: int + """ + run_in_thread(self.librados.rados_write_op_omap_clear, + (c_void_p(write_op),)) + @requires(('key', str), ('name', str), ('cookie', str), ('desc', str), ('duration', opt(int)), ('flags', int)) def lock_exclusive(self, key, name, cookie, desc="", duration=None, flags=0): diff --git a/src/test/pybind/test_rados.py b/src/test/pybind/test_rados.py index 2b5d1b0b51679..c0322785bff9b 100644 --- a/src/test/pybind/test_rados.py +++ b/src/test/pybind/test_rados.py @@ -1,7 +1,7 @@ from nose.tools import eq_ as eq, ok_ as ok, assert_raises from rados import (Rados, Error, RadosStateError, Object, ObjectExists, ObjectNotFound, ObjectBusy, requires, opt, - ANONYMOUS_AUID, ADMIN_AUID, LIBRADOS_ALL_NSPACES) + ANONYMOUS_AUID, ADMIN_AUID, LIBRADOS_ALL_NSPACES, WriteOpCtx, ReadOpCtx) import time import threading import json @@ -337,6 +337,54 @@ class TestIoctx(object): self.ioctx.remove_snap('foo') eq(list(self.ioctx.list_snaps()), []) + def test_set_omap(self): + keys = ("1", "2", "3") + values = ("aaa", "bbb", "ccc") + with WriteOpCtx(self.ioctx) as write_op: + self.ioctx.set_omap(write_op, keys, values) + self.ioctx.operate_write_op(write_op, "hw") + with ReadOpCtx(self.ioctx) as read_op: + iter, ret = self.ioctx.get_omap_vals(read_op, "", "", 3) + self.ioctx.operate_read_op(read_op, "hw") + iter.next() + eq(list(iter), [("2", "bbb"), ("3", "ccc")]) + + def test_get_omap_vals_by_keys(self): + keys = ("1", "2", "3") + values = ("aaa", "bbb", "ccc") + with WriteOpCtx(self.ioctx) as write_op: + self.ioctx.set_omap(write_op, keys, values) + self.ioctx.operate_write_op(write_op, "hw") + with ReadOpCtx(self.ioctx) as read_op: + iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("3",)) + self.ioctx.operate_read_op(read_op, "hw") + eq(list(iter), [("3", "ccc")]) + + def test_get_omap_keys(self): + keys = ("1", "2", "3") + values = ("aaa", "bbb", "ccc") + with WriteOpCtx(self.ioctx) as write_op: + self.ioctx.set_omap(write_op, keys, values) + self.ioctx.operate_write_op(write_op, "hw") + with ReadOpCtx(self.ioctx) as read_op: + iter, ret = self.ioctx.get_omap_keys(read_op,"",2) + self.ioctx.operate_read_op(read_op, "hw") + eq(list(iter), [("1", None), ("2", None)]) + + def test_clear_omap(self): + keys = ("1", "2", "3") + values = ("aaa", "bbb", "ccc") + with WriteOpCtx(self.ioctx) as write_op: + self.ioctx.set_omap(write_op, keys, values) + self.ioctx.operate_write_op(write_op, "hw") + with WriteOpCtx(self.ioctx) as write_op_1: + self.ioctx.clear_omap(write_op_1) + self.ioctx.operate_write_op(write_op_1, "hw") + with ReadOpCtx(self.ioctx) as read_op: + iter, ret = self.ioctx.get_omap_vals_by_keys(read_op,("1",)) + self.ioctx.operate_read_op(read_op, "hw") + eq(list(iter), []) + def test_locator(self): self.ioctx.set_locator_key("bar") self.ioctx.write('foo', 'contents1') -- 2.39.5