From: Mehdi Abaakouk Date: Thu, 25 Feb 2016 14:54:05 +0000 (+0100) Subject: pybind: move cephfs to Cython X-Git-Tag: v10.1.0~270^2~2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=10f125f765ccd2a32254150e49b77e1f4157f936;p=ceph.git pybind: move cephfs to Cython This change moves cephfs binding to Cython. Closes-bug: #14818 Signed-off-by: Mehdi Abaakouk --- diff --git a/admin/build-doc b/admin/build-doc index fe7cf97988d45..68e5772b1e97e 100755 --- a/admin/build-doc +++ b/admin/build-doc @@ -85,6 +85,7 @@ nm $vdir/lib/python*/*-packages/rados.so | grep 'U rados_' | \ # --global-option=build_ext --global-option="--cython-include-dirs $TOPDIR/src/pybind/rados/" # but that doesn't work, so copying the file in the rbd module directly, that's ok for docs cp -f $TOPDIR/src/pybind/rados/rados.pxd $TOPDIR/src/pybind/rbd/ +cp -f $TOPDIR/src/pybind/rados/rados.pxd $TOPDIR/src/pybind/cephfs/ ln -sf librbd.so.1 $vdir/lib/librbd.so gcc -shared -o $vdir/lib/librbd.so.1 -xc /dev/null @@ -96,7 +97,19 @@ nm $vdir/lib/python*/*-packages/rbd.so | grep 'U rbd_' | \ awk '{ print "void "$2"(void) {}" }' | \ gcc -shared -o $vdir/lib/librbd.so.1 -xc - -rm -f $TOPDIR/src/pybind/rbd/rados.pxd + +ln -sf libcephfs.so.1 $vdir/lib/libcephfs.so +gcc -shared -o $vdir/lib/libcephfs.so.1 -xc /dev/null +CFLAGS="-iquote $TOPDIR/src/include" \ + CPPFLAGS="-iquote $TOPDIR/src/include" \ + LDFLAGS="-L$vdir/lib -Wl,--no-as-needed" \ + $vdir/bin/pip install $TOPDIR/src/pybind/cephfs +nm $vdir/lib/python*/*-packages/cephfs.so | grep 'U cephfs_' | \ + awk '{ print "void "$2"(void) {}" }' | \ + gcc -shared -o $vdir/lib/libcephfs.so.1 -xc - + +rm -f $TOPDIR/src/pybind/rbd/rados.pxd $TOPDIR/src/pybind/cephfs/rados.pxd + $vdir/bin/sphinx-build -a -n -b dirhtml -d doctrees $TOPDIR/doc $TOPDIR/build-doc/output/html $vdir/bin/sphinx-build -a -b man -d doctrees $TOPDIR/doc $TOPDIR/build-doc/output/man diff --git a/ceph.spec.in b/ceph.spec.in index d385074f78a72..cf5f220c5b481 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -1254,7 +1254,8 @@ ln -sf %{_libdir}/librbd.so.1 /usr/lib64/qemu/librbd.so.1 ################################################################################# %files -n python-cephfs %defattr(-,root,root,-) -%{python_sitelib}/cephfs.py* +%{python_sitearch}/cephfs.so +%{python_sitearch}/cephfs-*.egg-info %{python_sitelib}/ceph_volume_client.py* ################################################################################# diff --git a/debian/python-cephfs.install b/debian/python-cephfs.install index 9001bc4b7b3d9..d21d1fe742c1a 100644 --- a/debian/python-cephfs.install +++ b/debian/python-cephfs.install @@ -1,2 +1,3 @@ -usr/lib/python*/dist-packages/cephfs.py* usr/lib/python*/dist-packages/ceph_volume_client.py* +usr/lib/python*/dist-packages/cephfs.so +usr/lib/python*/dist-packages/cephfs-*.egg-info diff --git a/src/Makefile-client.am b/src/Makefile-client.am index 07413eaebe7de..98e8ebb74d299 100644 --- a/src/Makefile-client.am +++ b/src/Makefile-client.am @@ -77,7 +77,6 @@ cephfs_SOURCES = cephfs.cc cephfs_LDADD = $(LIBCOMMON) bin_PROGRAMS += cephfs -python_PYTHON += pybind/cephfs.py python_PYTHON += pybind/ceph_volume_client.py # libcephfs (this should go somewhere else in the future) diff --git a/src/civetweb b/src/civetweb index 554fa96229bcb..8d271315a5412 160000 --- a/src/civetweb +++ b/src/civetweb @@ -1 +1 @@ -Subproject commit 554fa96229bcb2de08c9e9d976a3153d324126b3 +Subproject commit 8d271315a541218caada366f84a2690fdbd474a2 diff --git a/src/pybind/Makefile.am b/src/pybind/Makefile.am index fa5ff8d279e8c..23c5fde719c5c 100644 --- a/src/pybind/Makefile.am +++ b/src/pybind/Makefile.am @@ -1,7 +1,5 @@ if ENABLE_CLIENT -if WITH_RADOS -if WITH_RBD if WITH_CYTHON PY_DISTUTILS = \ @@ -11,10 +9,15 @@ PY_DISTUTILS = \ CYTHON_BUILD_DIR="$(shell readlink -f $(builddir))/build" \ ${PYTHON} ./setup.py -include pybind/rbd/Makefile.am +if WITH_RADOS include pybind/rados/Makefile.am +if WITH_RBD +include pybind/rbd/Makefile.am +endif # WITH_RBD +if WITH_CEPHFS +include pybind/cephfs/Makefile.am +endif # WITH_CEPHFS +endif # WITH_RADOS -endif -endif -endif -endif +endif # WITH_CYTHON +endif # ENABLE_CLIENT diff --git a/src/pybind/cephfs.py b/src/pybind/cephfs.py deleted file mode 100644 index 71c0d40e74a57..0000000000000 --- a/src/pybind/cephfs.py +++ /dev/null @@ -1,590 +0,0 @@ -""" -This module is a thin wrapper around libcephfs. -""" -from ctypes import CDLL, c_char_p, c_size_t, c_void_p, c_int, c_long, c_uint, c_ulong, \ - c_ushort, create_string_buffer, byref, Structure, pointer, c_char, POINTER, \ - c_uint8, c_int64 -from ctypes.util import find_library -from collections import namedtuple -import errno -import os - - -class Error(Exception): - pass - - -class PermissionError(Error): - pass - - -class ObjectNotFound(Error): - pass - - -class NoData(Error): - pass - - -class ObjectExists(Error): - pass - - -class IOError(Error): - pass - - -class NoSpace(Error): - pass - - -class InvalidValue(Error): - pass - - -class OperationNotSupported(Error): - pass - - -class IncompleteWriteError(Error): - pass - - -class LibCephFSStateError(Error): - pass - - -def make_ex(ret, msg): - """ - Translate a libcephfs return code into an exception. - - :param ret: the return code - :type ret: int - :param msg: the error message to use - :type msg: str - :returns: a subclass of :class:`Error` - """ - - errors = { - errno.EPERM : PermissionError, - errno.ENOENT : ObjectNotFound, - errno.EIO : IOError, - errno.ENOSPC : NoSpace, - errno.EEXIST : ObjectExists, - errno.ENODATA : NoData, - errno.EINVAL : InvalidValue, - errno.EOPNOTSUPP: OperationNotSupported, - } - ret = abs(ret) - if ret in errors: - return errors[ret](msg) - else: - return Error(msg + (": error code %d" % ret)) - - -class cephfs_statvfs(Structure): - _fields_ = [("f_bsize", c_ulong), - ("f_frsize", c_ulong), - ("f_blocks", c_ulong), - ("f_bfree", c_ulong), - ("f_bavail", c_ulong), - ("f_files", c_ulong), - ("f_ffree", c_ulong), - ("f_favail", c_ulong), - ("f_fsid", c_ulong), - ("f_flag", c_ulong), - ("f_namemax", c_ulong), - ("f_padding", c_ulong*32)] - - -class cephfs_dirent(Structure): - _fields_ = [("d_ino", c_long), - ("d_off", c_ulong), - ("d_reclen", c_ushort), - ("d_type", c_uint8), - ("d_name", c_char*256)] - -# struct timespec { -# long int tv_sec; -# long int tv_nsec; -# } -class cephfs_timespec(Structure): - _fields_ = [('tv_sec', c_long), - ('tv_nsec', c_long)] - - -# struct stat { -# unsigned long st_dev; -# unsigned long st_ino; -# unsigned long st_nlink; -# unsigned int st_mode; -# unsigned int st_uid; -# unsigned int st_gid; -# int __pad0; -# unsigned long st_rdev; -# long int st_size; -# long int st_blksize; -# long int st_blocks; -# struct timespec st_atim; -# struct timespec st_mtim; -# struct timespec st_ctim; -# long int __unused[3]; -# }; -class cephfs_stat(Structure): - _fields_ = [('st_dev', c_ulong), # ID of device containing file - ('st_ino', c_ulong), # inode number - ('st_nlink', c_ulong), # number of hard links - ('st_mode', c_uint), # protection - ('st_uid', c_uint), # user ID of owner - ('st_gid', c_uint), # group ID of owner - ('__pad0', c_int), - ('st_rdev', c_ulong), # device ID (if special file) - ('st_size', c_long), # total size, in bytes - ('st_blksize', c_long), # blocksize for file system I/O - ('st_blocks', c_long), # num of 512B blocks allocated - ('st_atime', cephfs_timespec), # time of last access - ('st_mtime', cephfs_timespec), # time of last modification - ('st_ctime', cephfs_timespec), # time of last status change - ('__unused1', c_long), - ('__unused2', c_long), - ('__unused3', c_long)] - - -class DirEntry(namedtuple('DirEntry', - ['d_ino', 'd_off', 'd_reclen', 'd_type', 'd_name'])): - DT_DIR = 0x4 - DT_REG = 0xA - DT_LNK = 0xC - def is_dir(self): - return self.d_type == self.DT_DIR - - def is_symbol_file(self): - return self.d_type == self.DT_LNK - - def is_file(self): - return self.d_type == self.DT_REG - -StatResult = namedtuple('StatResult', - ["st_dev", "st_ino", "st_mode", "st_nlink", "st_uid", - "st_gid", "st_rdev", "st_size", "st_blksize", - "st_blocks", "st_atime", "st_mtime", "st_ctime"]) - -def load_libcephfs(): - """ - Load the libcephfs shared library. - """ - libcephfs_path = find_library('cephfs') - if libcephfs_path: - return CDLL(libcephfs_path) - - # try harder, find_library() doesn't search LD_LIBRARY_PATH - # in addition, it doesn't seem work on centos 6.4 (see e46d2ca067b5) - try: - return CDLL('libcephfs.so.1') - except OSError as e: - raise EnvironmentError("Unable to load libcephfs: %s" % e) - - -class LibCephFS(object): - """libcephfs python wrapper""" - def require_state(self, *args): - for a in args: - if self.state == a: - return - raise LibCephFSStateError("You cannot perform that operation on a " - "CephFS object in state %s." % (self.state)) - - def __init__(self, conf=None, conffile=None, auth_id=None, rados_inst=None): - self.libcephfs = load_libcephfs() - self.cluster = c_void_p() - - self.state = "uninitialized" - if rados_inst is not None: - if auth_id is not None or conffile is not None or conf is not None: - raise InvalidValue("May not pass RADOS instance as well as other configuration") - - return self.create_with_rados(rados_inst) - else: - return self.create(conf, conffile, auth_id) - - def create_with_rados(self, rados_inst): - ret = self.libcephfs.ceph_create_from_rados( - byref(self.cluster), - rados_inst.cluster) - if ret != 0: - raise Error("libcephfs_initialize failed with error code: %d" % ret) - self.state = "configuring" - - def create(self, conf=None, conffile=None, auth_id=None, rados_inst=None): - if conffile is not None and not isinstance(conffile, basestring): - raise TypeError('conffile must be a string or None') - - if auth_id is not None and not isinstance(auth_id, basestring): - raise TypeError('auth_id must be a string or None') - - ret = self.libcephfs.ceph_create(byref(self.cluster), - c_char_p(auth_id) if auth_id else c_char_p(0)) - if ret != 0: - raise Error("libcephfs_initialize failed with error code: %d" % ret) - self.state = "configuring" - if conffile is not None: - # read the default conf file when '' is given - if conffile == '': - conffile = None - self.conf_read_file(conffile) - if conf is not None: - for key, value in conf.iteritems(): - self.conf_set(key, value) - - def conf_read_file(self, conffile=None): - if conffile is not None and not isinstance(conffile, basestring): - raise TypeError('conffile param must be a string') - ret = self.libcephfs.ceph_conf_read_file(self.cluster, c_char_p(conffile)) - if ret != 0: - raise make_ex(ret, "error calling conf_read_file") - - def conf_parse_argv(self, argv): - self.require_state("configuring") - c_argv = (c_char_p * len(argv))(*argv) - ret = self.libcephfs.ceph_conf_parse_argv(self.cluster, len(argv), - c_argv) - if ret != 0: - raise make_ex(ret, "error calling conf_parse_argv") - - def shutdown(self): - """ - Unmount and destroy the ceph mount handle. - """ - if self.state != "shutdown": - self.libcephfs.ceph_shutdown(self.cluster) - self.state = "shutdown" - - def __enter__(self): - self.mount() - return self - - def __exit__(self, type_, value, traceback): - self.shutdown() - return False - - def __del__(self): - self.shutdown() - - def version(self): - """ - Get the version number of the ``libcephfs`` C library. - - :returns: a tuple of ``(major, minor, extra)`` components of the - libcephfs version - """ - major = c_int(0) - minor = c_int(0) - extra = c_int(0) - self.libcephfs.ceph_version(byref(major), byref(minor), byref(extra)) - return (major.value, minor.value, extra.value) - - def conf_get(self, option): - self.require_state("configuring", "initialized", "mounted") - if not isinstance(option, basestring): - raise TypeError('option must be a string') - length = 20 - while True: - ret_buf = create_string_buffer(length) - ret = self.libcephfs.ceph_conf_get(self.cluster, option, - ret_buf, c_size_t(length)) - if ret == 0: - return ret_buf.value - elif ret == -errno.ENAMETOOLONG: - length = length * 2 - elif ret == -errno.ENOENT: - return None - else: - raise make_ex(ret, "error calling conf_get") - - def conf_set(self, option, val): - self.require_state("configuring", "initialized", "mounted") - if not isinstance(option, basestring): - raise TypeError('option must be a string') - if not isinstance(val, basestring): - raise TypeError('val must be a string') - ret = self.libcephfs.ceph_conf_set(self.cluster, c_char_p(option), - c_char_p(val)) - if ret != 0: - raise make_ex(ret, "error calling conf_set") - - def init(self): - self.require_state("configuring") - ret = self.libcephfs.ceph_init(self.cluster) - if ret != 0: - raise make_ex(ret, "error calling ceph_init") - self.state = "initialized" - - def mount(self): - if self.state == "configuring": - self.init() - self.require_state("initialized") - ret = self.libcephfs.ceph_mount(self.cluster, "/") - if ret != 0: - raise make_ex(ret, "error calling ceph_mount") - self.state = "mounted" - - def statfs(self, path): - if not isinstance(path, basestring): - raise TypeError('path must be a string') - self.require_state("mounted") - statbuf = cephfs_statvfs() - ret = self.libcephfs.ceph_statfs(self.cluster, c_char_p(path), byref(statbuf)) - if ret < 0: - raise make_ex(ret, "statfs failed: %s" % path) - return {'f_bsize': statbuf.f_bsize, - 'f_frsize': statbuf.f_frsize, - 'f_blocks': statbuf.f_blocks, - 'f_bfree': statbuf.f_bfree, - 'f_bavail': statbuf.f_bavail, - 'f_files': statbuf.f_files, - 'f_ffree': statbuf.f_ffree, - 'f_favail': statbuf.f_favail, - 'f_fsid': statbuf.f_fsid, - 'f_flag': statbuf.f_flag, - 'f_namemax': statbuf.f_namemax} - - def sync_fs(self): - self.require_state("mounted") - ret = self.libcephfs.ceph_sync_fs(self.cluster) - if ret < 0: - raise make_ex(ret, "sync_fs failed") - - def getcwd(self): - self.require_state("mounted") - self.libcephfs.ceph_getcwd.restype = c_char_p - return self.libcephfs.ceph_getcwd(self.cluster) - - def chdir(self, path): - self.require_state("mounted") - ret = self.libcephfs.ceph_chdir(self.cluster, c_char_p(path)) - if ret < 0: - raise make_ex(ret, "chdir failed") - - def opendir(self, path): - self.require_state("mounted") - if not isinstance(path, basestring): - raise TypeError('path must be a string') - dir_handler = c_void_p() - ret = self.libcephfs.ceph_opendir(self.cluster, c_char_p(path), - pointer(dir_handler)); - if ret < 0: - raise make_ex(ret, "opendir failed") - return dir_handler - - def readdir(self, dir_handler): - self.require_state("mounted") - self.libcephfs.ceph_readdir.restype = POINTER(cephfs_dirent) - while True: - dirent = self.libcephfs.ceph_readdir(self.cluster, dir_handler) - if not dirent: - return None - - return DirEntry(d_ino=dirent.contents.d_ino, - d_off=dirent.contents.d_off, - d_reclen=dirent.contents.d_reclen, - d_type=dirent.contents.d_type, - d_name=dirent.contents.d_name) - - def closedir(self, dir_handler): - self.require_state("mounted") - ret = self.libcephfs.ceph_closedir(self.cluster, dir_handler) - if ret < 0: - raise make_ex(ret, "closedir failed") - - def mkdir(self, path, mode): - self.require_state("mounted") - if not isinstance(path, basestring): - raise TypeError('path must be a string') - ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode)) - if ret < 0: - raise make_ex(ret, "error in mkdir '%s'" % path) - - def mkdirs(self, path, mode): - self.require_state("mounted") - if not isinstance(path, basestring): - raise TypeError('path must be a string') - if not isinstance(mode, basestring): - raise TypeError('mode must be an int') - ret = self.libcephfs.ceph_mkdir(self.cluster, c_char_p(path), c_int(mode)) - if ret < 0: - raise make_ex(ret, "error in mkdirs '%s'" % path) - - def rmdir(self, path): - self.require_state("mounted") - if not isinstance(path, basestring): - raise TypeError('path must be a string') - ret = self.libcephfs.ceph_rmdir(self.cluster, c_char_p(path)) - if ret < 0: - raise make_ex(ret, "error in rmdir '%s'" % path) - - def open(self, path, flags, mode=0): - self.require_state("mounted") - if not isinstance(path, basestring): - raise TypeError('path must be a string') - if not isinstance(flags, basestring): - raise TypeError('flags must be a string') - if not isinstance(mode, int): - raise TypeError('mode must be an int') - cephfs_flags = 0 - if flags == '': - cephfs_flags = os.O_RDONLY - else: - for c in flags: - if c == 'r': - cephfs_flags |= os.O_RDONLY - elif c == 'w': - cephfs_flags |= os.O_WRONLY | os.O_TRUNC | os.O_CREAT - elif c == '+': - cephfs_flags |= os.O_RDWR - else: - raise OperationNotSupported( - "open flags doesn't support %s" % c) - - ret = self.libcephfs.ceph_open(self.cluster, c_char_p(path), - c_int(cephfs_flags), c_int(mode)) - if ret < 0: - raise make_ex(ret, "error in open '%s'" % path) - return ret - - def close(self, fd): - self.require_state("mounted") - ret = self.libcephfs.ceph_close(self.cluster, c_int(fd)) - if ret < 0: - raise make_ex(ret, "error in close") - - def read(self, fd, offset, l): - self.require_state("mounted") - if not isinstance(offset, int): - raise TypeError('path must be an int') - if not isinstance(l, int): - raise TypeError('path must be an int') - - buf = create_string_buffer(l) - ret = self.libcephfs.ceph_read(self.cluster, c_int(fd), - buf, c_int64(l), c_int64(offset)) - if ret < 0: - raise make_ex(ret, "error in close") - return buf.value - - def write(self, fd, buf, offset): - self.require_state("mounted") - if not isinstance(buf, basestring): - raise TypeError('buf must be a string') - if not isinstance(offset, int): - raise TypeError('offset must be an int') - - ret = self.libcephfs.ceph_write(self.cluster, c_int(fd), - c_char_p(buf), c_int64(len(buf)), - c_int64(offset)) - if ret < 0: - raise make_ex(ret, "error in close") - return ret - - def getxattr(self, path, name): - if not isinstance(path, basestring): - raise TypeError('path must be a string') - if not isinstance(name, basestring): - raise TypeError('name must be a string') - - self.require_state("mounted") - l = 255 - buf = create_string_buffer(l) - actual_l = self.libcephfs.ceph_getxattr(self.cluster, path, name, buf, c_int(l)) - if actual_l > l: - buf = create_string_buffer(actual_) - self.libcephfs.ceph_getxattr(path, name, new_buf, actual_l) - return buf.value - - def setxattr(self, path, name, value, flags): - if not isinstance(path, basestring): - raise TypeError('path must be a string') - if not isinstance(name, basestring): - raise TypeError('name must be a string') - if not isinstance(value, basestring): - raise TypeError('value must be a string') - self.require_state("mounted") - ret = self.libcephfs.ceph_setxattr(self.cluster, c_char_p(path), - c_char_p(name), c_char_p(value), - c_size_t(len(value)), c_int(flags)) - if ret < 0: - raise make_ex(ret, "error in setxattr") - - def stat(self, path): - self.require_state("mounted") - if not isinstance(path, basestring): - raise TypeError('path must be a string') - statbuf = cephfs_stat() - ret = self.libcephfs.ceph_stat(self.cluster, c_char_p(path), - byref(statbuf)) - if ret < 0: - raise make_ex(ret, "error in stat: %s" % path) - return StatResult(st_dev=statbuf.st_dev, st_ino=statbuf.st_ino, - st_mode=statbuf.st_mode, st_nlink=statbuf.st_nlink, - st_uid=statbuf.st_uid, st_gid=statbuf.st_gid, - st_rdev=statbuf.st_rdev, st_size=statbuf.st_size, - st_blksize=statbuf.st_blksize, - st_blocks=statbuf.st_blocks, - st_atime=statbuf.st_atime, st_mtime=statbuf.st_mtime, - st_ctime=statbuf.st_ctime) - - def symlink(self, existing, newname): - if not isinstance(existing, str): - raise TypeError('existing must be a string') - if not isinstance(newname, str): - raise TypeError('newname must be a string') - self.require_state("mounted") - ret = self.libcephfs.ceph_symlink( - self.cluster, - c_char_p(existing), - c_char_p(newname)) - if ret < 0: - raise make_ex(ret, "error in symlink") - - def unlink(self, path): - self.require_state("mounted") - ret = self.libcephfs.ceph_unlink( - self.cluster, - c_char_p(path)) - if ret < 0: - raise make_ex(ret, "error in unlink: %s" % path) - - def rename(self, src, dst): - self.require_state("mounted") - if not isinstance(src, basestring) or not isinstance(dst, basestring): - raise TypeError('source and destination must be a string') - ret = self.libcephfs.ceph_rename(self.cluster, c_char_p(src), c_char_p(dst)) - if ret < 0: - raise make_ex(ret, "error in rename '%s' to '%s'" % (src, dst)) - - def mds_command(self, mds_spec, args, input_data): - """ - :return 3-tuple of output status int, output status string, output data - """ - - cmdarr = (c_char_p * len(args))(*args) - - outbufp = pointer(pointer(c_char())) - outbuflen = c_long() - outsp = pointer(pointer(c_char())) - outslen = c_long() - - ret = self.libcephfs.ceph_mds_command(self.cluster, c_char_p(mds_spec), - cmdarr, len(args), - c_char_p(input_data), - len(input_data), outbufp, - byref(outbuflen), outsp, - byref(outslen)) - - my_outbuf = outbufp.contents[:(outbuflen.value)] - my_outs = outsp.contents[:(outslen.value)] - if outbuflen.value: - self.libcephfs.ceph_buffer_free(outbufp.contents) - if outslen.value: - self.libcephfs.ceph_buffer_free(outsp.contents) - - return (ret, my_outbuf, my_outs) diff --git a/src/pybind/cephfs/Makefile.am b/src/pybind/cephfs/Makefile.am new file mode 100644 index 0000000000000..577431fdd0fbe --- /dev/null +++ b/src/pybind/cephfs/Makefile.am @@ -0,0 +1,34 @@ +EXTRA_DIST += $(srcdir)/pybind/cephfs/setup.py $(srcdir)/pybind/cephfs/cephfs.pyx + +cephfs-pybind-all: libcephfs.la ${srcdir}/ceph_ver.h + cd $(srcdir)/pybind/cephfs; $(PY_DISTUTILS) build \ + --build-base $(shell readlink -f $(builddir))/build \ + --verbose + +cephfs-pybind-clean: ${srcdir}/ceph_ver.h + cd $(srcdir)/pybind/cephfs; $(PY_DISTUTILS) clean \ + --build-base $(shell readlink -f $(builddir))/build \ + --verbose + +cephfs-pybind-install-exec: ${srcdir}/ceph_ver.h + if test "$(DESTDIR)" ; then \ + if lsb_release -si | grep --quiet 'Ubuntu\|Debian\|Devuan' ; then \ + options=--install-layout=deb ; \ + else \ + options=--prefix=/usr ; \ + fi ; \ + root="--root=$(DESTDIR)" ; \ + else \ + options=--prefix=$(prefix) ; \ + fi ; \ + cd $(srcdir)/pybind/cephfs; $(PY_DISTUTILS) build \ + --build-base $(shell readlink -f $(builddir))/build \ + install \ + $$options $$root \ + --single-version-externally-managed \ + --record /dev/null \ + --verbose + +LOCAL_ALL += cephfs-pybind-all +LOCAL_CLEAN += cephfs-pybind-clean +LOCAL_INSTALLEXEC += cephfs-pybind-install-exec diff --git a/src/pybind/cephfs/cephfs.pyx b/src/pybind/cephfs/cephfs.pyx new file mode 100644 index 0000000000000..95413fce0a6c9 --- /dev/null +++ b/src/pybind/cephfs/cephfs.pyx @@ -0,0 +1,841 @@ +""" +This module is a thin wrapper around libcephfs. +""" + +from cpython cimport PyObject, ref, exc +from libc cimport errno +from libc.stdint cimport * +from libc.stdlib cimport malloc, realloc, free + +cimport rados + +from collections import namedtuple +from datetime import datetime +import errno +import os +import sys + +# Are we running Python 2.x +_python2 = sys.hexversion < 0x03000000 + +if _python2: + str_type = basestring +else: + str_type = str + + +cdef extern from "Python.h": + # These are in cpython/string.pxd, but use "object" types instead of + # PyObject*, which invokes assumptions in cpython that we need to + # legitimately break to implement zero-copy string buffers in Image.read(). + # This is valid use of the Python API and documented as a special case. + PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL + char* PyBytes_AsString(PyObject *string) except NULL + int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1 + void PyEval_InitThreads() + + +cdef extern from "sys/statvfs.h": + cdef struct statvfs: + unsigned long int f_bsize + unsigned long int f_frsize + unsigned long int f_blocks + unsigned long int f_bfree + unsigned long int f_bavail + unsigned long int f_files + unsigned long int f_ffree + unsigned long int f_favail + unsigned long int f_fsid + unsigned long int f_flag + unsigned long int f_namemax + unsigned long int f_padding[32] + + +cdef extern from "dirent.h": + cdef struct dirent: + long int d_ino + unsigned long int d_off + unsigned short int d_reclen + unsigned char d_type + char d_name[256] + + +cdef extern from "time.h": + ctypedef long int time_t + + +cdef extern from "sys/types.h": + ctypedef unsigned long mode_t + + +cdef extern from "sys/stat.h": + cdef struct stat: + unsigned long st_dev + unsigned long st_ino + unsigned long st_nlink + unsigned int st_mode + unsigned int st_uid + unsigned int st_gid + int __pad0 + unsigned long st_rdev + long int st_size + long int st_blksize + long int st_blocks + time_t st_atime + time_t st_mtime + time_t st_ctime + + +cdef extern from "cephfs/libcephfs.h" nogil: + cdef struct ceph_mount_info: + pass + + cdef struct ceph_dir_result: + pass + + ctypedef void* rados_t + + const char *ceph_version(int *major, int *minor, int *patch) + + int ceph_create(ceph_mount_info **cmount, const char * const id) + int ceph_create_from_rados(ceph_mount_info **cmount, rados_t cluster) + int ceph_init(ceph_mount_info *cmount) + void ceph_shutdown(ceph_mount_info *cmount) + + int ceph_conf_read_file(ceph_mount_info *cmount, const char *path_list) + int ceph_conf_parse_argv(ceph_mount_info *cmount, int argc, const char **argv) + int ceph_conf_get(ceph_mount_info *cmount, const char *option, char *buf, size_t len) + int ceph_conf_set(ceph_mount_info *cmount, const char *option, const char *value) + + int ceph_mount(ceph_mount_info *cmount, const char *root) + int ceph_stat(ceph_mount_info *cmount, const char *path, stat *stbuf) + int ceph_statfs(ceph_mount_info *cmount, const char *path, statvfs *stbuf) + + int ceph_mds_command(ceph_mount_info *cmount, const char *mds_spec, const char **cmd, size_t cmdlen, + const char *inbuf, size_t inbuflen, char **outbuf, size_t *outbuflen, + char **outs, size_t *outslen) + int ceph_rename(ceph_mount_info *cmount, const char *from_, const char *to) + int ceph_unlink(ceph_mount_info *cmount, const char *path) + int ceph_symlink(ceph_mount_info *cmount, const char *existing, const char *newname) + int ceph_setxattr(ceph_mount_info *cmount, const char *path, const char *name, + const void *value, size_t size, int flags) + int ceph_getxattr(ceph_mount_info *cmount, const char *path, const char *name, + void *value, size_t size) + int ceph_write(ceph_mount_info *cmount, int fd, const char *buf, int64_t size, int64_t offset) + int ceph_read(ceph_mount_info *cmount, int fd, char *buf, int64_t size, int64_t offset) + int ceph_close(ceph_mount_info *cmount, int fd) + int ceph_open(ceph_mount_info *cmount, const char *path, int flags, mode_t mode) + int ceph_mkdir(ceph_mount_info *cmount, const char *path, mode_t mode) + int ceph_mkdirs(ceph_mount_info *cmount, const char *path, mode_t mode) + int ceph_closedir(ceph_mount_info *cmount, ceph_dir_result *dirp) + int ceph_opendir(ceph_mount_info *cmount, const char *name, ceph_dir_result **dirpp) + int ceph_chdir(ceph_mount_info *cmount, const char *path) + dirent * ceph_readdir(ceph_mount_info *cmount, ceph_dir_result *dirp) + int ceph_rmdir(ceph_mount_info *cmount, const char *path) + const char* ceph_getcwd(ceph_mount_info *cmount) + int ceph_sync_fs(ceph_mount_info *cmount) + int ceph_conf_parse_argv(ceph_mount_info *cmount, int argc, const char **argv) + void ceph_buffer_free(char *buf) + + + +class Error(Exception): + pass + + +class PermissionError(Error): + pass + + +class ObjectNotFound(Error): + pass + + +class NoData(Error): + pass + + +class ObjectExists(Error): + pass + + +class IOError(Error): + pass + + +class NoSpace(Error): + pass + + +class InvalidValue(Error): + pass + + +class OperationNotSupported(Error): + pass + + +class IncompleteWriteError(Error): + pass + + +class LibCephFSStateError(Error): + pass + + +cdef errno_to_exception = { + errno.EPERM : PermissionError, + errno.ENOENT : ObjectNotFound, + errno.EIO : IOError, + errno.ENOSPC : NoSpace, + errno.EEXIST : ObjectExists, + errno.ENODATA : NoData, + errno.EINVAL : InvalidValue, + errno.EOPNOTSUPP: OperationNotSupported, +} + + +cdef make_ex(ret, msg): + """ + Translate a librados return code into an exception. + + :param ret: the return code + :type ret: int + :param msg: the error message to use + :type msg: str + :returns: a subclass of :class:`Error` + """ + ret = abs(ret) + if ret in errno_to_exception: + return errno_to_exception[ret](msg) + else: + return Error(msg + (": error code %d" % ret)) + + +class DirEntry(namedtuple('DirEntry', + ['d_ino', 'd_off', 'd_reclen', 'd_type', 'd_name'])): + DT_DIR = 0x4 + DT_REG = 0xA + DT_LNK = 0xC + def is_dir(self): + return self.d_type == self.DT_DIR + + def is_symbol_file(self): + return self.d_type == self.DT_LNK + + def is_file(self): + return self.d_type == self.DT_REG + +StatResult = namedtuple('StatResult', + ["st_dev", "st_ino", "st_mode", "st_nlink", "st_uid", + "st_gid", "st_rdev", "st_size", "st_blksize", + "st_blocks", "st_atime", "st_mtime", "st_ctime"]) + +cdef class DirResult(object): + cdef ceph_dir_result *handler + + +def cstr(val, name, encoding="utf-8", opt=False): + """ + Create a byte string from a Python string + + :param basestring val: Python string + :param str name: Name of the string parameter, for exceptions + :param str encoding: Encoding to use + :param bool opt: If True, None is allowed + :rtype: bytes + :raises: :class:`InvalidArgument` + """ + if opt and val is None: + return None + if isinstance(val, bytes): + return val + elif isinstance(val, unicode): + return val.encode(encoding) + else: + raise TypeError('%s must be a string' % name) + + +def cstr_list(list_str, name, encoding="utf-8"): + return [cstr(s, name) for s in list_str] + + +def decode_cstr(val, encoding="utf-8"): + """ + Decode a byte string into a Python string. + + :param bytes val: byte string + :rtype: unicode or None + """ + if val is None: + return None + + return val.decode(encoding) + + +cdef char* opt_str(s) except? NULL: + if s is None: + return NULL + return s + + +cdef char ** to_bytes_array(list_bytes): + cdef char **ret = malloc(len(list_bytes) * sizeof(char *)) + if ret == NULL: + raise MemoryError("malloc failed") + for i in xrange(len(list_bytes)): + ret[i] = list_bytes[i] + return ret + + +cdef void* realloc_chk(void* ptr, size_t size) except NULL: + cdef void *ret = realloc(ptr, size) + if ret == NULL: + raise MemoryError("realloc failed") + return ret + + +cdef class LibCephFS(object): + """libcephfs python wrapper""" + + cdef public object state + cdef ceph_mount_info *cluster + + def require_state(self, *args): + if self.state in args: + return + raise LibCephFSStateError("You cannot perform that operation on a " + "CephFS object in state %s." % (self.state)) + + def __cinit__(self, conf=None, conffile=None, auth_id=None, rados_inst=None): + PyEval_InitThreads() + self.state = "uninitialized" + if rados_inst is not None: + if auth_id is not None or conffile is not None or conf is not None: + raise InvalidValue("May not pass RADOS instance as well as other configuration") + + self.create_with_rados(rados_inst) + else: + self.create(conf, conffile, auth_id) + + def create_with_rados(self, rados.Rados rados_inst): + cdef int ret + with nogil: + ret = ceph_create_from_rados(&self.cluster, rados_inst.cluster) + if ret != 0: + raise Error("libcephfs_initialize failed with error code: %d" % ret) + self.state = "configuring" + + def create(self, conf=None, conffile=None, auth_id=None): + if conf is not None and not isinstance(conf, dict): + raise TypeError("conf must be dict or None") + cstr(conffile, 'configfile', opt=True) + auth_id = cstr(auth_id, 'configfile', opt=True) + + cdef: + char* _auth_id = opt_str(auth_id) + int ret + + with nogil: + ret = ceph_create(&self.cluster, _auth_id) + if ret != 0: + raise Error("libcephfs_initialize failed with error code: %d" % ret) + + self.state = "configuring" + if conffile is not None: + # read the default conf file when '' is given + if conffile == '': + conffile = None + self.conf_read_file(conffile) + if conf is not None: + for key, value in conf.iteritems(): + self.conf_set(key, value) + + def conf_read_file(self, conffile=None): + conffile = cstr(conffile, 'conffile', opt=True) + cdef: + char *_conffile = opt_str(conffile) + with nogil: + ret = ceph_conf_read_file(self.cluster, _conffile) + if ret != 0: + raise make_ex(ret, "error calling conf_read_file") + + def conf_parse_argv(self, argv): + self.require_state("configuring") + cargv = cstr_list(argv, 'argv') + cdef: + int _argc = len(argv) + char **_argv = to_bytes_array(cargv) + + try: + with nogil: + ret = ceph_conf_parse_argv(self.cluster, _argc, + _argv) + if ret != 0: + raise make_ex(ret, "error calling conf_parse_argv") + finally: + free(_argv) + + def shutdown(self): + """ + Unmount and destroy the ceph mount handle. + """ + if self.state in ["initialized", "mounted"]: + with nogil: + ceph_shutdown(self.cluster) + self.state = "shutdown" + + def __enter__(self): + self.mount() + return self + + def __exit__(self, type_, value, traceback): + self.shutdown() + return False + + def __dealloc__(self): + self.shutdown() + + def version(self): + """ + Get the version number of the ``libcephfs`` C library. + + :returns: a tuple of ``(major, minor, extra)`` components of the + libcephfs version + """ + cdef: + int major = 0 + int minor = 0 + int extra = 0 + with nogil: + ceph_version(&major, &minor, &extra) + return (major, minor, extra) + + def conf_get(self, option): + self.require_state("configuring", "initialized", "mounted") + + option = cstr(option, 'option') + cdef: + char *_option = option + size_t length = 20 + char *ret_buf = NULL + + try: + while True: + ret_buf = realloc_chk(ret_buf, length) + with nogil: + ret = ceph_conf_get(self.cluster, _option, ret_buf, length) + if (ret == 0): + return decode_cstr(ret_buf) + elif (ret == -errno.ENAMETOOLONG): + length = length * 2 + elif (ret == -errno.ENOENT): + return None + else: + raise make_ex(ret, "error calling conf_get") + finally: + free(ret_buf) + + def conf_set(self, option, val): + self.require_state("configuring", "initialized", "mounted") + + option = cstr(option, 'option') + val = cstr(val, 'val') + cdef: + char *_option = option + char *_val = val + + with nogil: + ret = ceph_conf_set(self.cluster, _option, _val) + if (ret != 0): + raise make_ex(ret, "error calling conf_set") + + def init(self): + self.require_state("configuring") + with nogil: + ret = ceph_init(self.cluster) + if ret != 0: + raise make_ex(ret, "error calling ceph_init") + self.state = "initialized" + + def mount(self): + if self.state == "configuring": + self.init() + self.require_state("initialized") + with nogil: + ret = ceph_mount(self.cluster, "/") + if ret != 0: + raise make_ex(ret, "error calling ceph_mount") + self.state = "mounted" + + def statfs(self, path): + self.require_state("mounted") + path = cstr(path, 'path') + cdef: + char* _path = path + statvfs statbuf + + with nogil: + ret = ceph_statfs(self.cluster, _path, &statbuf) + if ret < 0: + raise make_ex(ret, "statfs failed: %s" % path) + return {'f_bsize': statbuf.f_bsize, + 'f_frsize': statbuf.f_frsize, + 'f_blocks': statbuf.f_blocks, + 'f_bfree': statbuf.f_bfree, + 'f_bavail': statbuf.f_bavail, + 'f_files': statbuf.f_files, + 'f_ffree': statbuf.f_ffree, + 'f_favail': statbuf.f_favail, + 'f_fsid': statbuf.f_fsid, + 'f_flag': statbuf.f_flag, + 'f_namemax': statbuf.f_namemax} + + def sync_fs(self): + self.require_state("mounted") + with nogil: + ret = ceph_sync_fs(self.cluster) + if ret < 0: + raise make_ex(ret, "sync_fs failed") + + def getcwd(self): + self.require_state("mounted") + with nogil: + ret = ceph_getcwd(self.cluster) + return ret + + def chdir(self, path): + self.require_state("mounted") + + path = cstr(path, 'path') + cdef char* _path = path + with nogil: + ret = ceph_chdir(self.cluster, _path) + if ret < 0: + raise make_ex(ret, "chdir failed") + + def opendir(self, path): + self.require_state("mounted") + + path = cstr(path, 'path') + cdef: + char* _path = path + ceph_dir_result *dir_handler + with nogil: + ret = ceph_opendir(self.cluster, _path, &dir_handler); + if ret < 0: + raise make_ex(ret, "opendir failed") + d = DirResult() + d.handler = dir_handler + return d + + def readdir(self, DirResult dir_handler): + self.require_state("mounted") + + cdef ceph_dir_result *_dir_handler = dir_handler.handler + with nogil: + dirent = ceph_readdir(self.cluster, _dir_handler) + if not dirent: + return None + + return DirEntry(d_ino=dirent.d_ino, + d_off=dirent.d_off, + d_reclen=dirent.d_reclen, + d_type=dirent.d_type, + d_name=dirent.d_name) + + def closedir(self, DirResult dir_handler): + self.require_state("mounted") + cdef: + ceph_dir_result *_dir_handler = dir_handler.handler + + with nogil: + ret = ceph_closedir(self.cluster, _dir_handler) + if ret < 0: + raise make_ex(ret, "closedir failed") + + def mkdir(self, path, mode): + self.require_state("mounted") + path = cstr(path, 'path') + if not isinstance(mode, int): + raise TypeError('mode must be an int') + cdef: + char* _path = path + int _mode = mode + with nogil: + ret = ceph_mkdir(self.cluster, _path, _mode) + if ret < 0: + raise make_ex(ret, "error in mkdir '%s'" % path) + + def mkdirs(self, path, mode): + self.require_state("mounted") + path = cstr(path, 'path') + if not isinstance(mode, int): + raise TypeError('mode must be an int') + cdef: + char* _path = path + int _mode = mode + + with nogil: + ret = ceph_mkdirs(self.cluster, _path, _mode) + if ret < 0: + raise make_ex(ret, "error in mkdirs '%s'" % path) + + def rmdir(self, path): + self.require_state("mounted") + path = cstr(path, 'path') + cdef char* _path = path + ret = ceph_rmdir(self.cluster, _path) + if ret < 0: + raise make_ex(ret, "error in rmdir '%s'" % path) + + def open(self, path, flags, mode=0): + self.require_state("mounted") + + path = cstr(path, 'path') + flags = cstr(flags, 'flags') + if not isinstance(mode, int): + raise TypeError('mode must be an int') + cephfs_flags = 0 + if flags == '': + cephfs_flags = os.O_RDONLY + else: + for c in flags: + if c == 'r': + cephfs_flags |= os.O_RDONLY + elif c == 'w': + cephfs_flags |= os.O_WRONLY | os.O_TRUNC | os.O_CREAT + elif c == '+': + cephfs_flags |= os.O_RDWR + else: + raise OperationNotSupported( + "open flags doesn't support %s" % c) + + cdef: + char* _path = path + int _flags = cephfs_flags + int _mode = mode + + with nogil: + ret = ceph_open(self.cluster, _path, _flags, _mode) + if ret < 0: + raise make_ex(ret, "error in open '%s'" % path) + return ret + + def close(self, fd): + self.require_state("mounted") + if not isinstance(fd, int): + raise TypeError('fd must be an int') + cdef int _fd = fd + with nogil: + ret = ceph_close(self.cluster, _fd) + if ret < 0: + raise make_ex(ret, "error in close") + + def read(self, fd, offset, l): + self.require_state("mounted") + if not isinstance(offset, int): + raise TypeError('offset must be an int') + if not isinstance(l, int): + raise TypeError('l must be an int') + if not isinstance(fd, int): + raise TypeError('fd must be an int') + cdef: + int _fd = fd + int64_t _offset = offset + int64_t _length = l + + char *ret_buf + PyObject* ret_s = NULL + + ret_s = PyBytes_FromStringAndSize(NULL, _length) + try: + ret_buf = PyBytes_AsString(ret_s) + with nogil: + ret = ceph_read(self.cluster, _fd, ret_buf, _length, _offset) + if ret < 0: + raise make_ex(ret, "error in read") + + if ret != _length: + _PyBytes_Resize(&ret_s, ret) + + return ret_s + finally: + # We DECREF unconditionally: the cast to object above will have + # INCREFed if necessary. This also takes care of exceptions, + # including if _PyString_Resize fails (that will free the string + # itself and set ret_s to NULL, hence XDECREF). + ref.Py_XDECREF(ret_s) + + def write(self, fd, buf, offset): + self.require_state("mounted") + if not isinstance(fd, int): + raise TypeError('fd must be an int') + if not isinstance(buf, bytes): + raise TypeError('buf must be a bytes') + if not isinstance(offset, int): + raise TypeError('offset must be an int') + + cdef: + int _fd = fd + char *_data = buf + int64_t _offset = offset + + size_t length = len(buf) + + with nogil: + ret = ceph_write(self.cluster, _fd, _data, length, _offset) + if ret < 0: + raise make_ex(ret, "error in write") + return ret + + def getxattr(self, path, name): + self.require_state("mounted") + + path = cstr(path, 'path') + name = cstr(name, 'name') + + cdef: + char* _path = path + char* _name = name + + size_t ret_length = 255 + char *ret_buf = NULL + + try: + while True: + ret_buf = realloc_chk(ret_buf, ret_length) + with nogil: + ret = ceph_getxattr(self.cluster, _path, _name, ret_buf, ret_length) + if ret < 0: + raise make_ex(ret, "error in getxattr") + elif ret > ret_length: + ret_length = ret + else: + return ret_buf[:ret] + finally: + free(ret_buf) + + def setxattr(self, path, name, value, flags): + self.require_state("mounted") + + name = cstr(name, 'name') + path = cstr(path, 'path') + if not isinstance(flags, int): + raise TypeError('flags must be a int') + if not isinstance(value, bytes): + raise TypeError('value must be a bytes') + + cdef: + char *_path = path + char *_name = name + char *_value = value + size_t _value_len = len(value) + int _flags = flags + + with nogil: + ret = ceph_setxattr(self.cluster, _path, _name, + _value, _value_len, _flags) + if ret < 0: + raise make_ex(ret, "error in setxattr") + + def stat(self, path): + self.require_state("mounted") + path = cstr(path, 'path') + + cdef: + char* _path = path + stat statbuf + + with nogil: + ret = ceph_stat(self.cluster, _path, &statbuf) + if ret < 0: + raise make_ex(ret, "error in stat: %s" % path) + return StatResult(st_dev=statbuf.st_dev, st_ino=statbuf.st_ino, + st_mode=statbuf.st_mode, st_nlink=statbuf.st_nlink, + st_uid=statbuf.st_uid, st_gid=statbuf.st_gid, + st_rdev=statbuf.st_rdev, st_size=statbuf.st_size, + st_blksize=statbuf.st_blksize, + st_blocks=statbuf.st_blocks, + st_atime=datetime.fromtimestamp(statbuf.st_atime), + st_mtime=datetime.fromtimestamp(statbuf.st_mtime), + st_ctime=datetime.fromtimestamp(statbuf.st_ctime)) + + def symlink(self, existing, newname): + self.require_state("mounted") + existing = cstr(existing, 'existing') + newname = cstr(newname, 'newname') + cdef: + char* _existing = existing + char* _newname = newname + + with nogil: + ret = ceph_symlink(self.cluster, _existing, _newname) + if ret < 0: + raise make_ex(ret, "error in symlink") + + def unlink(self, path): + self.require_state("mounted") + path = cstr(path, 'path') + cdef char* _path = path + with nogil: + ret = ceph_unlink(self.cluster, _path) + if ret < 0: + raise make_ex(ret, "error in unlink: %s" % path) + + def rename(self, src, dst): + self.require_state("mounted") + + src = cstr(src, 'source') + dst = cstr(dst, 'destination') + + cdef: + char* _src = src + char* _dst = dst + + with nogil: + ret = ceph_rename(self.cluster, _src, _dst) + if ret < 0: + raise make_ex(ret, "error in rename '%s' to '%s'" % (src, dst)) + + def mds_command(self, mds_spec, args, input_data): + """ + :return 3-tuple of output status int, output status string, output data + """ + mds_spec = cstr(mds_spec, 'mds_spec') + args = cstr_list(args, 'args') + input_data = cstr(input_data, 'input_data') + + cdef: + char *_mds_spec = opt_str(mds_spec) + char **_cmd = to_bytes_array(args) + size_t _cmdlen = len(args) + + char *_inbuf = input_data + size_t _inbuf_len = len(input_data) + + char *_outbuf + size_t _outbuf_len + char *_outs + size_t _outs_len + + try: + with nogil: + ret = ceph_mds_command(self.cluster, _mds_spec, + _cmd, _cmdlen, + _inbuf, _inbuf_len, + &_outbuf, &_outbuf_len, + &_outs, &_outs_len) + if ret == 0: + my_outs = decode_cstr(_outs[:_outs_len]) + my_outbuf = _outbuf[:_outbuf_len] + if _outs_len: + ceph_buffer_free(_outs) + if _outbuf_len: + ceph_buffer_free(_outbuf) + return (ret, my_outbuf, my_outs) + else: + return (ret, b"", "") + finally: + free(_cmd) + diff --git a/src/pybind/cephfs/setup.py b/src/pybind/cephfs/setup.py new file mode 100755 index 0000000000000..cf29229dc3c33 --- /dev/null +++ b/src/pybind/cephfs/setup.py @@ -0,0 +1,53 @@ +# Largely taken from +# https://blog.kevin-brown.com/programming/2014/09/24/combining-autotools-and-setuptools.html +import os, sys, os.path + +from setuptools.command.egg_info import egg_info +from distutils.core import setup +from distutils.extension import Extension +from Cython.Build import cythonize + +def get_version(): + try: + for line in open(os.path.join(os.path.dirname(__file__), "..", "ceph_ver.h")): + if "CEPH_GIT_NICE_VER" in line: + return line.split()[2][1:-1] + else: + return "0" + except IOError: + return "0" + +class EggInfoCommand(egg_info): + def finalize_options(self): + egg_info.finalize_options(self) + if "build" in self.distribution.command_obj: + build_command = self.distribution.command_obj["build"] + self.egg_base = build_command.build_base + self.egg_info = os.path.join(self.egg_base, os.path.basename(self.egg_info)) + +# Disable cythonification if we're not really building anything +if (len(sys.argv) >= 2 and + any(i in sys.argv[1:] for i in ('--help', 'clean', 'egg_info', '--version') + )): + def cythonize(x, **kwargs): + return x + +setup( + name = 'cephfs', + version = get_version(), + description = "Python libraries for the Ceph libcephfs library", + long_description = ( + "This package contains Python libraries for interacting with Ceph's " + "cephfs library."), + ext_modules = cythonize([ + Extension("cephfs", + ["cephfs.pyx"], + libraries=["cephfs"] + ) + ], build_dir=os.environ.get("CYTHON_BUILD_DIR", None), include_path=[ + os.path.join(os.path.dirname(__file__), "..", "rados")] + ), + cmdclass={ + "egg_info": EggInfoCommand, + }, +) diff --git a/src/pybind/rados/rados.pyx b/src/pybind/rados/rados.pyx index ad40e4bddd27a..7ffdbbc2e3f84 100644 --- a/src/pybind/rados/rados.pyx +++ b/src/pybind/rados/rados.pyx @@ -13,13 +13,11 @@ method. # Copyright 2015 Hector Martin # Copyright 2016 Mehdi Abaakouk -from cpython cimport PyObject, ref, exc, array -from cpython.dict cimport PyDict_New +from cpython cimport PyObject, ref from libc cimport errno from libc.stdint cimport * from libc.stdlib cimport malloc, realloc, free -import array import sys import threading import time diff --git a/src/test/pybind/test_cephfs.py b/src/test/pybind/test_cephfs.py index bd813adb829ec..43a3646ad05d1 100644 --- a/src/test/pybind/test_cephfs.py +++ b/src/test/pybind/test_cephfs.py @@ -13,6 +13,10 @@ def teardown_module(): global cephfs cephfs.shutdown() +def test_conf_get(): + fsid = cephfs.conf_get("fsid") + assert(fsid != "") + def test_version(): cephfs.version() @@ -25,8 +29,11 @@ def test_syncfs(): def test_directory(): cephfs.mkdir("/temp-directory", 0755) + cephfs.mkdirs("/temp-directory/foo/bar", 0755) cephfs.chdir("/temp-directory") assert_equal(cephfs.getcwd(), "/temp-directory") + cephfs.rmdir("/temp-directory/foo/bar") + cephfs.rmdir("/temp-directory/foo") cephfs.rmdir("/temp-directory") assert_raises(libcephfs.ObjectNotFound, cephfs.chdir, "/temp-directory") @@ -52,6 +59,12 @@ def test_xattr(): assert_raises(libcephfs.OperationNotSupported, cephfs.setxattr, "/", "key", "value", 0) cephfs.setxattr("/", "user.key", "value", 0) assert_equal("value", cephfs.getxattr("/", "user.key")) + cephfs.setxattr("/", "user.big", "" * 300, 0) + # NOTE(sileht): this actually doesn't work, cephfs returns errno -34 + # when we retrieve big xattr value, at least on my setup + # The old python binding was not checking the return code and was + # returning a empty string in that case. + assert_equal(300, len(cephfs.getxattr("/", "user.big"))) def test_rename(): cephfs.mkdir("/a", 0755)