export LD_LIBRARY_PATH="$vdir/lib"
export PYTHONPATH=$TOPDIR/src/pybind
-ln -sf librados.so.2 $vdir/lib/librados.so
-gcc -shared -o $vdir/lib/librados.so.2 -xc /dev/null
-BUILD_DOC=1 \
- CFLAGS="-iquote$TOPDIR/src/include" \
- CPPFLAGS="-iquote$TOPDIR/src/include" \
- LDFLAGS="-L$vdir/lib -Wl,--no-as-needed" \
- $vdir/bin/pip install $TOPDIR/src/pybind/rados
-nm $vdir/lib/python*/*-packages/rados.so | grep 'U rados_' | \
- awk '{ print "void "$2"(void) {}" }' | \
- gcc -shared -o $vdir/lib/librados.so.2 -xc -
-
# FIXME(sileht): I dunno how to pass the include-dirs correctly with pip
# for build_ext step, it should be:
# --global-option=build_ext --global-option="--cython-include-dirs $TOPDIR/src/pybind/rados/"
# but that doesn't work, so copying the file in the rbd module directly, that's ok for docs
-cp -f $TOPDIR/src/pybind/rados/rados.pxd $TOPDIR/src/pybind/rbd/
-cp -f $TOPDIR/src/pybind/rados/rados.pxd $TOPDIR/src/pybind/cephfs/
-
-ln -sf librbd.so.1 $vdir/lib/librbd.so
-gcc -shared -o $vdir/lib/librbd.so.1 -xc /dev/null
-BUILD_DOC=1 \
- CFLAGS="-iquote$TOPDIR/src/include" \
- CPPFLAGS="-iquote$TOPDIR/src/include" \
- LDFLAGS="-L$vdir/lib -Wl,--no-as-needed" \
- $vdir/bin/pip install $TOPDIR/src/pybind/rbd
-nm $vdir/lib/python*/*-packages/rbd.so | grep 'U rbd_' | \
- awk '{ print "void "$2"(void) {}" }' | \
- gcc -shared -o $vdir/lib/librbd.so.1 -xc -
-
-
-ln -sf libcephfs.so.1 $vdir/lib/libcephfs.so
-gcc -shared -o $vdir/lib/libcephfs.so.1 -xc /dev/null
-BUILD_DOC=1 \
- CFLAGS="-iquote$TOPDIR/src/include" \
- CPPFLAGS="-iquote$TOPDIR/src/include" \
- LDFLAGS="-L$vdir/lib -Wl,--no-as-needed" \
- $vdir/bin/pip install $TOPDIR/src/pybind/cephfs
-nm $vdir/lib/python*/*-packages/cephfs.so | grep 'U cephfs_' | \
- awk '{ print "void "$2"(void) {}" }' | \
- gcc -shared -o $vdir/lib/libcephfs.so.1 -xc -
-
-rm -f $TOPDIR/src/pybind/rbd/rados.pxd $TOPDIR/src/pybind/cephfs/rados.pxd
-
+for bind in rados rbd cephfs rgw; do
+ if [ ${bind} != rados ]; then
+ cp -f $TOPDIR/src/pybind/rados/rados.pxd $TOPDIR/src/pybind/${bind}/
+ fi
+ ln -sf lib${bind}.so.1 $vdir/lib/lib${bind}.so
+ gcc -shared -o $vdir/lib/lib${bind}.so.1 -xc /dev/null
+ BUILD_DOC=1 \
+ CFLAGS="-iquote$TOPDIR/src/include" \
+ CPPFLAGS="-iquote$TOPDIR/src/include" \
+ LDFLAGS="-L$vdir/lib -Wl,--no-as-needed" \
+ $vdir/bin/pip install $TOPDIR/src/pybind/${bind}
+ # rgwfile_version(), librgw_create(), rgw_mount()
+ nm $vdir/lib/python*/*-packages/${bind}.so | grep -E "U (lib)?${bind}" | \
+ awk '{ print "void "$2"(void) {}" }' | \
+ gcc -shared -o $vdir/lib/lib${bind}.so.1 -xc -
+ if [ ${bind} != rados ]; then
+ rm -f $TOPDIR/src/pybind/${bind}/rados.pxd
+ fi
+done
$vdir/bin/sphinx-build -a -n -b dirhtml -d doctrees $TOPDIR/doc $TOPDIR/build-doc/output/html
$vdir/bin/sphinx-build -a -b man -t man -d doctrees $TOPDIR/doc $TOPDIR/build-doc/output/man
.. _librbdpy: ../rbd/librbdpy
+Ceph RADOS Gateway APIs
+=======================
+
+See `librgw-py`_.
+
+.. _librgw-py: ../radosgw/api
Calamari APIs
=============
--- /dev/null
+===============
+librgw (Python)
+===============
+
+.. highlight:: python
+
+The `rgw` python module provides file-like access to rgw.
+
+API Reference
+=============
+
+.. automodule:: rgw
+ :members: LibRGWFS, FileHandle
+
--- /dev/null
+"""
+This module is a thin wrapper around rgw_file.
+"""
+
+
+from cpython cimport PyObject, ref, exc, array
+from libc.stdint cimport *
+from libcpp cimport bool
+from libc.stdlib cimport malloc, realloc, free
+
+cimport rados
+
+from collections import namedtuple
+from datetime import datetime
+import errno
+
+
+cdef extern from "Python.h":
+ # These are in cpython/string.pxd, but use "object" types instead of
+ # PyObject*, which invokes assumptions in cpython that we need to
+ # legitimately break to implement zero-copy string buffers in Image.read().
+ # This is valid use of the Python API and documented as a special case.
+ PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL
+ char* PyBytes_AsString(PyObject *string) except NULL
+ int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1
+ void PyEval_InitThreads()
+
+
+cdef extern from "time.h":
+ ctypedef long int time_t
+
+
+cdef extern from "sys/stat.h":
+ cdef struct stat:
+ unsigned long st_dev
+ unsigned long st_ino
+ unsigned long st_nlink
+ unsigned int st_mode
+ unsigned int st_uid
+ unsigned int st_gid
+ int __pad0
+ unsigned long st_rdev
+ long int st_size
+ long int st_blksize
+ long int st_blocks
+ time_t st_atime
+ time_t st_mtime
+ time_t st_ctime
+
+
+cdef extern from "rados/librgw.h" nogil:
+ ctypedef void* librgw_t
+
+ int librgw_create(librgw_t *rgw, int argc, char **argv)
+ void librgw_shutdown(librgw_t rgw)
+
+
+cdef extern from "rados/rgw_file.h" nogil:
+ enum:
+ RGW_FS_TYPE_FILE
+ RGW_FS_TYPE_DIRECTORY
+
+ RGW_LOOKUP_FLAG_CREATE
+
+ RGW_SETATTR_MODE
+ RGW_SETATTR_UID
+ RGW_SETATTR_GID
+ RGW_SETATTR_MTIME
+ RGW_SETATTR_ATIME
+ RGW_SETATTR_SIZE
+ RGW_SETATTR_CTIME
+
+ RGW_READDIR_FLAG_NONE
+ RGW_READDIR_FLAG_DOTDOT
+
+ RGW_OPEN_FLAG_CREATE
+ RGW_OPEN_FLAG_V3 # ops have v3 semantics
+ RGW_OPEN_FLAG_STATELESS # alias it
+
+ RGW_CLOSE_FLAG_RELE
+
+
+ ctypedef void *rgw_fh_hk
+ cdef struct rgw_file_handle:
+ pass
+
+ cdef struct rgw_fs:
+ librgw_t rgw
+ void *fs_private
+ void *root_fh
+
+ # mount info hypothetical--emulate Unix, support at least UUID-length fsid
+ cdef struct rgw_statvfs:
+ uint64_t f_bsize # file system block size
+ uint64_t f_frsize # fragment size
+ uint64_t f_blocks # size of fs in f_frsize units
+ uint64_t f_bfree # free blocks
+ uint64_t f_bavail # free blocks for unprivileged users
+ uint64_t f_files # inodes
+ uint64_t f_ffree # free inodes
+ uint64_t f_favail # free inodes for unprivileged users
+ uint64_t f_fsid[2] # /* file system ID
+ uint64_t f_flag # mount flags
+ uint64_t f_namemax # maximum filename length
+
+ void rgwfile_version(int *major, int *minor, int *extra)
+
+ int rgw_lookup(rgw_fs *fs,
+ rgw_file_handle *parent_fh, const char *path,
+ rgw_file_handle **fh, uint32_t flags)
+
+ int rgw_lookup_handle(rgw_fs *fs, rgw_fh_hk *fh_hk,
+ rgw_file_handle **fh, uint32_t flags)
+
+ int rgw_fh_rele(rgw_fs *fs, rgw_file_handle *fh,
+ uint32_t flags)
+
+ int rgw_mount(librgw_t rgw, const char *uid, const char *key,
+ const char *secret, rgw_fs **fs, uint32_t flags)
+
+ int rgw_umount(rgw_fs *fs, uint32_t flags)
+
+ int rgw_statfs(rgw_fs *fs, rgw_file_handle *parent_fh,
+ rgw_statvfs *vfs_st, uint32_t flags)
+
+ int rgw_create(rgw_fs *fs, rgw_file_handle *parent_fh,
+ const char *name, stat *st, uint32_t mask,
+ rgw_file_handle **fh, uint32_t posix_flags,
+ uint32_t flags)
+
+ int rgw_mkdir(rgw_fs *fs,
+ rgw_file_handle *parent_fh,
+ const char *name, stat *st, uint32_t mask,
+ rgw_file_handle **fh, uint32_t flags)
+
+ int rgw_rename(rgw_fs *fs,
+ rgw_file_handle *olddir, const char* old_name,
+ rgw_file_handle *newdir, const char* new_name,
+ uint32_t flags)
+
+ int rgw_unlink(rgw_fs *fs,
+ rgw_file_handle *parent_fh, const char* path,
+ uint32_t flags)
+
+ int rgw_readdir(rgw_fs *fs,
+ rgw_file_handle *parent_fh, uint64_t *offset,
+ bool (*cb)(const char *name, void *arg, uint64_t offset) nogil except? -9000,
+ void *cb_arg, bool *eof, uint32_t flags) except? -9000
+
+ int rgw_getattr(rgw_fs *fs,
+ rgw_file_handle *fh, stat *st,
+ uint32_t flags)
+
+ int rgw_setattr(rgw_fs *fs, rgw_file_handle *fh, stat *st,
+ uint32_t mask, uint32_t flags)
+
+ int rgw_truncate(rgw_fs *fs, rgw_file_handle *fh, uint64_t size, uint32_t flags)
+
+ int rgw_open(rgw_fs *fs, rgw_file_handle *parent_fh,
+ uint32_t posix_flags, uint32_t flags)
+
+ int rgw_close(rgw_fs *fs, rgw_file_handle *fh,
+ uint32_t flags)
+
+ int rgw_read(rgw_fs *fs,
+ rgw_file_handle *fh, uint64_t offset,
+ size_t length, size_t *bytes_read, void *buffer,
+ uint32_t flags)
+
+ int rgw_write(rgw_fs *fs,
+ rgw_file_handle *fh, uint64_t offset,
+ size_t length, size_t *bytes_written, void *buffer,
+ uint32_t flags)
+
+ int rgw_fsync(rgw_fs *fs, rgw_file_handle *fh,
+ uint32_t flags)
+
+ int rgw_commit(rgw_fs *fs, rgw_file_handle *fh,
+ uint64_t offset, uint64_t length, uint32_t flags)
+
+
+class Error(Exception):
+ pass
+
+
+class PermissionError(Error):
+ pass
+
+
+class ObjectNotFound(Error):
+ pass
+
+
+class NoData(Error):
+ pass
+
+
+class ObjectExists(Error):
+ pass
+
+
+class IOError(Error):
+ pass
+
+
+class NoSpace(Error):
+ pass
+
+
+class InvalidValue(Error):
+ pass
+
+
+class OperationNotSupported(Error):
+ pass
+
+
+class IncompleteWriteError(Error):
+ pass
+
+
+class LibCephFSStateError(Error):
+ pass
+
+class WouldBlock(Error):
+ pass
+
+class OutOfRange(Error):
+ pass
+
+cdef errno_to_exception = {
+ errno.EPERM : PermissionError,
+ errno.ENOENT : ObjectNotFound,
+ errno.EIO : IOError,
+ errno.ENOSPC : NoSpace,
+ errno.EEXIST : ObjectExists,
+ errno.ENODATA : NoData,
+ errno.EINVAL : InvalidValue,
+ errno.EOPNOTSUPP : OperationNotSupported,
+ errno.ERANGE : OutOfRange,
+ errno.EWOULDBLOCK: WouldBlock,
+}
+
+
+cdef class FileHandle(object):
+ cdef rgw_file_handle *handler
+
+
+StatResult = namedtuple('StatResult',
+ ["st_dev", "st_ino", "st_mode", "st_nlink", "st_uid",
+ "st_gid", "st_rdev", "st_size", "st_blksize",
+ "st_blocks", "st_atime", "st_mtime", "st_ctime"])
+
+
+def cstr(val, name, encoding="utf-8", opt=False):
+ """
+ Create a byte string from a Python string
+
+ :param basestring val: Python string
+ :param str name: Name of the string parameter, for exceptions
+ :param str encoding: Encoding to use
+ :param bool opt: If True, None is allowed
+ :rtype: bytes
+ :raises: :class:`InvalidArgument`
+ """
+ if opt and val is None:
+ return None
+ if isinstance(val, bytes):
+ return val
+ elif isinstance(val, unicode):
+ return val.encode(encoding)
+ else:
+ raise TypeError('%s must be a string' % name)
+
+
+cdef make_ex(ret, msg):
+ """
+ Translate a librados return code into an exception.
+
+ :param ret: the return code
+ :type ret: int
+ :param msg: the error message to use
+ :type msg: str
+ :returns: a subclass of :class:`Error`
+ """
+ ret = abs(ret)
+ if ret in errno_to_exception:
+ return errno_to_exception[ret](msg)
+ else:
+ return Error(msg + (": error code %d" % ret))
+
+
+cdef bool readdir_cb(const char *name, void *arg, uint64_t offset) \
+except? -9000 with gil:
+ if exc.PyErr_Occurred():
+ return False
+ (<object>arg)(name, offset)
+ return True
+
+
+class LibCephFSStateError(Error):
+ pass
+
+
+cdef class LibRGWFS(object):
+ """librgwfs python wrapper"""
+
+ cdef public object state
+ cdef public object uid
+ cdef public object key
+ cdef public object secret
+ cdef librgw_t cluster
+ cdef rgw_fs *fs
+
+ def require_state(self, *args):
+ if self.state in args:
+ return
+ raise LibCephFSStateError("You cannot perform that operation on a "
+ "RGWFS object in state %s." % (self.state))
+
+ def __cinit__(self, uid, key, secret):
+ PyEval_InitThreads()
+ self.state = "umounted"
+ ret = librgw_create(&self.cluster, 0, NULL)
+ if ret != 0:
+ raise make_ex(ret, "error calling librgw_create")
+ self.uid = cstr(uid, "uid")
+ self.key = cstr(key, "key")
+ self.secret = cstr(secret, "secret")
+
+ def shutdown(self):
+ """
+ Unmount and destroy the ceph mount handle.
+ """
+ if self.state in ["mounted"]:
+ with nogil:
+ ret = rgw_umount(self.fs, 0);
+ if ret != 0:
+ raise make_ex(ret, "error calling rgw_unmount")
+ self.state = "shutdown"
+
+ def __enter__(self):
+ self.mount()
+ return self
+
+ def __exit__(self, type_, value, traceback):
+ self.shutdown()
+
+ def __dealloc__(self):
+ self.shutdown()
+
+ def version(self):
+ """
+ Get the version number of the ``librgwfile`` C library.
+
+ :returns: a tuple of ``(major, minor, extra)`` components of the
+ libcephfs version
+ """
+ cdef:
+ int major = 0
+ int minor = 0
+ int extra = 0
+ with nogil:
+ rgwfile_version(&major, &minor, &extra)
+ return (major, minor, extra)
+
+ def mount(self):
+ self.require_state("umounted")
+ cdef:
+ char *_uid = self.uid
+ char *_key = self.key
+ char *_secret = self.secret
+ with nogil:
+ ret = rgw_mount(self.cluster, <const char*>_uid, <const char*>_key,
+ <const char*>_secret, &self.fs, 0)
+ if ret != 0:
+ raise make_ex(ret, "error calling rgw_mount")
+ self.state = "mounted"
+ dir_handler = FileHandle()
+ dir_handler.handler = <rgw_file_handle*>self.fs.root_fh
+ return dir_handler
+
+ def unmount(self):
+ self.require_state("mounted")
+ with nogil:
+ ret = rgw_umount(self.fs, 0)
+ if ret != 0:
+ raise make_ex(ret, "error calling rgw_umount")
+ self.state = "umounted"
+
+ def statfs(self):
+ self.require_state("mounted")
+ cdef:
+ rgw_statvfs statbuf
+
+ with nogil:
+ ret = rgw_statfs(self.fs, <rgw_file_handle*>self.fs.root_fh, &statbuf, 0)
+ if ret < 0:
+ raise make_ex(ret, "statfs failed")
+ cdef uint64_t[:] fsid = statbuf.f_fsid
+ return {'f_bsize': statbuf.f_bsize,
+ 'f_frsize': statbuf.f_frsize,
+ 'f_blocks': statbuf.f_blocks,
+ 'f_bfree': statbuf.f_bfree,
+ 'f_bavail': statbuf.f_bavail,
+ 'f_files': statbuf.f_files,
+ 'f_ffree': statbuf.f_ffree,
+ 'f_favail': statbuf.f_favail,
+ 'f_fsid': fsid,
+ 'f_flag': statbuf.f_flag,
+ 'f_namemax': statbuf.f_namemax}
+
+
+ def create(self, FileHandle dir_handler, filename, flags = 0):
+ self.require_state("mounted")
+
+ if not isinstance(flags, int):
+ raise TypeError("flags must be an integer")
+
+ filename = cstr(filename, 'filename')
+
+ cdef:
+ rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
+ rgw_file_handle *_file_handler
+ int _flags = flags
+ char* _filename = filename
+ stat statbuf
+
+ with nogil:
+ ret = rgw_create(self.fs, _dir_handler, _filename, &statbuf, 0,
+ &_file_handler, 0, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in create '%s'" % filename)
+ with nogil:
+ ret = rgw_open(self.fs, _file_handler, 0, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in open '%s'" % filename)
+
+ file_handler = FileHandle()
+ file_handler.handler = _file_handler
+ return file_handler
+
+ def mkdir(self, FileHandle dir_handler, dirname, flags = 0):
+ self.require_state("mounted")
+ dirname = cstr(dirname, 'dirname')
+ new_dir_handler = FileHandle()
+ cdef:
+ rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
+ rgw_file_handle *_new_dir_handler
+ char* _dirname = dirname
+ int _flags = flags
+ stat statbuf
+ with nogil:
+ ret = rgw_mkdir(self.fs, _dir_handler, _dirname, &statbuf,
+ 0, &_new_dir_handler, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in mkdir '%s'" % dirname)
+ new_dir_handler.handler = _new_dir_handler
+ return new_dir_handler
+
+ def rename(self, FileHandle src_handler, src_name, FileHandle dst_handler, dst_name, flags = 0):
+ self.require_state("mounted")
+
+ src_name = cstr(src_name, 'src_name')
+ dst_name = cstr(dst_name, 'dst_name')
+
+ cdef:
+ rgw_file_handle *_src_dir_handler = <rgw_file_handle*>src_handler.handler
+ rgw_file_handle *_dst_dir_handler = <rgw_file_handle*>dst_handler.handler
+ char* _src_name = src_name
+ char* _dst_name = dst_name
+ int _flags = flags
+
+ with nogil:
+ ret = rgw_rename(self.fs, _src_dir_handler, _src_name,
+ _dst_dir_handler, _dst_name, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in rename '%s' to '%s'" % (src_name,
+ dst_name))
+ return ret
+
+ def unlink(self, FileHandle handler, name, flags = 0):
+ self.require_state("mounted")
+ name = cstr(name, 'name')
+ cdef:
+ rgw_file_handle *_handler = <rgw_file_handle*>handler.handler
+ int _flags = flags
+ char* _name = name
+ with nogil:
+ ret = rgw_unlink(self.fs, _handler, _name, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in unlink")
+ return ret
+
+ def readdir(self, FileHandle dir_handler, iterate_cb, offset, flags = 0):
+ self.require_state("mounted")
+
+ cdef:
+ rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
+ uint64_t _offset = offset
+ bool _eof
+ uint32_t _flags = flags
+ with nogil:
+ ret = rgw_readdir(self.fs, _dir_handler, &_offset, &readdir_cb,
+ <void *>iterate_cb, &_eof, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in readdir")
+
+ return (_offset, _eof)
+
+ def fstat(self, FileHandle file_handler):
+ self.require_state("mounted")
+
+ cdef:
+ rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
+ stat statbuf
+
+ with nogil:
+ ret = rgw_getattr(self.fs, _file_handler, &statbuf, 0)
+ if ret < 0:
+ raise make_ex(ret, "error in getattr")
+ return StatResult(st_dev=statbuf.st_dev, st_ino=statbuf.st_ino,
+ st_mode=statbuf.st_mode, st_nlink=statbuf.st_nlink,
+ st_uid=statbuf.st_uid, st_gid=statbuf.st_gid,
+ st_rdev=statbuf.st_rdev, st_size=statbuf.st_size,
+ st_blksize=statbuf.st_blksize,
+ st_blocks=statbuf.st_blocks,
+ st_atime=datetime.fromtimestamp(statbuf.st_atime),
+ st_mtime=datetime.fromtimestamp(statbuf.st_mtime),
+ st_ctime=datetime.fromtimestamp(statbuf.st_ctime))
+
+ def opendir(self, FileHandle dir_handler, dirname, flags = 0):
+ self.require_state("mounted")
+
+ if not isinstance(flags, int):
+ raise TypeError("flags must be an integer")
+
+ dirname = cstr(dirname, 'dirname')
+
+ cdef:
+ rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
+ rgw_file_handle *_file_handler
+ int _flags = flags
+ char* _dirname = dirname
+
+ with nogil:
+ ret = rgw_lookup(self.fs, _dir_handler, _dirname,
+ &_file_handler, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in open '%s'" % dirname)
+
+ file_handler = FileHandle()
+ file_handler.handler = _file_handler
+ return file_handler
+
+ def open(self, FileHandle dir_handler, filename, flags = 0):
+ self.require_state("mounted")
+
+ if not isinstance(flags, int):
+ raise TypeError("flags must be an integer")
+
+ filename = cstr(filename, 'filename')
+
+ cdef:
+ rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
+ rgw_file_handle *_file_handler
+ int _flags = flags
+ char* _filename = filename
+
+ with nogil:
+ ret = rgw_lookup(self.fs, _dir_handler, _filename,
+ &_file_handler, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in open '%s'" % filename)
+ with nogil:
+ ret = rgw_open(self.fs, _file_handler, 0, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in open '%s'" % filename)
+
+ file_handler = FileHandle()
+ file_handler.handler = _file_handler
+ return file_handler
+
+ def close(self, FileHandle file_handler, flags = 0):
+ self.require_state("mounted")
+ cdef:
+ rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
+ int _flags = flags
+ with nogil:
+ ret = rgw_close(self.fs, _file_handler, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in close")
+
+ def read(self, FileHandle file_handler, offset, l, flags = 0):
+ self.require_state("mounted")
+ if not isinstance(offset, int):
+ raise TypeError('offset must be an int')
+ if not isinstance(l, int):
+ raise TypeError('l must be an int')
+ cdef:
+ rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
+ int64_t _offset = offset
+ size_t _length = l
+ size_t _got
+ int _flags = flags
+
+ char *ret_buf
+ PyObject* ret_s = NULL
+
+ ret_s = PyBytes_FromStringAndSize(NULL, _length)
+ try:
+ ret_buf = PyBytes_AsString(ret_s)
+ with nogil:
+ ret = rgw_read(self.fs, _file_handler, _offset, _length,
+ &_got, ret_buf, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in read")
+
+ if _got < _length:
+ _PyBytes_Resize(&ret_s, _got)
+
+ return <object>ret_s
+ finally:
+ # We DECREF unconditionally: the cast to object above will have
+ # INCREFed if necessary. This also takes care of exceptions,
+ # including if _PyString_Resize fails (that will free the string
+ # itself and set ret_s to NULL, hence XDECREF).
+ ref.Py_XDECREF(ret_s)
+
+
+ def write(self, FileHandle file_handler, offset, buf, flags = 0):
+ self.require_state("mounted")
+ if not isinstance(buf, bytes):
+ raise TypeError('buf must be a bytes')
+ if not isinstance(offset, int):
+ raise TypeError('offset must be an int')
+
+ cdef:
+ rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
+ char *_data = buf
+ int64_t _offset = offset
+
+ size_t length = len(buf)
+ int _flags = flags
+ size_t _written
+
+ with nogil:
+ ret = rgw_write(self.fs, _file_handler, _offset, length, &_written,
+ _data, _flags)
+ if ret < 0:
+ raise make_ex(ret, "error in write")
+ return ret
+
+
+ def fsync(self, FileHandle handler, flags = 0):
+ self.require_state("mounted")
+
+ cdef:
+ rgw_file_handle *_file_handler = <rgw_file_handle*>handler.handler
+ int _flags = flags
+ with nogil:
+ ret = rgw_fsync(self.fs, _file_handler, _flags)
+
+ if ret < 0:
+ raise make_ex(ret, "fsync failed")
+++ /dev/null
-"""
-This module is a thin wrapper around rgw_file.
-"""
-
-
-from cpython cimport PyObject, ref, exc, array
-from libc.stdint cimport *
-from libcpp cimport bool
-from libc.stdlib cimport malloc, realloc, free
-
-cimport rados
-
-from collections import namedtuple
-from datetime import datetime
-import errno
-
-
-cdef extern from "Python.h":
- # These are in cpython/string.pxd, but use "object" types instead of
- # PyObject*, which invokes assumptions in cpython that we need to
- # legitimately break to implement zero-copy string buffers in Image.read().
- # This is valid use of the Python API and documented as a special case.
- PyObject *PyBytes_FromStringAndSize(char *v, Py_ssize_t len) except NULL
- char* PyBytes_AsString(PyObject *string) except NULL
- int _PyBytes_Resize(PyObject **string, Py_ssize_t newsize) except -1
- void PyEval_InitThreads()
-
-
-cdef extern from "time.h":
- ctypedef long int time_t
-
-
-cdef extern from "sys/stat.h":
- cdef struct stat:
- unsigned long st_dev
- unsigned long st_ino
- unsigned long st_nlink
- unsigned int st_mode
- unsigned int st_uid
- unsigned int st_gid
- int __pad0
- unsigned long st_rdev
- long int st_size
- long int st_blksize
- long int st_blocks
- time_t st_atime
- time_t st_mtime
- time_t st_ctime
-
-
-cdef extern from "rados/librgw.h" nogil:
- ctypedef void* librgw_t
-
- int librgw_create(librgw_t *rgw, int argc, char **argv)
- void librgw_shutdown(librgw_t rgw)
-
-
-cdef extern from "rados/rgw_file.h" nogil:
- enum:
- RGW_FS_TYPE_FILE
- RGW_FS_TYPE_DIRECTORY
-
- RGW_LOOKUP_FLAG_CREATE
-
- RGW_SETATTR_MODE
- RGW_SETATTR_UID
- RGW_SETATTR_GID
- RGW_SETATTR_MTIME
- RGW_SETATTR_ATIME
- RGW_SETATTR_SIZE
- RGW_SETATTR_CTIME
-
- RGW_READDIR_FLAG_NONE
- RGW_READDIR_FLAG_DOTDOT
-
- RGW_OPEN_FLAG_CREATE
- RGW_OPEN_FLAG_V3 # ops have v3 semantics
- RGW_OPEN_FLAG_STATELESS # alias it
-
- RGW_CLOSE_FLAG_RELE
-
-
- ctypedef void *rgw_fh_hk
- cdef struct rgw_file_handle:
- pass
-
- cdef struct rgw_fs:
- librgw_t rgw
- void *fs_private
- void *root_fh
-
- # mount info hypothetical--emulate Unix, support at least UUID-length fsid
- cdef struct rgw_statvfs:
- uint64_t f_bsize # file system block size
- uint64_t f_frsize # fragment size
- uint64_t f_blocks # size of fs in f_frsize units
- uint64_t f_bfree # free blocks
- uint64_t f_bavail # free blocks for unprivileged users
- uint64_t f_files # inodes
- uint64_t f_ffree # free inodes
- uint64_t f_favail # free inodes for unprivileged users
- uint64_t f_fsid[2] # /* file system ID
- uint64_t f_flag # mount flags
- uint64_t f_namemax # maximum filename length
-
- void rgwfile_version(int *major, int *minor, int *extra)
-
- int rgw_lookup(rgw_fs *fs,
- rgw_file_handle *parent_fh, const char *path,
- rgw_file_handle **fh, uint32_t flags)
-
- int rgw_lookup_handle(rgw_fs *fs, rgw_fh_hk *fh_hk,
- rgw_file_handle **fh, uint32_t flags)
-
- int rgw_fh_rele(rgw_fs *fs, rgw_file_handle *fh,
- uint32_t flags)
-
- int rgw_mount(librgw_t rgw, const char *uid, const char *key,
- const char *secret, rgw_fs **fs, uint32_t flags)
-
- int rgw_umount(rgw_fs *fs, uint32_t flags)
-
- int rgw_statfs(rgw_fs *fs, rgw_file_handle *parent_fh,
- rgw_statvfs *vfs_st, uint32_t flags)
-
- int rgw_create(rgw_fs *fs, rgw_file_handle *parent_fh,
- const char *name, stat *st, uint32_t mask,
- rgw_file_handle **fh, uint32_t posix_flags,
- uint32_t flags)
-
- int rgw_mkdir(rgw_fs *fs,
- rgw_file_handle *parent_fh,
- const char *name, stat *st, uint32_t mask,
- rgw_file_handle **fh, uint32_t flags)
-
- int rgw_rename(rgw_fs *fs,
- rgw_file_handle *olddir, const char* old_name,
- rgw_file_handle *newdir, const char* new_name,
- uint32_t flags)
-
- int rgw_unlink(rgw_fs *fs,
- rgw_file_handle *parent_fh, const char* path,
- uint32_t flags)
-
- int rgw_readdir(rgw_fs *fs,
- rgw_file_handle *parent_fh, uint64_t *offset,
- bool (*cb)(const char *name, void *arg, uint64_t offset) nogil except? -9000,
- void *cb_arg, bool *eof, uint32_t flags) except? -9000
-
- int rgw_getattr(rgw_fs *fs,
- rgw_file_handle *fh, stat *st,
- uint32_t flags)
-
- int rgw_setattr(rgw_fs *fs, rgw_file_handle *fh, stat *st,
- uint32_t mask, uint32_t flags)
-
- int rgw_truncate(rgw_fs *fs, rgw_file_handle *fh, uint64_t size, uint32_t flags)
-
- int rgw_open(rgw_fs *fs, rgw_file_handle *parent_fh,
- uint32_t posix_flags, uint32_t flags)
-
- int rgw_close(rgw_fs *fs, rgw_file_handle *fh,
- uint32_t flags)
-
- int rgw_read(rgw_fs *fs,
- rgw_file_handle *fh, uint64_t offset,
- size_t length, size_t *bytes_read, void *buffer,
- uint32_t flags)
-
- int rgw_write(rgw_fs *fs,
- rgw_file_handle *fh, uint64_t offset,
- size_t length, size_t *bytes_written, void *buffer,
- uint32_t flags)
-
- int rgw_fsync(rgw_fs *fs, rgw_file_handle *fh,
- uint32_t flags)
-
- int rgw_commit(rgw_fs *fs, rgw_file_handle *fh,
- uint64_t offset, uint64_t length, uint32_t flags)
-
-
-class Error(Exception):
- pass
-
-
-class PermissionError(Error):
- pass
-
-
-class ObjectNotFound(Error):
- pass
-
-
-class NoData(Error):
- pass
-
-
-class ObjectExists(Error):
- pass
-
-
-class IOError(Error):
- pass
-
-
-class NoSpace(Error):
- pass
-
-
-class InvalidValue(Error):
- pass
-
-
-class OperationNotSupported(Error):
- pass
-
-
-class IncompleteWriteError(Error):
- pass
-
-
-class LibCephFSStateError(Error):
- pass
-
-class WouldBlock(Error):
- pass
-
-class OutOfRange(Error):
- pass
-
-cdef errno_to_exception = {
- errno.EPERM : PermissionError,
- errno.ENOENT : ObjectNotFound,
- errno.EIO : IOError,
- errno.ENOSPC : NoSpace,
- errno.EEXIST : ObjectExists,
- errno.ENODATA : NoData,
- errno.EINVAL : InvalidValue,
- errno.EOPNOTSUPP : OperationNotSupported,
- errno.ERANGE : OutOfRange,
- errno.EWOULDBLOCK: WouldBlock,
-}
-
-
-cdef class FileHandle(object):
- cdef rgw_file_handle *handler
-
-
-StatResult = namedtuple('StatResult',
- ["st_dev", "st_ino", "st_mode", "st_nlink", "st_uid",
- "st_gid", "st_rdev", "st_size", "st_blksize",
- "st_blocks", "st_atime", "st_mtime", "st_ctime"])
-
-
-def cstr(val, name, encoding="utf-8", opt=False):
- """
- Create a byte string from a Python string
-
- :param basestring val: Python string
- :param str name: Name of the string parameter, for exceptions
- :param str encoding: Encoding to use
- :param bool opt: If True, None is allowed
- :rtype: bytes
- :raises: :class:`InvalidArgument`
- """
- if opt and val is None:
- return None
- if isinstance(val, bytes):
- return val
- elif isinstance(val, unicode):
- return val.encode(encoding)
- else:
- raise TypeError('%s must be a string' % name)
-
-
-cdef make_ex(ret, msg):
- """
- Translate a librados return code into an exception.
-
- :param ret: the return code
- :type ret: int
- :param msg: the error message to use
- :type msg: str
- :returns: a subclass of :class:`Error`
- """
- ret = abs(ret)
- if ret in errno_to_exception:
- return errno_to_exception[ret](msg)
- else:
- return Error(msg + (": error code %d" % ret))
-
-
-cdef bool readdir_cb(const char *name, void *arg, uint64_t offset) \
-except? -9000 with gil:
- if exc.PyErr_Occurred():
- return False
- (<object>arg)(name, offset)
- return True
-
-
-class LibCephFSStateError(Error):
- pass
-
-
-cdef class LibRGWFS(object):
- """librgwfs python wrapper"""
-
- cdef public object state
- cdef public object uid
- cdef public object key
- cdef public object secret
- cdef librgw_t cluster
- cdef rgw_fs *fs
-
- def require_state(self, *args):
- if self.state in args:
- return
- raise LibCephFSStateError("You cannot perform that operation on a "
- "RGWFS object in state %s." % (self.state))
-
- def __cinit__(self, uid, key, secret):
- PyEval_InitThreads()
- self.state = "umounted"
- ret = librgw_create(&self.cluster, 0, NULL)
- if ret != 0:
- raise make_ex(ret, "error calling librgw_create")
- self.uid = cstr(uid, "uid")
- self.key = cstr(key, "key")
- self.secret = cstr(secret, "secret")
-
- def shutdown(self):
- """
- Unmount and destroy the ceph mount handle.
- """
- if self.state in ["mounted"]:
- with nogil:
- ret = rgw_umount(self.fs, 0);
- if ret != 0:
- raise make_ex(ret, "error calling rgw_unmount")
- self.state = "shutdown"
-
- def __enter__(self):
- self.mount()
- return self
-
- def __exit__(self, type_, value, traceback):
- self.shutdown()
-
- def __dealloc__(self):
- self.shutdown()
-
- def version(self):
- """
- Get the version number of the ``librgwfile`` C library.
-
- :returns: a tuple of ``(major, minor, extra)`` components of the
- libcephfs version
- """
- cdef:
- int major = 0
- int minor = 0
- int extra = 0
- with nogil:
- rgwfile_version(&major, &minor, &extra)
- return (major, minor, extra)
-
- def mount(self):
- self.require_state("umounted")
- cdef:
- char *_uid = self.uid
- char *_key = self.key
- char *_secret = self.secret
- with nogil:
- ret = rgw_mount(self.cluster, <const char*>_uid, <const char*>_key,
- <const char*>_secret, &self.fs, 0)
- if ret != 0:
- raise make_ex(ret, "error calling rgw_mount")
- self.state = "mounted"
- dir_handler = FileHandle()
- dir_handler.handler = <rgw_file_handle*>self.fs.root_fh
- return dir_handler
-
- def unmount(self):
- self.require_state("mounted")
- with nogil:
- ret = rgw_umount(self.fs, 0)
- if ret != 0:
- raise make_ex(ret, "error calling rgw_umount")
- self.state = "umounted"
-
- def statfs(self):
- self.require_state("mounted")
- cdef:
- rgw_statvfs statbuf
-
- with nogil:
- ret = rgw_statfs(self.fs, <rgw_file_handle*>self.fs.root_fh, &statbuf, 0)
- if ret < 0:
- raise make_ex(ret, "statfs failed")
- cdef uint64_t[:] fsid = statbuf.f_fsid
- return {'f_bsize': statbuf.f_bsize,
- 'f_frsize': statbuf.f_frsize,
- 'f_blocks': statbuf.f_blocks,
- 'f_bfree': statbuf.f_bfree,
- 'f_bavail': statbuf.f_bavail,
- 'f_files': statbuf.f_files,
- 'f_ffree': statbuf.f_ffree,
- 'f_favail': statbuf.f_favail,
- 'f_fsid': fsid,
- 'f_flag': statbuf.f_flag,
- 'f_namemax': statbuf.f_namemax}
-
-
- def create(self, FileHandle dir_handler, filename, flags = 0):
- self.require_state("mounted")
-
- if not isinstance(flags, int):
- raise TypeError("flags must be an integer")
-
- filename = cstr(filename, 'filename')
-
- cdef:
- rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
- rgw_file_handle *_file_handler
- int _flags = flags
- char* _filename = filename
- stat statbuf
-
- with nogil:
- ret = rgw_create(self.fs, _dir_handler, _filename, &statbuf, 0,
- &_file_handler, 0, _flags)
- if ret < 0:
- raise make_ex(ret, "error in create '%s'" % filename)
- with nogil:
- ret = rgw_open(self.fs, _file_handler, 0, _flags)
- if ret < 0:
- raise make_ex(ret, "error in open '%s'" % filename)
-
- file_handler = FileHandle()
- file_handler.handler = _file_handler
- return file_handler
-
- def mkdir(self, FileHandle dir_handler, dirname, flags = 0):
- self.require_state("mounted")
- dirname = cstr(dirname, 'dirname')
- new_dir_handler = FileHandle()
- cdef:
- rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
- rgw_file_handle *_new_dir_handler
- char* _dirname = dirname
- int _flags = flags
- stat statbuf
- with nogil:
- ret = rgw_mkdir(self.fs, _dir_handler, _dirname, &statbuf,
- 0, &_new_dir_handler, _flags)
- if ret < 0:
- raise make_ex(ret, "error in mkdir '%s'" % dirname)
- new_dir_handler.handler = _new_dir_handler
- return new_dir_handler
-
- def rename(self, FileHandle src_handler, src_name, FileHandle dst_handler, dst_name, flags = 0):
- self.require_state("mounted")
-
- src_name = cstr(src_name, 'src_name')
- dst_name = cstr(dst_name, 'dst_name')
-
- cdef:
- rgw_file_handle *_src_dir_handler = <rgw_file_handle*>src_handler.handler
- rgw_file_handle *_dst_dir_handler = <rgw_file_handle*>dst_handler.handler
- char* _src_name = src_name
- char* _dst_name = dst_name
- int _flags = flags
-
- with nogil:
- ret = rgw_rename(self.fs, _src_dir_handler, _src_name,
- _dst_dir_handler, _dst_name, _flags)
- if ret < 0:
- raise make_ex(ret, "error in rename '%s' to '%s'" % (src_name,
- dst_name))
- return ret
-
- def unlink(self, FileHandle handler, name, flags = 0):
- self.require_state("mounted")
- name = cstr(name, 'name')
- cdef:
- rgw_file_handle *_handler = <rgw_file_handle*>handler.handler
- int _flags = flags
- char* _name = name
- with nogil:
- ret = rgw_unlink(self.fs, _handler, _name, _flags)
- if ret < 0:
- raise make_ex(ret, "error in unlink")
- return ret
-
- def readdir(self, FileHandle dir_handler, iterate_cb, offset, flags = 0):
- self.require_state("mounted")
-
- cdef:
- rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
- uint64_t _offset = offset
- bool _eof
- uint32_t _flags = flags
- with nogil:
- ret = rgw_readdir(self.fs, _dir_handler, &_offset, &readdir_cb,
- <void *>iterate_cb, &_eof, _flags)
- if ret < 0:
- raise make_ex(ret, "error in readdir")
-
- return (_offset, _eof)
-
- def fstat(self, FileHandle file_handler):
- self.require_state("mounted")
-
- cdef:
- rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
- stat statbuf
-
- with nogil:
- ret = rgw_getattr(self.fs, _file_handler, &statbuf, 0)
- if ret < 0:
- raise make_ex(ret, "error in getattr")
- return StatResult(st_dev=statbuf.st_dev, st_ino=statbuf.st_ino,
- st_mode=statbuf.st_mode, st_nlink=statbuf.st_nlink,
- st_uid=statbuf.st_uid, st_gid=statbuf.st_gid,
- st_rdev=statbuf.st_rdev, st_size=statbuf.st_size,
- st_blksize=statbuf.st_blksize,
- st_blocks=statbuf.st_blocks,
- st_atime=datetime.fromtimestamp(statbuf.st_atime),
- st_mtime=datetime.fromtimestamp(statbuf.st_mtime),
- st_ctime=datetime.fromtimestamp(statbuf.st_ctime))
-
- def opendir(self, FileHandle dir_handler, dirname, flags = 0):
- self.require_state("mounted")
-
- if not isinstance(flags, int):
- raise TypeError("flags must be an integer")
-
- dirname = cstr(dirname, 'dirname')
-
- cdef:
- rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
- rgw_file_handle *_file_handler
- int _flags = flags
- char* _dirname = dirname
-
- with nogil:
- ret = rgw_lookup(self.fs, _dir_handler, _dirname,
- &_file_handler, _flags)
- if ret < 0:
- raise make_ex(ret, "error in open '%s'" % dirname)
-
- file_handler = FileHandle()
- file_handler.handler = _file_handler
- return file_handler
-
- def open(self, FileHandle dir_handler, filename, flags = 0):
- self.require_state("mounted")
-
- if not isinstance(flags, int):
- raise TypeError("flags must be an integer")
-
- filename = cstr(filename, 'filename')
-
- cdef:
- rgw_file_handle *_dir_handler = <rgw_file_handle*>dir_handler.handler
- rgw_file_handle *_file_handler
- int _flags = flags
- char* _filename = filename
-
- with nogil:
- ret = rgw_lookup(self.fs, _dir_handler, _filename,
- &_file_handler, _flags)
- if ret < 0:
- raise make_ex(ret, "error in open '%s'" % filename)
- with nogil:
- ret = rgw_open(self.fs, _file_handler, 0, _flags)
- if ret < 0:
- raise make_ex(ret, "error in open '%s'" % filename)
-
- file_handler = FileHandle()
- file_handler.handler = _file_handler
- return file_handler
-
- def close(self, FileHandle file_handler, flags = 0):
- self.require_state("mounted")
- cdef:
- rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
- int _flags = flags
- with nogil:
- ret = rgw_close(self.fs, _file_handler, _flags)
- if ret < 0:
- raise make_ex(ret, "error in close")
-
- def read(self, FileHandle file_handler, offset, l, flags = 0):
- self.require_state("mounted")
- if not isinstance(offset, int):
- raise TypeError('offset must be an int')
- if not isinstance(l, int):
- raise TypeError('l must be an int')
- cdef:
- rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
- int64_t _offset = offset
- size_t _length = l
- size_t _got
- int _flags = flags
-
- char *ret_buf
- PyObject* ret_s = NULL
-
- ret_s = PyBytes_FromStringAndSize(NULL, _length)
- try:
- ret_buf = PyBytes_AsString(ret_s)
- with nogil:
- ret = rgw_read(self.fs, _file_handler, _offset, _length,
- &_got, ret_buf, _flags)
- if ret < 0:
- raise make_ex(ret, "error in read")
-
- if _got < _length:
- _PyBytes_Resize(&ret_s, _got)
-
- return <object>ret_s
- finally:
- # We DECREF unconditionally: the cast to object above will have
- # INCREFed if necessary. This also takes care of exceptions,
- # including if _PyString_Resize fails (that will free the string
- # itself and set ret_s to NULL, hence XDECREF).
- ref.Py_XDECREF(ret_s)
-
-
- def write(self, FileHandle file_handler, offset, buf, flags = 0):
- self.require_state("mounted")
- if not isinstance(buf, bytes):
- raise TypeError('buf must be a bytes')
- if not isinstance(offset, int):
- raise TypeError('offset must be an int')
-
- cdef:
- rgw_file_handle *_file_handler = <rgw_file_handle*>file_handler.handler
- char *_data = buf
- int64_t _offset = offset
-
- size_t length = len(buf)
- int _flags = flags
- size_t _written
-
- with nogil:
- ret = rgw_write(self.fs, _file_handler, _offset, length, &_written,
- _data, _flags)
- if ret < 0:
- raise make_ex(ret, "error in write")
- return ret
-
-
- def fsync(self, FileHandle handler, flags = 0):
- self.require_state("mounted")
-
- cdef:
- rgw_file_handle *_file_handler = <rgw_file_handle*>handler.handler
- int _flags = flags
- with nogil:
- ret = rgw_fsync(self.fs, _file_handler, _flags)
-
- if ret < 0:
- raise make_ex(ret, "fsync failed")
projects / ceph.git / commitdiff