From: Xiaoxi Chen Date: Wed, 29 Apr 2015 08:24:37 +0000 (+0800) Subject: Kill Flat_index. X-Git-Tag: v9.1.0~421^2~4 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=5df688a5df9f6879ee3c266166ddc2d2d43a8ae4;p=ceph.git Kill Flat_index. Too old, remove it entirely. Signed-off-by: Xiaoxi Chen --- diff --git a/src/os/FlatIndex.cc b/src/os/FlatIndex.cc deleted file mode 100644 index c6045228be86..000000000000 --- a/src/os/FlatIndex.cc +++ /dev/null @@ -1,426 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2004-2006 Sage Weil - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#if defined(__FreeBSD__) -#include -#include -#endif - -#include "FlatIndex.h" -#include "common/ceph_crypto.h" -#include "osd/osd_types.h" -#include - -#include "chain_xattr.h" - -using ceph::crypto::SHA1; - -/* - * long file names will have the following format: - * - * prefix_hash_index_cookie - * - * The prefix will just be the first X bytes of the original file name. - * The cookie is a constant string that shows whether this file name - * is hashed - */ - -#define FILENAME_LFN_DIGEST_SIZE CEPH_CRYPTO_SHA1_DIGESTSIZE - -#define FILENAME_MAX_LEN 4096 // the long file name size -#define FILENAME_SHORT_LEN 255 // the short file name size -#define FILENAME_COOKIE "long" // ceph long file name -#define FILENAME_HASH_LEN FILENAME_LFN_DIGEST_SIZE -#define FILENAME_EXTRA 4 // underscores and digit - -#define LFN_ATTR "user.cephos.lfn" - -#define FILENAME_PREFIX_LEN (FILENAME_SHORT_LEN - FILENAME_HASH_LEN - (sizeof(FILENAME_COOKIE) - 1) - FILENAME_EXTRA) - - -int FlatIndex::cleanup() { - return 0; -} - -static inline void buf_to_hex(const unsigned char *buf, int len, char *str) -{ - int i; - str[0] = '\0'; - for (i = 0; i < len; i++) { - sprintf(&str[i*2], "%02x", (int)buf[i]); - } -} - -static int hash_filename(const char *filename, char *hash, int buf_len) -{ - if (buf_len < FILENAME_HASH_LEN + 1) - return -EINVAL; - - char buf[FILENAME_LFN_DIGEST_SIZE]; - char hex[FILENAME_LFN_DIGEST_SIZE * 2]; - - SHA1 h; - h.Update((const byte *)filename, strlen(filename)); - h.Final((byte *)buf); - - buf_to_hex((byte *)buf, (FILENAME_HASH_LEN + 1) / 2, hex); - strncpy(hash, hex, FILENAME_HASH_LEN); - hash[FILENAME_HASH_LEN] = '\0'; - return 0; -} - -static void build_filename(char *filename, int len, const char *old_filename, int i) -{ - char hash[FILENAME_HASH_LEN + 1]; - - assert(len >= FILENAME_SHORT_LEN + 4); - - strncpy(filename, old_filename, FILENAME_PREFIX_LEN); - filename[FILENAME_PREFIX_LEN] = '\0'; - if (strlen(filename) < FILENAME_PREFIX_LEN) - return; - if (old_filename[FILENAME_PREFIX_LEN] == '\0') - return; - - hash_filename(old_filename, hash, sizeof(hash)); - int ofs = FILENAME_PREFIX_LEN; - while (1) { - int suffix_len = sprintf(filename + ofs, "_%s_%d_" FILENAME_COOKIE, hash, i); - if (ofs + suffix_len <= FILENAME_SHORT_LEN || !ofs) - break; - ofs--; - } -} - -/* is this a candidate? */ -static int lfn_is_hashed_filename(const char *filename) -{ - int len = strlen(filename); - if (len < FILENAME_SHORT_LEN) - return 0; - return (strcmp(filename + len - (sizeof(FILENAME_COOKIE) - 1), FILENAME_COOKIE) == 0); -} - -static void lfn_translate(const char *path, const char *name, char *new_name, int len) -{ - if (!lfn_is_hashed_filename(name)) { - strncpy(new_name, name, len); - return; - } - - char buf[PATH_MAX]; - - snprintf(buf, sizeof(buf), "%s/%s", path, name); - int r = chain_getxattr(buf, LFN_ATTR, new_name, len - 1); - if (r < 0) - strncpy(new_name, name, len); - else - new_name[r] = '\0'; - return; -} - -static int append_oname(const ghobject_t &oid, char *s, int len) -{ - //assert(sizeof(oid) == 28); - char *end = s + len; - char *t = s + strlen(s); - - const char *i = oid.hobj.oid.name.c_str(); - while (*i && t < end) { - if (*i == '\\') { - *t++ = '\\'; - *t++ = '\\'; - } else if (*i == '.' && i == oid.hobj.oid.name.c_str()) { // only escape leading . - *t++ = '\\'; - *t++ = '.'; - } else if (*i == '/') { - *t++ = '\\'; - *t++ = 's'; - } else - *t++ = *i; - i++; - } - - int size = t - s; - - if (oid.hobj.snap == CEPH_NOSNAP) - size += snprintf(t, end - t, "_head"); - else if (oid.hobj.snap == CEPH_SNAPDIR) - size += snprintf(t, end - t, "_snapdir"); - else - size += snprintf(t, end - t, "_%llx", (long long unsigned)oid.hobj.snap); - - return size; -} - -static bool parse_object(char *s, ghobject_t& oid) -{ - sobject_t o; - char *bar = s + strlen(s) - 1; - while (*bar != '_' && - bar > s) - bar--; - if (*bar == '_') { - char buf[bar-s + 1]; - char *t = buf; - char *i = s; - while (i < bar) { - if (*i == '\\') { - i++; - switch (*i) { - case '\\': *t++ = '\\'; break; - case '.': *t++ = '.'; break; - case 's': *t++ = '/'; break; - default: assert(0); - } - } else { - *t++ = *i; - } - i++; - } - *t = 0; - o.oid.name = string(buf, t-buf); - if (strcmp(bar+1, "head") == 0) - o.snap = CEPH_NOSNAP; - else if (strcmp(bar+1, "snapdir") == 0) - o.snap = CEPH_SNAPDIR; - else - o.snap = strtoull(bar+1, &s, 16); - oid = ghobject_t(hobject_t(o)); - return true; - } - return false; -} - -static int lfn_get(const char *coll_path, const ghobject_t& oid, char *pathname, int len, char *lfn, int lfn_len, int *exist, int *is_lfn) -{ - int i = 0; - strncpy(pathname, coll_path, len); - size_t path_len = strlen(coll_path); - pathname[path_len] = '/'; - path_len++; - pathname[path_len] = '\0'; - char *filename = pathname + path_len; - - *lfn = '\0'; - int actual_len = append_oname(oid, lfn, lfn_len); - - if (actual_len < (int)FILENAME_PREFIX_LEN) { - /* not a long file name, just build it as it is */ - strncpy(filename, lfn, len - path_len); - *is_lfn = 0; - struct stat buf; - int r = ::stat(pathname, &buf); - if (r < 0) { - if (errno == ENOENT) { - *exist = 0; - } else { - return -errno; - } - } else { - *exist = 1; - } - - return 0; - } - - *is_lfn = 1; - *exist = 0; - - while (1) { - char buf[PATH_MAX]; - int r; - - build_filename(filename, len - path_len, lfn, i); - r = chain_getxattr(pathname, LFN_ATTR, buf, sizeof(buf)); - if (r < 0) - r = -errno; - if (r > 0) { - buf[MIN((int)sizeof(buf)-1, r)] = '\0'; - if (strcmp(buf, lfn) == 0) { // a match? - *exist = 1; - return i; - } - } - switch (r) { - case -ENOENT: - return i; - case -ERANGE: - assert(0); // shouldn't happen - default: - break; - } - if (r < 0) - break; - i++; - } - - return 0; // unreachable anyway -} - -int FlatIndex::init() { - return 0; -} - -int FlatIndex::created(const ghobject_t &hoid, const char *path) { - char long_name[PATH_MAX]; - long_name[0] = '\0'; - int actual_len = append_oname(hoid, long_name, sizeof(long_name)); - if (actual_len < (int)FILENAME_PREFIX_LEN) { - return 0; - } - assert(long_name[actual_len] == '\0'); - assert(long_name[actual_len - 1] != '\0'); - int r = chain_setxattr(path, LFN_ATTR, long_name, actual_len); - if (r < 0) - return r; - return 0; -} - -int FlatIndex::unlink(const ghobject_t &o) { - char long_fn[PATH_MAX]; - char short_fn[PATH_MAX]; - char short_fn2[PATH_MAX]; - int r, i, exist, err; - int path_len; - int is_lfn; - - r = lfn_get(base_path.c_str(), o, short_fn, sizeof(short_fn), - long_fn, sizeof(long_fn), &exist, &is_lfn); - if (r < 0) - return r; - if (!is_lfn) { - r = ::unlink(short_fn); - if (r < 0) { - return -errno; - } - return 0; - } - if (!exist) - return -ENOENT; - - const char *next = strncpy(short_fn2, base_path.c_str(), sizeof(short_fn2)); - path_len = next - short_fn2; - short_fn2[path_len] = '/'; - path_len++; - short_fn2[path_len] = '\0'; - - for (i = r + 1; ; i++) { - struct stat buf; - int ret; - - build_filename(&short_fn2[path_len], sizeof(short_fn2) - path_len, long_fn, i); - ret = ::stat(short_fn2, &buf); - if (ret < 0) { - if (i == r + 1) { - err = ::unlink(short_fn); - if (err < 0) - return err; - return 0; - } - break; - } - } - - build_filename(&short_fn2[path_len], sizeof(short_fn2) - path_len, long_fn, i - 1); - - if (rename(short_fn2, short_fn) < 0) { - assert(0); - } - - return 0; -} - -int FlatIndex::lookup(const ghobject_t &hoid, IndexedPath *path, int *exist) { - char long_fn[PATH_MAX]; - char short_fn[PATH_MAX]; - int r; - int is_lfn; - r = lfn_get(base_path.c_str(), hoid, - short_fn, sizeof(short_fn), long_fn, - sizeof(long_fn), exist, &is_lfn); - if (r < 0) - return r; - *path = IndexedPath(new Path(string(short_fn), this)); - return 0; -} - -static int get_hobject_from_oinfo(const char *dir, const char *file, - ghobject_t *o) { - char path[PATH_MAX]; - bufferptr bp(PATH_MAX); - snprintf(path, sizeof(path), "%s/%s", dir, file); - // Hack, user.ceph._ is the attribute used to store the object info - int r = chain_getxattr(path, "user.ceph._", bp.c_str(), bp.length()); - if (r < 0) - return r; - bufferlist bl; - bl.push_back(bp); - object_info_t oi(bl); - *o = ghobject_t(oi.soid); - return 0; -} - -int FlatIndex::collection_list_partial(const ghobject_t &start, - const ghobject_t end, - int max_count, - snapid_t seq, - vector *ls, - ghobject_t *next) { - assert(0); // Should not be called - return 0; -} - -int FlatIndex::collection_list(vector *ls) { - char buf[offsetof(struct dirent, d_name) + PATH_MAX + 1]; - char dir_name[PATH_MAX], new_name[PATH_MAX]; - strncpy(dir_name, base_path.c_str(), sizeof(dir_name)); - dir_name[sizeof(dir_name)-1]='\0'; - - DIR *dir = ::opendir(dir_name); - if (!dir) - return -errno; - - // first, build (ino, object) list - vector< pair > inolist; - - struct dirent *de; - while (::readdir_r(dir, (struct dirent *)buf, &de) == 0) { - if (!de) - break; - // parse - if (de->d_name[0] == '.') - continue; - //cout << " got object " << de->d_name << std::endl; - ghobject_t o; - lfn_translate(dir_name, de->d_name, new_name, sizeof(new_name)); - if (parse_object(new_name, o)) { - get_hobject_from_oinfo(dir_name, de->d_name, &o); - inolist.push_back(pair(de->d_ino, o)); - ls->push_back(o); - } - } - - // sort - sort(inolist.begin(), inolist.end()); - - // build final list - ls->resize(inolist.size()); - int i = 0; - for (vector< pair >::iterator p = inolist.begin(); p != inolist.end(); ++p) - (*ls)[i++].swap(p->second); - - ::closedir(dir); - return 0; -} diff --git a/src/os/FlatIndex.h b/src/os/FlatIndex.h deleted file mode 100644 index 087799e6d937..000000000000 --- a/src/os/FlatIndex.h +++ /dev/null @@ -1,85 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2004-2006 Sage Weil - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - -#ifndef CEPH_FLATINDEX_H -#define CEPH_FLATINDEX_H - -#include -#include -#include -#include -#include "include/memory.h" - -#include "CollectionIndex.h" - -/** - * FlatIndex implements the collection layout prior to CollectionIndex - * - * This class should only be used for converting old filestores. - */ -class FlatIndex : public CollectionIndex { - string base_path; - coll_t collection; -public: - FlatIndex(coll_t collection, string base_path) : - CollectionIndex(collection), - base_path(base_path), - collection(collection) {} - - /// @see CollectionIndex - uint32_t collection_version() { return FLAT_INDEX_TAG; } - - coll_t coll() const { return collection; } - - /// @see CollectionIndex - int cleanup(); - - /// @see CollectionIndex - int init(); - - /// @see CollectionIndex - int created( - const ghobject_t &oid, - const char *path - ); - - /// @see CollectionIndex - int unlink( - const ghobject_t &oid - ); - - /// @see CollectionIndex - int lookup( - const ghobject_t &oid, - IndexedPath *path, - int *exist - ); - - /// @see CollectionIndex - int collection_list( - vector *ls - ); - - /// @see CollectionIndex - int collection_list_partial( - const ghobject_t &start, - const ghobject_t end, - int max_count, - snapid_t seq, - vector *ls, - ghobject_t *next - ); -}; - -#endif diff --git a/src/os/IndexManager.cc b/src/os/IndexManager.cc index 7f999a1822ae..6a9f040d106a 100644 --- a/src/os/IndexManager.cc +++ b/src/os/IndexManager.cc @@ -28,7 +28,6 @@ #include "include/buffer.h" #include "IndexManager.h" -#include "FlatIndex.h" #include "HashIndex.h" #include "CollectionIndex.h" @@ -95,10 +94,7 @@ int IndexManager::build_index(coll_t c, const char *path, CollectionIndex **inde return r; switch (version) { - case CollectionIndex::FLAT_INDEX_TAG: { - *index = new FlatIndex(c, path); - return 0; - } + case CollectionIndex::FLAT_INDEX_TAG: case CollectionIndex::HASH_INDEX_TAG: // fall through case CollectionIndex::HASH_INDEX_TAG_2: // fall through case CollectionIndex::HOBJECT_WITH_POOL: { diff --git a/src/os/IndexManager.h b/src/os/IndexManager.h index 6feafacd0185..b167e7de28f8 100644 --- a/src/os/IndexManager.h +++ b/src/os/IndexManager.h @@ -24,7 +24,6 @@ #include "CollectionIndex.h" #include "HashIndex.h" -#include "FlatIndex.h" /// Public type for Index diff --git a/src/os/Makefile.am b/src/os/Makefile.am index 2638810edede..ac7452d78c0b 100644 --- a/src/os/Makefile.am +++ b/src/os/Makefile.am @@ -11,7 +11,6 @@ libos_la_SOURCES = \ os/GenericObjectMap.cc \ os/FileJournal.cc \ os/FileStore.cc \ - os/FlatIndex.cc \ os/GenericFileStoreBackend.cc \ os/HashIndex.cc \ os/IndexManager.cc \ @@ -55,7 +54,6 @@ noinst_HEADERS += \ os/GenericObjectMap.h \ os/FileJournal.h \ os/FileStore.h \ - os/FlatIndex.h \ os/FDCache.h \ os/GenericFileStoreBackend.h \ os/HashIndex.h \ diff --git a/src/test/Makefile-server.am b/src/test/Makefile-server.am index e1ecab4aa559..a10f146f80dd 100644 --- a/src/test/Makefile-server.am +++ b/src/test/Makefile-server.am @@ -221,11 +221,6 @@ unittest_chain_xattr_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) unittest_chain_xattr_CXXFLAGS = $(UNITTEST_CXXFLAGS) check_TESTPROGRAMS += unittest_chain_xattr -unittest_flatindex_SOURCES = test/os/TestFlatIndex.cc -unittest_flatindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) -unittest_flatindex_CXXFLAGS = $(UNITTEST_CXXFLAGS) -check_TESTPROGRAMS += unittest_flatindex - unittest_lfnindex_SOURCES = test/os/TestLFNIndex.cc unittest_lfnindex_LDADD = $(LIBOS) $(UNITTEST_LDADD) $(CEPH_GLOBAL) unittest_lfnindex_CXXFLAGS = $(UNITTEST_CXXFLAGS) diff --git a/src/test/os/TestFlatIndex.cc b/src/test/os/TestFlatIndex.cc deleted file mode 100644 index ea1d24f3766c..000000000000 --- a/src/test/os/TestFlatIndex.cc +++ /dev/null @@ -1,139 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2013 Cloudwatt - * - * Author: Loic Dachary - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU Library Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Library Public License for more details. - * - */ - -#include -#include -#include "os/FlatIndex.h" -#include "os/CollectionIndex.h" -#include "os/chain_xattr.h" -#include "common/ceph_argparse.h" -#include "global/global_init.h" -#include - -TEST(FlatIndex, FlatIndex) { - coll_t collection; - const std::string base_path("PATH"); - FlatIndex index(collection, base_path); - EXPECT_EQ(collection, index.coll()); - EXPECT_EQ((unsigned)0, index.collection_version()); - // - // checking placeholders - // - EXPECT_EQ(0, index.init()); - EXPECT_EQ(0, index.cleanup()); -} - -TEST(FlatIndex, collection) { - coll_t collection; - const std::string base_path("PATH"); - FlatIndex index(collection, base_path); - const std::string key("KEY"); - uint64_t hash = 111; - uint64_t pool = 222; - const std::string object_name(10, 'A'); - ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, "")); - vector ls; - ASSERT_THROW(index.collection_list_partial(hoid, ghobject_t::get_max(), 0, 0, &ls, &hoid), FailedAssertion); -} - -TEST(FlatIndex, created_unlink) { - coll_t collection; - const std::string base_path("PATH"); - EXPECT_EQ(0, ::system("rm -fr PATH")); - EXPECT_EQ(0, ::mkdir("PATH", 0700)); - ceph::shared_ptr index(new FlatIndex(collection, base_path)); - const std::string key("KEY"); - uint64_t hash = 111; - uint64_t pool = 222; - // - // short object name - // - { - CollectionIndex::IndexedPath indexed_path; - const std::string object_name(10, 'A'); - ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, "")); - int exists; - EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists)); - EXPECT_EQ(0, exists); - EXPECT_EQ(0, ::close(::creat(indexed_path->path(), 0600))); - EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists)); - EXPECT_EQ(1, exists); - EXPECT_EQ(0, index->unlink(hoid)); - EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists)); - EXPECT_EQ(0, exists); - } - // - // long object name - // - { - CollectionIndex::IndexedPath indexed_path; - const std::string object_name(1024, 'A'); - ghobject_t hoid(hobject_t(object_t(object_name), key, CEPH_NOSNAP, hash, pool, "")); - int exists; - EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists)); - EXPECT_EQ(0, exists); - EXPECT_EQ(0, ::close(::creat(indexed_path->path(), 0600))); - EXPECT_EQ(0, index->created(hoid, indexed_path->path())); - EXPECT_EQ(0, index->unlink(hoid)); - EXPECT_EQ(0, index->lookup(hoid, &indexed_path, &exists)); - EXPECT_EQ(0, exists); - } - EXPECT_EQ(0, ::system("rm -fr PATH")); -} - -TEST(FlatIndex, collection_list) { - coll_t collection; - const std::string base_path("PATH"); - EXPECT_EQ(0, ::system("rm -fr PATH")); - EXPECT_EQ(0, ::mkdir("PATH", 0700)); - const std::string object_name("ABC"); - const std::string filename("PATH/" + object_name + "_head"); - EXPECT_EQ(0, ::close(::creat(filename.c_str(), 0600))); - ceph::shared_ptr index(new FlatIndex(collection, base_path)); - vector ls; - index->collection_list(&ls); - EXPECT_EQ((unsigned)1, ls.size()); - EXPECT_EQ(object_name, ls[0].hobj.oid.name); - EXPECT_EQ(0, ::system("rm -fr PATH")); -} - -int main(int argc, char **argv) { - int fd = ::creat("detect", 0600); - int ret = chain_fsetxattr(fd, "user.test", "A", 1); - ::close(fd); - ::unlink("detect"); - if (ret < 0) { - cerr << "SKIP FlatIndex because unable to test for xattr" << std::endl; - } else { - vector args; - argv_to_vec(argc, (const char **)argv, args); - - global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, CODE_ENVIRONMENT_UTILITY, 0); - common_init_finish(g_ceph_context); - - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); - } -} - -// Local Variables: -// compile-command: "cd ../.. ; make unittest_flatindex ; ./unittest_flatindex # --gtest_filter=FlatIndexTest.FlatIndex --log-to-stderr=true --debug-filestore=20" -// End: