Merge pull request #10098 from dzafman/wip-16568

tools: New "removeall" used to remove head with snapshots

Reviewed-by: Kefu Chai <kchai@redhat.com>

diff --cc src/test/ceph_objectstore_tool.py
index 60b69f4da736de2db8332663b6d344302281cf9a,2e2e12d5163fd9a05ea711b8291409672f624768..eee6662a743ada8b51f3b77db648539f764f2299
--- 1/src/test/ceph_objectstore_tool.py
--- 2/src/test/ceph_objectstore_tool.py
+++ b/src/test/ceph_objectstore_tool.py
@@@ -587,15 -577,81 +589,84 @@@ def test_get_set_inc_osdmap(CFSD_PREFIX
      return errors
  
  
+ def test_removeall(CFSD_PREFIX, db, OBJREPPGS, REP_POOL, CEPH_BIN, OSDDIR, REP_NAME, NUM_CLONED_REP_OBJECTS):
+     # Test removeall
+     TMPFILE = r"/tmp/tmp.{pid}".format(pid=os.getpid())
+     nullfd = open(os.devnull, "w")
+     errors=0
+     print "Test removeall"
+     kill_daemons()
+     for nspace in db.keys():
+         for basename in db[nspace].keys():
+             JSON = db[nspace][basename]['json']
+             for pg in OBJREPPGS:
+                 OSDS = get_osds(pg, OSDDIR)
+                 for osd in OSDS:
+                     DIR = os.path.join(OSDDIR, os.path.join(osd, os.path.join("current", "{pg}_head".format(pg=pg))))
+                     fnames = [f for f in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, f))
+                               and f.split("_")[0] == basename and f.split("_")[4] == nspace]
+                     if not fnames:
+                         continue
+ 
+                     if int(basename.split(REP_NAME)[1]) <= int(NUM_CLONED_REP_OBJECTS):
+                         cmd = (CFSD_PREFIX + "'{json}' remove").format(osd=osd, json=JSON)
+                         errors += test_failure(cmd, "Snapshots are present, use removeall to delete everything")
+ 
+                     cmd = (CFSD_PREFIX + " --force --dry-run '{json}' remove").format(osd=osd, json=JSON)
+                     logging.debug(cmd)
+                     ret = call(cmd, shell=True, stdout=nullfd, stderr=nullfd)
+                     if ret != 0:
+                         logging.error("remove with --force failed for {json}".format(json=JSON))
+                         errors += 1
+ 
+                     cmd = (CFSD_PREFIX + " --dry-run '{json}' removeall").format(osd=osd, json=JSON)
+                     logging.debug(cmd)
+                     ret = call(cmd, shell=True, stdout=nullfd, stderr=nullfd)
+                     if ret != 0:
+                         logging.error("removeall failed for {json}".format(json=JSON))
+                         errors += 1
+ 
+                     cmd = (CFSD_PREFIX + " '{json}' removeall").format(osd=osd, json=JSON)
+                     logging.debug(cmd)
+                     ret = call(cmd, shell=True, stdout=nullfd, stderr=nullfd)
+                     if ret != 0:
+                         logging.error("removeall failed for {json}".format(json=JSON))
+                         errors += 1
+ 
+                     tmpfd = open(TMPFILE, "w")
+                     cmd = (CFSD_PREFIX + "--op list --pgid {pg} --namespace {ns} {name}").format(osd=osd, pg=pg, ns=nspace, name=basename)
+                     logging.debug(cmd)
+                     ret = call(cmd, shell=True, stdout=tmpfd)
+                     if ret != 0:
+                         logging.error("Bad exit status {ret} from {cmd}".format(ret=ret, cmd=cmd))
+                         errors += 1
+                     tmpfd.close()
+                     lines = get_lines(TMPFILE)
+                     if len(lines) != 0:
+                         logging.error("Removeall didn't remove all objects {ns}/{name} : {lines}".format(ns=nspace, name=basename, lines=lines))
+                         errors += 1
+     vstart(new=False)
+     wait_for_health()
+     cmd = "{path}/rados -p {pool} rmsnap snap1".format(pool=REP_POOL, path=CEPH_BIN)
+     logging.debug(cmd)
+     ret = call(cmd, shell=True, stdout=nullfd, stderr=nullfd)
+     if ret != 0:
+         logging.error("rados rmsnap failed")
+         errors += 1
+     time.sleep(2)
+     wait_for_health()
+     return errors
+ 
+ 
  def main(argv):
 -    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
 +    if sys.version_info[0] < 3:
 +        sys.stdout = stdout = os.fdopen(sys.stdout.fileno(), 'wb', 0)
 +    else:
 +        stdout = sys.stdout.buffer
      if len(argv) > 1 and argv[1] == "debug":
 -        nullfd = sys.stdout
 +        nullfd = stdout
      else:
 -        nullfd = open(os.devnull, "w")
 +        nullfd = DEVNULL
  
      call("rm -fr {dir}; mkdir {dir}".format(dir=CEPH_DIR), shell=True)
      os.environ["CEPH_DIR"] = CEPH_DIR
@@@ -1877,14 -1937,16 +1952,16 @@@
      call("/bin/rm -rf {dir}".format(dir=TESTDIR), shell=True)
      call("/bin/rm -rf {dir}".format(dir=DATADIR), shell=True)
  
+     ERRORS += test_removeall(CFSD_PREFIX, db, OBJREPPGS, REP_POOL, CEPH_BIN, OSDDIR, REP_NAME, NUM_CLONED_REP_OBJECTS)
+ 
      # vstart() starts 4 OSDs
 -    ERRORS += test_get_set_osdmap(CFSD_PREFIX, range(4), ALLOSDS)
 +    ERRORS += test_get_set_osdmap(CFSD_PREFIX, list(range(4)), ALLOSDS)
      ERRORS += test_get_set_inc_osdmap(CFSD_PREFIX, ALLOSDS[0])
      if ERRORS == 0:
 -        print "TEST PASSED"
 +        print("TEST PASSED")
          return 0
      else:
 -        print "TEST FAILED WITH {errcount} ERRORS".format(errcount=ERRORS)
 +        print("TEST FAILED WITH {errcount} ERRORS".format(errcount=ERRORS))
          return 1
  
  

diff --cc src/tools/ceph_objectstore_tool.cc
Simple merge