Merge remote-tracking branch 'gh/wip-hitset'

Reviewed-by: Greg Farnum <greg@inktank.com>
Conflicts:
	src/common/config_opts.h
	src/osd/ReplicatedPG.cc
	src/osdc/Objecter.cc
	src/vstart.sh

diff --cc ceph.spec.in
Simple merge

diff --cc debian/ceph-test.install
Simple merge

diff --cc qa/workunits/cephtool/test.sh
index 94f37dd118d62fab34c74b0595236a21f8a4e334,cfa58f1e88f8ae24e53d7bb8f64015097e57ac8a..2335d485674e7d44e0efaa1510d9aa0465409750
--- 1/qa/workunits/cephtool/test.sh
--- 2/qa/workunits/cephtool/test.sh
+++ b/qa/workunits/cephtool/test.sh
@@@ -330,11 -329,15 +330,19 @@@ ceph osd pool set data size 
  
  ceph osd pool set data hashpspool true
  ceph osd pool set data hashpspool false
 +ceph osd pool set data hashpspool 0
 +ceph osd pool set data hashpspool 1
 +expect_false ceph osd pool set data hashpspool asdf
 +expect_false ceph osd pool set data hashpspool 2
  
+ ceph osd pool set rbd hit_set_type explicit_hash
+ ceph osd pool set rbd hit_set_type explicit_object
+ ceph osd pool set rbd hit_set_type bloom
+ expect_false ceph osd pool set rbd hit_set_type i_dont_exist
+ ceph osd pool set rbd hit_set_period 123
+ ceph osd pool set rbd hit_set_count 12
+ ceph osd pool set rbd hit_set_fpp .01
+ 
  ceph osd pool get rbd crush_ruleset | grep 'crush_ruleset: 2'
  
  ceph osd thrash 10

diff --cc src/common/config_opts.h
index 9a9509e88352c2970f9e6e3fabae0e1a087188dc,1efa6670ae0ac94d7af82b03f721fb39234abcf1..551c5066ee3e55ff396bec0cbdd341f7e6a98693
--- 1/src/common/config_opts.h
--- 2/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@@ -396,7 -392,9 +396,9 @@@ OPTION(osd_pool_default_min_size, OPT_I
  OPTION(osd_pool_default_pg_num, OPT_INT, 8) // number of PGs for new pools. Configure in global or mon section of ceph.conf
  OPTION(osd_pool_default_pgp_num, OPT_INT, 8) // number of PGs for placement purposes. Should be equal to pg_num
  OPTION(osd_pool_default_flags, OPT_INT, 0)   // default flags for new pools
 -OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, false)   // use new pg hashing to prevent pool/pg overlap
 +OPTION(osd_pool_default_flag_hashpspool, OPT_BOOL, true)   // use new pg hashing to prevent pool/pg overlap
+ OPTION(osd_hit_set_min_size, OPT_INT, 1000)  // min target size for a HitSet
+ OPTION(osd_hit_set_namespace, OPT_STR, ".ceph-internal") // rados namespace for hit_set tracking
  OPTION(osd_map_dedup, OPT_BOOL, true)
  OPTION(osd_map_cache_size, OPT_INT, 500)
  OPTION(osd_map_message_max, OPT_INT, 100)  // max maps per MOSDMap message

diff --cc src/include/encoding.h
Simple merge

diff --cc src/include/rados.h
Simple merge

diff --cc src/include/rados/librados.h
Simple merge

diff --cc src/mon/MonCommands.h
Simple merge

diff --cc src/mon/OSDMonitor.cc
index 377682aa8c08ecd964f496e4f86e9df122776a24,7d2ca25b9dbc05e3271267f47d284c63fd2a6cf5..a98e00909ee056069fb2197ff6e945a659705495
--- 1/src/mon/OSDMonitor.cc
--- 2/src/mon/OSDMonitor.cc
+++ b/src/mon/OSDMonitor.cc
@@@ -2826,22 -2734,18 +2826,25 @@@ int OSDMonitor::prepare_command_pool_se
    if (pending_inc.new_pools.count(pool))
      p = pending_inc.new_pools[pool];
  
 -  // accept val as a json string or int, and parse out int or float
 -  // values from the string as needed
 +  // accept val as a json string in the normal case (current
 +  // generation monitor).  parse out int or float values from the
 +  // string as needed.  however, if it is not a string, try to pull
 +  // out an int, in case an older monitor with an older json schema is
 +  // forwarding a request.
    string val;
 -  cmd_getval(g_ceph_context, cmdmap, "val", val);
--  string interr;
++  string interr, floaterr;
    int64_t n = 0;
 -  if (!cmd_getval(g_ceph_context, cmdmap, "val", n))
++  double f = 0;
 +  if (!cmd_getval(g_ceph_context, cmdmap, "val", val)) {
 +    // wasn't a string; maybe an older mon forwarded json with an int?
 +    if (!cmd_getval(g_ceph_context, cmdmap, "val", n))
 +      return -EINVAL;  // no value!
 +  } else {
 +    // we got a string.  see if it contains an int.
      n = strict_strtoll(val.c_str(), 10, &interr);
 -  string floaterr;
 -  float f;
 -  if (!cmd_getval(g_ceph_context, cmdmap, "val", f))
++    // or a float
+     f = strict_strtod(val.c_str(), &floaterr);
 +  }
  
    if (var == "size") {
      if (interr.length()) {

diff --cc src/osd/OSDMap.cc
Simple merge

diff --cc src/osd/PG.cc
Simple merge

diff --cc src/osd/PG.h
Simple merge

diff --cc src/osd/ReplicatedPG.cc
index c4edafe50ae5f4e1db57b2613c8d4fdb02d94953,d94668a248fdf13de2ab924325f220e5dc182588..acf32e4f0e2ca4739e09478d849aa9e17cc23586
--- 1/src/osd/ReplicatedPG.cc
--- 2/src/osd/ReplicatedPG.cc
+++ b/src/osd/ReplicatedPG.cc
@@@ -7381,16 -7402,19 +7432,18 @@@ void ReplicatedPG::on_shutdown(
    cancel_recovery();
  }
  
 +// For now only care about a single backfill at a time
  void ReplicatedPG::on_activate()
  {
 -  for (unsigned i = 1; i<acting.size(); i++) {
 -    if (peer_info[acting[i]].last_backfill != hobject_t::get_max()) {
 -      assert(backfill_target == -1);
 -      backfill_target = acting[i];
 -      last_backfill_started = peer_info[acting[i]].last_backfill;
 -      dout(10) << " chose backfill target osd." << backfill_target
 -             << " from " << last_backfill_started << dendl;
 -    }
 -  }
 +  int backfill_target = get_backfill_target();
 +  if (backfill_target == -1)
 +    return;
 +  last_backfill_started = peer_info[backfill_target].last_backfill;
 +  assert(last_backfill_started != hobject_t::get_max());
 +  dout(10) << " chose backfill target osd." << backfill_target
 +         << " from " << last_backfill_started << dendl;
+ 
+   hit_set_setup();
  }
  
  void ReplicatedPG::on_change(ObjectStore::Transaction *t)

diff --cc src/osd/ReplicatedPG.h
Simple merge

diff --cc src/osd/osd_types.cc
Simple merge

diff --cc src/osd/osd_types.h
Simple merge

diff --cc src/osdc/Objecter.cc
index 1af2f4d3a354ed2ccd81169510ce8313702c852c,f004a5c6f355fb5f34a7312ff878994e75cb38e1..d4a0581233b647f8d82d6ef0d31b69e2b632b716
--- 1/src/osdc/Objecter.cc
--- 2/src/osdc/Objecter.cc
+++ b/src/osdc/Objecter.cc
@@@ -1302,20 -1357,26 +1339,35 @@@ bool Objecter::is_pg_changed(vector<int
    return false;      // same primary (tho replicas may have changed)
  }
  
- int Objecter::recalc_op_target(Op *op)
 +bool Objecter::op_should_be_paused(Op *op)
 +{
 +  bool pauserd = osdmap->test_flag(CEPH_OSDMAP_PAUSERD);
 +  bool pausewr = osdmap->test_flag(CEPH_OSDMAP_PAUSEWR) || osdmap->test_flag(CEPH_OSDMAP_FULL);
 +
 +  return (op->flags & CEPH_OSD_FLAG_READ && pauserd) ||
 +         (op->flags & CEPH_OSD_FLAG_WRITE && pausewr);
 +}
 +
+ int64_t Objecter::get_object_hash_position(int64_t pool, const string& key,
+                                          const string& ns)
  {
-   vector<int> acting;
-   pg_t pgid = op->pgid;
+   const pg_pool_t *p = osdmap->get_pg_pool(pool);
+   if (!p)
+     return -ENOENT;
+   return p->hash_key(key, ns);
+ }
  
+ int64_t Objecter::get_object_pg_hash_position(int64_t pool, const string& key,
+                                             const string& ns)
+ {
+   const pg_pool_t *p = osdmap->get_pg_pool(pool);
+   if (!p)
+     return -ENOENT;
+   return p->raw_hash_to_pg(p->hash_key(key, ns));
+ }
+ 
+ int Objecter::recalc_op_target(Op *op)
+ {
    bool is_read = op->flags & CEPH_OSD_FLAG_READ;
    bool is_write = op->flags & CEPH_OSD_FLAG_WRITE;
  
@@@ -1349,16 -1414,9 +1405,17 @@@
      if (ret == -ENOENT)
        return RECALC_OP_TARGET_POOL_DNE;
    }
+   vector<int> acting;
    osdmap->pg_to_acting_osds(pgid, acting);
  
 +  bool need_resend = false;
 +
 +  bool paused = op_should_be_paused(op);
 +  if (!paused && paused != op->paused) {
 +    op->paused = false;
 +    need_resend = true;
 +  }
 +
    if (op->pgid != pgid || is_pg_changed(op->acting, acting, op->used_replica)) {
      op->pgid = pgid;
      op->acting = acting;

diff --cc src/osdc/Objecter.h
Simple merge

diff --cc src/test/Makefile.am
Simple merge

diff --cc src/test/encoding/ceph_dencoder.cc
Simple merge

diff --cc src/test/encoding/check-generated.sh
Simple merge

diff --cc src/vstart.sh
index c10ed8c4c6d9e6c0914bbfea20ad483c50f0fec6,86f969a76d6a85887f15f0de9cf3ab7db4ed1a86..ae3002f7307569aae9a8947d0faf67e6aeda0c32
--- 1/src/vstart.sh
--- 2/src/vstart.sh
+++ b/src/vstart.sh
@@@ -35,9 -34,9 +35,10 @@@ start_rgw=
  ip=""
  nodaemon=0
  smallmds=0
+ hitset=""
  overwrite_conf=1
  cephx=1 #turn cephx on by default
 +memstore=0
  
  MON_ADDR=""
  
@@@ -143,9 -143,11 +145,14 @@@ case $1 i
      -k )
            overwrite_conf=0
            ;;
 +    --memstore )
 +          memstore=1
 +          ;;
+     --hitset )
+           hitset="$hitset $2 $3"
+           shift
+           shift
+           ;;
      -o )
            extra_conf="$extra_conf     $2
  "