]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
*** empty log message ***
authorsage <sage@29311d96-e01e-0410-9327-a35deaab8ce9>
Tue, 20 Sep 2005 20:29:07 +0000 (20:29 +0000)
committersage <sage@29311d96-e01e-0410-9327-a35deaab8ce9>
Tue, 20 Sep 2005 20:29:07 +0000 (20:29 +0000)
git-svn-id: https://ceph.svn.sf.net/svnroot/ceph@499 29311d96-e01e-0410-9327-a35deaab8ce9

ceph/TODO
ceph/crush/Bucket.h
ceph/crush/Hash.h
ceph/crush/crush.h
ceph/test/testcrush.cc

index 6a3210ca853cc0a5d7a4120d79ef30881fd1e81f..7aa3f39092408555af329e8ec83ebc7909d319ca 100644 (file)
--- a/ceph/TODO
+++ b/ceph/TODO
@@ -60,120 +60,6 @@ osd fun
 
 
 
-RUSH_P : prime numbers
- draw K from expanding set of clusters
-
-RUSH_R : removal
- draw K from expanding or contracting set of clusters
-
-RUSH_T : tree
-
-
-two types of buckets: 
-
- edge bucket .. list of identical disks
- bucket      .. list of whatever, structured as a tree.
-
-struct disk_set {
-  int id;
-  float disk_weight;
-  int ndisks;
-  vector<int> disks;
-};
-
-struct bucket {
-  int         id;
-  vector<int> contents;
-  map<int, float>  weight;
-};
-
-
-
-struct bucket {
-  int             id;
-  int             type;    // disk, shelf, etc.
-  vector<int>     contents;
-  float           fixed_weight;  // homogenous bucket, or
-  map<int, float> weight;        // mixed bucket
-}
-
-give each bucket "type" has a numerical id, whatever you want.  eg
-
-0 disk
-1 shelf
-2 cabinet
-3 row
-4 room
-
-mirror(4)
-take(root)
-choose(2, room)
-take_vert
-choose(2, cab)
-take_vert
-choose(1, disk)
-
-
-=> 0
-takeb root
-choose_r 0 room -> w[2]
-takew 2
-choose_r 0 cab -> w[1]
-takew 1
-choose_r 0 disk -> 0
-
-=> 1
-takeb root
-choose_r 0 room -> w[2]
-takew 2
-choose_r 1 cab -> w[1]
-takew 1
-choose_r 0 disk -> 0
-
-=> 2
-takeb root
-choose_r 1 room -> w[2]
-takew 2
-choose_r 0 cab -> w[1]
-takew 1
-choose_r 0 disk -> 0
-
-=> 3
-takeb root
-choose_r 1 room -> w[2]
-takew 2
-choose_r 1 cab -> w[1]
-takew 1
-choose_r 0 disk -> 0
-
-
-mirror(3)
-choose(1, room, root)
-choose(3, cab, _)
-vert(_)
-choose(1, disk, _)
-
-
-=> 0
-takeb root
-choose_r 0/1 room 
-distinct
- choose_r r/3 cab
-choose_r 0/1 disk
-takeb backup
-choose_r 0/1 disk
-
-=> 1
-takeb root
-choose_r 0/1 room
-choose_r 1/3 cab
-choose_r 0/1 disk
-
-=> 2
-takeb root
-choose_r 0/1 room
-choose_r 2/3 cab
-choose_r 0/1 disk
 
 
 
index a28f49a93ee8b5d04e7357b6baad40948ef94392..e07e6c98d518a49343ffb6b47b09db0385b96e88 100644 (file)
@@ -88,8 +88,8 @@ namespace crush {
          //cout << "uniformbucket.choose_r(" << x << ", " << r << ")" << endl;
          //if (r >= get_size()) cout << "warning: r " << r << " >= " << get_size() << " uniformbucket.size" << endl;
          
-         int v = (h(x, get_id(), 1) % get_size()) * get_size();
-         int p = get_prime( h(x, 2) );  // choose a prime based on hash(x, get_id(), 2)
+         int v = h(x, get_id(), 1) % get_size();
+         int p = get_prime( h(x, get_id(), 2) );  // choose a prime based on hash(x, get_id(), 2)
          int s = (x + v + (r+1)*p) % get_size();
          return items[s];
        }
@@ -115,32 +115,6 @@ namespace crush {
        bool        is_uniform() const { return false; }
        int get_size() const { return node_map.size(); }
 
-       /*
-       float calc_weight() {
-         weight = 0;
-         for (unsigned i=0; i<items.size(); i++) {
-               weight += get_item_weight(i);
-         }
-         return weight;
-       }
-       */
-
-       /*
-       void make_new_tree(vector<int>& _items) {
-         assert(items.empty());
-         assert(tree.empty());
-         
-         items = _items;
-         
-         for (unsigned i=0; i<items.size(); i++) {
-               int n = tree.add_node(item_weight[i]);
-               node_map[n] = items[i];
-         }
-
-         //calc_weight();
-       }
-       */
-
        void add_item(int item, float w) {
          int n = tree.add_node(w);
          node_map[n] = item;
@@ -153,7 +127,7 @@ namespace crush {
          while (!tree.terminal(n)) {
                // pick a point in [0,w)
                float w = tree.weight(n);
-               float f = (float)(h(x, n, r) % 1000) * w / 1000.0;
+               float f = (float)(h(x, n, r, get_id()) % 1000) * w / 1000.0;
 
                // left or right?
                int l = tree.left(n);
index 57d36fa4486ea1fe203a0264c68269cd59575aa2..796d7112f7b7c824cc489f26109632b864b7a297 100644 (file)
@@ -15,8 +15,69 @@ namespace crush {
          if (0) 
                return (n ^ 0xdead1234) * (884811920 * 3  + 1);
          
-         // RS Hash function, from Robert Sedgwicks Algorithms in C book, w/ some changes.
+         // djb2
+         if (0) {
+               unsigned int hash = 5381;
+               for (int i=0; i<4; i++) {
+                 hash = ((hash << 5) + hash) + ((n&255) ^ 123);
+                 n = n >> 8;
+               }
+               return hash;
+         }
+
+         // JS
+         //  a little better than RS
          if (1) {
+               unsigned int hash = 1315423911;
+               
+               for(unsigned int i = 0; i < 4; i++)
+                 {
+                       hash ^= ((hash << 5) + (n&255) + (hash >> 2));
+                       n = n >> 8;
+                 }
+               
+               return (hash & 0x7FFFFFFF);
+         }
+
+         // SDBM
+         if (1) {
+               unsigned int hash = 0;
+               
+               for(unsigned int i = 0; i < 4; i++)
+                 {
+                       hash = (n&255) + (hash << 6) + (hash << 16) - hash;
+                       n = n >> 8;
+                 }
+               
+               return (hash & 0x7FFFFFFF);
+         }
+
+         // PJW
+         //  horrid
+         if (0) {
+               unsigned int BitsInUnsignedInt = (unsigned int)(sizeof(unsigned int) * 8);
+               unsigned int ThreeQuarters     = (unsigned int)((BitsInUnsignedInt  * 3) / 4);
+               unsigned int OneEighth         = (unsigned int)(BitsInUnsignedInt / 8);
+               unsigned int HighBits          = (unsigned int)(0xFFFFFFFF) << (BitsInUnsignedInt - OneEighth);
+               unsigned int hash              = 0;
+               unsigned int test              = 0;
+               
+               for(unsigned int i = 0; i < 4; i++)
+                 {
+                       hash = (hash << OneEighth) + (n&255);
+                       
+                       if((test = hash & HighBits)  != 0)
+                         {
+                               hash = (( hash ^ (test >> ThreeQuarters)) & (~HighBits));
+                         }
+                       n = n >> 8;
+                 }
+               
+               return (hash & 0x7FFFFFFF);
+         }
+
+         // RS Hash function, from Robert Sedgwicks Algorithms in C book, w/ some changes.
+         if (0) {
                unsigned int b    = 378551;
                unsigned int a    = 63689;
                unsigned int hash = 0;
@@ -31,6 +92,36 @@ namespace crush {
                return (hash & 0x7FFFFFFF);
          }
 
+         // DJB
+         //  worse than rs
+         if (0) {
+               unsigned int hash = 5381;
+               
+               for(unsigned int i = 0; i < 4; i++)
+                 {
+                       hash = ((hash << 5) + hash) + (n&255);
+                       n = n >> 8;
+                 }
+               
+               return (hash & 0x7FFFFFFF);
+         }
+
+         // AP
+         //  even worse
+         if (1) {
+               unsigned int hash = 0;
+               
+               for(unsigned int i = 0; i < 4; i++)
+                 {
+                       hash ^= ((i & 1) == 0) ? (  (hash <<  7) ^ (n&255) ^ (hash >> 3)) :
+                         (~((hash << 11) ^ (n&255) ^ (hash >> 5)));
+                       n = n >> 8;
+                 }
+               
+               return (hash & 0x7FFFFFFF);
+         }
+
+
        }
        
 
@@ -40,10 +131,13 @@ namespace crush {
          return myhash(a) ^ seed;
        }
        int operator()(int a, int b) {
-         return myhash(a) ^ myhash(b) ^ seed;
+         return myhash( myhash(a) ^ myhash(b) ^ seed );
        }
        int operator()(int a, int b, int c) {
-         return myhash(a) ^ myhash(b) ^ myhash(c) ^ seed;
+         return myhash( myhash(a ^ seed) ^ myhash(b ^ seed) ^ myhash(c ^ seed) ^ seed );
+       }
+       int operator()(int a, int b, int c, int d) {
+         return myhash( myhash(a ^ seed) ^ myhash(b ^ seed) ^ myhash(c ^ seed) ^ myhash(d ^ seed) ^ seed );
        }
   };
 
index 46626893342e0aa89b2479ee2333a33fa0281a78..90c519e7adea75e2d35bea8955b758d711c608db 100644 (file)
@@ -3,6 +3,7 @@
 
 #include <iostream>
 #include <list>
+#include <vector>
 using namespace std;
 
 ostream& operator<<(ostream& out, vector<int>& v)
@@ -49,29 +50,13 @@ namespace crush {
   const int CRUSH_RULE_TAKE = 0;
   const int CRUSH_RULE_CHOOSE = 1;
   const int CRUSH_RULE_VERT = 2;
+  const int CRUSH_RULE_EMIT = 3;
 
   class Rule {
   public:
        vector< RuleStep > steps;
   };
 
-  // CRule
-  const int CRULE_TAKEB = 1;
-  const int CRULE_CHOOSER = 2;
-  const int CRULE_CHOOSE_R = 10;
-  const int CRULE_CHOOSE_D = 11;
-
-  class CRule {
-  public:
-       vector< vector<RuleStep> > steps;
-       int nchoose;
-       int nrep;
-
-       CRule() {}
-       CRule(int nr) : steps(nr), nrep(nr) {}
-  };
-
-
 
 
 
@@ -81,13 +66,19 @@ namespace crush {
   protected:
        map<int, Bucket*>  buckets;
        map<int, Rule>     rules;
-       map<int, CRule>     crules;
        Hash h;
 
        set<int>           failed;
+  public:
+       map<int, float>    overload;
+       
 
   public:
-       Crush(int seed=123) : h(seed) {}
+       Crush(int seed=123) : h(seed), collisions(20), bumps(20) {}
+
+       vector<int> collisions;
+       vector<int> bumps;
+
 
        void add_bucket( Bucket *b ) {
          buckets[b->get_id()] = b;
@@ -95,107 +86,13 @@ namespace crush {
        void add_rule( int id, Rule& r ) {
          rules[id] = r;
        }
-       void add_crule( int id, CRule& r ) {
-         crules[id] = r;
-       }
-
-       void crule_choose(int ruleno, int x, vector<int>& result) {
-         CRule& rule = crules[ruleno];
-         
-         // input
-         Bucket *in = 0;
-         int out = -1;
-         
-         // for replicas
-         for (int rep=0; rep<rule.nrep; rep++) {
-               // initially zero
-               vector<int> add_r(rule.nchoose);
-               
-               for (int attempt=0; ; attempt++) {
-                 
-                 // steps
-                 int nchoose = 0;
-                 for (int pc=0; pc<rule.steps[rep].size(); pc++) {
-                       switch (rule.steps[rep][pc].cmd) {
-                       case CRULE_TAKEB:
-                         {
-                               const int arg = rule.steps[rep][pc].args[0];
-                               assert(buckets.count(arg));
-                               in = buckets[arg];
-                         }
-                         break;
-                         
-                       case CRULE_CHOOSER:
-                         {
-                               int r = rule.steps[rep][pc].args[0];
-                               const int rperiod = rule.steps[rep][pc].args[1];
-                               const int type = rule.steps[rep][pc].args[2];
-                               
-                               // adjust to skip unusable.  nonlinearly.
-                               r += (rperiod + add_r[nchoose]) * add_r[nchoose];
-                               nchoose++;
-                               
-                               //cout << "choose_r " << r << " type " << type << endl;
-                               
-                               // choose through any intervening buckets
-                               while (1) {
-                                 // choose in this bucket
-                                 out = in->choose_r(x, r, h);
-
-                                 if (in->is_uniform() && 
-                                         ((UniformBucket*)in)->get_item_type() == type)
-                                       break;
-
-                                 int itemtype = 0;  // 0 is a terminal type
-                                 if (buckets.count(out)) {
-                                       in = buckets[out];
-                                       itemtype = in->get_type();
-                                 } 
-                                 if (itemtype == type) break;  // this is what we want!
-                               }                               
-                               
-                               if (type != 0) {  // translate back into a bucket
-                                 assert(buckets.count(out));
-                                 in = buckets[out];
-                               }
-                         }
-                         break;
-                         
-                       default:
-                         assert(0);
-                       }
-                 }
 
-                 // disk failed?
-                 bool bad = false;
-                 if (failed.count(out)) 
-                       bad = true;
-                 
-                 for (int prep=0; prep<rep; prep++) {
-                       if (result[prep] == out) 
-                         bad = true;
-                 }
-
-                 if (bad) {
-                       // bump an 'r'
-                       //cout << "failed|repeat " << attempt << endl;
-                       add_r[ rule.nchoose - 1 - ((attempt/2)%rule.nchoose) ]++;
-                       continue;
-                 }
-
-                 break;                  // disk is fine.
-               }
-               
-               // ok!
-               result[rep] = out;
-         }
-       }
 
-
-       void newchoose(int ruleno, int x, vector<int>& result) {
+       void choose(int ruleno, int x, vector<int>& result) {
          assert(rules.count(ruleno));
          Rule& rule = rules[ruleno];
 
+         int numresult = 0;
          int maxdepth = 20;
 
          // working variable
@@ -213,12 +110,13 @@ namespace crush {
                case CRUSH_RULE_TAKE:
                  {
                        const int arg = pc->args[0];
-                       cout << "take " << arg << endl;
+                       //cout << "take " << arg << endl;
 
                        // input is some explicit existing bucket
                        assert(buckets.count(arg));
                        
-                       if (out.empty()) {
+                       in.clear();
+                       if (true || out.empty()) {
                          // 1 row
                          in.push_back( buckets[arg] );
                        } else {
@@ -252,7 +150,7 @@ namespace crush {
                        const int numrep = pc->args[0];
                        const int type = pc->args[1];
 
-                       cout << "choose " << numrep << " of " << type << endl;
+                       //cout << "choose " << numrep << " of type " << type << endl;
 
                        // reset output
                        out.clear();
@@ -262,7 +160,7 @@ namespace crush {
                                 inrow != in.end();
                                 inrow++) {
                          // make new output row
-                         out.push_back( vector<int>() );
+                         out.push_back( vector<int>(numrep) );
                          vector<int>& outrow = out.back();
                          
                          // for each replica
@@ -279,10 +177,30 @@ namespace crush {
 
                                  // choose through intervening buckets
                                  while (1) {
-                                       // r may be twiddled to avoid collision
-                                       int r = rep + (numrep + add_r[depth]) * add_r[depth];
-                                       out = in->choose_r(x, r, h); 
+                                       // r may be twiddled to (try to) avoid past collisions
+                                       int r = rep;
+                                       if (in->is_uniform()) {
+                                         // uniform bucket; be careful!
+                                         if (numrep >= in->get_size()) {
+                                               // uniform bucket is too small; just walk thru elements
+                                               r += add_r[depth];
+                                         } else {
+                                               // make sure numrep is not a multple of bucket size
+                                               int add = numrep*add_r[depth];
+                                               if (in->get_size() % numrep == 0) {
+                                                 add += add/in->get_size();         // shift seq once per pass through the bucket
+                                               }
+                                               r += add;
+                                         }
+                                       } else {
+                                         // mixed bucket; just make a distinct-ish r sequence
+                                         r += numrep * add_r[depth];
+                                       }
+
+                                       // choose
+                                       out = in->choose_r(x, r, h);                                    
                                        
+                                       // did we get the type we want?
                                        if (in->is_uniform() && 
                                                ((UniformBucket*)in)->get_item_type() == type)
                                          break;
@@ -297,196 +215,69 @@ namespace crush {
                                  }
                                  
                                  // ok choice?
-                                 bool bad = false;
-                                 if (failed.count(out)) 
-                                       bad = true;
+                                 int bad_localness = 0;     // 1 -> no locality in replacement, >1 more locality
+                                 if (type == 0 && failed.count(out)) 
+                                       bad_localness = 1;         // no locality
+
+                                 if (overload.count(out)) {
+                                       float f = (float)(h(x, out) % 1000) / 1000.0;
+                                       if (f > overload[out])
+                                         bad_localness = 1;       // no locality
+                                 }
+
                                  for (int prep=0; prep<rep; prep++) 
                                        if (outrow[prep] == out) 
-                                         bad = true;
-                                 if (bad) {
+                                         bad_localness = 4;       // local is better
+
+                                 if (bad_localness) {
                                        // bump an 'r'
-                                       int d = depth - 1 - ((attempt/2)%depth);
-                                       cout << "failed|repeat " << attempt << ", bumping r at depth " << d << endl;
-                                       add_r[d]++;
+                                       depth++;   // want actual depth, not array index
+                                       bad_localness--;
+                                       if (bad_localness) {
+                                         // we want locality
+                                         int d = depth - 1 - ((attempt/bad_localness)%(depth));
+                                         add_r[d]++;
+                                         collisions[attempt]++;
+                                         bumps[d]++;
+                                       } else {
+                                         // uniformly try a new disk; bump all r's
+                                         for (int j=0; j<depth; j++) {
+                                               add_r[j]++;
+                                               bumps[j]++;
+                                         }
+                                         collisions[attempt]++;
+                                       }
                                        continue;
                                  }
                                  break;
                                }
 
                                outrow[rep] = out;
+                               //cout << "outrow[" << rep << "] = " << out << endl;
                          } // for rep
-                         cout << "outrow is " << outrow << endl;
+                         //cout << "outrow is " << outrow << endl;
                        } // for inrow
                  }
                  break;
 
-               default:
-                 assert(0);
-               }
-         }
-
-         // assemble result
-         int o = 0;
-         for (int i=0; i<out.size(); i++)
-               for (int j=0; j<out[i].size(); j++)
-                 result[o++] = out[i][j];
-       }
-
-
-
-       void choose(int rno, int x, vector<int>& result) {
-         assert(rules.count(rno));
-         Rule& r = rules[rno];
-
-         // working variable
-         vector< Bucket* >       in;
-         vector< vector<int>  >  out;
-
-         list< MixedBucket >     temp_buckets;
-
-         // go through each statement
-         for (vector<RuleStep>::iterator pc = r.steps.begin();
-                  pc != r.steps.end();
-                  pc++) {
-               // move input?
-               
-               // do it
-               switch (pc->cmd) {
-               case CRUSH_RULE_TAKE:
-                 {
-                       const int arg = pc->args[0];
-                       //cout << "take " << arg << endl;
-
-                       in.clear();
-                       temp_buckets.clear();
-
-                       if (arg == 0) {  
-                         // input is old output
-
-                         for (vector< vector<int> >::iterator row = out.begin();
-                                  row != out.end();
-                                  row++) {
-                               
-                               if (row->size() == 1) {
-                                 in.push_back( buckets[ (*row)[0] ] );
-                               } else {
-                                 // make a temp bucket!
-                                 temp_buckets.push_back( MixedBucket( rno, -1 ) );
-                                 in.push_back( &temp_buckets.back() );
-                                 
-                                 // put everything in.
-                                 for (int j=0; j<row->size(); j++)
-                                       temp_buckets.back().add_item( (*row)[j],
-                                                                                                 buckets[ (*row)[j] ]->get_weight() );
-                               }
-                         }
-                         
-                         // reset output variable
-                         //out.clear();
-                         out = vector< vector<int> >(in.size());
-
-                       } else {         
-                         // input is some explicit existing bucket
-                         assert(buckets.count(arg));
-
-                         if (out.empty()) {
-                               // 1 row
-                               in.push_back( buckets[arg] );
-                         } else {
-                               // match rows in output?
-                               for (vector< vector<int> >::iterator row = out.begin();
-                                        row != out.end();
-                                        row++) 
-                                 in.push_back( buckets[arg] );
-                         }
-                       }
-                       
-                       /*
-                       cout << "take: in is [";
-                       for (int i=0; i<in.size(); i++) 
-                         cout << " " << in[i]->get_id();
-                       cout << "]" << endl;                      
-                       */
-                 }
-                 break;
-
-               case CRUSH_RULE_VERT:
+               case CRUSH_RULE_EMIT:
                  {
-                       in.clear();
-                       temp_buckets.clear();
-                       
-                       // input is (currently always) old output
-
-                       for (vector< vector<int> >::iterator row = out.begin();
-                                row != out.end();
-                                row++) {
-                         for (int i=0; i<row->size(); i++) {
-                               in.push_back( buckets[ (*row)[i] ] );
-                         }
-                       }
-                 }
-                 break;
-                 
-               case CRUSH_RULE_CHOOSE:
-                 {
-                       const int num = pc->args[0];
-                       const int type = pc->args[1];
-                       
-                       // reset output
-                       out.clear();
-                       
-                       // do each row independently
-                       for (vector< Bucket* >::iterator row = in.begin();
-                                row != in.end();
-                                row++) {
-                         // make new output row
-                         out.push_back( vector<int>() );
-                         vector<int>& outrow = out.back();
-                         
-                         // for each replica
-                         for (int r=0; r<num; r++) {
-                               // start with input bucket
-                               const Bucket *b = *row;
-                               
-                               // choose through any intervening buckets
-                               while (1) {
-                                 if (b->is_uniform() && 
-                                         ((UniformBucket*)b)->get_item_type() == type) 
-                                       break;
-                                 
-                                 int next = b->choose_r(x, r, h);
-                                 int itemtype = 0;  // 0 is a terminal type
-                                 if (buckets.count(next)) {
-                                       b = buckets[next];
-                                       itemtype = b->get_type();
-                                 } 
-                                 if (itemtype == type) break;  // this is what we want!
-                               }
-
-                               // choose in this bucket!
-                               int item = b->choose_r(x, r, h);
-                               outrow.push_back(item);
-                         }
-                       }
+                       int o = 0;
+                       for (int i=0; i<out.size(); i++)
+                         for (int j=0; j<out[i].size(); j++)
+                               result[numresult++] = out[i][j];
                  }
                  break;
 
                default:
                  assert(0);
                }
-               
-               
          }
 
-         // assemble result
-         int o = 0;
-         for (int i=0; i<out.size(); i++)
-               for (int j=0; j<out[i].size(); j++)
-                 result[o++] = out[i][j];
        }
-       
+
   };
-  
+
 }
 
 #endif
index 29e21d3b7f4889e80a67c1f9c6e91592a9946c9e..2229c6b3d2d99213736b916091ab0def52dceccd 100644 (file)
@@ -3,11 +3,13 @@
 #include "../crush/crush.h"
 using namespace crush;
 
+#include <math.h>
+
 #include <iostream>
 #include <vector>
 using namespace std;
 
-
+/*
 ostream& operator<<(ostream& out, vector<int>& v)
 {
   out << "[";
@@ -18,7 +20,7 @@ ostream& operator<<(ostream& out, vector<int>& v)
   out << "]";
   return out;
 }
-
+*/
 
 void make_disks(int n, int& no, vector<int>& d) 
 {
@@ -32,76 +34,112 @@ void make_disks(int n, int& no, vector<int>& d)
 
 int main() 
 {
-  Hash h(73);
-  int numrep = 5;
+  Hash h(73232313);
+
+  // crush
+  Crush c;
 
 
   // buckets
   vector<int> disks;
   int ndisks = 0;
   
-  make_disks(12, ndisks, disks);
-  UniformBucket ub1(1, 1, 0, 30, disks);
-  ub1.make_primes(h);
-  cout << "ub1 primes are " << ub1.primes << endl;
-  
-  make_disks(17, ndisks, disks);
-  UniformBucket ub2(2, 1, 0, 30, disks);
-  ub2.make_primes(h);  
-  cout << "ub2 primes are " << ub2.primes << endl;
-
-  make_disks(4, ndisks, disks);
-  UniformBucket ub3(3, 1, 0, 30, disks);
-  ub3.make_primes(h);  
-  cout << "ub3 primes are " << ub3.primes << endl;
-
-  /*
-  make_disks(20, ndisks, disks);
-  MixedBucket umb1(4, 1);
-  for (int i=0; i<20; i++)
-       umb1.add_item(disks[ndisks-i-1], 30);
-  */
-
-  MixedBucket b(100, 1);
-  b.add_item(1, ub1.get_weight());
-  b.add_item(2, ub2.get_weight());
-  b.add_item(3, ub3.get_weight());
-  //b.add_item(4, umb1.get_weight());
+  if (0) {
+       make_disks(12, ndisks, disks);
+       UniformBucket ub1(-1, 1, 0, 30, disks);
+       ub1.make_primes(h);
+       cout << "ub1 primes are " << ub1.primes << endl;
+       c.add_bucket(&ub1);
+       
+       make_disks(17, ndisks, disks);
+       UniformBucket ub2(-2, 1, 0, 30, disks);
+       ub2.make_primes(h);  
+       cout << "ub2 primes are " << ub2.primes << endl;
+       c.add_bucket(&ub2);
+       
+       make_disks(4, ndisks, disks);
+       UniformBucket ub3(-3, 1, 0, 30, disks);
+       ub3.make_primes(h);  
+       cout << "ub3 primes are " << ub3.primes << endl;
+       c.add_bucket(&ub3);
+       
+       make_disks(20, ndisks, disks);
+       MixedBucket umb1(-4, 1);
+       for (int i=0; i<20; i++)
+         umb1.add_item(disks[i], 30);
+       c.add_bucket(&umb1);
+       
+       MixedBucket b(-100, 1);
+       //b.add_item(-2, ub1.get_weight());
+       b.add_item(-4, umb1.get_weight());
+       //b.add_item(-2, ub2.get_weight());
+       //b.add_item(-3, ub3.get_weight());
+  }
+
+  if (1) {
+       int bucket = -1;
+       MixedBucket *root = new MixedBucket(bucket--, 2);
+
+       for (int i=0; i<5; i++) {
+         MixedBucket *b = new MixedBucket(bucket--, 1);
+
+         int n = 5;
+         for (int j=0; j<n; j++) {
+
+               MixedBucket *d = new MixedBucket(bucket--, 1);
+
+               make_disks(n, ndisks, disks);
+               for (int k=0; k<n; k++)
+                 d->add_item(disks[k], 10);
+               
+               //b->add_item(disks[j], 10);
+               c.add_bucket(d);
+               b->add_item(d->get_id(), d->get_weight());
+         }
+
+         c.add_bucket(b);
+         root->add_item(b->get_id(), b->get_weight());
+       }
+
+       c.add_bucket(root);
+  }
+
+
 
   // rule
+  int numrep = 1;
+
   Rule rule;
-  rule.steps.push_back(RuleStep(CRUSH_RULE_TAKE, 100));
-  rule.steps.push_back(RuleStep(CRUSH_RULE_CHOOSE, numrep, 0));
-
-  CRule crule(numrep);
-  crule.nchoose = 2;
-  for (int j=0; j<numrep; j++) {
-       crule.steps[j].push_back(RuleStep(CRULE_TAKEB, 100));
-       crule.steps[j].push_back(RuleStep(CRULE_CHOOSER, j, numrep, 1));
-       crule.steps[j].push_back(RuleStep(CRULE_CHOOSER, j, numrep, 0));
+  if (0) {
+       rule.steps.push_back(RuleStep(CRUSH_RULE_TAKE, -100));
+       rule.steps.push_back(RuleStep(CRUSH_RULE_CHOOSE, numrep, 0));
+       c.add_rule(numrep, rule);
+  }
+  if (1) {
+       /*
+       rule.steps.push_back(RuleStep(CRUSH_RULE_TAKE, -4));
+       rule.steps.push_back(RuleStep(CRUSH_RULE_CHOOSE, 2, 0));
+       rule.steps.push_back(RuleStep(CRUSH_RULE_EMIT));
+       */
+       rule.steps.push_back(RuleStep(CRUSH_RULE_TAKE, -1));
+       rule.steps.push_back(RuleStep(CRUSH_RULE_CHOOSE, 1, 0));
+       rule.steps.push_back(RuleStep(CRUSH_RULE_EMIT));
+       c.add_rule(numrep, rule);
   }
 
-  // crush
-  Crush c;
-  c.add_bucket(&ub1);
-  c.add_bucket(&ub2);
-  c.add_bucket(&ub3);
-  //c.add_bucket(&umb1);
-  c.add_bucket(&b);
-  c.add_rule(numrep, rule);
-  c.add_crule(numrep, crule);
-
+  c.overload[10] = .1;
 
   
   vector<int> ocount(ndisks);
 
   vector<int> v(numrep);
-  int numo = 1000*ndisks/numrep;
+  int numo = 10000*ndisks/numrep;
+  cout << "nrep is " << numrep << endl;
   cout << "placing " << numo << " logical,  " << numo*numrep << " total" << endl;
   for (int x=1; x<numo; x++) {
        //cout << H(x) << "\t" << h(x) << endl;
-       c.newchoose(numrep, x, v);
-       cout << "v = " << v << endl;// " " << v[0] << " " << v[1] << "  " << v[2] << endl;
+       c.choose(numrep, x, v);
+       //cout << "v = " << v << endl;// " " << v[0] << " " << v[1] << "  " << v[2] << endl;
 
        bool bad = false;
        for (int i=0; i<numrep; i++) {
@@ -123,4 +161,20 @@ int main()
        cout << "disk " << i << " has " << ocount[i] << endl;
   }
 
+  cout << "collisions: " << c.collisions << endl;
+  cout << "r bumps: " << c.bumps << endl;
+
+  
+  float avg = 0.0;
+  for (int i=0; i<ocount.size(); i++)
+       avg += ocount[i];
+  avg /= ocount.size();
+  float var = 0.0;
+  for (int i=0; i<ocount.size(); i++)
+       var += (ocount[i] - avg) * (ocount[i] - avg);
+  var /= ocount.size();
+
+  cout << "avg " << avg << "  var " << var << "   sd " << sqrt(var) << endl;
+
+
 }