Fix issues with 64 bit keyspace

main issues were:
* --requests was only 32bit
* --key-pattern=P:P was incompatible with large keyspace
* random number generator was unfair (some of the upper bits were constant)

Author: oranagra

client.cpp

@@ -108,9 +108,9 @@ bool client::setup_client(benchmark_config *config, abstract_protocol *protocol,
         m_obj_gen->set_random_seed(config->next_client_idx);
 
     if (config->key_pattern[0]=='P') {
-        int range = (config->key_maximum - config->key_minimum)/(config->clients*config->threads) + 1;
-        int min = config->key_minimum + range*config->next_client_idx;
-        int max = min+range;
+        unsigned long long range = (config->key_maximum - config->key_minimum)/(config->clients*config->threads) + 1;
+        unsigned long long min = config->key_minimum + range*config->next_client_idx;
+        unsigned long long max = min+range;
         if(config->next_client_idx==(int)(config->clients*config->threads)-1)
             max = config->key_maximum; //the last clients takes the leftover
         m_obj_gen->set_key_range(min, max);

client.h

@@ -152,13 +152,13 @@ class client : public connections_manager {
     object_generator* m_obj_gen;
     run_stats m_stats;
 
-    unsigned int m_reqs_processed;      // requests processed (responses received)
-    unsigned int m_reqs_generated;      // requests generated (wait for responses)
+    unsigned long long m_reqs_processed;      // requests processed (responses received)
+    unsigned long long m_reqs_generated;      // requests generated (wait for responses)
     unsigned int m_set_ratio_count;     // number of sets counter (overlaps on ratio)
     unsigned int m_get_ratio_count;     // number of gets counter (overlaps on ratio)
 
-    unsigned long m_tot_set_ops;        // Total number of SET ops
-    unsigned long m_tot_wait_ops;       // Total number of WAIT ops
+    unsigned long long m_tot_set_ops;        // Total number of SET ops
+    unsigned long long m_tot_wait_ops;       // Total number of WAIT ops
 
     keylist *m_keylist;                 // used to construct multi commands
 
@@ -174,15 +174,15 @@ class client : public connections_manager {
     run_stats* get_stats(void) { return &m_stats; }
 
     // client manager api's
-    unsigned int get_reqs_processed() {
+    unsigned long long get_reqs_processed() {
         return m_reqs_processed;
     }
 
     void inc_reqs_processed() {
         m_reqs_processed++;
     }
 
-    unsigned int get_reqs_generated() {
+    unsigned long long get_reqs_generated() {
         return m_reqs_generated;
     }
 

connections_manager.h

@@ -21,9 +21,9 @@
 
 class connections_manager {
 public:
-    virtual unsigned int get_reqs_processed(void) = 0;
+    virtual unsigned long long get_reqs_processed(void) = 0;
     virtual void inc_reqs_processed(void) = 0;
-    virtual unsigned int get_reqs_generated(void) = 0;
+    virtual unsigned long long get_reqs_generated(void) = 0;
     virtual void inc_reqs_generated(void) = 0;
     virtual bool finished(void) = 0;
 

memtier_benchmark.cpp

@@ -84,7 +84,7 @@ static void config_print(FILE *file, struct benchmark_config *cfg)
         "client_stats = %s\n"
         "run_count = %u\n"
         "debug = %u\n"
-        "requests = %u\n"
+        "requests = %llu\n"
         "clients = %u\n"
         "threads = %u\n"
         "test_time = %u\n"
@@ -172,7 +172,7 @@ static void config_print_to_json(json_handler * jsonhandler, struct benchmark_co
     jsonhandler->write_obj("client_stats"      ,"\"%s\"",      	cfg->client_stats);
     jsonhandler->write_obj("run_count"         ,"%u",          	cfg->run_count);
     jsonhandler->write_obj("debug"             ,"%u",          	cfg->debug);
-    jsonhandler->write_obj("requests"          ,"%u",          	cfg->requests);
+    jsonhandler->write_obj("requests"          ,"%llu",        	cfg->requests);
     jsonhandler->write_obj("clients"           ,"%u",          	cfg->clients);
     jsonhandler->write_obj("threads"           ,"%u",          	cfg->threads);
     jsonhandler->write_obj("test_time"         ,"%u",          	cfg->test_time);
@@ -237,11 +237,11 @@ static void config_init_defaults(struct benchmark_config *cfg)
         cfg->key_pattern = "R:R";
     if (!cfg->data_size_pattern)
         cfg->data_size_pattern = "R";
-    if (cfg->requests == (unsigned int)-1) {
+    if (cfg->requests == (unsigned long long)-1) {
         cfg->requests = cfg->key_maximum - cfg->key_minimum;
         if (strcmp(cfg->key_pattern, "P:P")==0)
             cfg->requests = cfg->requests / (cfg->clients * cfg->threads) + 1;
-        printf("setting requests to %d\n", cfg->requests);
+        printf("setting requests to %llu\n", cfg->requests);
     }
     if (!cfg->requests && !cfg->test_time)
         cfg->requests = 10000;
@@ -443,7 +443,7 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
                     if (strcmp(optarg, "allkeys")==0)
                         cfg->requests = -1;
                     else {
-                        cfg->requests = (unsigned int) strtoul(optarg, &endptr, 10);
+                        cfg->requests = (unsigned long long) strtoull(optarg, &endptr, 10);
                         if (!cfg->requests || !endptr || *endptr != '\0') {
                             fprintf(stderr, "error: requests must be greater than zero.\n");
                             return -1;
@@ -1165,16 +1165,16 @@ int main(int argc, char *argv[])
         fprintf(outfile,
                "%-9u Threads\n"
                "%-9u Connections per thread\n"
-               "%-9u %s\n",
+               "%-9llu %s\n",
                cfg.threads, cfg.clients, 
-               cfg.requests > 0 ? cfg.requests : cfg.test_time,
+               (unsigned long long)(cfg.requests > 0 ? cfg.requests : cfg.test_time),
                cfg.requests > 0 ? "Requests per thread"  : "Seconds");
         if (jsonhandler != NULL){
             jsonhandler->open_nesting("run information");
             jsonhandler->write_obj("Threads","%u",cfg.threads);
             jsonhandler->write_obj("Connections per thread","%u",cfg.clients);
-            jsonhandler->write_obj(cfg.requests > 0 ? "Requests per thread"  : "Seconds","%u",
-                                   cfg.requests > 0 ? cfg.requests : cfg.test_time);
+            jsonhandler->write_obj(cfg.requests > 0 ? "Requests per thread"  : "Seconds","%llu",
+                                   cfg.requests > 0 ? cfg.requests : (unsigned long long)cfg.test_time);
             jsonhandler->close_nesting();
         }
 

memtier_benchmark.h

@@ -46,7 +46,7 @@ struct benchmark_config {
     int distinct_client_seed;
     int randomize;
     int next_client_idx;
-    unsigned int requests;
+    unsigned long long requests;
     unsigned int clients;
     unsigned int threads;
     unsigned int test_time;

obj_gen.cpp

@@ -50,31 +50,39 @@ void random_generator::set_seed(int seed)
     assert(ret == 0);
 #elif (defined HAVE_DRAND48)
     memset(&m_data_blob, 0, sizeof(m_data_blob));
-    memcpy(&m_data_blob, &seed, MIN(sizeof(seed), sizeof(m_data_blob)));
+    size_t seed_size = sizeof(seed); //get MIN size between seed and m_data_blob
+    if (seed_size > sizeof(m_data_blob))
+        seed_size = sizeof(m_data_blob);
+    memcpy(&m_data_blob, &seed, seed_size);
 #endif
 }
 
 unsigned long long random_generator::get_random()
 {
-    int rn;
     unsigned long long llrn;
 #ifdef HAVE_RANDOM_R
-    int ret = random_r(&m_data_blob, &rn);//max is RAND_MAX
+    int32_t rn;
+    // max is RAND_MAX, which is usually 2^31-1 (although can be as low as 2^16-1, which we ignore now)
+    // this is fine, especially considering that random_r is a nonstandard glibc extension
+    // it returns a positive int32_t, so either way the MSB is off
+    int ret = random_r(&m_data_blob, &rn);
     assert(ret == 0);
-
     llrn = rn;
-    llrn = llrn << 32; // move to upper 32bits
+    llrn = llrn << 31;
 
-    ret = random_r(&m_data_blob, &rn);//max is RAND_MAX
+    ret = random_r(&m_data_blob, &rn);
     assert(ret == 0);
-    llrn |= rn; // set lower 32bits
+    llrn |= rn;
 #elif (defined HAVE_DRAND48)
-    rn = nrand48(m_data_blob); // max is 1<<31
+    long rn;
+    // jrand48's range is -2^31..+2^31 (i.e. all 32 bits)
+    rn = jrand48(m_data_blob);
     llrn = rn;
-    llrn = llrn << 32; // move to upper 32bits
+    llrn = llrn << 32;
 
-    rn = nrand48(m_data_blob); // max is 1<<31
-    llrn |= rn; // set lower 32bits
+    rn = jrand48(m_data_blob);
+    llrn |= rn & 0xffffffff; // reset the sign extension bits of negative numbers
+    llrn &= 0x8000000000000000; // avoid any trouble from sign mismatch and negative numbers
 #else
     #error no random function
 #endif
@@ -84,12 +92,9 @@ unsigned long long random_generator::get_random()
 unsigned long long random_generator::get_random_max() const
 {
 #ifdef HAVE_RANDOM_R
-    unsigned long long rand_max = RAND_MAX;
-    return (rand_max << 32) | RAND_MAX;
+    return 0x3fffffffffffffff;//62 bits
 #elif (defined HAVE_DRAND48)
-    return ((1<<31) << 32) | (1<<31);
-#else
-    #error no random function
+    return 0x7fffffffffffffff;//63 bits
 #endif
 }