Adding support for Redisson

core/README.md

@@ -14,13 +14,14 @@ A prototype for generating and running ctests. Below are the projects we current
 - Hbase 2.2.2: `hbase-server`.
 - ZooKeeper 3.5.6: `zookeeper-server`.
 - Alluxio 2.1.0: `core`.
-
+- Redisson 3.18.0: `redisson`.
 We also provided our instrumented versions of the above projects:
 
 - Hadoop 2.8.5: https://github.com/xlab-uiuc/hadoop
 - Hbase 2.2.2: https://github.com/xlab-uiuc/hbase
 - ZooKeeper 3.5.6: https://github.com/xlab-uiuc/zookeeper
 - Alluxio 2.1.0: https://github.com/xlab-uiuc/alluxio
+- Redisson 3.18.0: https://github.com/kfadillah/redisson
 
 Our instrumented version projects have two branches: 
 - `ctest-injection`: branch with "Intercept Configuration API" instrumentation (See `ADDING_NEW_PROJECT.md`). This branch is used by `generate_ctest` and `run_ctest`.
@@ -59,7 +60,7 @@ To generate ctests or run ctest, you need to first clone the target project.
 1. In `openctest/core`, run `./add_project.sh <main project>` to clone the project, switch to and build the branch `ctest-injection`. This branch will be later used by `generate_ctest` and `run_ctest`.
 2. In `openctest/core/identify_param`, run `./add_project.sh <main project>` to clone the project, switch to and build the branch `ctest-logging`. This branch will be later used by `identify_param`.
 
-`<main project>` can be `hadoop`, `hbase`, `zookeeper` or `alluxio`.
+`<main project>` can be `hadoop`, `hbase`, `zookeeper`, `redisson` or `alluxio`.
 
 ## Usage
 

core/add_project.sh

@@ -48,6 +48,13 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_redisson() {
+    [ ! -d "app/ctest-redissonredisson" ] && git clone https://github.com/kfadillah/redisson.git app/ctest-redisson
+    cd app/ctest-redisson
+    git fetch && git checkout ctest-injection
+    mvn clean install -pl redisson -am -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +71,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            redisson) setup_redisson ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, redisson and alluxio." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+REDISSON = "redisson"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_REDISSON_DIR = os.path.join(APP_DIR, "ctest-redisson")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    REDISSON: CTEST_REDISSON_DIR
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    REDISSON: "redisson"
 }
 
 
@@ -58,6 +62,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    REDISSON: [os.path.join(CTEST_REDISSON_DIR, MODULE_SUBDIR[REDISSON], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,10 +79,10 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    REDISSON: os.path.join(DEFAULT_CONF_DIR, REDISSON + "-default.tsv"),
 }
 
-
 # injecting config file location
 INJECTION_PATH = {
     HCOMMON: [
@@ -96,6 +101,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    REDISSON: [
+        os.path.join(CTEST_REDISSON_DIR, MODULE_SUBDIR[REDISSON],"src/test/resources/org/redisson/ctest-injection.yaml")
     ]
 }
 
@@ -108,9 +116,9 @@
 FAIL = "f" # test failed
 PASS = "p" # test passed
 GOOD_VAL = "GOOD"
+
 BAD_VAL = "BAD"
 SKIP_VAL = "SKIP"
-
 CTESTS_DIR = os.path.join(GEN_CTEST_DIR, "ctest_mapping")
 os.makedirs(CTESTS_DIR, exist_ok=True)
 CTESTS_FILE = "ctests-{project}.json"

core/default_configs/redisson-default.tsv

@@ -0,0 +1,11 @@
+nettythreads	16	32
+referenceEnabled	true	false
+lockWatchdogTimeout	30000	35000
+checkLockSyncedSlaves	true	false
+reliableTopicWatchdogTimeout	0.05	1.0
+keepPubSubOrder	true	false
+useScriptCache	true	false
+minCleanUpDelay	5	10
+maxCleanUpDelay	1800	1000
+cleanUpKeysAmount	100	50
+useThreadClassLoader	true	false

core/generate_ctest/inject.py

@@ -35,6 +35,13 @@ def inject_config(param_value_pairs):
             file.write(str.encode("<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n"))
             file.write(ET.tostring(conf))
             file.close()
+    elif project == 'redisson':
+        for inject_path in INJECTION_PATH[project]:
+            print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
+            file = open(inject_path, "w")
+            for p, v in param_value_pairs.items():
+                file.write(p + ": " + v + "\n")
+            file.close()
     else:
         sys.exit(">>>>[ctest_core] value injection for {} is not supported yet".format(project))
 
@@ -53,5 +60,10 @@ def clean_conf_file(project):
             file.write(str.encode("<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n"))
             file.write(ET.tostring(conf))
             file.close()
+    elif project == 'redisson':
+        for inject_path in INJECTION_PATH[project]:
+            file = open(inject_path, "w")
+            file.write("\n")
+            file.close()
     else:
         sys.exit(">>>>[ctest_core] value injection for {} is not supported yet".format(project))

core/generate_ctest/main.py

@@ -31,7 +31,9 @@ def test_value_pair(test_input):
     for param, values in test_input.items():
         tr_file = open(os.path.join(GENCTEST_TR_DIR, project, TR_FILE.format(id=param)), "w")
         mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")
-
+        if param not in mapping:
+            print(">>>>[ctest_core] param not in mapping" + param)
+            continue
         associated_tests = mapping[param] if param in mapping else []
         if len(mapping[param]) != 0:
             for value in values:

core/generate_ctest/program_input.py

@@ -4,17 +4,17 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "redisson", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-redisson.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "../default_configs/redisson-default.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test
     "use_surefire": True, # bool
     # additional maven options to pass to `mvn surefire:test -Dtest=...`
-    "maven_args": [], # list of strings, each element is an option
+    "maven_args": ["-DredisBinary=/usr/bin/redis-server"], # list of strings, each element is an option
     # timeout on the mvn test command
     "cmd_timeout": None, # int
 }

core/generate_value/value_generation.py

@@ -25,6 +25,9 @@ def read_tsv(module):
     if module == "zookeeper-server":
         assert len(params) == 32
         return 32
+    elif module == "redisson":
+        assert len(params) == 11
+        return 32
     else:
         assert len(params) == 90
         return 90
@@ -105,6 +108,8 @@ def print_params(module):
     f = open(module + output, "w")
     if module == "zookeeper-server":
         assert len(params) == 32
+    elif module == "redisson":
+        assert len(params) == 11
     else:
         assert len(params) >= 90
     for param in params:

core/identify_param/add_project.sh

@@ -47,6 +47,13 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_redisson() {
+    [ ! -d "app/ctest-redissonredisson" ] && git clone https://github.com/kfadillah/redisson.git app/ctest-redisson
+    cd app/ctest-redisson
+    git fetch && git checkout ctest-logging
+    mvn clean install -pl redisson -am -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -63,7 +70,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            redisson) setup_redisson ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, redisson and alluxio." ;;
         esac
     fi
 }

core/identify_param/constant.py

@@ -7,20 +7,22 @@
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZOOKEEPER_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
-
+CTEST_REDISSON_DIR = os.path.join(APP_DIR, "ctest-redisson")
 MODULE_PATH = {
     "hadoop-common": CTEST_HADOOP_DIR,
     "hadoop-hdfs": CTEST_HADOOP_DIR,
     "hbase-server": CTEST_HBASE_DIR,
-    "alluxio-core": CTEST_ALLUXIO_DIR
+    "alluxio-core": CTEST_ALLUXIO_DIR,
+    "redisson": CTEST_REDISSON_DIR
 }
 
 SRC_SUBDIR = {
     "hadoop-common": "hadoop-common-project/hadoop-common",
     "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs",
     "hbase-server": "hbase-server",
     "zookeeper-server": "zookeeper-server",
-    "alluxio-core": "core"
+    "alluxio-core": "core",
+    "redisson": "redisson"
 }
 
 MVN_TEST_PATH = {
@@ -29,14 +31,16 @@
     "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]),
     "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]),
     "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]),
+    "redisson": os.path.join(CTEST_REDISSON_DIR, SRC_SUBDIR["redisson"]),
 }
 
 LOCAL_CONF_PATH = {
     "hadoop-common": "results/hadoop-common/conf_params.txt",
     "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt",
     "hbase-server": "results/hbase-server/conf_params.txt",
     "zookeeper-server": "results/zookeeper-server/conf_params.txt",
-    "alluxio-core": "results/alluxio-core/conf_params.txt"
+    "alluxio-core": "results/alluxio-core/conf_params.txt",
+    "redisson": "results/redisson/conf_params.txt"
 }
 
 SUREFIRE_SUBDIR = "target/surefire-reports/*"
@@ -63,7 +67,10 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR)
-    ]
+    ],
+    "redisson": [
+        os.path.join(CTEST_REDISSON_DIR, "redisson", SUREFIRE_SUBDIR)
+    ],
 }
 
 LOCAL_SUREFIRE_SUFFIX = "surefire-reports/*"
@@ -83,5 +90,8 @@
     ],
     "alluxio-core": [
         os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX)
+    ],
+    "redisson": [
+        os.path.join("surefire-reports/redisson", LOCAL_SUREFIRE_SUFFIX)
     ]
 }

core/identify_param/identify_param.sh

@@ -12,9 +12,9 @@ function main() {
       usage
     else
         case $project in
-            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;;
+            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | redisson) python3 runner.py $project; python3 collector.py $project ;;
             -h | --help) usage ;;
-            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;;
+            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, redisson, zookeeper-server and alluxio-core." ;;
         esac
     fi
 }