Adding support for netty/transport-udt

core/add_project.sh

@@ -48,6 +48,17 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_netty_udt() {
+    [ ! -d "app/ctest-netty-udt" ] && git clone https://github.com/HongxuMeng/netty.git app/ctest-netty-udt
+    cd app/ctest-netty-udt
+    git fetch && git checkout ctest-injection
+    home_dir=$PWD
+    cd $home_dir/transport
+    mvn clean package -DskipTests
+    cd $home_dir/transport-udt
+    mvn clean install -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +75,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            netty-transport-udt) setup_netty_udt ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and netty-transport-udt." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+NETTY_UDT = "netty-transport-udt"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_NETTY_UDT_DIR = os.path.join(APP_DIR, "ctest-netty-udt")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    NETTY_UDT: CTEST_NETTY_UDT_DIR,
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    NETTY_UDT: "transport-udt",
 }
 
 
@@ -58,6 +62,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    NETTY_UDT: [os.path.join(CTEST_NETTY_UDT_DIR, MODULE_SUBDIR[NETTY_UDT], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +79,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    NETTY_UDT: os.path.join(DEFAULT_CONF_DIR, NETTY_UDT + "-default.tsv"),
 }
 
 
@@ -96,7 +102,10 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
-    ]
+    ],
+    NETTY_UDT: [
+        os.path.join(CTEST_NETTY_UDT_DIR, "transport-udt/ctest.json")
+    ],
 }
 
 

core/default_configs/netty-transport-udt-default.tsv

@@ -0,0 +1,22 @@
+connectTimeoutMillis	30000	
+maxMessagesPerRead	1	
+maxMessagesPerWrite	2147483647	
+writeSpinCount	16	
+allocator	io.netty.buffer.PooledByteBufAllocator	
+recvByteBufAllocator	io.netty.channel.AdaptiveRecvByteBufAllocator	
+autoRead	1	
+autoClose	true	
+writeBufferHighWaterMark	65536	
+writeBufferLowWaterMark	32768	
+writeBufferWaterMark	io.netty.channel.WriteBufferWaterMark	Obj param must be (int, int)
+messageSizeEstimator	io.netty.channel.DefaultMessageSizeEstimator	Obj param must be (int)
+pinEventExecutorPerGroup	true	
+protocolReceiveBufferSize	10485760	
+receiveBufferSize	131072	
+sendBufferSize	131072	
+soLinger	0	
+reuseAddress	true	
+systemReceiveBufferSize	1048576	
+protocolSendBufferSize	10485760	
+systemSendBufferSize	1048576	
+backlog	64	

core/generate_ctest/ctest_mapping/ctests-netty-transport-udt.json

@@ -0,0 +1 @@
+{"soLinger": ["io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho", "io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory"], "sendBufferSize": ["io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory", "io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "systemReceiveBufferSize": ["io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory", "io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho", "io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata"], "writeBufferHighWaterMark": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "protocolSendBufferSize": ["io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory", "io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata"], "recvByteBufAllocator": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "connectTimeoutMillis": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "reuseAddress": ["io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "writeBufferLowWaterMark": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "messageSizeEstimator": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "writeSpinCount": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "allocator": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "autoRead": ["io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"], "systemSendBufferSize": ["io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory", "io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata"], "protocolReceiveBufferSize": ["io.netty.test.udt.nio.NioUdtMessageRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtProviderTest#provideFactory", "io.netty.test.udt.nio.NioUdtMessageConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteConnectorChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#metadata", "io.netty.test.udt.nio.NioUdtByteRendezvousChannelTest#basicEcho"]}

core/generate_ctest/identify.py

@@ -43,6 +43,8 @@ def identify_ctest(project):
             for test, pass_good_values in testinfo[param]["tests"].items():
                 if len(good_values) == len(pass_good_values):
                     ctests[param].append(test)
+                else:
+                    print(param, good_values, pass_good_values)
             print(">>>>[ctest_core] param {} has {} ctests".format(
                 param, len(ctests[param])))
 

core/generate_ctest/inject.py

@@ -2,6 +2,7 @@
 
 import sys
 import xml.etree.ElementTree as ET
+import json
 
 sys.path.append("..")
 from ctest_const import *
@@ -13,8 +14,15 @@
 def inject_config(param_value_pairs):
     for p, v in param_value_pairs.items():
         print(">>>>[ctest_core] injecting {} with value {}".format(p, v))
-
-    if project in [ZOOKEEPER, ALLUXIO]:
+
+    if project in [NETTY_UDT]:
+        for inject_path in INJECTION_PATH[project]:
+            print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
+            file = open(inject_path, "w")
+            json_object = json.dumps(param_value_pairs, indent=4)
+            file.write(json_object)
+            file.close()
+    elif project in [ZOOKEEPER, ALLUXIO]:
         for inject_path in INJECTION_PATH[project]:
             print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
             file = open(inject_path, "w")
@@ -41,7 +49,12 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [NETTY_UDT]:
+        for inject_path in INJECTION_PATH[project]:
+            file = open(inject_path, "w")
+            file.write("{}\n")
+            file.close()
+    elif project in [ZOOKEEPER, ALLUXIO]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")

core/generate_ctest/main.py

@@ -33,7 +33,7 @@ def test_value_pair(test_input):
         mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")
 
         associated_tests = mapping[param] if param in mapping else []
-        if len(mapping[param]) != 0:
+        if param in mapping and len(mapping[param]) != 0:
             for value in values:
                 tr = run_test_seperate(param, value, associated_tests)
 

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "netty-transport-udt", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-netty-transport-udt.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "../generate_value/netty-transport-udt-generated-values.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_value/netty-transport-udt-generated-values.tsv

@@ -0,0 +1,22 @@
+connectTimeoutMillis	15000	60000
+maxMessagesPerRead	2	3
+maxMessagesPerWrite	1073741823	4294967294
+writeSpinCount	8	32
+allocator	io.netty.buffer.UnpooledByteBufAllocator	io.netty.buffer.PooledByteBufAllocator
+recvByteBufAllocator	io.netty.channel.ServerChannelRecvByteBufAllocator	SKIP
+autoRead	1	SKIP
+autoClose	false	SKIP
+writeBufferHighWaterMark	32768	131072
+writeBufferLowWaterMark	16384	65536
+writeBufferWaterMark	SKIP	SKIP
+messageSizeEstimator	io.netty.channel.DefaultMessageSizeEstimator	SKIP
+pinEventExecutorPerGroup	false	SKIP
+protocolReceiveBufferSize	5242880	20971520
+receiveBufferSize	65536	262144
+sendBufferSize	65536	262144
+soLinger	1	2
+reuseAddress	false	SKIP
+systemReceiveBufferSize	524288	2097152
+protocolSendBufferSize	5242880	20971520
+systemSendBufferSize	524288	2097152
+backlog	32	48

core/generate_value/value_generation.py

@@ -25,6 +25,9 @@ def read_tsv(module):
     if module == "zookeeper-server":
         assert len(params) == 32
         return 32
+    elif module == "netty-transport-udt":
+        assert len(params) == 22
+        return 22
     else:
         assert len(params) == 90
         return 90
@@ -105,6 +108,8 @@ def print_params(module):
     f = open(module + output, "w")
     if module == "zookeeper-server":
         assert len(params) == 32
+    elif module == "netty-transport-udt":
+        assert len(params) == 22
     else:
         assert len(params) >= 90
     for param in params:

core/identify_param/add_project.sh

@@ -47,6 +47,17 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_netty_udt() {
+    [ ! -d "app/ctest-netty-udt" ] && git clone https://github.com/HongxuMeng/netty.git app/ctest-netty-udt
+    cd app/ctest-netty-udt
+    git fetch && git checkout ctest-logging
+    home_dir=$PWD
+    cd $home_dir/transport
+    mvn clean package -DskipTests
+    cd $home_dir/transport-udt
+    mvn clean package -DskipTests
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -63,6 +74,7 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
+            netty-transport-udt) setup_netty_udt ;;
             *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
         esac
     fi

core/identify_param/constant.py

@@ -7,20 +7,23 @@
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZOOKEEPER_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_NETTY_UDT_DIR = os.path.join(APP_DIR, "ctest-netty-udt")
 
 MODULE_PATH = {
     "hadoop-common": CTEST_HADOOP_DIR,
     "hadoop-hdfs": CTEST_HADOOP_DIR,
     "hbase-server": CTEST_HBASE_DIR,
-    "alluxio-core": CTEST_ALLUXIO_DIR
+    "alluxio-core": CTEST_ALLUXIO_DIR,
+    "netty-transport-udt": CTEST_NETTY_UDT_DIR
 }
 
 SRC_SUBDIR = {
     "hadoop-common": "hadoop-common-project/hadoop-common",
     "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs",
     "hbase-server": "hbase-server",
     "zookeeper-server": "zookeeper-server",
-    "alluxio-core": "core"
+    "alluxio-core": "core",
+    "netty-transport-udt": "transport-udt",
 }
 
 MVN_TEST_PATH = {
@@ -29,14 +32,16 @@
     "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]),
     "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]),
     "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]),
+    "netty-transport-udt": os.path.join(CTEST_NETTY_UDT_DIR, SRC_SUBDIR["netty-transport-udt"]),
 }
 
 LOCAL_CONF_PATH = {
     "hadoop-common": "results/hadoop-common/conf_params.txt",
     "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt",
     "hbase-server": "results/hbase-server/conf_params.txt",
     "zookeeper-server": "results/zookeeper-server/conf_params.txt",
-    "alluxio-core": "results/alluxio-core/conf_params.txt"
+    "alluxio-core": "results/alluxio-core/conf_params.txt",
+    "netty-transport-udt": "results/netty-transport-udt/conf_params.txt",
 }
 
 SUREFIRE_SUBDIR = "target/surefire-reports/*"
@@ -63,7 +68,10 @@
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR)
-    ]
+    ],
+    "netty-transport-udt": [
+        os.path.join(CTEST_NETTY_UDT_DIR, SRC_SUBDIR["netty-transport-udt"], SUREFIRE_SUBDIR)
+    ],
 }
 
 LOCAL_SUREFIRE_SUFFIX = "surefire-reports/*"
@@ -83,5 +91,8 @@
     ],
     "alluxio-core": [
         os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX)
-    ]
-}
+    ],
+    "netty-transport-udt": [
+        os.path.join("surefire-reports/netty/transport-udt", LOCAL_SUREFIRE_SUFFIX)
+    ],
+}

core/identify_param/identify_param.sh

@@ -5,16 +5,25 @@ function usage() {
   exit 1
 }
 
+function setup_netty_udt() {
+    [ ! -d "app/ctest-netty-udt" ] && git clone https://github.com/HongxuMeng/netty.git app/ctest-netty-udt
+    cd app/ctest-netty-udt
+    git fetch && git checkout ctest-logging
+    home_dir=$PWD
+    cd $home_dir/transport-udt
+    mvn clean install -DskipTests
+}
+
 project=$1
 function main() {
     if [ -z $project ]
     then
       usage
     else
         case $project in
-            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;;
+            hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | netty-transport-udt) python3 runner.py $project; python3 collector.py $project ;;
             -h | --help) usage ;;
-            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;;
+            *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server, alluxio-core and netty-transport-udt." ;;
         esac
     fi
 }

core/identify_param/results/netty-transport-udt/conf_params.txt

@@ -0,0 +1,22 @@
+connectTimeoutMillis
+maxMessagesPerRead
+maxMessagesPerWrite
+writeSpinCount
+allocator
+recvByteBufAllocator
+autoRead
+autoClose
+writeBufferHighWaterMark
+writeBufferLowWaterMark
+writeBufferWaterMark
+messageSizeEstimator
+pinEventExecutorPerGroup
+protocolReceiveBufferSize
+receiveBufferSize
+sendBufferSize
+soLinger
+reuseAddress
+systemReceiveBufferSize
+protocolSendBufferSize
+systemSendBufferSize
+backlog

core/identify_param/results/netty-transport-udt/logs/failure.json

@@ -0,0 +1 @@
+[]