Adding Hadoop hadoop-hdfs-rbf into the openctest framework

core/add_project.sh

@@ -3,7 +3,7 @@
 function setup_hadoop() {
     [ ! -d "app/ctest-hadoop" ] && git clone https://github.com/xlab-uiuc/hadoop.git app/ctest-hadoop
     cd app/ctest-hadoop
-    git fetch && git checkout ctest-injection
+    git fetch && git checkout trunk
     home_dir=$PWD
     cd $home_dir/hadoop-common-project/hadoop-common
     mvn clean install -DskipTests

core/ctest_const.py

@@ -9,18 +9,21 @@
 
 HCOMMON = "hadoop-common"
 HDFS = "hadoop-hdfs"
+HDFSRBF = "hadoop-hdfs-rbf"
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
 
-CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
+# CTEST_HADOOP_DIR = os.path.join(APP_DIR, "../../../hadoop")
+CTEST_HADOOP_DIR = os.path.join(CUR_DIR, "../../hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
+    HDFSRBF: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
@@ -31,6 +34,7 @@
 MODULE_SUBDIR = {
     HCOMMON: "hadoop-common-project/hadoop-common",
     HDFS: "hadoop-hdfs-project/hadoop-hdfs",
+    HDFSRBF: "hadoop-hdfs-project/hadoop-hdfs-rbf",
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
@@ -46,6 +50,7 @@
 SUREFIRE_DIR = {
     HCOMMON: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HCOMMON], SUREFIRE_SUBDIR)],
     HDFS: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HDFS], SUREFIRE_SUBDIR)],
+    HDFSRBF: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HDFSRBF], SUREFIRE_SUBDIR)],
     HBASE: [os.path.join(CTEST_HBASE_DIR, MODULE_SUBDIR[HBASE], SUREFIRE_SUBDIR)],
     ZOOKEEPER: [os.path.join(CTEST_ZK_DIR, MODULE_SUBDIR[ZOOKEEPER], SUREFIRE_SUBDIR)],
     ALLUXIO: [
@@ -72,6 +77,7 @@
 DEFAULT_CONF_FILE = {
     HCOMMON: os.path.join(DEFAULT_CONF_DIR, HCOMMON + "-default.tsv"),
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
+    HDFSRBF: os.path.join(DEFAULT_CONF_DIR, HDFSRBF + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
     ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
@@ -87,6 +93,10 @@
         os.path.join(CTEST_HADOOP_DIR, "hadoop-hdfs-project/hadoop-hdfs/target/classes/core-ctest.xml"),
         os.path.join(CTEST_HADOOP_DIR, "hadoop-hdfs-project/hadoop-hdfs/target/classes/hdfs-ctest.xml")
     ],
+    HDFSRBF: [
+        os.path.join(CTEST_HADOOP_DIR, "hadoop-hdfs-project/hadoop-hdfs-rbf/target/classes/core-ctest.xml"),
+        os.path.join(CTEST_HADOOP_DIR, "hadoop-hdfs-project/hadoop-hdfs-rbf/target/classes/hdfs-rbf-ctest.xml")
+    ],
     HBASE: [
         os.path.join(CTEST_HBASE_DIR, "hbase-server/target/classes/core-ctest.xml"),
         os.path.join(CTEST_HBASE_DIR, "hbase-server/target/classes/hbase-ctest.xml")

core/default_configs/hadoop-hdfs-rbf-default.tsv

@@ -0,0 +1,66 @@
+dfs.federation.router.default.nameserviceId		Nameservice identifier of the default subcluster to monitor.
+dfs.federation.router.default.nameservice.enable	true	The default subcluster is enabled to read and write files.
+dfs.federation.router.rpc.enable	true	If true, the RPC service to handle client requests in the router is enabled.
+dfs.federation.router.rpc-address	0.0.0.0:8888	RPC address that handles all clients requests. The value of this property will take the form of router-host1:rpc-port.
+dfs.federation.router.rpc-bind-host		The actual address the RPC server will bind to. If this optional address is set, it overrides only the hostname portion of dfs.federation.router.rpc-address. This is useful for making the name node listen on all interfaces by setting it to 0.0.0.0.
+dfs.federation.router.handler.count	10	The number of server threads for the router to handle RPC requests from clients.
+dfs.federation.router.handler.queue.size	100	The size of the queue for the number of handlers to handle RPC client requests.
+dfs.federation.router.reader.count	1	The number of readers for the router to handle RPC client requests.
+dfs.federation.router.reader.queue.size	100	The size of the queue for the number of readers for the router to handle RPC client requests.
+dfs.federation.router.connection.creator.queue-size	100	Size of async connection creator queue.
+dfs.federation.router.connection.pool-size	1	Size of the pool of connections from the router to namenodes.
+dfs.federation.router.connection.min-active-ratio	0.5f	Minimum active ratio of connections from the router to namenodes.
+dfs.federation.router.connection.clean.ms	10000	Time interval, in milliseconds, to check if the connection pool should remove unused connections.
+dfs.federation.router.connection.pool.clean.ms	60000	Time interval, in milliseconds, to check if the connection manager should remove unused connection pools.
+dfs.federation.router.metrics.enable	true	If the metrics in the router are enabled.
+dfs.federation.router.dn-report.time-out	1000	Time out, in milliseconds for getDatanodeReport.
+dfs.federation.router.dn-report.cache-expire	10s	Expiration time in seconds for datanodereport.
+dfs.federation.router.metrics.class	org.apache.hadoop.hdfs.server.federation.metrics.FederationRPCPerformanceMonitor	Class to monitor the RPC system in the router. It must implement the RouterRpcMonitor interface.
+dfs.federation.router.admin.enable	true	If true, the RPC admin service to handle client requests in the router is enabled.
+dfs.federation.router.admin-address	0.0.0.0:8111	RPC address that handles the admin requests. The value of this property will take the form of router-host1:rpc-port.
+dfs.federation.router.admin-bind-host		The actual address the RPC admin server will bind to. If this optional address is set, it overrides only the hostname portion of dfs.federation.router.admin-address. This is useful for making the name node listen on all interfaces by setting it to 0.0.0.0.
+dfs.federation.router.admin.handler.count	1	The number of server threads for the router to handle RPC requests from admin.
+dfs.federation.router.http-address	0.0.0.0:50071	HTTP address that handles the web requests to the Router. The value of this property will take the form of router-host1:http-port.
+dfs.federation.router.http-bind-host		The actual address the HTTP server will bind to. If this optional address is set, it overrides only the hostname portion of dfs.federation.router.http-address. This is useful for making the name node listen on all interfaces by setting it to 0.0.0.0.
+dfs.federation.router.https-address	0.0.0.0:50072	HTTPS address that handles the web requests to the Router. The value of this property will take the form of router-host1:https-port.
+dfs.federation.router.https-bind-host		The actual address the HTTPS server will bind to. If this optional address is set, it overrides only the hostname portion of dfs.federation.router.https-address. This is useful for making the name node listen on all interfaces by setting it to 0.0.0.0.
+dfs.federation.router.http.enable	true	If the HTTP service to handle client requests in the router is enabled.
+dfs.federation.router.file.resolver.client.class	org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver	Class to resolve files to subclusters. To enable multiple subclusters for a mount point, set to org.apache.hadoop.hdfs.server.federation.resolver.MultipleDestinationMountTableResolver.
+dfs.federation.router.namenode.resolver.client.class	org.apache.hadoop.hdfs.server.federation.resolver.MembershipNamenodeResolver	Class to resolve the namenode for a subcluster.
+dfs.federation.router.store.enable	true	If true, the Router connects to the State Store.
+dfs.federation.router.store.serializer	org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreSerializerPBImpl	Class to serialize State Store records.
+dfs.federation.router.store.driver.class	org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl	Class to implement the State Store. There are three implementation classes currently being supported: org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileImpl, org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreFileSystemImpl and org.apache.hadoop.hdfs.server.federation.store.driver.impl.StateStoreZooKeeperImpl. These implementation classes use the local file, filesystem and ZooKeeper as a backend respectively. By default it uses the ZooKeeper as the default State Store.
+dfs.federation.router.store.connection.test	60000	How often to check for the connection to the State Store in milliseconds.
+dfs.federation.router.cache.ttl	1m	How often to refresh the State Store caches in milliseconds. This setting supports multiple time unit suffixes as described in dfs.heartbeat.interval. If no suffix is specified then milliseconds is assumed.
+dfs.federation.router.store.membership.expiration	300000	Expiration time in milliseconds for a membership record.
+dfs.federation.router.store.membership.expiration.deletion	-1	Deletion time in milliseconds for a membership record. If an expired membership record exists beyond this time, it will be deleted. If this value is negative, the deletion is disabled.
+dfs.federation.router.heartbeat.enable	true	If true, the Router heartbeats into the State Store.
+dfs.federation.router.heartbeat.interval	5000	How often the Router should heartbeat into the State Store in milliseconds.
+dfs.federation.router.heartbeat-state.interval	5s	How often the Router should heartbeat its state into the State Store in milliseconds. This setting supports multiple time unit suffixes as described in dfs.federation.router.quota-cache.update.interval.
+dfs.federation.router.namenode.heartbeat.enable		If true, get namenode heartbeats and send into the State Store. If not explicitly specified takes the same value as for dfs.federation.router.heartbeat.enable.
+dfs.federation.router.store.router.expiration	5m	Expiration time in milliseconds for a router state record. This setting supports multiple time unit suffixes as described in dfs.federation.router.quota-cache.update.interval.
+dfs.federation.router.store.router.expiration.deletion	-1	Deletion time in milliseconds for a router state record. If an expired router state record exists beyond this time, it will be deleted. If this value is negative, the deletion is disabled.
+dfs.federation.router.safemode.enable	true	
+dfs.federation.router.safemode.extension	30s	Time after startup that the Router is in safe mode. This setting supports multiple time unit suffixes as described in dfs.heartbeat.interval. If no suffix is specified then milliseconds is assumed.
+dfs.federation.router.safemode.expiration	3m	Time without being able to reach the State Store to enter safe mode. This setting supports multiple time unit suffixes as described in dfs.heartbeat.interval. If no suffix is specified then milliseconds is assumed.
+dfs.federation.router.monitor.namenode		The identifier of the namenodes to monitor and heartbeat.
+dfs.federation.router.monitor.localnamenode.enable	true	If true, the Router should monitor the namenode in the local machine.
+dfs.federation.router.mount-table.max-cache-size	10000	Maximum number of mount table cache entries to have. By default, remove cache entries if we have more than 10k.
+dfs.federation.router.mount-table.cache.enable	true	Set to true to enable mount table cache (Path to Remote Location cache). Disabling the cache is recommended when a large amount of unique paths are queried.
+dfs.federation.router.quota.enable	false	Set to true to enable quota system in Router. When it's enabled, setting or clearing sub-cluster's quota directly is not recommended since Router Admin server will override sub-cluster's quota with global quota.
+dfs.federation.router.quota-cache.update.interval	60s	Interval time for updating quota usage cache in Router. This property is used only if the value of dfs.federation.router.quota.enable is true. This setting supports multiple time unit suffixes as described in dfs.heartbeat.interval. If no suffix is specified then milliseconds is assumed.
+dfs.federation.router.client.thread-size	32	Max threads size for the RouterClient to execute concurrent requests.
+dfs.federation.router.client.retry.max.attempts	3	Max retry attempts for the RouterClient talking to the Router.
+dfs.federation.router.client.reject.overload	false	Set to true to reject client requests when we run out of RPC client threads.
+dfs.federation.router.client.allow-partial-listing	true	If the Router can return a partial list of files in a multi-destination mount point when one of the subclusters is unavailable. True may return a partial list of files if a subcluster is down. False will fail the request if one is unavailable.
+dfs.federation.router.client.mount-status.time-out	1s	Set a timeout for the Router when listing folders containing mount points. In this process, the Router checks the mount table and then it checks permissions in the subcluster. After the time out, we return the default values.
+dfs.federation.router.connect.max.retries.on.timeouts	0	Maximum number of retries for the IPC Client when connecting to the subclusters. By default, it doesn't let the IPC retry and the Router handles it.
+dfs.federation.router.connect.timeout	2s	Time out for the IPC client connecting to the subclusters. This should be short as the Router has knowledge of the state of the Routers.
+dfs.federation.router.keytab.file		The keytab file used by router to login as its service principal. The principal name is configured with dfs.federation.router.kerberos.principal.
+dfs.federation.router.kerberos.principal		The Router service principal. This is typically set to router/_HOST@REALM.TLD. Each Router will substitute _HOST with its own fully qualified hostname at startup. The _HOST placeholder allows using the same configuration setting on both Router in an HA setup.
+dfs.federation.router.kerberos.principal.hostname		Optional.  The hostname for the Router containing this configuration file.  Will be different for each machine. Defaults to current hostname.
+dfs.federation.router.kerberos.internal.spnego.principal	${dfs.web.authentication.kerberos.principal}	The server principal used by the Router for web UI SPNEGO authentication when Kerberos security is enabled. This is typically set to HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix HTTP/ by convention.  If the value is '*', the web server will attempt to login with every principal specified in the keytab file dfs.web.authentication.kerberos.keytab.
+dfs.federation.router.mount-table.cache.update	false	Set true to enable MountTableRefreshService. This service updates mount table cache immediately after adding, modifying or deleting the mount table entries. If this service is not enabled mount table cache are refreshed periodically by StateStoreCacheUpdateService
+dfs.federation.router.mount-table.cache.update.timeout	1m	This property defines how long to wait for all the admin servers to finish their mount table cache update. This setting supports multiple time unit suffixes as described in dfs.federation.router.safemode.extension.
+dfs.federation.router.mount-table.cache.update.client.max.time	5m	Remote router mount table cache is updated through RouterClient(RPC call). To improve performance, RouterClient connections are cached but it should not be kept in cache forever. This property defines the max time a connection can be cached. This setting supports multiple time unit suffixes as described in dfs.federation.router.safemode.extension.
+dfs.federation.router.secret.manager.class	org.apache.hadoop.hdfs.server.federation.router.security.token.ZKDelegationTokenSecretManagerImpl	Class to implement state store to delegation tokens. Default implementation uses zookeeper as the backend to store delegation tokens.

core/generate_ctest/inject.py

@@ -21,7 +21,7 @@ def inject_config(param_value_pairs):
             for p, v in param_value_pairs.items():
                 file.write(p + "=" + v + "\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HDFSRBF, HBASE]:
         conf = ET.Element("configuration")
         for p, v in param_value_pairs.items():
             prop = ET.SubElement(conf, "property")
@@ -46,7 +46,7 @@ def clean_conf_file(project):
             file = open(inject_path, "w")
             file.write("\n")
             file.close()
-    elif project in [HCOMMON, HDFS, HBASE]:
+    elif project in [HCOMMON, HDFS, HDFSRBF, HBASE]:
         conf = ET.Element("configuration")
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "wb")

core/generate_ctest/program_input.py

@@ -4,11 +4,11 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "hadoop-hdfs-rbf", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-hdfs-rbf.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "hadoop-hdfs-rbf.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test

core/generate_ctest/run_test.py

@@ -59,5 +59,5 @@ def run_test_seperate(param, value, associated_tests):
     os.chdir(CUR_DIR)
     print(">>>>[ctest_core] chdir to {}".format(CUR_DIR))
     print(">>>>[ctest_core] python-timed for running config pair: {}".format(duration))
-    clean_conf_file(project)
+    # clean_conf_file(project)
     return tr