Skip to content

Adding hadoop-yarn-timelineservice into the openctest framework #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion core/add_project.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,17 @@ function setup_hadoop() {
mvn package -DskipTests
}

function setup_hadoop_yarn_tls() {
[ ! -d "app/ctest-hadoop" ] && git clone git@github.com:KobeNorris/hadoop.git app/ctest-hadoop
cd app/ctest-hadoop
git fetch && git checkout ctest-interception
home_dir=$PWD
cd $home_dir/hadoop-common-project/hadoop-common
mvn clean install -DskipTests
cd $home_dir/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice
mvn package -DskipTests
}

function setup_hbase() {
old_dir=$PWD
[ ! -d "app/ctest-hadoop" ] && git clone https://github.com/xlab-uiuc/hadoop.git app/ctest-hadoop
Expand Down Expand Up @@ -64,7 +75,8 @@ function main() {
hbase) setup_hbase ;;
zookeeper) setup_zookeeper ;;
alluxio) setup_alluxio ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
hadoop-yarn-tls) setup_hadoop_yarn_tls ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and hadoop-yarn-tls." ;;
esac
fi
}
Expand Down
9 changes: 9 additions & 0 deletions core/ctest_const.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,19 @@
HCOMMON = "hadoop-common"
HDFS = "hadoop-hdfs"
HBASE = "hbase-server"
HYARNTLS = "hadoop-yarn-tls"
ZOOKEEPER = "zookeeper-server"
ALLUXIO = "alluxio-core"

CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
CTEST_HYARNTLS_DIR = os.path.join(APP_DIR, "ctest-hadoop")
CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")

PROJECT_DIR = {
HCOMMON: CTEST_HADOOP_DIR,
HYARNTLS: CTEST_HADOOP_DIR,
HDFS: CTEST_HADOOP_DIR,
HBASE: CTEST_HBASE_DIR,
ZOOKEEPER: CTEST_ZK_DIR,
Expand All @@ -31,6 +34,7 @@
MODULE_SUBDIR = {
HCOMMON: "hadoop-common-project/hadoop-common",
HDFS: "hadoop-hdfs-project/hadoop-hdfs",
HYARNTLS: "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice",
HBASE: "hbase-server",
ZOOKEEPER: "zookeeper-server",
ALLUXIO: "core",
Expand All @@ -47,6 +51,7 @@
HCOMMON: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HCOMMON], SUREFIRE_SUBDIR)],
HDFS: [os.path.join(CTEST_HADOOP_DIR, MODULE_SUBDIR[HDFS], SUREFIRE_SUBDIR)],
HBASE: [os.path.join(CTEST_HBASE_DIR, MODULE_SUBDIR[HBASE], SUREFIRE_SUBDIR)],
HYARNTLS: [os.path.join(CTEST_HYARNTLS_DIR, MODULE_SUBDIR[HYARNTLS], SUREFIRE_SUBDIR)],
ZOOKEEPER: [os.path.join(CTEST_ZK_DIR, MODULE_SUBDIR[ZOOKEEPER], SUREFIRE_SUBDIR)],
ALLUXIO: [
os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "base", SUREFIRE_SUBDIR),
Expand All @@ -73,6 +78,7 @@
HCOMMON: os.path.join(DEFAULT_CONF_DIR, HCOMMON + "-default.tsv"),
HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
HYARNTLS: os.path.join(DEFAULT_CONF_DIR, HYARNTLS + "-default.tsv"),
ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
}
Expand All @@ -91,6 +97,9 @@
os.path.join(CTEST_HBASE_DIR, "hbase-server/target/classes/core-ctest.xml"),
os.path.join(CTEST_HBASE_DIR, "hbase-server/target/classes/hbase-ctest.xml")
],
HYARNTLS: [
os.path.join(CTEST_HADOOP_DIR, "hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/target/classes/core-ctest.xml")
],
ZOOKEEPER: [
os.path.join(CTEST_ZK_DIR, "zookeeper-server/ctest.cfg")
],
Expand Down
214 changes: 214 additions & 0 deletions core/default_configs/hadoop-core-default.tsv

Large diffs are not rendered by default.

222 changes: 222 additions & 0 deletions core/default_configs/hadoop-yarn-tls-default.tsv

Large diffs are not rendered by default.

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions core/generate_ctest/inject.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def inject_config(param_value_pairs):
for p, v in param_value_pairs.items():
file.write(p + "=" + v + "\n")
file.close()
elif project in [HCOMMON, HDFS, HBASE]:
elif project in [HCOMMON, HDFS, HBASE, HYARNTLS]:
conf = ET.Element("configuration")
for p, v in param_value_pairs.items():
prop = ET.SubElement(conf, "property")
Expand All @@ -46,7 +46,7 @@ def clean_conf_file(project):
file = open(inject_path, "w")
file.write("\n")
file.close()
elif project in [HCOMMON, HDFS, HBASE]:
elif project in [HCOMMON, HDFS, HBASE, HYARNTLS]:
conf = ET.Element("configuration")
for inject_path in INJECTION_PATH[project]:
file = open(inject_path, "wb")
Expand Down
2 changes: 1 addition & 1 deletion core/generate_ctest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def test_value_pair(test_input):
mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")

associated_tests = mapping[param] if param in mapping else []
if len(mapping[param]) != 0:
if param in mapping.keys() and len(mapping[param]) != 0:
for value in values:
tr = run_test_seperate(param, value, associated_tests)

Expand Down
6 changes: 3 additions & 3 deletions core/generate_ctest/program_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
# run mode
"run_mode": "generate_ctest", # string
# name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
"project": "hadoop-common", # string
"project": "hadoop-yarn-tls", # string
# path to param -> tests json mapping
"mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
"mapping_path": "../../data/ctest_mapping/opensource-hadoop-yarn-tls.json", # string
# good values of params tests will be run against
"param_value_tsv": "sample-hadoop-common.tsv", # string
"param_value_tsv": "sample-hadoop-yarn-tls.tsv", # string
# display the terminal output live, without saving any results
"display_mode": False, # bool
# whether to use mvn test or mvn surefire:test
Expand Down
2 changes: 0 additions & 2 deletions core/generate_ctest/sample-hadoop-common.tsv

This file was deleted.

221 changes: 221 additions & 0 deletions core/generate_ctest/sample-hadoop-yarn-tls.tsv
Original file line number Diff line number Diff line change
@@ -0,0 +1,221 @@
hadoop.security.authentication simple kerberos
hadoop.security.auth_to_local DEFAULT SKIP
hadoop.security.auth_to_local.mechanism hadoop mit
hadoop.user.group.static.mapping.overrides dr.who=; user1=group1,group2;user2=;user3=group2
yarn.timeline-service.fs-writer.root-dir * .
fs.defaultFS file:/// SKIP
yarn.timeline-service.collector.bind-port-ranges SKIP SKIP
0.0.0.0 SKIP SKIP
yarn.http.policy HTTP HTTPS_ONLY
hadoop.http.staticuser.user dr.who xdsuper
yarn.resourcemanager.cluster-id yarn-cluster cluster1
yarn.admin.acl * ACL YARN
yarn.timeline-service.read.allowed.users xdsuper samsuper
yarn.ipc.client.factory.class SKIP SKIP
yarn.ipc.server.factory.class SKIP SKIP
yarn.ipc.record.factory.class SKIP SKIP
yarn.ipc.rpc.class SKIP SKIP
yarn.resourcemanager.hostname 127.0.0.1 SKIP
yarn.resourcemanager.address 127.0.0.1 SKIP
yarn.resourcemanager.bind-host SKIP SKIP
yarn.resourcemanager.client.thread-count 25 100
yarn.resourcemanager.amlauncher.thread-count 25 100
yarn.resourcemanager.nodemanager-connect-retries 1 20
yarn.dispatcher.drain-events.timeout 150000 600000
yarn.am.liveness-monitor.expiry-interval-ms 300000 1200000
yarn.resourcemanager.principal SKIP SKIP
yarn.resourcemanager.scheduler.address 127.0.0.1 SKIP
yarn.resourcemanager.scheduler.client.thread-count 25 100
yarn.resourcemanager.webapp.address 127.0.0.1 SKIP
yarn.resourcemanager.webapp.https.address 127.0.0.1 SKIP
yarn.resourcemanager.resource-tracker.address 127.0.0.1 SKIP
yarn.acl.enable true SKIP
yarn.resourcemanager.admin.address 127.0.0.1 SKIP
yarn.resourcemanager.admin.client.thread-count 0 2
yarn.resourcemanager.connect.max-wait.ms 450000 1800000
yarn.resourcemanager.connect.retry-interval.ms 15000 60000
yarn.resourcemanager.am.max-attempts 1 4
yarn.resourcemanager.container.liveness-monitor.interval-ms 300000 1200000
yarn.resourcemanager.keytab /valid/file1 /valid/file2
yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled false SKIP
yarn.resourcemanager.webapp.cross-origin.enabled true SKIP
yarn.nm.liveness-monitor.expiry-interval-ms 300000 1200000
yarn.resourcemanager.nodes.include-path SKIP SKIP
yarn.resourcemanager.nodes.exclude-path SKIP SKIP
yarn.resourcemanager.resource-tracker.client.thread-count 25 100
yarn.resourcemanager.scheduler.class SKIP SKIP
yarn.scheduler.minimum-allocation-mb 512 2048
yarn.scheduler.maximum-allocation-mb 4096 16384
yarn.scheduler.minimum-allocation-vcores 0 2
yarn.scheduler.maximum-allocation-vcores 16 64
yarn.resourcemanager.recovery.enabled true SKIP
yarn.resourcemanager.fail-fast SKIP SKIP
yarn.fail-fast true SKIP
yarn.resourcemanager.work-preserving-recovery.enabled false SKIP
yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms 5000 20000
yarn.resourcemanager.store.class SKIP SKIP
yarn.resourcemanager.state-store.max-completed-applications SKIP SKIP
yarn.resourcemanager.zk-address 127.0.0.1 SKIP
yarn.resourcemanager.zk-num-retries 500 2000
yarn.resourcemanager.zk-retry-interval-ms 500 2000
yarn.resourcemanager.zk-state-store.parent-path /valid/file1 /valid/file2
yarn.resourcemanager.zk-timeout-ms 5000 20000
yarn.resourcemanager.zk-acl SKIP SKIP
yarn.resourcemanager.zk-state-store.root-node.acl SKIP SKIP
yarn.resourcemanager.zk-auth SKIP SKIP
yarn.resourcemanager.fs.state-store.uri /valid/file1 /valid/file2
yarn.resourcemanager.fs.state-store.retry-policy-spec [1000, 250] [4000, 1000]
yarn.resourcemanager.fs.state-store.num-retries 1 -1
yarn.resourcemanager.fs.state-store.retry-interval-ms 500 2000
yarn.resourcemanager.leveldb-state-store.path /valid/file1 /valid/file2
yarn.resourcemanager.leveldb-state-store.compaction-interval-secs 1800 7200
yarn.resourcemanager.ha.enabled true SKIP
yarn.resourcemanager.ha.automatic-failover.enabled false SKIP
yarn.resourcemanager.ha.automatic-failover.embedded false SKIP
yarn.resourcemanager.ha.automatic-failover.zk-base-path /valid/file1 /valid/file2
yarn.resourcemanager.ha.rm-ids SKIP SKIP
yarn.resourcemanager.ha.id SKIP SKIP
yarn.client.failover-proxy-provider SKIP SKIP
yarn.client.failover-max-attempts SKIP SKIP
yarn.client.failover-sleep-base-ms SKIP SKIP
yarn.client.failover-sleep-max-ms SKIP SKIP
yarn.client.failover-retries 1 -1
yarn.client.failover-retries-on-socket-timeouts 1 -1
yarn.resourcemanager.max-completed-applications 5000 20000
yarn.resourcemanager.delayed.delegation-token.removal-interval-ms 15000 60000
yarn.resourcemanager.proxy-user-privileges.enabled true SKIP
yarn.resourcemanager.am-rm-tokens.master-key-rolling-interval-secs 43200 172800
yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs 43200 172800
yarn.resourcemanager.nodemanagers.heartbeat-interval-ms 500 2000
yarn.resourcemanager.nodemanager.minimum.version SKIP SKIP
yarn.resourcemanager.scheduler.monitor.enable true SKIP
yarn.resourcemanager.scheduler.monitor.policies SKIP SKIP
yarn.resourcemanager.configuration.provider-class SKIP SKIP
yarn.resourcemanager.system-metrics-publisher.enabled true SKIP
yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size 1 20
yarn.nodemanager.hostname 127.0.0.1 SKIP
yarn.nodemanager.address 127.0.0.1 SKIP
yarn.nodemanager.bind-host SKIP SKIP
yarn.nodemanager.admin-env SKIP SKIP
yarn.nodemanager.env-whitelist JAVA_HOME HADOOP_COMMON_HOME
yarn.nodemanager.container-executor.class SKIP SKIP
yarn.nodemanager.container-manager.thread-count 10 40
yarn.nodemanager.delete.thread-count 1 8
yarn.nodemanager.delete.debug-delay-sec 1 -1
yarn.nodemanager.keytab /valid/file1 /valid/file2
yarn.nodemanager.local-dirs /valid/file1 /valid/file2
yarn.nodemanager.local-cache.max-files-per-directory 4096 16384
yarn.nodemanager.localizer.address 127.0.0.1 SKIP
yarn.nodemanager.localizer.cache.cleanup.interval-ms 300000 1200000
yarn.nodemanager.localizer.cache.target-size-mb 5120 20480
yarn.nodemanager.localizer.client.thread-count 1 10
yarn.nodemanager.localizer.fetch.thread-count 1 8
yarn.nodemanager.log-dirs /valid/file1 /valid/file2
yarn.log-aggregation-enable true SKIP
yarn.log-aggregation.retain-seconds 0 -2
yarn.log-aggregation.retain-check-interval-seconds 0 -2
yarn.nodemanager.log.retain-seconds 5400 21600
yarn.nodemanager.remote-app-log-dir /valid/file1 /valid/file2
yarn.nodemanager.remote-app-log-dir-suffix SKIP SKIP
yarn.nodemanager.resource.memory-mb 4096 16384
yarn.nodemanager.pmem-check-enabled false SKIP
yarn.nodemanager.vmem-check-enabled false SKIP
yarn.nodemanager.vmem-pmem-ratio 1.05 4.2
yarn.nodemanager.resource.cpu-vcores 1 16
yarn.nodemanager.resource.percentage-physical-cpu-limit 50 200
yarn.nodemanager.webapp.address 127.0.0.1 SKIP
yarn.nodemanager.container-monitor.interval-ms 1500 6000
yarn.nodemanager.container-monitor.resource-calculator.class SKIP SKIP
yarn.nodemanager.health-checker.interval-ms 300000 1200000
yarn.nodemanager.health-checker.script.timeout-ms 600000 2400000
yarn.nodemanager.health-checker.script.path /valid/file1 /valid/file2
yarn.nodemanager.health-checker.script.opts SKIP SKIP
yarn.nodemanager.disk-health-checker.interval-ms 60000 240000
yarn.nodemanager.disk-health-checker.min-healthy-disks 0.125 0.5
yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage 45.0 180.0
yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb 1 -1
yarn.nodemanager.linux-container-executor.path /valid/file1 /valid/file2
yarn.nodemanager.linux-container-executor.resources-handler.class SKIP SKIP
yarn.nodemanager.linux-container-executor.cgroups.hierarchy /valid/file1 /valid/file2
yarn.nodemanager.linux-container-executor.cgroups.mount true SKIP
yarn.nodemanager.linux-container-executor.cgroups.mount-path SKIP SKIP
yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users false SKIP
yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user xdsuper samsuper
yarn.nodemanager.linux-container-executor.nonsecure-mode.user-pattern ^[_.A-Za-z0-9][-@_.A-Za-z0-9]{0 255}?[$]?$
yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage true SKIP
yarn.nodemanager.windows-container.memory-limit.enabled true SKIP
yarn.nodemanager.windows-container.cpu-limit.enabled true SKIP
yarn.nodemanager.log-aggregation.compression-type SKIP SKIP
yarn.nodemanager.principal SKIP SKIP
yarn.nodemanager.aux-services SKIP SKIP
yarn.nodemanager.sleep-delay-before-sigkill.ms 125 500
yarn.nodemanager.process-kill-wait.ms 1000 4000
yarn.nodemanager.resourcemanager.minimum.version SKIP SKIP
yarn.client.nodemanager-client-async.thread-pool-max-size 250 1000
yarn.client.nodemanager-connect.max-wait-ms 90000 360000
yarn.client.nodemanager-connect.retry-interval-ms 5000 20000
yarn.client.max-cached-nodemanagers-proxies 1 -1
yarn.nodemanager.recovery.enabled true SKIP
yarn.nodemanager.recovery.dir /valid/file1 /valid/file2
yarn.nodemanager.recovery.compaction-interval-secs 1800 7200
yarn.nodemanager.container-metrics.unregister-delay-ms 5000 20000
yarn.nodemanager.docker-container-executor.exec-name /valid/file1 /valid/file2
yarn.nodemanager.aux-services.mapreduce_shuffle.class SKIP SKIP
mapreduce.job.jar SKIP SKIP
mapreduce.job.hdfs-servers SKIP SKIP
yarn.web-proxy.principal SKIP SKIP
yarn.web-proxy.keytab /valid/file1 /valid/file2
yarn.web-proxy.address 127.0.0.1 SKIP
yarn.application.classpath SKIP SKIP
yarn.timeline-service.enabled true SKIP
yarn.timeline-service.hostname 127.0.0.1 SKIP
yarn.timeline-service.address 127.0.0.1 SKIP
yarn.timeline-service.webapp.address 127.0.0.1 SKIP
yarn.timeline-service.webapp.https.address 127.0.0.1 SKIP
yarn.timeline-service.bind-host SKIP SKIP
yarn.timeline-service.generic-application-history.max-applications 5000 20000
yarn.timeline-service.store-class SKIP SKIP
yarn.timeline-service.ttl-enable false SKIP
yarn.timeline-service.ttl-ms 302400000 1209600000
yarn.timeline-service.leveldb-timeline-store.path /valid/file1 /valid/file2
yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms 150000 600000
yarn.timeline-service.leveldb-timeline-store.read-cache-size 52428800 209715200
yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size 5000 20000
yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size 5000 20000
yarn.timeline-service.handler-thread-count 1 20
yarn.timeline-service.http-authentication.type SKIP SKIP
yarn.timeline-service.http-authentication.simple.anonymous.allowed false SKIP
yarn.timeline-service.principal SKIP SKIP
yarn.timeline-service.keytab /valid/file1 /valid/file2
yarn.timeline-service.ui-names SKIP SKIP
yarn.timeline-service.client.max-retries 15 60
yarn.timeline-service.client.best-effort true SKIP
yarn.timeline-service.client.retry-interval-ms 500 2000
yarn.timeline-service.recovery.enabled true SKIP
yarn.timeline-service.state-store-class SKIP SKIP
yarn.timeline-service.leveldb-state-store.path /valid/file1 /valid/file2
yarn.sharedcache.enabled true SKIP
yarn.sharedcache.root-dir /valid/file1 /valid/file2
yarn.sharedcache.nested-level 1 6
yarn.sharedcache.store.class SKIP SKIP
yarn.sharedcache.app-checker.class SKIP SKIP
yarn.sharedcache.store.in-memory.staleness-period-mins 5040 20160
yarn.sharedcache.store.in-memory.initial-delay-mins 1 20
yarn.sharedcache.store.in-memory.check-period-mins 360 1440
yarn.sharedcache.admin.address 0.0.0.0:3000 0.0.0.0:3001
yarn.sharedcache.admin.thread-count 0 2
yarn.sharedcache.webapp.address 0.0.0.0:3000 0.0.0.0:3001
yarn.sharedcache.cleaner.period-mins 720 2880
yarn.sharedcache.cleaner.initial-delay-mins 1 20
yarn.sharedcache.cleaner.resource-sleep-ms 1 -1
yarn.sharedcache.uploader.server.address 0.0.0.0:3000 0.0.0.0:3001
yarn.sharedcache.uploader.server.thread-count 25 100
yarn.sharedcache.client-server.address 0.0.0.0:3000 0.0.0.0:3001
yarn.sharedcache.client-server.thread-count 25 100
yarn.sharedcache.checksum.algo.impl SKIP SKIP
yarn.sharedcache.nm.uploader.replication.factor 1 20
yarn.sharedcache.nm.uploader.thread-count 10 40
yarn.client.application-client-protocol.poll-interval-ms 100 400
yarn.nodemanager.container-monitor.procfs-tree.smaps-based-rss.enabled true SKIP
yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds 0 -2
yarn.nodemanager.webapp.cross-origin.enabled true SKIP
5 changes: 5 additions & 0 deletions core/generate_value/value_generation.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ def read_tsv(module):
if module == "zookeeper-server":
assert len(params) == 32
return 32
elif module == "hadoop-yarn-tls":
assert len(params) == 222
return 16
else:
assert len(params) == 90
return 90
Expand Down Expand Up @@ -105,6 +108,8 @@ def print_params(module):
f = open(module + output, "w")
if module == "zookeeper-server":
assert len(params) == 32
elif module == "hadoop-yarn-tls":
assert len(params) == 222
else:
assert len(params) >= 90
for param in params:
Expand Down
14 changes: 13 additions & 1 deletion core/identify_param/add_project.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,17 @@ function setup_hadoop() {
mvn package -DskipTests
}

function setup_hadoop_yarn_tls() {
[ ! -d "app/ctest-hadoop" ] && git clone git@github.com:KobeNorris/hadoop.git app/ctest-hadoop
cd app/ctest-hadoop
git fetch && git checkout ctest-logging
home_dir=$PWD
cd $home_dir/hadoop-common-project/hadoop-common
mvn clean install -DskipTests
cd $home_dir/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice
mvn package -DskipTests
}

function setup_hbase() {
old_dir=$PWD
[ ! -d "app/ctest-hadoop" ] && git clone https://github.com/xlab-uiuc/hadoop.git app/ctest-hadoop
Expand Down Expand Up @@ -63,7 +74,8 @@ function main() {
hbase) setup_hbase ;;
zookeeper) setup_zookeeper ;;
alluxio) setup_alluxio ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
hadoop-yarn-tls) setup_hadoop_yarn_tls ;;
*) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and hadoop-yarn-tls." ;;
esac
fi
}
Expand Down
Loading