diff --git a/core/add_project.sh b/core/add_project.sh index e5640839..2a5aab56 100755 --- a/core/add_project.sh +++ b/core/add_project.sh @@ -47,6 +47,12 @@ function setup_alluxio() { cd core mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true } +function setup_nifi(){ + [ ! -d "app/ctest-nifi" ] && git clone https://github.com/lilacyl/nifi.git app/ctest-nifi + cd app/ctest-nifi + git fetch && git checkout ctest-injection + mvn clean install -pl nifi-commons/ -DskipTest +} function usage() { echo "Usage: add_project.sh
" @@ -64,7 +70,8 @@ function main() { hbase) setup_hbase ;; zookeeper) setup_zookeeper ;; alluxio) setup_alluxio ;; - *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;; + nifi) setup_nifi ;; + *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, nifi, and alluxio." ;; esac fi } diff --git a/core/ctest_const.py b/core/ctest_const.py index d0820f43..ff73ab60 100644 --- a/core/ctest_const.py +++ b/core/ctest_const.py @@ -12,11 +12,13 @@ HBASE = "hbase-server" ZOOKEEPER = "zookeeper-server" ALLUXIO = "alluxio-core" +NIFI = "nifi-commons" CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop") CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase") CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper") CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio") +CTEST_NIFI_DIR = os.path.join(APP_DIR, "ctest-nifi") PROJECT_DIR = { HCOMMON: CTEST_HADOOP_DIR, @@ -24,6 +26,7 @@ HBASE: CTEST_HBASE_DIR, ZOOKEEPER: CTEST_ZK_DIR, ALLUXIO: CTEST_ALLUXIO_DIR, + NIFI: CTEST_NIFI_DIR, } @@ -34,12 +37,14 @@ HBASE: "hbase-server", ZOOKEEPER: "zookeeper-server", ALLUXIO: "core", + NIFI: "nifi-commons", } # surefire report SUREFIRE_SUBDIR = "target/surefire-reports/" SUREFIRE_XML = "TEST-{}.xml" # slot is the classname +SUREFIRE_XML_NIFI = "TEST-org.apache.nifi.util.{}.xml" # slot is the classname SUREFIRE_TXT = "{}.txt" # testclass SUREFIRE_OUTTXT = "{}-output.txt" #testclass @@ -58,6 +63,7 @@ os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR), os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR), ], + NIFI: [os.path.join(CTEST_NIFI_DIR, MODULE_SUBDIR[NIFI],"nifi-properties", SUREFIRE_SUBDIR)], } # default or deprecate conf path @@ -74,7 +80,8 @@ HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"), HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"), ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"), - ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv") + ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"), + NIFI: os.path.join(DEFAULT_CONF_DIR, NIFI + "-default.tsv"), } @@ -96,6 +103,9 @@ ], ALLUXIO: [ os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties") + ], + NIFI: [ + os.path.join(CTEST_NIFI_DIR, "nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties") ] } diff --git a/core/default_configs/nifi-commons-default.tsv b/core/default_configs/nifi-commons-default.tsv new file mode 100644 index 00000000..f1c4b1bf --- /dev/null +++ b/core/default_configs/nifi-commons-default.tsv @@ -0,0 +1,229 @@ +nifi.properties.file.path +nifi.flow.configuration.file +nifi.flow.configuration.json.file +nifi.flow.configuration.archive.enabled "true" +nifi.flow.configuration.archive.dir +nifi.flow.configuration.archive.max.time "30 days" +nifi.flow.configuration.archive.max.storage "500 MB" +nifi.flow.configuration.archive.max.count +nifi.authorizer.configuration.file "conf/authorizers.xml" +nifi.login.identity.provider.configuration.file "conf/login-identity-providers.xml" +nifi.database.directory +nifi.restore.directory +nifi.flowservice.writedelay.interval +nifi.flowcontroller.autoResumeState true +nifi.flowcontroller.graceful.shutdown.period +nifi.nar.library.directory +nifi.nar.library.directory. +nifi.nar.library.autoload.directory +nifi.nar.working.directory +nifi.nar.unpack.uber.jar false +nifi.documentation.working.directory "./work/docs/components" +nifi.sensitive.props.key +nifi.sensitive.props.algorithm +nifi.h2.url.append +nifi.remote.input.host +nifi.remote.input.socket.port null +nifi.remote.input.secure +nifi.remote.input.http.enabled +nifi.remote.input.http.transaction.ttl "30 secs" +nifi.remote.contents.cache.expiration +nifi.templates.directory Paths.get("conf", "templates") +nifi.administrative.yield.duration "30 sec" +nifi.bored.yield.duration "10 millis" +nifi.processor.scheduling.timeout +nifi.queue.backpressure.count 10_000L +nifi.queue.backpressure.size "1 GB" +nifi.listener.bootstrap.port 0 +nifi.repository.encryption.protocol.version +nifi.repository.encryption.key.id +nifi.repository.encryption.key.provider +nifi.repository.encryption.key.provider.keystore.location +nifi.repository.encryption.key.provider.keystore.password +nifi.content.repository.directory. +nifi.content.repository.implementation +nifi.content.claim.max.appendable.size "1 MB" +nifi.content.claim.max.flow.files 100 +nifi.content.repository.archive.max.retention.period +nifi.content.repository.archive.max.usage.percentage +nifi.content.repository.archive.backpressure.percentage +nifi.content.repository.archive.enabled +nifi.content.repository.archive.cleanup.frequency +nifi.content.viewer.url +nifi.content.repository.encryption.key +nifi.content.repository.encryption.key.id +nifi.content.repository.encryption.key.provider.implementation +nifi.content.repository.encryption.key.provider.location +nifi.content.repository.encryption.key.provider.password +nifi.flowfile.repository.implementation +nifi.flowfile.repository.wal.implementation +nifi.flowfile.repository.always.sync +nifi.flowfile.repository.directory +nifi.flowfile.repository.checkpoint.interval +nifi.flowfile.repository.encryption.key +nifi.flowfile.repository.encryption.key.id +nifi.flowfile.repository.encryption.key.provider.implementation +nifi.flowfile.repository.encryption.key.provider.location +nifi.flowfile.repository.encryption.key.provider.password +nifi.swap.manager.implementation +nifi.queue.swap.threshold 20000 +nifi.provenance.repository.implementation +nifi.provenance.repository.directory. +nifi.provenance.repository.max.storage.time +nifi.provenance.repository.max.storage.size +nifi.provenance.repository.rollover.time +nifi.provenance.repository.rollover.size +nifi.provenance.repository.rollover.events +nifi.provenance.repository.query.threads +nifi.provenance.repository.index.threads +nifi.provenance.repository.compress.on.rollover +nifi.provenance.repository.indexed.fields +nifi.provenance.repository.indexed.attributes +nifi.provenance.repository.index.shard.size +nifi.provenance.repository.journal.count +nifi.provenance.repository.encryption.key +nifi.provenance.repository.encryption.key.id +nifi.provenance.repository.encryption.key.provider.implementation +nifi.provenance.repository.encryption.key.provider.location +nifi.provenance.repository.encryption.key.provider.password +nifi.provenance.repository.debug.frequency +nifi.components.status.repository.implementation +nifi.components.status.snapshot.frequency "5 mins" +nifi.status.repository.questdb.persist.node.days +nifi.status.repository.questdb.persist.component.days +nifi.status.repository.questdb.persist.location +nifi.security.keystore +nifi.security.keystoreType +nifi.security.keystorePasswd +nifi.security.keyPasswd +nifi.security.truststore +nifi.security.truststoreType +nifi.security.truststorePasswd +nifi.security.autoreload.enabled +nifi.security.autoreload.interval "10 secs" +nifi.security.user.authorizer +nifi.security.allow.anonymous.authentication +nifi.security.user.login.identity.provider +nifi.security.ocsp.responder.url +nifi.security.ocsp.responder.certificate +nifi.security.identity.mapping.pattern. +nifi.security.identity.mapping.value. +nifi.security.identity.mapping.transform. +nifi.security.group.mapping.pattern. +nifi.security.group.mapping.value. +nifi.security.group.mapping.transform. +nifi.security.user.jws.key.rotation.period +nifi.security.user.oidc.discovery.url +nifi.security.user.oidc.connect.timeout "5 secs" +nifi.security.user.oidc.read.timeout "5 secs" +nifi.security.user.oidc.client.id +nifi.security.user.oidc.client.secret +nifi.security.user.oidc.truststore.strategy "JDK" +nifi.security.user.oidc.preferred.jwsalgorithm +nifi.security.user.oidc.additional.scopes +nifi.security.user.oidc.claim.identifying.user +nifi.security.user.oidc.claim.groups +nifi.security.user.oidc.fallback.claims.identifying.user +nifi.security.user.knox.url +nifi.security.user.knox.publicKey +nifi.security.user.knox.cookieName +nifi.security.user.knox.audiences +nifi.security.user.saml.idp.metadata.url +nifi.security.user.saml.sp.entity.id +nifi.security.user.saml.identity.attribute.name +nifi.security.user.saml.group.attribute.name +nifi.security.user.saml.metadata.signing.enabled "false" +nifi.security.user.saml.request.signing.enabled "false" +nifi.security.user.saml.want.assertions.signed "true" +nifi.security.user.saml.signature.algorithm "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256" +nifi.security.user.saml.signature.digest.algorithm +nifi.security.user.saml.message.logging.enabled "false" +nifi.security.user.saml.authentication.expiration "12 hours" +nifi.security.user.saml.single.logout.enabled "false" +nifi.security.user.saml.http.client.truststore.strategy "JDK" +nifi.security.user.saml.http.client.connect.timeout "30 secs" +nifi.security.user.saml.http.client.read.timeout "30 secs" +nifi.web.http.port +nifi.web.http.port.forwarding +nifi.web.http.host +nifi.web.http.network.interface. +nifi.web.https.port +nifi.web.https.port.forwarding +nifi.web.https.host +nifi.web.https.application.protocols +nifi.web.https.ciphersuites.include +nifi.web.https.ciphersuites.exclude +nifi.web.https.network.interface. +nifi.web.jetty.working.directory "./work/jetty" +nifi.web.jetty.threads 200 +nifi.web.max.header.size "16 KB" +nifi.web.proxy.context.path +nifi.web.proxy.host +nifi.web.max.content.size "20 MB" +nifi.web.max.requests.per.second 30000 +nifi.web.max.access.token.requests.per.second 25 +nifi.web.request.timeout "60 secs" +nifi.web.request.ip.whitelist +nifi.web.should.send.server.version "true" +nifi.web.request.log.format +nifi.ui.banner.text +nifi.ui.autorefresh.interval +nifi.cluster.protocol.heartbeat.interval "5 sec" +nifi.cluster.protocol.heartbeat.missable.max 8 +nifi.cluster.protocol.is.secure +nifi.cluster.is.node +nifi.cluster.node.address +nifi.cluster.node.protocol.port +nifi.cluster.node.protocol.max.threads 50 +nifi.cluster.node.connection.timeout "5 sec" +nifi.cluster.node.read.timeout "5 sec" +nifi.cluster.node.max.concurrent.requests 100 +nifi.cluster.firewall.file +nifi.cluster.flow.election.max.wait.time "5 mins" +nifi.cluster.flow.election.max.candidates +nifi.cluster.load.balance.host +nifi.cluster.load.balance.port 6342 +nifi.cluster.load.balance.connections.per.node 4 +nifi.cluster.load.balance.max.thread.count 8 +nifi.cluster.load.balance.comms.timeout "30 sec" +nifi.zookeeper.connect.string +nifi.zookeeper.connect.timeout "3 secs" +nifi.zookeeper.session.timeout "3 secs" +nifi.zookeeper.root.node "/nifi" +nifi.zookeeper.client.secure false +nifi.zookeeper.security.keystore +nifi.zookeeper.security.keystoreType +nifi.zookeeper.security.keystorePasswd +nifi.zookeeper.security.truststore +nifi.zookeeper.security.truststoreType +nifi.zookeeper.security.truststorePasswd +nifi.zookeeper.auth.type "default" +nifi.zookeeper.kerberos.removeHostFromPrincipal "true" +nifi.zookeeper.kerberos.removeRealmFromPrincipal "true" +nifi.zookeeper.jute.maxbuffer 1048575 +nifi.kerberos.krb5.file +nifi.kerberos.service.principal +nifi.kerberos.service.keytab.location +nifi.kerberos.spnego.principal +nifi.kerberos.spnego.keytab.location +nifi.kerberos.spnego.authentication.expiration "12 hours" +nifi.state.management.configuration.file "conf/state-management.xml" +nifi.state.management.provider.local +nifi.state.management.provider.cluster +nifi.state.management.embedded.zookeeper.start +nifi.state.management.embedded.zookeeper.properties +nifi.variable.registry.properties +nifi.analytics.predict.enabled "false" +nifi.analytics.predict.interval "3 mins" +nifi.analytics.query.interval "3 mins" +nifi.analytics.connection.model.implementation +nifi.analytics.connection.model.score.name +nifi.analytics.connection.model.score.threshold +nifi.monitor.long.running.task.schedule +nifi.monitor.long.running.task.threshold +nifi.diagnostics.on.shutdown.enabled +nifi.diagnostics.on.shutdown.verbose +nifi.diagnostics.on.shutdown.directory "./diagnostics" +nifi.diagnostics.on.shutdown.max.filecount 10 +nifi.diagnostics.on.shutdown.max.directory.size "10 MB" +nifi.performance.tracking.percentage 0 diff --git a/core/generate_ctest/ctest_mapping/ctests-nifi-commons.json b/core/generate_ctest/ctest_mapping/ctests-nifi-commons.json new file mode 100644 index 00000000..c16c2183 --- /dev/null +++ b/core/generate_ctest/ctest_mapping/ctests-nifi-commons.json @@ -0,0 +1 @@ +{"nifi.ui.banner.text": ["NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidPortValue", "NiFiPropertiesTest#testProperties", "NiFiPropertiesTest#testBlankProperties", "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", "NiFiPropertiesTest#testValidateProperties", "NiFiPropertiesTest#testMissingProperties"], "nifi.nar.library.directory": ["NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", "NiFiPropertiesTest#testShouldVerifyValidPortValue", "NiFiPropertiesTest#testProperties", "NiFiPropertiesTest#testBlankProperties", "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", "NiFiPropertiesTest#testValidateProperties", "NiFiPropertiesTest#testMissingProperties"]} \ No newline at end of file diff --git a/core/generate_ctest/inject.py b/core/generate_ctest/inject.py index 75f5b443..503962f4 100644 --- a/core/generate_ctest/inject.py +++ b/core/generate_ctest/inject.py @@ -14,7 +14,7 @@ def inject_config(param_value_pairs): for p, v in param_value_pairs.items(): print(">>>>[ctest_core] injecting {} with value {}".format(p, v)) - if project in [ZOOKEEPER, ALLUXIO]: + if project in [ZOOKEEPER, ALLUXIO, NIFI]: for inject_path in INJECTION_PATH[project]: print(">>>>[ctest_core] injecting into file: {}".format(inject_path)) file = open(inject_path, "w") @@ -41,7 +41,7 @@ def inject_config(param_value_pairs): def clean_conf_file(project): print(">>>> cleaning injected configuration from file") - if project in [ZOOKEEPER, ALLUXIO]: + if project in [ZOOKEEPER, ALLUXIO, NIFI]: for inject_path in INJECTION_PATH[project]: file = open(inject_path, "w") file.write("\n") diff --git a/core/generate_ctest/main.py b/core/generate_ctest/main.py index a53169d6..c65c2276 100644 --- a/core/generate_ctest/main.py +++ b/core/generate_ctest/main.py @@ -33,7 +33,8 @@ def test_value_pair(test_input): mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w") associated_tests = mapping[param] if param in mapping else [] - if len(mapping[param]) != 0: + print(param) + if param in mapping and len(mapping[param]) != 0: for value in values: tr = run_test_seperate(param, value, associated_tests) diff --git a/core/generate_ctest/parse_output.py b/core/generate_ctest/parse_output.py index c750f398..d3438f32 100644 --- a/core/generate_ctest/parse_output.py +++ b/core/generate_ctest/parse_output.py @@ -19,7 +19,10 @@ def parse_surefire(clsname, expected_methods): try: fpath = None for surefire_path in SUREFIRE_DIR[project]: - xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname)) + if project == "nifi-commons": + xml_path = os.path.join(surefire_path, SUREFIRE_XML_NIFI.format(clsname)) + else: + xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname)) if os.path.exists(xml_path): print(">>>>[ctest_core] surefire report path: " + xml_path) fpath = open(xml_path) diff --git a/core/generate_ctest/program_input.py b/core/generate_ctest/program_input.py index 9147868a..37d43d4d 100644 --- a/core/generate_ctest/program_input.py +++ b/core/generate_ctest/program_input.py @@ -4,11 +4,11 @@ # run mode "run_mode": "generate_ctest", # string # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py - "project": "hadoop-common", # string + "project": "nifi-commons", # string # path to param -> tests json mapping - "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string + "mapping_path": "../../data/ctest_mapping/opensource-nifi-commons.json", # string # good values of params tests will be run against - "param_value_tsv": "sample-hadoop-common.tsv", # string + "param_value_tsv": "sample-nifi-commons.tsv", # string # display the terminal output live, without saving any results "display_mode": False, # bool # whether to use mvn test or mvn surefire:test diff --git a/core/generate_ctest/sample-nifi-commons.tsv b/core/generate_ctest/sample-nifi-commons.tsv new file mode 100644 index 00000000..f6a9134e --- /dev/null +++ b/core/generate_ctest/sample-nifi-commons.tsv @@ -0,0 +1,229 @@ +nifi.properties.file.path /valid/file1 /valid/file2 +nifi.flow.configuration.file /valid/file1 /valid/file2 +nifi.flow.configuration.json.file /valid/file1 /valid/file2 +nifi.flow.configuration.archive.enabled false SKIP +nifi.flow.configuration.archive.dir /valid/dir1 /valid/dir2 +nifi.flow.configuration.archive.max.time SKIP SKIP +nifi.flow.configuration.archive.max.storage 1MB 1000MB +nifi.flow.configuration.archive.max.count SKIP SKIP +nifi.authorizer.configuration.file /valid/file1 /valid/file2 +nifi.login.identity.provider.configuration.file /valid/file1 /valid/file2 +nifi.database.directory SKIP SKIP +nifi.restore.directory SKIP SKIP +nifi.flowservice.writedelay.interval SKIP SKIP +nifi.flowcontroller.autoResumeState false SKIP +nifi.flowcontroller.graceful.shutdown.period SKIP SKIP +nifi.nar.library.directory ./lib ./lib +nifi.nar.library.directory. ./lib/nar ./lib +nifi.nar.library.autoload.directory SKIP SKIP +nifi.nar.working.directory SKIP SKIP +nifi.nar.unpack.uber.jar true SKIP +nifi.documentation.working.directory SKIP SKIP +nifi.sensitive.props.key SKIP SKIP +nifi.sensitive.props.algorithm SKIP SKIP +nifi.h2.url.append SKIP SKIP +nifi.remote.input.host SKIP SKIP +nifi.remote.input.socket.port SKIP SKIP +nifi.remote.input.secure SKIP SKIP +nifi.remote.input.http.enabled SKIP SKIP +nifi.remote.input.http.transaction.ttl 1s 60s +nifi.remote.contents.cache.expiration SKIP SKIP +nifi.templates.directory Paths.get("conf" "templates") +nifi.administrative.yield.duration 1sec 60sec +nifi.bored.yield.duration SKIP SKIP +nifi.processor.scheduling.timeout SKIP SKIP +nifi.queue.backpressure.count SKIP SKIP +nifi.queue.backpressure.size 0.1 0.5 +nifi.listener.bootstrap.port 3000 3001 +nifi.repository.encryption.protocol.version SKIP SKIP +nifi.repository.encryption.key.id SKIP SKIP +nifi.repository.encryption.key.provider SKIP SKIP +nifi.repository.encryption.key.provider.keystore.location SKIP SKIP +nifi.repository.encryption.key.provider.keystore.password SKIP SKIP +nifi.content.repository.directory. SKIP SKIP +nifi.content.repository.implementation SKIP SKIP +nifi.content.claim.max.appendable.size 10MB 2MB +nifi.content.claim.max.flow.files 50 200 +nifi.content.repository.archive.max.retention.period SKIP SKIP +nifi.content.repository.archive.max.usage.percentage SKIP SKIP +nifi.content.repository.archive.backpressure.percentage SKIP SKIP +nifi.content.repository.archive.enabled SKIP SKIP +nifi.content.repository.archive.cleanup.frequency SKIP SKIP +nifi.content.viewer.url SKIP SKIP +nifi.content.repository.encryption.key SKIP SKIP +nifi.content.repository.encryption.key.id SKIP SKIP +nifi.content.repository.encryption.key.provider.implementation SKIP SKIP +nifi.content.repository.encryption.key.provider.location SKIP SKIP +nifi.content.repository.encryption.key.provider.password SKIP SKIP +nifi.flowfile.repository.implementation SKIP SKIP +nifi.flowfile.repository.wal.implementation SKIP SKIP +nifi.flowfile.repository.always.sync SKIP SKIP +nifi.flowfile.repository.directory SKIP SKIP +nifi.flowfile.repository.checkpoint.interval SKIP SKIP +nifi.flowfile.repository.encryption.key SKIP SKIP +nifi.flowfile.repository.encryption.key.id SKIP SKIP +nifi.flowfile.repository.encryption.key.provider.implementation SKIP SKIP +nifi.flowfile.repository.encryption.key.provider.location SKIP SKIP +nifi.flowfile.repository.encryption.key.provider.password SKIP SKIP +nifi.swap.manager.implementation SKIP SKIP +nifi.queue.swap.threshold 10000 40000 +nifi.provenance.repository.implementation SKIP SKIP +nifi.provenance.repository.directory. SKIP SKIP +nifi.provenance.repository.max.storage.time SKIP SKIP +nifi.provenance.repository.max.storage.size 0.1 0.5 +nifi.provenance.repository.rollover.time SKIP SKIP +nifi.provenance.repository.rollover.size 0.1 0.5 +nifi.provenance.repository.rollover.events SKIP SKIP +nifi.provenance.repository.query.threads SKIP SKIP +nifi.provenance.repository.index.threads SKIP SKIP +nifi.provenance.repository.compress.on.rollover SKIP SKIP +nifi.provenance.repository.indexed.fields SKIP SKIP +nifi.provenance.repository.indexed.attributes SKIP SKIP +nifi.provenance.repository.index.shard.size 0.1 0.5 +nifi.provenance.repository.journal.count SKIP SKIP +nifi.provenance.repository.encryption.key SKIP SKIP +nifi.provenance.repository.encryption.key.id SKIP SKIP +nifi.provenance.repository.encryption.key.provider.implementation SKIP SKIP +nifi.provenance.repository.encryption.key.provider.location SKIP SKIP +nifi.provenance.repository.encryption.key.provider.password SKIP SKIP +nifi.provenance.repository.debug.frequency SKIP SKIP +nifi.components.status.repository.implementation SKIP SKIP +nifi.components.status.snapshot.frequency SKIP SKIP +nifi.status.repository.questdb.persist.node.days SKIP SKIP +nifi.status.repository.questdb.persist.component.days SKIP SKIP +nifi.status.repository.questdb.persist.location SKIP SKIP +nifi.security.keystore SKIP SKIP +nifi.security.keystoreType SKIP SKIP +nifi.security.keystorePasswd SKIP SKIP +nifi.security.keyPasswd SKIP SKIP +nifi.security.truststore SKIP SKIP +nifi.security.truststoreType SKIP SKIP +nifi.security.truststorePasswd SKIP SKIP +nifi.security.autoreload.enabled SKIP SKIP +nifi.security.autoreload.interval 1s 20s +nifi.security.user.authorizer SKIP SKIP +nifi.security.allow.anonymous.authentication SKIP SKIP +nifi.security.user.login.identity.provider SKIP SKIP +nifi.security.ocsp.responder.url SKIP SKIP +nifi.security.ocsp.responder.certificate SKIP SKIP +nifi.security.identity.mapping.pattern. SKIP SKIP +nifi.security.identity.mapping.value. SKIP SKIP +nifi.security.identity.mapping.transform. SKIP SKIP +nifi.security.group.mapping.pattern. SKIP SKIP +nifi.security.group.mapping.value. SKIP SKIP +nifi.security.group.mapping.transform. SKIP SKIP +nifi.security.user.jws.key.rotation.period SKIP SKIP +nifi.security.user.oidc.discovery.url SKIP SKIP +nifi.security.user.oidc.connect.timeout 1s 10s +nifi.security.user.oidc.read.timeout 1s 10s +nifi.security.user.oidc.client.id SKIP SKIP +nifi.security.user.oidc.client.secret SKIP SKIP +nifi.security.user.oidc.truststore.strategy SKIP SKIP +nifi.security.user.oidc.preferred.jwsalgorithm SKIP SKIP +nifi.security.user.oidc.additional.scopes SKIP SKIP +nifi.security.user.oidc.claim.identifying.user xdsuper samsuper +nifi.security.user.oidc.claim.groups xdgroup samgroup +nifi.security.user.oidc.fallback.claims.identifying.user xdsuper samsuper +nifi.security.user.knox.url SKIP SKIP +nifi.security.user.knox.publicKey SKIP SKIP +nifi.security.user.knox.cookieName SKIP SKIP +nifi.security.user.knox.audiences SKIP SKIP +nifi.security.user.saml.idp.metadata.url SKIP SKIP +nifi.security.user.saml.sp.entity.id SKIP SKIP +nifi.security.user.saml.identity.attribute.name SKIP SKIP +nifi.security.user.saml.group.attribute.name SKIP SKIP +nifi.security.user.saml.metadata.signing.enabled true SKIP +nifi.security.user.saml.request.signing.enabled true SKIP +nifi.security.user.saml.want.assertions.signed false SKIP +nifi.security.user.saml.signature.algorithm SKIP SKIP +nifi.security.user.saml.signature.digest.algorithm SKIP SKIP +nifi.security.user.saml.message.logging.enabled true SKIP +nifi.security.user.saml.authentication.expiration SKIP SKIP +nifi.security.user.saml.single.logout.enabled true SKIP +nifi.security.user.saml.http.client.truststore.strategy SKIP SKIP +nifi.security.user.saml.http.client.connect.timeout 1s 60s +nifi.security.user.saml.http.client.read.timeout 1s 60s +nifi.web.http.port 3000 3001 +nifi.web.http.port.forwarding SKIP SKIP +nifi.web.http.host SKIP SKIP +nifi.web.http.network.interface. SKIP SKIP +nifi.web.https.port 3000 3001 +nifi.web.https.port.forwarding SKIP SKIP +nifi.web.https.host SKIP SKIP +nifi.web.https.application.protocols SKIP SKIP +nifi.web.https.ciphersuites.include SKIP SKIP +nifi.web.https.ciphersuites.exclude SKIP SKIP +nifi.web.https.network.interface. SKIP SKIP +nifi.web.jetty.working.directory SKIP SKIP +nifi.web.jetty.threads 100 400 +nifi.web.max.header.size 0.1 0.5 +nifi.web.proxy.context.path /valid/file1 /valid/file2 +nifi.web.proxy.host SKIP SKIP +nifi.web.max.content.size 1MB 40MB +nifi.web.max.requests.per.second 15000 60000 +nifi.web.max.access.token.requests.per.second 12 50 +nifi.web.request.timeout 1s 120s +nifi.web.request.ip.whitelist SKIP SKIP +nifi.web.should.send.server.version false SKIP +nifi.web.request.log.format SKIP SKIP +nifi.ui.banner.text UI Banner Text text +nifi.ui.autorefresh.interval SKIP SKIP +nifi.cluster.protocol.heartbeat.interval 1sec 10sec +nifi.cluster.protocol.heartbeat.missable.max 1 16 +nifi.cluster.protocol.is.secure SKIP SKIP +nifi.cluster.is.node SKIP SKIP +nifi.cluster.node.address 127.0.0.1 SKIP +nifi.cluster.node.protocol.port 3000 3001 +nifi.cluster.node.protocol.max.threads 25 100 +nifi.cluster.node.connection.timeout 1sec 10sec +nifi.cluster.node.read.timeout 1sec 10sec +nifi.cluster.node.max.concurrent.requests 50 200 +nifi.cluster.firewall.file /valid/file1 /valid/file2 +nifi.cluster.flow.election.max.wait.time SKIP SKIP +nifi.cluster.flow.election.max.candidates SKIP SKIP +nifi.cluster.load.balance.host SKIP SKIP +nifi.cluster.load.balance.port 3000 3001 +nifi.cluster.load.balance.connections.per.node 1 8 +nifi.cluster.load.balance.max.thread.count 1 16 +nifi.cluster.load.balance.comms.timeout 1sec 60sec +nifi.zookeeper.connect.string SKIP SKIP +nifi.zookeeper.connect.timeout 1s 6s +nifi.zookeeper.session.timeout 1s 6s +nifi.zookeeper.root.node /valid/file1 /valid/file2 +nifi.zookeeper.client.secure true SKIP +nifi.zookeeper.security.keystore SKIP SKIP +nifi.zookeeper.security.keystoreType SKIP SKIP +nifi.zookeeper.security.keystorePasswd SKIP SKIP +nifi.zookeeper.security.truststore SKIP SKIP +nifi.zookeeper.security.truststoreType SKIP SKIP +nifi.zookeeper.security.truststorePasswd SKIP SKIP +nifi.zookeeper.auth.type SKIP SKIP +nifi.zookeeper.kerberos.removeHostFromPrincipal false SKIP +nifi.zookeeper.kerberos.removeRealmFromPrincipal false SKIP +nifi.zookeeper.jute.maxbuffer 524287 2097150 +nifi.kerberos.krb5.file /valid/file1 /valid/file2 +nifi.kerberos.service.principal SKIP SKIP +nifi.kerberos.service.keytab.location SKIP SKIP +nifi.kerberos.spnego.principal SKIP SKIP +nifi.kerberos.spnego.keytab.location SKIP SKIP +nifi.kerberos.spnego.authentication.expiration SKIP SKIP +nifi.state.management.configuration.file /valid/file1 /valid/file2 +nifi.state.management.provider.local SKIP SKIP +nifi.state.management.provider.cluster SKIP SKIP +nifi.state.management.embedded.zookeeper.start SKIP SKIP +nifi.state.management.embedded.zookeeper.properties SKIP SKIP +nifi.variable.registry.properties SKIP SKIP +nifi.analytics.predict.enabled true SKIP +nifi.analytics.predict.interval SKIP SKIP +nifi.analytics.query.interval SKIP SKIP +nifi.analytics.connection.model.implementation SKIP SKIP +nifi.analytics.connection.model.score.name SKIP SKIP +nifi.analytics.connection.model.score.threshold SKIP SKIP +nifi.monitor.long.running.task.schedule SKIP SKIP +nifi.monitor.long.running.task.threshold SKIP SKIP +nifi.diagnostics.on.shutdown.enabled SKIP SKIP +nifi.diagnostics.on.shutdown.verbose SKIP SKIP +nifi.diagnostics.on.shutdown.directory SKIP SKIP +nifi.diagnostics.on.shutdown.max.filecount 1 20 +nifi.diagnostics.on.shutdown.max.directory.size 1MB 20MB +nifi.performance.tracking.percentage 1 -1 diff --git a/core/generate_value/value_generation.py b/core/generate_value/value_generation.py index 82a5bde8..ab51a0fd 100644 --- a/core/generate_value/value_generation.py +++ b/core/generate_value/value_generation.py @@ -25,6 +25,9 @@ def read_tsv(module): if module == "zookeeper-server": assert len(params) == 32 return 32 + if module == "nifi-commons": + assert len(params) >= 200 + return len(params) else: assert len(params) == 90 return 90 diff --git a/core/identify_param/add_project.sh b/core/identify_param/add_project.sh index 954dbab8..4ec6dfeb 100755 --- a/core/identify_param/add_project.sh +++ b/core/identify_param/add_project.sh @@ -47,6 +47,14 @@ function setup_alluxio() { mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true } +function setup_nifi(){ + [ ! -d "app/ctest-nifi" ] && git clone https://github.com/lilacyl/nifi.git app/ctest-nifi + cd app/ctest-nifi + git fetch && git checkout ctest-logging + mvn clean install -pl nifi-commons/ -DskipTest + +} + function usage() { echo "Usage: add_project.sh
" exit 1 @@ -63,7 +71,8 @@ function main() { hbase) setup_hbase ;; zookeeper) setup_zookeeper ;; alluxio) setup_alluxio ;; - *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;; + nifi) setup_nifi ;; + *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, nifi, and alluxio." ;; esac fi } diff --git a/core/identify_param/constant.py b/core/identify_param/constant.py index a0b2d067..ca3ef864 100644 --- a/core/identify_param/constant.py +++ b/core/identify_param/constant.py @@ -7,12 +7,14 @@ CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase") CTEST_ZOOKEEPER_DIR = os.path.join(APP_DIR, "ctest-zookeeper") CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio") +CTEST_NIFI_DIR = os.path.join(APP_DIR, "ctest-nifi") MODULE_PATH = { "hadoop-common": CTEST_HADOOP_DIR, "hadoop-hdfs": CTEST_HADOOP_DIR, "hbase-server": CTEST_HBASE_DIR, - "alluxio-core": CTEST_ALLUXIO_DIR + "alluxio-core": CTEST_ALLUXIO_DIR, + "nifi-commons": CTEST_NIFI_DIR } SRC_SUBDIR = { @@ -20,7 +22,8 @@ "hadoop-hdfs": "hadoop-hdfs-project/hadoop-hdfs", "hbase-server": "hbase-server", "zookeeper-server": "zookeeper-server", - "alluxio-core": "core" + "alluxio-core": "core", + "nifi-commons": "" } MVN_TEST_PATH = { @@ -29,6 +32,7 @@ "hbase-server": os.path.join(CTEST_HBASE_DIR, SRC_SUBDIR["hbase-server"]), "zookeeper-server": os.path.join(CTEST_ZOOKEEPER_DIR, SRC_SUBDIR["zookeeper-server"]), "alluxio-core": os.path.join(CTEST_ALLUXIO_DIR, SRC_SUBDIR["alluxio-core"]), + "nifi-commons":os.path.join(CTEST_NIFI_DIR, SRC_SUBDIR["nifi-commons"]), } LOCAL_CONF_PATH = { @@ -36,7 +40,8 @@ "hadoop-hdfs": "results/hadoop-hdfs/conf_params.txt", "hbase-server": "results/hbase-server/conf_params.txt", "zookeeper-server": "results/zookeeper-server/conf_params.txt", - "alluxio-core": "results/alluxio-core/conf_params.txt" + "alluxio-core": "results/alluxio-core/conf_params.txt", + "nifi-commons": "results/nifi-commons/conf_params.txt" } SUREFIRE_SUBDIR = "target/surefire-reports/*" @@ -63,6 +68,9 @@ os.path.join(CTEST_ALLUXIO_DIR, "core/server/proxy", SUREFIRE_SUBDIR), os.path.join(CTEST_ALLUXIO_DIR, "core/server/worker", SUREFIRE_SUBDIR), os.path.join(CTEST_ALLUXIO_DIR, "core/server/master", SUREFIRE_SUBDIR) + ], + "nifi-commons": [ + os.path.join(CTEST_NIFI_DIR, "nifi-commons/nifi-properties", SUREFIRE_SUBDIR) ] } @@ -83,5 +91,8 @@ ], "alluxio-core": [ os.path.join("surefire-reports/alluxio-core", LOCAL_SUREFIRE_SUFFIX) - ] + ], + "nifi-commons":[ + os.path.join("surefire-reports/nifi/nifi-commons", LOCAL_SUREFIRE_SUFFIX) + ], } diff --git a/core/identify_param/identify_param.sh b/core/identify_param/identify_param.sh index 76963419..cd7faae1 100755 --- a/core/identify_param/identify_param.sh +++ b/core/identify_param/identify_param.sh @@ -12,9 +12,9 @@ function main() { usage else case $project in - hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core) python3 runner.py $project; python3 collector.py $project ;; + hadoop-common | hadoop-hdfs | hbase-server | zookeeper-server | alluxio-core | nifi-commons) python3 runner.py $project; python3 collector.py $project ;; -h | --help) usage ;; - *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server and alluxio-core." ;; + *) echo "Unexpected project: $project - only support hadoop-common, hadoop-hdfs, hbase-server, zookeeper-server, nifi-commons, and alluxio-core." ;; esac fi } diff --git a/core/identify_param/results/nifi-commons/conf_params.txt b/core/identify_param/results/nifi-commons/conf_params.txt new file mode 100644 index 00000000..f6bcc418 --- /dev/null +++ b/core/identify_param/results/nifi-commons/conf_params.txt @@ -0,0 +1,229 @@ +nifi.properties.file.path +nifi.flow.configuration.file +nifi.flow.configuration.json.file +nifi.flow.configuration.archive.enabled +nifi.flow.configuration.archive.dir +nifi.flow.configuration.archive.max.time +nifi.flow.configuration.archive.max.storage +nifi.flow.configuration.archive.max.count +nifi.authorizer.configuration.file +nifi.login.identity.provider.configuration.file +nifi.database.directory +nifi.restore.directory +nifi.flowservice.writedelay.interval +nifi.flowcontroller.autoResumeState +nifi.flowcontroller.graceful.shutdown.period +nifi.nar.library.directory +nifi.nar.library.directory. +nifi.nar.library.autoload.directory +nifi.nar.working.directory +nifi.nar.unpack.uber.jar +nifi.documentation.working.directory +nifi.sensitive.props.key +nifi.sensitive.props.algorithm +nifi.h2.url.append +nifi.remote.input.host +nifi.remote.input.socket.port +nifi.remote.input.secure +nifi.remote.input.http.enabled +nifi.remote.input.http.transaction.ttl +nifi.remote.contents.cache.expiration +nifi.templates.directory +nifi.administrative.yield.duration +nifi.bored.yield.duration +nifi.processor.scheduling.timeout +nifi.queue.backpressure.count +nifi.queue.backpressure.size +nifi.listener.bootstrap.port +nifi.repository.encryption.protocol.version +nifi.repository.encryption.key.id +nifi.repository.encryption.key.provider +nifi.repository.encryption.key.provider.keystore.location +nifi.repository.encryption.key.provider.keystore.password +nifi.content.repository.directory. +nifi.content.repository.implementation +nifi.content.claim.max.appendable.size +nifi.content.claim.max.flow.files +nifi.content.repository.archive.max.retention.period +nifi.content.repository.archive.max.usage.percentage +nifi.content.repository.archive.backpressure.percentage +nifi.content.repository.archive.enabled +nifi.content.repository.archive.cleanup.frequency +nifi.content.viewer.url +nifi.content.repository.encryption.key +nifi.content.repository.encryption.key.id +nifi.content.repository.encryption.key.provider.implementation +nifi.content.repository.encryption.key.provider.location +nifi.content.repository.encryption.key.provider.password +nifi.flowfile.repository.implementation +nifi.flowfile.repository.wal.implementation +nifi.flowfile.repository.always.sync +nifi.flowfile.repository.directory +nifi.flowfile.repository.checkpoint.interval +nifi.flowfile.repository.encryption.key +nifi.flowfile.repository.encryption.key.id +nifi.flowfile.repository.encryption.key.provider.implementation +nifi.flowfile.repository.encryption.key.provider.location +nifi.flowfile.repository.encryption.key.provider.password +nifi.swap.manager.implementation +nifi.queue.swap.threshold +nifi.provenance.repository.implementation +nifi.provenance.repository.directory. +nifi.provenance.repository.max.storage.time +nifi.provenance.repository.max.storage.size +nifi.provenance.repository.rollover.time +nifi.provenance.repository.rollover.size +nifi.provenance.repository.rollover.events +nifi.provenance.repository.query.threads +nifi.provenance.repository.index.threads +nifi.provenance.repository.compress.on.rollover +nifi.provenance.repository.indexed.fields +nifi.provenance.repository.indexed.attributes +nifi.provenance.repository.index.shard.size +nifi.provenance.repository.journal.count +nifi.provenance.repository.encryption.key +nifi.provenance.repository.encryption.key.id +nifi.provenance.repository.encryption.key.provider.implementation +nifi.provenance.repository.encryption.key.provider.location +nifi.provenance.repository.encryption.key.provider.password +nifi.provenance.repository.debug.frequency +nifi.components.status.repository.implementation +nifi.components.status.snapshot.frequency +nifi.status.repository.questdb.persist.node.days +nifi.status.repository.questdb.persist.component.days +nifi.status.repository.questdb.persist.location +nifi.security.keystore +nifi.security.keystoreType +nifi.security.keystorePasswd +nifi.security.keyPasswd +nifi.security.truststore +nifi.security.truststoreType +nifi.security.truststorePasswd +nifi.security.autoreload.enabled +nifi.security.autoreload.interval +nifi.security.user.authorizer +nifi.security.allow.anonymous.authentication +nifi.security.user.login.identity.provider +nifi.security.ocsp.responder.url +nifi.security.ocsp.responder.certificate +nifi.security.identity.mapping.pattern. +nifi.security.identity.mapping.value. +nifi.security.identity.mapping.transform. +nifi.security.group.mapping.pattern. +nifi.security.group.mapping.value. +nifi.security.group.mapping.transform. +nifi.security.user.jws.key.rotation.period +nifi.security.user.oidc.discovery.url +nifi.security.user.oidc.connect.timeout +nifi.security.user.oidc.read.timeout +nifi.security.user.oidc.client.id +nifi.security.user.oidc.client.secret +nifi.security.user.oidc.truststore.strategy +nifi.security.user.oidc.preferred.jwsalgorithm +nifi.security.user.oidc.additional.scopes +nifi.security.user.oidc.claim.identifying.user +nifi.security.user.oidc.claim.groups +nifi.security.user.oidc.fallback.claims.identifying.user +nifi.security.user.knox.url +nifi.security.user.knox.publicKey +nifi.security.user.knox.cookieName +nifi.security.user.knox.audiences +nifi.security.user.saml.idp.metadata.url +nifi.security.user.saml.sp.entity.id +nifi.security.user.saml.identity.attribute.name +nifi.security.user.saml.group.attribute.name +nifi.security.user.saml.metadata.signing.enabled +nifi.security.user.saml.request.signing.enabled +nifi.security.user.saml.want.assertions.signed +nifi.security.user.saml.signature.algorithm +nifi.security.user.saml.signature.digest.algorithm +nifi.security.user.saml.message.logging.enabled +nifi.security.user.saml.authentication.expiration +nifi.security.user.saml.single.logout.enabled +nifi.security.user.saml.http.client.truststore.strategy +nifi.security.user.saml.http.client.connect.timeout +nifi.security.user.saml.http.client.read.timeout +nifi.web.http.port +nifi.web.http.port.forwarding +nifi.web.http.host +nifi.web.http.network.interface. +nifi.web.https.port +nifi.web.https.port.forwarding +nifi.web.https.host +nifi.web.https.application.protocols +nifi.web.https.ciphersuites.include +nifi.web.https.ciphersuites.exclude +nifi.web.https.network.interface. +nifi.web.jetty.working.directory +nifi.web.jetty.threads +nifi.web.max.header.size +nifi.web.proxy.context.path +nifi.web.proxy.host +nifi.web.max.content.size +nifi.web.max.requests.per.second +nifi.web.max.access.token.requests.per.second +nifi.web.request.timeout +nifi.web.request.ip.whitelist +nifi.web.should.send.server.version +nifi.web.request.log.format +nifi.ui.banner.text +nifi.ui.autorefresh.interval +nifi.cluster.protocol.heartbeat.interval +nifi.cluster.protocol.heartbeat.missable.max +nifi.cluster.protocol.is.secure +nifi.cluster.is.node +nifi.cluster.node.address +nifi.cluster.node.protocol.port +nifi.cluster.node.protocol.max.threads +nifi.cluster.node.connection.timeout +nifi.cluster.node.read.timeout +nifi.cluster.node.max.concurrent.requests +nifi.cluster.firewall.file +nifi.cluster.flow.election.max.wait.time +nifi.cluster.flow.election.max.candidates +nifi.cluster.load.balance.host +nifi.cluster.load.balance.port +nifi.cluster.load.balance.connections.per.node +nifi.cluster.load.balance.max.thread.count +nifi.cluster.load.balance.comms.timeout +nifi.zookeeper.connect.string +nifi.zookeeper.connect.timeout +nifi.zookeeper.session.timeout +nifi.zookeeper.root.node +nifi.zookeeper.client.secure +nifi.zookeeper.security.keystore +nifi.zookeeper.security.keystoreType +nifi.zookeeper.security.keystorePasswd +nifi.zookeeper.security.truststore +nifi.zookeeper.security.truststoreType +nifi.zookeeper.security.truststorePasswd +nifi.zookeeper.auth.type +nifi.zookeeper.kerberos.removeHostFromPrincipal +nifi.zookeeper.kerberos.removeRealmFromPrincipal +nifi.zookeeper.jute.maxbuffer +nifi.kerberos.krb5.file +nifi.kerberos.service.principal +nifi.kerberos.service.keytab.location +nifi.kerberos.spnego.principal +nifi.kerberos.spnego.keytab.location +nifi.kerberos.spnego.authentication.expiration +nifi.state.management.configuration.file +nifi.state.management.provider.local +nifi.state.management.provider.cluster +nifi.state.management.embedded.zookeeper.start +nifi.state.management.embedded.zookeeper.properties +nifi.variable.registry.properties +nifi.analytics.predict.enabled +nifi.analytics.predict.interval +nifi.analytics.query.interval +nifi.analytics.connection.model.implementation +nifi.analytics.connection.model.score.name +nifi.analytics.connection.model.score.threshold +nifi.monitor.long.running.task.schedule +nifi.monitor.long.running.task.threshold +nifi.diagnostics.on.shutdown.enabled +nifi.diagnostics.on.shutdown.verbose +nifi.diagnostics.on.shutdown.directory +nifi.diagnostics.on.shutdown.max.filecount +nifi.diagnostics.on.shutdown.max.directory.size +nifi.performance.tracking.percentage \ No newline at end of file diff --git a/core/identify_param/results/nifi-commons/param_getter_map.json b/core/identify_param/results/nifi-commons/param_getter_map.json new file mode 100644 index 00000000..9e0ca3ea --- /dev/null +++ b/core/identify_param/results/nifi-commons/param_getter_map.json @@ -0,0 +1,22 @@ +{ + "nifi.ui.banner.text": [ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testValidateProperties" + ], + "nifi.nar.library.directory": [ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testValidateProperties" + ] +} \ No newline at end of file diff --git a/core/identify_param/results/nifi-commons/param_setter_map.json b/core/identify_param/results/nifi-commons/param_setter_map.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/core/identify_param/results/nifi-commons/param_setter_map.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/core/identify_param/results/nifi-commons/param_unset_getter_map.json b/core/identify_param/results/nifi-commons/param_unset_getter_map.json new file mode 100644 index 00000000..9e0ca3ea --- /dev/null +++ b/core/identify_param/results/nifi-commons/param_unset_getter_map.json @@ -0,0 +1,22 @@ +{ + "nifi.ui.banner.text": [ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testValidateProperties" + ], + "nifi.nar.library.directory": [ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testValidateProperties" + ] +} \ No newline at end of file diff --git a/core/identify_param/results/nifi-commons/test_method_list.json b/core/identify_param/results/nifi-commons/test_method_list.json new file mode 100644 index 00000000..0cf5b6af --- /dev/null +++ b/core/identify_param/results/nifi-commons/test_method_list.json @@ -0,0 +1,353 @@ +[ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testValidateProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInvalidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenPortValueIsZero", + "NiFiPropertiesTest#testShouldHaveReasonableMaxContentLengthValues", + "NiFiPropertiesTest#testIsZooKeeperTlsConfigurationPresent", + "NiFiPropertiesTest#testSomeZooKeeperTlsConfigurationIsMissing", + "NiFiPropertiesTest#testZooKeeperTlsPasswordsBlank", + "NiFiPropertiesTest#testKeystorePasswordIsMissing", + "NiFiPropertiesTest#testTlsConfigurationIsPresentWithEmptyPasswords", + "NiFiPropertiesTest#testTlsConfigurationIsNotPresentWithPropertiesMissing", + "NiFiPropertiesTest#testTlsConfigurationIsNotPresentWithNoProperties", + "NiFiPropertiesTest#testGetPropertiesWithPrefixWithoutDot", + "NiFiPropertiesTest#testGetPropertiesWithPrefixWithDot", + "NiFiPropertiesTest#testGetPropertiesWithPrefixWhenNoResult", + "NiFiPropertiesTest#testGetDirectSubsequentTokensWithoutDot", + "NiFiPropertiesTest#testGetDirectSubsequentTokensWithDot", + "NiFiPropertiesTest#testGetDirectSubsequentTokensWithNonExistingToken", + "NiFiPropertiesTest#testGetDirectSubsequentTokensWhenMoreTokensAfterward", + "StandardFlowEncryptorTest#testProcessEncrypted", + "StandardFlowEncryptorTest#testProcessNoEncrypted", + "StandardFlowEncryptorTest#testProcessJson", + "StandardFlowEncryptorTest#testProcessXml", + "AesGcmSensitivePropertyProviderTest#testInvalidKeyLength", + "AesGcmSensitivePropertyProviderTest#testIsSupported", + "AesGcmSensitivePropertyProviderTest#testGetIdentifierKeyAesGcm128", + "AesGcmSensitivePropertyProviderTest#testGetIdentifierKeyAesGcm256", + "AesGcmSensitivePropertyProviderTest#testProtectUnprotectSuccess", + "AesGcmSensitivePropertyProviderTest#testProtectUnprotectDifferentKeyFailed", + "AesGcmSensitivePropertyProviderTest#testUnprotectMinLengthRequired", + "AzureKeyVaultSecretSensitivePropertyProviderTest#testClientNull", + "AzureKeyVaultSecretSensitivePropertyProviderTest#testProtect", + "AzureKeyVaultSecretSensitivePropertyProviderTest#testProtectException", + "AzureKeyVaultSecretSensitivePropertyProviderTest#testUnprotect", + "AzureKeyVaultSecretSensitivePropertyProviderTest#testUnprotectResourceNotFoundException", + "AzureKeyVaultSecretSensitivePropertyProviderTest#testGetIdentifierKey", + "WebUtilsTest#testDeterminedProxiedSchemeWhenNoHeaders", + "WebUtilsTest#testDeterminedProxiedSchemeWhenXProxySchemeAvailable", + "WebUtilsTest#testDeterminedProxiedSchemeWhenXForwardedProtoAvailable", + "WebUtilsTest#testDetermineProxiedHostWhenNoHeaders", + "WebUtilsTest#testDetermineProxiedHostWhenXProxyHostAvailable", + "WebUtilsTest#testDetermineProxiedHostWhenXProxyHostAvailableWithPort", + "WebUtilsTest#testDetermineProxiedHostWhenXForwardedHostAvailable", + "WebUtilsTest#testDetermineProxiedHostWhenXForwardedHostAvailableWithPort", + "WebUtilsTest#testDetermineProxiedPortWhenNoHeaders", + "WebUtilsTest#testDetermineProxiedPortWhenXProxyPortAvailable", + "WebUtilsTest#testDetermineProxiedPortWhenPortInXProxyHost", + "WebUtilsTest#testDetermineProxiedPortWhenXProxyPortOverridesXProxy", + "WebUtilsTest#testDetermineProxiedPortWhenXForwardedPortAvailable", + "WebUtilsTest#testDetermineProxiedPortWhenPortInXForwardedHost", + "WebUtilsTest#testDetermineProxiedPortWhenXForwardedPortOverridesXForwardedHost", + "DeprecationExceptionTest#testGetMessageReferenceClassFound", + "AwsKmsSensitivePropertyProviderTest#testValidateClientNull", + "AwsKmsSensitivePropertyProviderTest#testValidateKeyDisabled", + "AwsKmsSensitivePropertyProviderTest#testCleanUp", + "AwsKmsSensitivePropertyProviderTest#testProtect", + "AwsKmsSensitivePropertyProviderTest#testUnprotect", + "AwsKmsSensitivePropertyProviderTest#testGetIdentifierKey", + "StandardDeprecationLoggerTest#testWarn", + "StandardDeprecationLoggerTest#testWarnArguments", + "StandardTlsConfigurationTest#testIsKeyStoreValid", + "StandardTlsConfigurationTest#testIsTrustStoreValid", + "StandardTlsConfigurationTest#testEqualsNullProperties", + "StandardTlsConfigurationTest#testHashCodeNullProperties", + "StandardTlsConfigurationTest#testFromNiFiPropertiesEmptyProperties", + "StandardTlsConfigurationTest#testFromNiFiProperties", + "StandardTlsConfigurationTest#testFromNiFiPropertiesTrustStoreProperties", + "StandardTlsConfigurationTest#testFunctionalKeyPasswordFromKeyStorePassword", + "StandardTlsConfigurationTest#testIsKeyStorePopulated", + "StandardTlsConfigurationTest#testIsKeyStorePopulatedMissingProperties", + "StandardTlsConfigurationTest#testIsTrustStorePopulated", + "StandardTlsConfigurationTest#testIsTrustStorePopulatedMissingProperties", + "StandardTlsConfigurationTest#testGetEnabledProtocolsVersionMatched", + "StandardTlsConfigurationTest#testGetEnabledProtocolsTlsProtocol", + "StandardTlsConfigurationTest#testGetEnabledProtocolsSslProtocol", + "StringUtilsTest#testIsBlank", + "StringUtilsTest#testStartsWith", + "StringUtilsTest#testPadRight", + "StringUtilsTest#testPadLeft", + "StringUtilsTest#testIsEmpty", + "StringUtilsTest#testSubstringAfter", + "StringUtilsTest#testJoin", + "StringUtilsTest#testShouldTitleCaseStrings", + "SetSensitivePropertiesAlgorithmTest#testMainNoArguments", + "SetSensitivePropertiesAlgorithmTest#testMainPopulatedKeyAndAlgorithm", + "StaticKeyProviderTest#testGetKey", + "StaticKeyProviderTest#testKeyExists", + "StaticKeyProviderTest#testGetAvailableKeys", + "StaticKeyProviderTest#testGetKeyNotFoundManagementException", + "StandardHttpUriBuilderTest#testBuildIllegalStateException", + "StandardHttpUriBuilderTest#testBuildSchemeHost", + "StandardHttpUriBuilderTest#testBuildSchemeHostPort", + "StandardHttpUriBuilderTest#testBuildSchemeHostPortEncodedPath", + "StandardHttpUriBuilderTest#testBuildSchemeHostPathSegment", + "StandardHttpUriBuilderTest#testBuildSchemeHostPathSegmentQueryParameter", + "StandardHttpUriBuilderTest#testBuildSchemeHostPathSegmentQueryParameterNullValue", + "HashiCorpVaultKeyValueSensitivePropertyProviderTest#testGetIdentifierKeyPropertiesNotFound", + "HashiCorpVaultKeyValueSensitivePropertyProviderTest#testIsSupportedPropertiesNotFound", + "StandardServerConnectorFactoryTest#testGetServerConnector", + "StandardServerConnectorFactoryTest#testGetServerConnectorSecured", + "StandardServerConnectorFactoryTest#testGetServerConnectorSecuredNeedClientAuthentication", + "StandardServerConnectorFactoryTest#testGetServerConnectorSecuredHttp2AndHttp1", + "StandardServerConnectorFactoryTest#testGetServerConnectorSecuredHttp2", + "StandardSensitivePropertyProviderFactoryTest#testGetPropertyContextNotConfigured", + "StandardSensitivePropertyProviderFactoryTest#testGetPropertyContext", + "StandardSensitivePropertyProviderFactoryTest#testGetSupportedProviders", + "StandardSensitivePropertyProviderFactoryTest#testAzureKeyVaultSecret", + "StandardSensitivePropertyProviderFactoryTest#testHashiCorpVaultKeyVaultSupported", + "StandardSensitivePropertyProviderFactoryTest#testHashiCorpVaultTransitSupported", + "StandardSensitivePropertyProviderFactoryTest#testHashiCorpVaultTransitExceptionWhenMissingProperties", + "StandardSensitivePropertyProviderFactoryTest#testAesGcmWithoutKeySizeSupported", + "StandardSensitivePropertyProviderFactoryTest#testAesGcm128Supported", + "TlsPlatformTest#testGetSupportedProtocolsFound", + "TlsPlatformTest#testGetPreferredProtocolsFound", + "TlsPlatformTest#testGetPreferredProtocolsShouldNotIncludeLegacyProtocols", + "TlsPlatformTest#testGetLatestProtocolFound", + "TlsPlatformTest#testGetLatestProtocolMostRecentVersion", + "StandardSchemaValidatorTest#testValidate", + "StandardSchemaValidatorTest#testValidateExternalEntityException", + "SSLSocketChannelTest#testClientConnectFailed", + "SSLSocketChannelTest#testClientConnectHandshakeFailed", + "SSLSocketChannelTest#testClientConnectWriteReadTls12", + "SSLSocketChannelTest#testClientConnectWriteReadTls13", + "SSLSocketChannelTest#testClientConnectWriteAvailableReadTls12", + "SSLSocketChannelTest#testClientConnectWriteAvailableReadTls13", + "ObjectTimestampFieldConverterTest#testConvertFieldNull", + "ObjectTimestampFieldConverterTest#testConvertFieldTimestamp", + "ObjectTimestampFieldConverterTest#testConvertFieldDate", + "ObjectTimestampFieldConverterTest#testConvertFieldLong", + "ObjectTimestampFieldConverterTest#testConvertFieldStringEmpty", + "ObjectTimestampFieldConverterTest#testConvertFieldStringFormatNull", + "ObjectTimestampFieldConverterTest#testConvertFieldStringFormatNullNumberFormatException", + "ObjectTimestampFieldConverterTest#testConvertFieldStringFormatDefault", + "ObjectTimestampFieldConverterTest#testConvertFieldStringFormatCustomNanoseconds", + "ObjectTimestampFieldConverterTest#testConvertFieldStringFormatCustomFormatterException", + "StandardXMLStreamReaderProviderTest#testGetStreamReaderStandard", + "StandardXMLStreamReaderProviderTest#testGetStreamReaderStandardDocumentTypeDeclaration", + "StandardXMLStreamReaderProviderTest#testGetStreamReaderStandardExternalEntityException", + "StandardFileBasedKeyReaderTest#testReadSecretKeys", + "StandardDocumentProviderTest#testNewDocument", + "StandardDocumentProviderTest#testParseStandard", + "StandardDocumentProviderTest#testParseDocumentTypeDeclarationException", + "StandardDocumentProviderTest#testParseExternalEntityException", + "StandardDocumentProviderTest#testParseNamespaceAwareSchemaConfiguredExternalEntityException", + "AzureCryptographyClientProviderTest#testClientPropertiesNull", + "AzureCryptographyClientProviderTest#testClientPropertiesKeyIdBlank", + "SetSensitivePropertiesKeyTest#testMainNoArguments", + "SetSensitivePropertiesKeyTest#testMainBlankKeyAndAlgorithm", + "DeprecationLoggerFactoryTest#testGetLogger", + "DeprecationLoggerFactoryTest#testGetLoggerClassRequired", + "SetSingleUserCredentialsTest#testMainNoArguments", + "SetSingleUserCredentialsTest#testMainUsernamePasswordUpdated", + "TextLineDemarcatorTest#nullStream", + "TextLineDemarcatorTest#illegalBufferSize", + "TextLineDemarcatorTest#emptyStreamNoStartWithFilter", + "TextLineDemarcatorTest#emptyStreamAndStartWithFilter", + "TextLineDemarcatorTest#endsWithCRWithBufferLengthEqualStringLengthA", + "TextLineDemarcatorTest#endsWithCRWithBufferLengthEqualStringLengthB", + "TextLineDemarcatorTest#singleCR", + "TextLineDemarcatorTest#singleLF", + "TextLineDemarcatorTest#validateNiFi_3495", + "TextLineDemarcatorTest#mixedCRLF", + "TextLineDemarcatorTest#consecutiveAndMixedCRLF", + "TextLineDemarcatorTest#startWithNoMatchOnWholeStream", + "TextLineDemarcatorTest#startWithSomeMatches", + "TextLineDemarcatorTest#testOnBufferSplitNoTrailingDelimiter", + "TextLineDemarcatorTest#validateStartsWithLongerThanLastToken", + "FileBasedKeyProviderTest#testGetKey", + "StandardPropertySecretKeyProviderTest#testPropertyEncryptionMethodSupported", + "StandardPropertySecretKeyProviderTest#testPasswordLengthLessThanRequired", + "FlowEncryptorCommandTest#testRunSystemPropertyNotDefined", + "FlowEncryptorCommandTest#testRunPropertiesKeyBlankProperties", + "FlowEncryptorCommandTest#testRunPropertiesKeyLegacyPropertiesWithoutFlowJson", + "FlowEncryptorCommandTest#testRunPropertiesAlgorithmWithPropertiesKeyPopulatedProperties", + "KeyProviderFactoryTest#testGetUnsupportedKeyProvider", + "KeyProviderFactoryTest#testGetStaticKeyProvider", + "KeyProviderFactoryTest#testGetFileBasedKeyProvider", + "KeyProviderFactoryTest#testGetKeyStoreKeyProvider", + "AesCtrByteArrayRepositoryEncryptorTest#testEncryptDecrypt", + "StandardInputSourceParserTest#testParseStandard", + "StandardInputSourceParserTest#testParseDocumentTypeDeclarationException", + "StandardInputSourceParserTest#testParseExternalEntityException", + "StandardInputSourceParserTest#testParseNamespaceAwareExternalEntityException", + "StreamDemarcatorTest#validateInitializationFailure", + "StreamDemarcatorTest#validateLargeBufferSmallMaxSize", + "StreamDemarcatorTest#vaidateOnPartialMatchThenSubsequentPartialMatch", + "StreamDemarcatorTest#validateNoDelimiter", + "StreamDemarcatorTest#validateNoDelimiterSmallInitialBuffer", + "StreamDemarcatorTest#validateSingleByteDelimiter", + "StreamDemarcatorTest#validateDelimiterAtTheBeginning", + "StreamDemarcatorTest#validateEmptyDelimiterSegments", + "StreamDemarcatorTest#validateSingleByteDelimiterSmallInitialBuffer", + "StreamDemarcatorTest#validateWithMultiByteDelimiter", + "StreamDemarcatorTest#validateWithMultiByteDelimiterAtTheBeginning", + "StreamDemarcatorTest#validateWithMultiByteDelimiterSmallInitialBuffer", + "StreamDemarcatorTest#validateWithMultiByteCharsNoDelimiter", + "StreamDemarcatorTest#validateWithMultiByteCharsNoDelimiterSmallInitialBuffer", + "StreamDemarcatorTest#validateWithComplexDelimiter", + "StreamDemarcatorTest#validateMaxBufferSize", + "StreamDemarcatorTest#validateScannerHandlesNegativeOneByteInputsNoDelimiter", + "StreamDemarcatorTest#validateScannerHandlesNegativeOneByteInputs", + "StreamDemarcatorTest#verifyScannerHandlesNegativeOneByteDelimiter", + "StreamDemarcatorTest#testWithoutTrailingDelimiter", + "StreamDemarcatorTest#testOnBufferSplitNoTrailingDelimiter", + "StreamDemarcatorTest#testOnBufferCorners", + "StreamDemarcatorTest#testOnBufferSplit", + "KeyStoreKeyProviderTest#testGetKey", + "KeyStoreKeyProviderTest#testKeyExists", + "KeyStoreKeyProviderTest#testGetAvailableKeys", + "KeyStoreKeyProviderTest#testGetKeyNotFoundManagementException", + "PasswordBasedCipherPropertyEncryptorTest#testEncryptDecrypt", + "PasswordBasedCipherPropertyEncryptorTest#testEncryptHexadecimalEncoded", + "PasswordBasedCipherPropertyEncryptorTest#testDecryptEncryptionException", + "PasswordBasedCipherPropertyEncryptorTest#testEqualsHashCode", + "PasswordBasedCipherPropertyEncryptorTest#testEqualsHashCodeDifferentPassword", + "StandardTransformProviderTest#testTransformStandard", + "StandardTransformProviderTest#testTransformStandardDocumentTypeDeclaration", + "StandardTransformProviderTest#testTransformStandardExternalEntityException", + "AzureSecretClientProviderTest#testClientPropertiesNull", + "AzureSecretClientProviderTest#testClientPropertiesUriBlank", + "RequestUriBuilderTest#testFromHttpServletRequestBuild", + "RequestUriBuilderTest#testFromHttpServletRequestPathBuild", + "RequestUriBuilderTest#testFromHttpServletRequestProxyHeadersBuild", + "RecordMetadataObjectInputStreamTest#testGetRecordMetadata", + "RecordMetadataObjectInputStreamTest#testGetRecordMetadataCompatibleClass", + "StringSelectorTest#testNull", + "StringSelectorTest#testEmpty", + "StringSelectorTest#testNull_Empty", + "StringSelectorTest#testNull_Empty_NonEmpty", + "StringSelectorTest#testNonEmpty_Null_NonEmpty", + "StringSelectorTest#testNonEmpty_Empty_NonEmpty", + "StringSelectorTest#testTwoNonEmpty", + "StringSelectorTest#testManyInputsWithNoExpected", + "StringSelectorTest#testManyInputsWithExpectedInFirstBatch", + "StringSelectorTest#testManyInputsWithExpectedInLaterBatch", + "SslSocketFactoryTest#testCreateSslContextNullTlsConfiguration", + "SslSocketFactoryTest#testCreateSslContextEmptyTlsConfiguration", + "SslSocketFactoryTest#testCreateSslContextEmptyKeyPassword", + "SslSocketFactoryTest#testCreateSslContextEmptyTrustStorePasswordJks", + "SslSocketFactoryTest#testCreateSslContext", + "SslSocketFactoryTest#testGetTrustManager", + "SslSocketFactoryTest#testGetTrustManagers", + "SslSocketFactoryTest#testGetTrustManagersEmptyTlsConfiguration", + "StandardRepositoryKeyProviderFactoryTest#testGetKeyProviderPropertyNotConfiguredException", + "StandardRepositoryKeyProviderFactoryTest#testGetKeyProviderNotFound", + "StandardRepositoryKeyProviderFactoryTest#testGetKeyProviderContentKeyStoreKeyProvider", + "StandardRepositoryKeyProviderFactoryTest#testGetKeyProviderContentStaticKeyProvider", + "StandardRepositoryKeyProviderFactoryTest#testGetKeyProviderFlowFileStaticKeyProvider", + "StandardRepositoryKeyProviderFactoryTest#testGetKeyProviderProvenanceStaticKeyProvider", + "StandardWebClientServiceTest#testSetTlsContext", + "StandardWebClientServiceTest#testSetTlsContextProtocolNotSupported", + "StandardWebClientServiceTest#testSocketTimeoutException", + "StandardWebClientServiceTest#testProxyAuthorization", + "StandardWebClientServiceTest#testRedirectHandlingFollowed", + "StandardWebClientServiceTest#testRedirectHandlingIgnored", + "StandardWebClientServiceTest#testDelete", + "StandardWebClientServiceTest#testDeleteMethodNotAllowed", + "StandardWebClientServiceTest#testGet", + "StandardWebClientServiceTest#testGetNotFound", + "StandardWebClientServiceTest#testGetInternalServerError", + "StandardWebClientServiceTest#testGetServiceUnavailable", + "StandardWebClientServiceTest#testPatch", + "StandardWebClientServiceTest#testPatchBadRequest", + "StandardWebClientServiceTest#testPost", + "StandardWebClientServiceTest#testPostUnauthorized", + "StandardWebClientServiceTest#testPut", + "StandardWebClientServiceTest#testPutForbidden", + "AwsSecretsManagerSensitivePropertyProviderTest#testValidateClientNull", + "AwsSecretsManagerSensitivePropertyProviderTest#testValidateKeyNoSecretString", + "AwsSecretsManagerSensitivePropertyProviderTest#testCleanUp", + "AwsSecretsManagerSensitivePropertyProviderTest#testProtectCreateSecret", + "AwsSecretsManagerSensitivePropertyProviderTest#testProtectExistingSecret", + "AwsSecretsManagerSensitivePropertyProviderTest#testUnprotect", + "AwsSecretsManagerSensitivePropertyProviderTest#testGetIdentifierKey", + "LimitingInputStreamTest#testReadLimitNotReached", + "LimitingInputStreamTest#testReadLimitMatched", + "LimitingInputStreamTest#testReadLimitExceeded", + "OSUtilsTest#testGetPid", + "OSUtilsTest#testParseJavaVersion8", + "OSUtilsTest#testParseJavaVersion11", + "PropertyEncryptorFactoryTest#testGetPropertyEncryptorUnsupportedEncryptionMethod", + "PropertyEncryptorFactoryTest#testGetPropertyEncryptorPropertiesBlankPassword", + "PropertyEncryptorFactoryTest#testGetPropertyEncryptorPropertiesKeyedCipherPropertyEncryptor", + "HashiCorpVaultTransitSensitivePropertyProviderTest#testGetIdentifierKeyPropertiesNotFound", + "HashiCorpVaultTransitSensitivePropertyProviderTest#testIsSupportedPropertiesNotFound", + "SecureHasherFactoryTest#testSecureHasherFactoryArgon2", + "SecureHasherFactoryTest#testSecureHasherFactoryPBKDF2", + "SecureHasherFactoryTest#testSecureHasherFactoryBCrypt", + "SecureHasherFactoryTest#testSecureHasherFactorySCrypt", + "SecureHasherFactoryTest#testSecureHasherFactoryArgon2ShortName", + "SecureHasherFactoryTest#testSecureHasherFactorySCryptShortName", + "SecureHasherFactoryTest#testSecureHasherFactoryLowerCaseName", + "SecureHasherFactoryTest#testSecureHasherFactoryArgon2SimilarName", + "SecureHasherFactoryTest#testSecureHasherFactoryFailsUnknownAlgorithmName", + "SecureHasherFactoryTest#testSecureHasherFactoryDefaultsToArgon2IfLongUnknownAlgorithmName", + "SecureHasherFactoryTest#testSecureHasherFactoryEmptyString", + "AesGcmByteArrayRepositoryEncryptorTest#testEncryptDecryptContentRecord", + "AesGcmByteArrayRepositoryEncryptorTest#testEncryptDecryptProvenanceRecord", + "AesGcmByteArrayRepositoryEncryptorTest#testDecryptEmptyByteArrayFailed", + "StandardLoginCredentialsWriterTest#testWriteLoginCredentialsProvidersNotFound", + "StandardLoginCredentialsWriterTest#testWriteLoginCredentialsBlankProviders", + "StandardLoginCredentialsWriterTest#testWriteLoginCredentialsPopulatedProviders", + "AzureKeyVaultKeySensitivePropertyProviderTest#testValidateClientNull", + "AzureKeyVaultKeySensitivePropertyProviderTest#testProtect", + "AzureKeyVaultKeySensitivePropertyProviderTest#testUnprotect", + "AzureKeyVaultKeySensitivePropertyProviderTest#testGetIdentifierKey", + "KeyStoreUtilsTest#testCreateTlsConfigAndNewKeystoreTruststore", + "KeyStoreUtilsTest#testKeystoreTypesPrivateKeyEntry", + "KeyStoreUtilsTest#testKeystoreTypesCertificateEntry", + "KeyStoreUtilsTest#testKeystoreTypesSecretKeyEntry", + "BCryptPasswordEncoderTest#testEncode", + "BCryptPasswordEncoderTest#testEncodeMatches", + "ByteCountingInputStreamTest#testReset", + "ByteCountingInputStreamTest#testAvailableShouldReturnCorrectCount", + "StandardXMLEventReaderProviderTest#testGetEventReaderStandard", + "StandardXMLEventReaderProviderTest#testGetEventReaderStandardDocumentTypeDeclaration", + "StandardXMLEventReaderProviderTest#testGetEventReaderStandardExternalEntityException", + "StandardProtectionSchemeResolverTest#getProtectionSchemeAesGcmFound", + "StandardProtectionSchemeResolverTest#getProtectionSchemeAesGcm256Found", + "StandardProtectionSchemeResolverTest#getProtectionSchemeUnknownNotFound", + "ResultSetRecordSetTest#testCreateSchema", + "ResultSetRecordSetTest#testCreateSchemaWhenNoRecordSchema", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherType", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherTypeUsingLogicalTypes", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherTypeAndNoLogicalTypes", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherTypeUsingLogicalTypesNoSchema", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherTypeAndNoLogicalTypesNoSchema", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherTypeUsingLogicalTypesWithRecord", + "ResultSetRecordSetTest#testCreateSchemaWhenOtherTypeWithoutSchema", + "ResultSetRecordSetTest#testCreateRecord", + "ResultSetRecordSetTest#testCreateSchemaArrayThrowsException", + "ResultSetRecordSetTest#testCreateSchemaThrowsExceptionSchemaCreationStillCalledConsideringLogicalTypeFlag", + "ResultSetRecordSetTest#testCreateSchemaArrayThrowsNotSupportedException", + "ResultSetRecordSetTest#testArrayTypeWithLogicalTypes", + "ResultSetRecordSetTest#testArrayTypeNoLogicalTypes", + "ResultSetRecordSetTest#testCreateSchemaWithLogicalTypes", + "ResultSetRecordSetTest#testCreateSchemaNoLogicalTypes", + "ResultSetRecordSetTest#testCreateSchemaWithLogicalTypesNoInputSchema", + "ResultSetRecordSetTest#testCreateSchemaNoLogicalTypesNoInputSchema", + "KeyedCipherPropertyEncryptorTest#testEncryptDecrypt", + "KeyedCipherPropertyEncryptorTest#testEncryptHexadecimalEncoded", + "KeyedCipherPropertyEncryptorTest#testDecryptEncryptionException", + "KeyedCipherPropertyEncryptorTest#testGetCipherEncryptionException", + "KeyedCipherPropertyEncryptorTest#testEqualsHashCode", + "KeyedCipherPropertyEncryptorTest#testEqualsHashCodeDifferentSecretKey" +] \ No newline at end of file diff --git a/core/identify_param/runner.py b/core/identify_param/runner.py index 2ce7fd99..a8bca15b 100644 --- a/core/identify_param/runner.py +++ b/core/identify_param/runner.py @@ -166,8 +166,9 @@ def run_individual_testmethod(self): if self.module == "alluxio-core": cmd = ["mvn", "surefire:test", "-Dtest=" + method, "-DfailIfNoTests=false"] else: - cmd = ["mvn", "surefire:test", "-Dtest=" + method] + cmd = ["mvn" ,"-pl", "nifi-commons/", "surefire:test", "-Dtest=" + method, "-DfailIfNoTests=false" , "-Dsurefire.failIfNoSpecifiedTests=false"] print ("mvn surefire:test -Dtest="+method) + child = subprocess.Popen(cmd, stdout=method_out, stderr=method_out) child.wait() diff --git a/core/patch/nifi-commons/injection.patch b/core/patch/nifi-commons/injection.patch new file mode 100644 index 00000000..d5062257 --- /dev/null +++ b/core/patch/nifi-commons/injection.patch @@ -0,0 +1,46 @@ +From bcfc0afd70f82ee9ea4ef67850284db657ae5712 Mon Sep 17 00:00:00 2001 +From: yinmeng2 +Date: Sat, 10 Dec 2022 02:35:13 -0600 +Subject: [PATCH] injection for NiFi Ctest -> Newest version + +--- + .../src/main/java/org/apache/nifi/util/NiFiProperties.java | 6 ++++++ + .../src/test/resources/NiFiProperties/conf/ctest.properties | 1 + + 2 files changed, 7 insertions(+) + create mode 100644 nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties + +diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +index 73c3ae20c0..078aa750ad 100644 +--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java ++++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +@@ -52,6 +52,9 @@ import java.util.stream.Stream; + public class NiFiProperties extends ApplicationProperties { + private static final Logger logger = LoggerFactory.getLogger(NiFiProperties.class); + ++ // ctest injection ++ public static final String CTEST_PROPERTIES_FILE_PATH = "src/test/resources/NiFiProperties/conf/ctest.properties"; ++ + // core properties + public static final String PROPERTIES_FILE_PATH = "nifi.properties.file.path"; + public static final String FLOW_CONFIGURATION_FILE = "nifi.flow.configuration.file"; +@@ -2142,6 +2145,9 @@ public class NiFiProperties extends ApplicationProperties { + readFromPropertiesFile(propertiesFilePath, properties); + } + ++ // ctest ++ readFromPropertiesFile(CTEST_PROPERTIES_FILE_PATH, properties); ++ + // The Properties(Properties) constructor does NOT inherit the provided values, just uses them as default values + if (additionalProperties != null) { + additionalProperties.forEach(properties::put); +diff --git a/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties +new file mode 100644 +index 0000000000..1aa10e1dda +--- /dev/null ++++ b/nifi-commons/nifi-properties/src/test/resources/NiFiProperties/conf/ctest.properties +@@ -0,0 +1 @@ ++nifi.flow.configuration.file=./target/flow.xml.gz +\ No newline at end of file +-- +2.25.1 + diff --git a/core/patch/nifi-commons/nifi-logging-patches/0001-finish-get.patch b/core/patch/nifi-commons/nifi-logging-patches/0001-finish-get.patch new file mode 100644 index 00000000..af79674f --- /dev/null +++ b/core/patch/nifi-commons/nifi-logging-patches/0001-finish-get.patch @@ -0,0 +1,667 @@ +From ec082191d7e4e7678f2120d0d70859eac35eef29 Mon Sep 17 00:00:00 2001 +From: xinyihe4 +Date: Sat, 10 Dec 2022 02:16:44 -0600 +Subject: [PATCH 1/2] finish get + +--- + .../org/apache/nifi/util/NiFiProperties.java | 116 +++++++++++++++++- + 1 file changed, 110 insertions(+), 6 deletions(-) + +diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +index 73c3ae20c0..2d271ef3bc 100644 +--- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java ++++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +@@ -450,6 +450,7 @@ public class NiFiProperties extends ApplicationProperties { + // getters for core properties // + public File getFlowConfigurationFile() { + try { ++ System.out.println("[CTEST][GET-PARAM] " + FLOW_CONFIGURATION_FILE); //add ctest + return new File(getProperty(FLOW_CONFIGURATION_FILE)); + } catch (Exception ex) { + return null; +@@ -458,6 +459,7 @@ public class NiFiProperties extends ApplicationProperties { + + public File getFlowConfigurationJsonFile() { + final String jsonFilename = getProperty(FLOW_CONFIGURATION_JSON_FILE); ++ System.out.println("[CTEST][GET-PARAM] " + FLOW_CONFIGURATION_JSON_FILE); //add ctest + if (jsonFilename != null) { + return new File(jsonFilename); + } +@@ -481,6 +483,7 @@ public class NiFiProperties extends ApplicationProperties { + + private Integer getPropertyAsPort(final String propertyName, final Integer defaultValue) { + final String port = getProperty(propertyName); ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); //add ctest + if (StringUtils.isEmpty(port)) { + return defaultValue; + } +@@ -497,6 +500,7 @@ public class NiFiProperties extends ApplicationProperties { + + public int getQueueSwapThreshold() { + final String thresholdValue = getProperty(QUEUE_SWAP_THRESHOLD); ++ System.out.println("[CTEST][GET-PARAM] " + QUEUE_SWAP_THRESHOLD); + if (thresholdValue == null) { + return DEFAULT_QUEUE_SWAP_THRESHOLD; + } +@@ -510,6 +514,7 @@ public class NiFiProperties extends ApplicationProperties { + + public Integer getIntegerProperty(final String propertyName, final Integer defaultValue) { + final String value = getProperty(propertyName); ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); + if (value == null || value.trim().isEmpty()) { + return defaultValue; + } +@@ -534,6 +539,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public String getRemoteInputHost() { + final String value = getProperty(REMOTE_INPUT_HOST); ++ System.out.println("[CTEST][GET-PARAM] " + REMOTE_INPUT_HOST); //add ctest + return StringUtils.isBlank(value) ? null : value; + } + +@@ -607,6 +613,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public Path getTemplateDirectory() { + final String strVal = getProperty(TEMPLATE_DIRECTORY); ++ System.out.println("[CTEST][GET-PARAM] " + TEMPLATE_DIRECTORY); + return (strVal == null) ? DEFAULT_TEMPLATE_DIRECTORY : Paths.get(strVal); + } + +@@ -616,6 +623,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return The write delay + */ + public String getFlowServiceWriteDelay() { ++ System.out.println("[CTEST][GET-PARAM] " + WRITE_DELAY_INTERVAL); + return getProperty(WRITE_DELAY_INTERVAL); + } + +@@ -647,6 +655,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public File getRestoreDirectory() { + final String value = getProperty(RESTORE_DIRECTORY); ++ System.out.println("[CTEST][GET-PARAM] " + RESTORE_DIRECTORY); + if (StringUtils.isBlank(value)) { + return null; + } else { +@@ -659,6 +668,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public File getAuthorizerConfigurationFile() { + final String value = getProperty(AUTHORIZER_CONFIGURATION_FILE); ++ System.out.println("[CTEST][GET-PARAM] " + AUTHORIZER_CONFIGURATION_FILE); + if (StringUtils.isBlank(value)) { + return new File(DEFAULT_AUTHORIZER_CONFIGURATION_FILE); + } else { +@@ -671,6 +681,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public File getLoginIdentityProviderConfigurationFile() { + final String value = getProperty(LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE); ++ System.out.println("[CTEST][GET-PARAM] " + LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE); + if (StringUtils.isBlank(value)) { + return new File(DEFAULT_LOGIN_IDENTITY_PROVIDER_CONFIGURATION_FILE); + } else { +@@ -681,6 +692,7 @@ public class NiFiProperties extends ApplicationProperties { + // getters for web properties // + public Integer getPort() { + Integer port = null; ++ System.out.println("[CTEST][GET-PARAM] " + WEB_HTTP_PORT); + try { + port = Integer.parseInt(getProperty(WEB_HTTP_PORT)); + } catch (NumberFormatException nfe) { +@@ -690,6 +702,7 @@ public class NiFiProperties extends ApplicationProperties { + + public Integer getSslPort() { + Integer sslPort = null; ++ System.out.println("[CTEST][GET-PARAM] " + WEB_HTTPS_PORT); + try { + sslPort = Integer.parseInt(getProperty(WEB_HTTPS_PORT)); + } catch (NumberFormatException nfe) { +@@ -747,6 +760,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the specified max content-length and units for an incoming HTTP request + */ + public String getWebMaxContentSize() { ++ System.out.println("[CTEST][GET-PARAM] " + WEB_MAX_CONTENT_SIZE); //add ctest + return getProperty(WEB_MAX_CONTENT_SIZE); + } + +@@ -763,6 +777,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getWebRequestIpWhitelist() { ++ System.out.println("[CTEST][GET-PARAM] " + WEB_REQUEST_IP_WHITELIST); //add ctest + return getProperty(WEB_REQUEST_IP_WHITELIST); + } + +@@ -788,6 +803,7 @@ public class NiFiProperties extends ApplicationProperties { + + public boolean isUnpackNarsToUberJar() { + final String propertyValue = getProperty(UNPACK_NARS_TO_UBER_JAR); ++ System.out.println("[CTEST][GET-PARAM] " + UNPACK_NARS_TO_UBER_JAR); //add ctest + if (propertyValue == null) { + return DEFAULT_UNPACK_NARS_TO_UBER_JAR; + } +@@ -814,6 +830,7 @@ public class NiFiProperties extends ApplicationProperties { + || NAR_LIBRARY_AUTOLOAD_DIRECTORY.equals(propertyName)) { + // attempt to resolve the path specified + String narLib = getProperty(propertyName); ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); //add ctest + if (!StringUtils.isBlank(narLib)) { + narLibraryPaths.add(Paths.get(narLib)); + } +@@ -848,6 +865,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the interval over which the properties should auto refresh + */ + public String getAutoRefreshInterval() { ++ System.out.println("[CTEST][GET-PARAM] " + UI_AUTO_REFRESH_INTERVAL); //add ctest + return getProperty(UI_AUTO_REFRESH_INTERVAL); + } + +@@ -894,12 +912,14 @@ public class NiFiProperties extends ApplicationProperties { + + // getters for cluster node properties // + public boolean isNode() { ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_IS_NODE); //add ctest + return Boolean.parseBoolean(getProperty(CLUSTER_IS_NODE)); + } + + public InetSocketAddress getClusterNodeProtocolAddress() { + try { + String socketAddress = getProperty(CLUSTER_NODE_ADDRESS); ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_NODE_ADDRESS); //add ctest + if (StringUtils.isBlank(socketAddress)) { + socketAddress = "localhost"; + } +@@ -915,6 +935,7 @@ public class NiFiProperties extends ApplicationProperties { + + public InetSocketAddress getClusterLoadBalanceAddress() { + try { ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_NODE_ADDRESS); //add ctest + String host = getProperty(LOAD_BALANCE_HOST); + if (StringUtils.isBlank(host)) { + host = getProperty(CLUSTER_NODE_ADDRESS); +@@ -932,6 +953,7 @@ public class NiFiProperties extends ApplicationProperties { + + public Integer getClusterNodeProtocolPort() { + try { ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_NODE_PROTOCOL_PORT); //add ctest + return Integer.parseInt(getProperty(CLUSTER_NODE_PROTOCOL_PORT)); + } catch (NumberFormatException nfe) { + return null; +@@ -948,6 +970,7 @@ public class NiFiProperties extends ApplicationProperties { + + public int getClusterNodeProtocolMaxPoolSize() { + try { ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_NODE_PROTOCOL_MAX_THREADS); //add ctest + return Integer.parseInt(getProperty(CLUSTER_NODE_PROTOCOL_MAX_THREADS)); + } catch (NumberFormatException nfe) { + return DEFAULT_CLUSTER_NODE_PROTOCOL_MAX_THREADS; +@@ -955,10 +978,12 @@ public class NiFiProperties extends ApplicationProperties { + } + + public boolean isClustered() { ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_IS_NODE); //add ctest + return Boolean.parseBoolean(getProperty(CLUSTER_IS_NODE)); + } + + public File getClusterNodeFirewallFile() { ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_FIREWALL_FILE); //add ctest + final String firewallFile = getProperty(CLUSTER_FIREWALL_FILE); + if (StringUtils.isBlank(firewallFile)) { + return null; +@@ -969,6 +994,7 @@ public class NiFiProperties extends ApplicationProperties { + + public String getClusterProtocolManagerToNodeApiScheme() { + final String isSecureProperty = getProperty(CLUSTER_PROTOCOL_IS_SECURE); ++ System.out.println("[CTEST][GET-PARAM] " + CLUSTER_PROTOCOL_IS_SECURE); //add ctest + if (Boolean.valueOf(isSecureProperty)) { + return "https"; + } else { +@@ -978,6 +1004,7 @@ public class NiFiProperties extends ApplicationProperties { + + public File getKerberosConfigurationFile() { + final String krb5File = getProperty(KERBEROS_KRB5_FILE); ++ System.out.println("[CTEST][GET-PARAM] " + KERBEROS_KRB5_FILE); + if (krb5File != null && krb5File.trim().length() > 0) { + return new File(krb5File.trim()); + } else { +@@ -987,6 +1014,7 @@ public class NiFiProperties extends ApplicationProperties { + + public String getKerberosServicePrincipal() { + final String servicePrincipal = getProperty(KERBEROS_SERVICE_PRINCIPAL); ++ System.out.println("[CTEST][GET-PARAM] " + KERBEROS_SERVICE_PRINCIPAL); + if (!StringUtils.isBlank(servicePrincipal)) { + return servicePrincipal.trim(); + } else { +@@ -996,6 +1024,7 @@ public class NiFiProperties extends ApplicationProperties { + + public String getKerberosServiceKeytabLocation() { + final String keytabLocation = getProperty(KERBEROS_SERVICE_KEYTAB_LOCATION); ++ System.out.println("[CTEST][GET-PARAM] " + KERBEROS_SERVICE_KEYTAB_LOCATION); + if (!StringUtils.isBlank(keytabLocation)) { + return keytabLocation.trim(); + } else { +@@ -1005,6 +1034,7 @@ public class NiFiProperties extends ApplicationProperties { + + public String getKerberosSpnegoPrincipal() { + final String spengoPrincipal = getProperty(KERBEROS_SPNEGO_PRINCIPAL); ++ System.out.println("[CTEST][GET-PARAM] " + KERBEROS_SPNEGO_PRINCIPAL); + if (!StringUtils.isBlank(spengoPrincipal)) { + return spengoPrincipal.trim(); + } else { +@@ -1014,6 +1044,7 @@ public class NiFiProperties extends ApplicationProperties { + + public String getKerberosSpnegoKeytabLocation() { + final String keytabLocation = getProperty(KERBEROS_SPNEGO_KEYTAB_LOCATION); ++ System.out.println("[CTEST][GET-PARAM] " + KERBEROS_SPNEGO_KEYTAB_LOCATION); + if (!StringUtils.isBlank(keytabLocation)) { + return keytabLocation.trim(); + } else { +@@ -1046,6 +1077,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return true if the login identity provider has been configured + */ + public boolean isLoginIdentityProviderEnabled() { ++ System.out.println("[CTEST][GET-PARAM] " + NiFiProperties.SECURITY_USER_LOGIN_IDENTITY_PROVIDER); + return !StringUtils.isBlank(getProperty(NiFiProperties.SECURITY_USER_LOGIN_IDENTITY_PROVIDER)); + } + +@@ -1073,6 +1105,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return OIDC discovery url + */ + public String getOidcDiscoveryUrl() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_OIDC_DISCOVERY_URL); + return getProperty(SECURITY_USER_OIDC_DISCOVERY_URL); + } + +@@ -1100,6 +1133,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return OIDC client id + */ + public String getOidcClientId() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_OIDC_CLIENT_ID); + return getProperty(SECURITY_USER_OIDC_CLIENT_ID); + } + +@@ -1109,6 +1143,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return OIDC client secret + */ + public String getOidcClientSecret() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_OIDC_CLIENT_SECRET); + return getProperty(SECURITY_USER_OIDC_CLIENT_SECRET); + } + +@@ -1118,6 +1153,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return OIDC preferred json web signature algorithm + */ + public String getOidcPreferredJwsAlgorithm() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM); + return getProperty(SECURITY_USER_OIDC_PREFERRED_JWSALGORITHM); + } + +@@ -1183,10 +1219,12 @@ public class NiFiProperties extends ApplicationProperties { + // Automatic diagnostic getters + + public boolean isDiagnosticsOnShutdownEnabled() { ++ System.out.println("[CTEST][GET-PARAM] " + DIAGNOSTICS_ON_SHUTDOWN_ENABLED); + return Boolean.parseBoolean(getProperty(DIAGNOSTICS_ON_SHUTDOWN_ENABLED)); + } + + public boolean isDiagnosticsOnShutdownVerbose() { ++ System.out.println("[CTEST][GET-PARAM] " + DIAGNOSTICS_ON_SHUTDOWN_VERBOSE); + return Boolean.parseBoolean(getProperty(DIAGNOSTICS_ON_SHUTDOWN_VERBOSE)); + } + +@@ -1196,6 +1234,7 @@ public class NiFiProperties extends ApplicationProperties { + + public int getDiagnosticsOnShutdownMaxFileCount() { + try { ++ System.out.println("[CTEST][GET-PARAM] " + DIAGNOSTICS_ON_SHUTDOWN_MAX_FILE_COUNT); + return Integer.parseInt(getProperty(DIAGNOSTICS_ON_SHUTDOWN_MAX_FILE_COUNT)); + } catch (NumberFormatException e) { + return DEFAULT_DIAGNOSTICS_ON_SHUTDOWN_MAX_FILE_COUNT; +@@ -1221,6 +1260,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return Knox URL + */ + public String getKnoxUrl() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_KNOX_URL); + return getProperty(SECURITY_USER_KNOX_URL); + } + +@@ -1230,6 +1270,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return Knox audiences + */ + public Set getKnoxAudiences() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_KNOX_AUDIENCES); + final String rawAudiences = getProperty(SECURITY_USER_KNOX_AUDIENCES); + if (StringUtils.isBlank(rawAudiences)) { + return null; +@@ -1245,6 +1286,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return path to the Knox public key + */ + public Path getKnoxPublicKeyPath() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_KNOX_PUBLIC_KEY); + return Paths.get(getProperty(SECURITY_USER_KNOX_PUBLIC_KEY)); + } + +@@ -1254,6 +1296,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return name of the Knox cookie + */ + public String getKnoxCookieName() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_KNOX_COOKIE_NAME); + return getProperty(SECURITY_USER_KNOX_COOKIE_NAME); + } + +@@ -1273,6 +1316,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the url to obtain the identity provider metadata + */ + public String getSamlIdentityProviderMetadataUrl() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_SAML_IDP_METADATA_URL); + return getProperty(SECURITY_USER_SAML_IDP_METADATA_URL); + } + +@@ -1282,6 +1326,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the service provider entity id + */ + public String getSamlServiceProviderEntityId() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_SAML_SP_ENTITY_ID); + return getProperty(SECURITY_USER_SAML_SP_ENTITY_ID); + } + +@@ -1293,6 +1338,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the attribute name containing the user identity + */ + public String getSamlIdentityAttributeName() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_SAML_IDENTITY_ATTRIBUTE_NAME); + return getProperty(SECURITY_USER_SAML_IDENTITY_ATTRIBUTE_NAME); + } + +@@ -1302,6 +1348,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the attribute name containing user groups + */ + public String getSamlGroupAttributeName() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_USER_SAML_GROUP_ATTRIBUTE_NAME); + return getProperty(SECURITY_USER_SAML_GROUP_ATTRIBUTE_NAME); + } + +@@ -1433,6 +1480,8 @@ public class NiFiProperties extends ApplicationProperties { + + final String host; + final Integer port; ++ System.out.println("[CTEST][GET-PARAM] " + WEB_HTTP_HOST); ++ System.out.println("[CTEST][GET-PARAM] " + WEB_HTTPS_HOST); + if ("http".equalsIgnoreCase(scheme)) { + // get host + if (StringUtils.isBlank(getProperty(WEB_HTTP_HOST))) { +@@ -1473,6 +1522,7 @@ public class NiFiProperties extends ApplicationProperties { + * @throws InvalidPathException If the configured path is invalid + */ + public Path getDatabaseRepositoryPath() { ++ System.out.println("[CTEST][GET-PARAM] " + REPOSITORY_DATABASE_DIRECTORY); + return Paths.get(getProperty(REPOSITORY_DATABASE_DIRECTORY)); + } + +@@ -1484,6 +1534,7 @@ public class NiFiProperties extends ApplicationProperties { + * @throws InvalidPathException If the configured path is invalid + */ + public Path getFlowFileRepositoryPath() { ++ System.out.println("[CTEST][GET-PARAM] " + FLOWFILE_REPOSITORY_DIRECTORY); + return Paths.get(getProperty(FLOWFILE_REPOSITORY_DIRECTORY)); + } + +@@ -1502,8 +1553,10 @@ public class NiFiProperties extends ApplicationProperties { + // go through each property + for (String propertyName : getPropertyKeys()) { + // determine if the property is a file repository path ++ + if (StringUtils.startsWith(propertyName, REPOSITORY_CONTENT_PREFIX)) { + // get the repository key ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); + final String key = StringUtils.substringAfter(propertyName, + REPOSITORY_CONTENT_PREFIX); + +@@ -1527,8 +1580,10 @@ public class NiFiProperties extends ApplicationProperties { + + // go through each property + for (String propertyName : getPropertyKeys()) { ++ + // determine if the property is a file repository path + if (StringUtils.startsWith(propertyName, PROVENANCE_REPO_DIRECTORY_PREFIX)) { ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); + // get the repository key + final String key = StringUtils.substringAfter(propertyName, + PROVENANCE_REPO_DIRECTORY_PREFIX); +@@ -1551,6 +1606,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public int getMaxFlowFilesPerClaim() { + try { ++ System.out.println("[CTEST][GET-PARAM] " + MAX_FLOWFILES_PER_CLAIM); + return Integer.parseInt(getProperty(MAX_FLOWFILES_PER_CLAIM)); + } catch (NumberFormatException nfe) { + return DEFAULT_MAX_FLOWFILES_PER_CLAIM; +@@ -1570,6 +1626,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getProperty(final String key, final String defaultValue) { ++ System.out.println("[CTEST][GET-PARAM] " + key); + final String value = getProperty(key); + return (value == null || value.trim().isEmpty()) ? defaultValue : value; + } +@@ -1583,10 +1640,12 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getLocalStateProviderId() { ++ System.out.println("[CTEST][GET-PARAM] " + STATE_MANAGEMENT_LOCAL_PROVIDER_ID); + return getProperty(STATE_MANAGEMENT_LOCAL_PROVIDER_ID); + } + + public String getClusterStateProviderId() { ++ System.out.println("[CTEST][GET-PARAM] " + STATE_MANAGEMENT_CLUSTER_PROVIDER_ID); + return getProperty(STATE_MANAGEMENT_CLUSTER_PROVIDER_ID); + } + +@@ -1603,11 +1662,13 @@ public class NiFiProperties extends ApplicationProperties { + } + + public File getEmbeddedZooKeeperPropertiesFile() { ++ System.out.println("[CTEST][GET-PARAM] " + STATE_MANAGEMENT_ZOOKEEPER_PROPERTIES); + final String filename = getProperty(STATE_MANAGEMENT_ZOOKEEPER_PROPERTIES); + return filename == null ? null : new File(filename); + } + + public boolean isStartEmbeddedZooKeeper() { ++ System.out.println("[CTEST][GET-PARAM] " + STATE_MANAGEMENT_START_EMBEDDED_ZOOKEEPER); + return Boolean.parseBoolean(getProperty(STATE_MANAGEMENT_START_EMBEDDED_ZOOKEEPER)); + } + +@@ -1616,6 +1677,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getFlowConfigurationArchiveDir() { ++ System.out.println("[CTEST][GET-PARAM] " + FLOW_CONFIGURATION_ARCHIVE_DIR); + return getProperty(FLOW_CONFIGURATION_ARCHIVE_DIR); + } + +@@ -1640,6 +1702,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getVariableRegistryProperties() { ++ System.out.println("[CTEST][GET-PARAM] " + VARIABLE_REGISTRY_PROPERTIES); + return getProperty(VARIABLE_REGISTRY_PROPERTIES); + } + +@@ -1672,8 +1735,10 @@ public class NiFiProperties extends ApplicationProperties { + + // go through each property + for (String propertyName : getPropertyKeys()) { ++ + // determine if the property is a network interface name + if (StringUtils.startsWith(propertyName, WEB_HTTP_NETWORK_INTERFACE_PREFIX)) { ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); + // get the network interface property key + final String key = StringUtils.substringAfter(propertyName, + WEB_HTTP_NETWORK_INTERFACE_PREFIX); +@@ -1694,8 +1759,10 @@ public class NiFiProperties extends ApplicationProperties { + + // go through each property + for (String propertyName : getPropertyKeys()) { ++ + // determine if the property is a network interface name + if (StringUtils.startsWith(propertyName, WEB_HTTPS_NETWORK_INTERFACE_PREFIX)) { ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); + // get the network interface property key + final String key = StringUtils.substringAfter(propertyName, + WEB_HTTPS_NETWORK_INTERFACE_PREFIX); +@@ -1720,6 +1787,11 @@ public class NiFiProperties extends ApplicationProperties { + } + + public boolean isZooKeeperTlsConfigurationPresent() { ++ System.out.println("[CTEST][GET-PARAM] " + ZOOKEEPER_CLIENT_SECURE); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + ZOOKEEPER_SECURITY_KEYSTORE); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + ZOOKEEPER_SECURITY_KEYSTORE_PASSWD); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + ZOOKEEPER_SECURITY_TRUSTSTORE); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + ZOOKEEPER_SECURITY_TRUSTSTORE_PASSWD); //add ctest + return StringUtils.isNotBlank(getProperty(NiFiProperties.ZOOKEEPER_CLIENT_SECURE)) + && StringUtils.isNotBlank(getProperty(NiFiProperties.ZOOKEEPER_SECURITY_KEYSTORE)) + && getProperty(NiFiProperties.ZOOKEEPER_SECURITY_KEYSTORE_PASSWD) != null +@@ -1728,6 +1800,10 @@ public class NiFiProperties extends ApplicationProperties { + } + + public boolean isTlsConfigurationPresent() { ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_KEYSTORE); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_KEYSTORE_PASSWD); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_TRUSTSTORE); //add ctest ++ System.out.println("[CTEST][GET-PARAM] " + SECURITY_TRUSTSTORE_PASSWD); //add ctest + return StringUtils.isNotBlank(getProperty(SECURITY_KEYSTORE)) + && getProperty(SECURITY_KEYSTORE_PASSWD) != null + && StringUtils.isNotBlank(getProperty(SECURITY_TRUSTSTORE)) +@@ -1735,6 +1811,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getFlowFileRepoEncryptionKeyId() { ++ System.out.println("[CTEST][GET-PARAM] " + REPOSITORY_ENCRYPTION_KEY_ID); //add ctest + return getProperty(FLOWFILE_REPOSITORY_ENCRYPTION_KEY_ID, getProperty(REPOSITORY_ENCRYPTION_KEY_ID)); + } + +@@ -1750,6 +1827,7 @@ public class NiFiProperties extends ApplicationProperties { + * @return the flowfile repository encryption key in hex form + */ + public String getFlowFileRepoEncryptionKey() { ++ System.out.println("[CTEST][GET-PARAM] " + FLOWFILE_REPOSITORY_ENCRYPTION_KEY); //add ctest + String keyId = getFlowFileRepoEncryptionKeyId(); + String keyKey = StringUtils.isBlank(keyId) ? FLOWFILE_REPOSITORY_ENCRYPTION_KEY : FLOWFILE_REPOSITORY_ENCRYPTION_KEY + ".id." + keyId; + return getProperty(keyKey, getProperty(FLOWFILE_REPOSITORY_ENCRYPTION_KEY)); +@@ -1784,6 +1862,7 @@ public class NiFiProperties extends ApplicationProperties { + + // Retrieve the actual key values and store non-empty values in the map + for (final String keyProperty : keyProperties) { ++ System.out.println("[CTEST][GET-PARAM] " + keyProperty); //add ctest + final String keyValue = getProperty(keyProperty); + if (StringUtils.isNotBlank(keyValue)) { + // If this property is .key (the actual hex key), put it in the map under the value of .key.id (i.e. key1) +@@ -1864,6 +1943,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getProvenanceRepoEncryptionKeyId() { ++ System.out.println("[CTEST][GET-PARAM] " + REPOSITORY_ENCRYPTION_KEY_ID); + return getProperty(PROVENANCE_REPO_ENCRYPTION_KEY_ID, getProperty(REPOSITORY_ENCRYPTION_KEY_ID)); + } + +@@ -1880,6 +1960,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public String getProvenanceRepoEncryptionKey() { + String keyId = getProvenanceRepoEncryptionKeyId(); ++ System.out.println("[CTEST][GET-PARAM] " + PROVENANCE_REPO_ENCRYPTION_KEY); + String keyKey = StringUtils.isBlank(keyId) ? PROVENANCE_REPO_ENCRYPTION_KEY : PROVENANCE_REPO_ENCRYPTION_KEY + ".id." + keyId; + return getProperty(keyKey, getProperty(PROVENANCE_REPO_ENCRYPTION_KEY)); + } +@@ -1895,6 +1976,7 @@ public class NiFiProperties extends ApplicationProperties { + } + + public String getContentRepositoryEncryptionKeyId() { ++ System.out.println("[CTEST][GET-PARAM] " + REPOSITORY_ENCRYPTION_KEY_ID); + return getProperty(CONTENT_REPOSITORY_ENCRYPTION_KEY_ID, getProperty(REPOSITORY_ENCRYPTION_KEY_ID)); + } + +@@ -1911,6 +1993,7 @@ public class NiFiProperties extends ApplicationProperties { + */ + public String getContentRepositoryEncryptionKey() { + String keyId = getContentRepositoryEncryptionKeyId(); ++ System.out.println("[CTEST][GET-PARAM] " + CONTENT_REPOSITORY_ENCRYPTION_KEY); + String keyKey = StringUtils.isBlank(keyId) ? CONTENT_REPOSITORY_ENCRYPTION_KEY : CONTENT_REPOSITORY_ENCRYPTION_KEY + ".id." + keyId; + return getProperty(keyKey, getProperty(CONTENT_REPOSITORY_ENCRYPTION_KEY)); + } +@@ -1995,28 +2078,41 @@ public class NiFiProperties extends ApplicationProperties { + + private List getFlowFileRepositoryEncryptionKeyProperties() { + // Filter all the property keys that define a key +- return getPropertyKeys().stream().filter(k -> +- k.startsWith(FLOWFILE_REPOSITORY_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(FLOWFILE_REPOSITORY_ENCRYPTION_KEY) +- ).collect(Collectors.toList()); ++ List properties = getPropertyKeys().stream().filter(k -> ++ k.startsWith(FLOWFILE_REPOSITORY_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(FLOWFILE_REPOSITORY_ENCRYPTION_KEY) ++).collect(Collectors.toList()); ++ for (String propertyName: properties){ ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); ++ } ++ return properties; + } + + private List getProvenanceRepositoryEncryptionKeyProperties() { + // Filter all the property keys that define a key +- return getPropertyKeys().stream().filter(k -> ++ List properties = getPropertyKeys().stream().filter(k -> + k.startsWith(PROVENANCE_REPO_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(PROVENANCE_REPO_ENCRYPTION_KEY) + ).collect(Collectors.toList()); ++ for (String propertyName: properties){ ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); ++ } ++ return properties; + } + + private List getContentRepositoryEncryptionKeyProperties() { + // Filter all the property keys that define a key +- return getPropertyKeys().stream().filter(k -> ++ List properties = getPropertyKeys().stream().filter(k -> + k.startsWith(CONTENT_REPOSITORY_ENCRYPTION_KEY_ID + ".") || k.equalsIgnoreCase(CONTENT_REPOSITORY_ENCRYPTION_KEY) + ).collect(Collectors.toList()); ++ for (String propertyName: properties){ ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); ++ } ++ return properties; + } + + public Long getDefaultBackPressureObjectThreshold() { + long backPressureCount; + try { ++ System.out.println("[CTEST][GET-PARAM] " + BACKPRESSURE_COUNT); + String backPressureCountStr = getProperty(BACKPRESSURE_COUNT); + if (backPressureCountStr == null || backPressureCountStr.trim().isEmpty()) { + backPressureCount = DEFAULT_BACKPRESSURE_COUNT; +@@ -2054,6 +2150,10 @@ public class NiFiProperties extends ApplicationProperties { + * @return A map of properties starting with the prefix. + */ + public Map getPropertiesWithPrefix(final String prefix) { ++ List properties = getPropertyKeys().stream().filter(key -> key.startsWith(prefix)).collect(Collectors.toList()); ++ for (String propertyName: properties){ ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); ++ } + return getPropertyKeys().stream().filter(key -> key.startsWith(prefix)).collect(Collectors.toMap(key -> key, key -> getProperty(key))); + } + +@@ -2070,11 +2170,15 @@ public class NiFiProperties extends ApplicationProperties { + public Set getDirectSubsequentTokens(final String prefix) { + final String fixedPrefix = prefix.endsWith(".") ? prefix : prefix + "."; + +- return getPropertyKeys().stream() ++ Set tokens = getPropertyKeys().stream() + .filter(key -> key.startsWith(fixedPrefix)) + .map(key -> key.substring(fixedPrefix.length())) + .map(key -> key.indexOf('.') == -1 ? key : key.substring(0, key.indexOf('.'))) + .collect(Collectors.toSet()); ++ for (String propertyName : tokens) { ++ System.out.println("[CTEST][GET-PARAM] " + propertyName); ++ } ++ return tokens; + } + + /** +-- +2.25.1 + diff --git a/core/patch/nifi-commons/nifi-logging-patches/0002-modify-add.sh-build.patch b/core/patch/nifi-commons/nifi-logging-patches/0002-modify-add.sh-build.patch new file mode 100644 index 00000000..f5fe2c54 --- /dev/null +++ b/core/patch/nifi-commons/nifi-logging-patches/0002-modify-add.sh-build.patch @@ -0,0 +1,24 @@ +From 491bb92f79b0e862327ebff7c30dc50c6f0fe1f7 Mon Sep 17 00:00:00 2001 +From: xinyihe4 +Date: Sun, 11 Dec 2022 16:06:26 -0600 +Subject: [PATCH 2/2] modify add.sh build + +--- + pom.xml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/pom.xml b/pom.xml +index 73433e2fcc..89d4ca3b7d 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -757,6 +757,7 @@ + maven-surefire-plugin + ${surefire.version} + ++ true + + true + +-- +2.25.1 + diff --git a/core/run_ctest/inject.py b/core/run_ctest/inject.py index 75f5b443..503962f4 100644 --- a/core/run_ctest/inject.py +++ b/core/run_ctest/inject.py @@ -14,7 +14,7 @@ def inject_config(param_value_pairs): for p, v in param_value_pairs.items(): print(">>>>[ctest_core] injecting {} with value {}".format(p, v)) - if project in [ZOOKEEPER, ALLUXIO]: + if project in [ZOOKEEPER, ALLUXIO, NIFI]: for inject_path in INJECTION_PATH[project]: print(">>>>[ctest_core] injecting into file: {}".format(inject_path)) file = open(inject_path, "w") @@ -41,7 +41,7 @@ def inject_config(param_value_pairs): def clean_conf_file(project): print(">>>> cleaning injected configuration from file") - if project in [ZOOKEEPER, ALLUXIO]: + if project in [ZOOKEEPER, ALLUXIO, NIFI]: for inject_path in INJECTION_PATH[project]: file = open(inject_path, "w") file.write("\n") diff --git a/core/run_ctest/parse_output.py b/core/run_ctest/parse_output.py index c750f398..9b857706 100644 --- a/core/run_ctest/parse_output.py +++ b/core/run_ctest/parse_output.py @@ -19,8 +19,13 @@ def parse_surefire(clsname, expected_methods): try: fpath = None for surefire_path in SUREFIRE_DIR[project]: - xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname)) + if project == "nifi-commons": + xml_path = os.path.join(surefire_path, SUREFIRE_XML_NIFI.format(clsname)) + else: + xml_path = os.path.join(surefire_path, SUREFIRE_XML.format(clsname)) + print(xml_path) if os.path.exists(xml_path): + print(">>>>[ctest_core] surefire report path: " + xml_path) fpath = open(xml_path) tree = ET.parse(fpath) diff --git a/core/run_ctest/program_input.py b/core/run_ctest/program_input.py index 2b971f1c..9c500854 100644 --- a/core/run_ctest/program_input.py +++ b/core/run_ctest/program_input.py @@ -4,11 +4,11 @@ # run mode "run_mode": "run_ctest", # string # name of the project, i.e. hadoop-common, hadoop-hdfs - "project": "hadoop-common", # string + "project": "nifi-commons", # string # path to param -> tests json mapping - "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string + "mapping_path": "../../data/ctest_mapping/opensource-nifi-commons.json", # string # input directory hosting configuration files to be test, target-project-format specific - "conf_file_dir": "sample-hadoop-common", # string + "conf_file_dir": "sample-nifi-commons", # string # display the terminal output live, without saving any results "display_mode": False, # bool # whether to use mvn test or mvn surefire:test diff --git a/core/run_ctest/run_single_ctest.py b/core/run_ctest/run_single_ctest.py index 7d299e3a..ad6ac723 100755 --- a/core/run_ctest/run_single_ctest.py +++ b/core/run_ctest/run_single_ctest.py @@ -2,7 +2,7 @@ import sys, time from program_input import p_input -from main import test_conf_file +# from main import test_conf_file from parse_input import * project = p_input["project"] diff --git a/core/run_ctest/run_test.py b/core/run_ctest/run_test.py index e156567f..736b2ba3 100644 --- a/core/run_ctest/run_test.py +++ b/core/run_ctest/run_test.py @@ -38,6 +38,7 @@ def run_test_batch(param_values, associated_test_map): if display_mode: os.system(" ".join(cmd)) continue + process = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout = "" diff --git a/core/run_ctest/run_test_utils.py b/core/run_ctest/run_test_utils.py index 12dfa13c..1189635d 100644 --- a/core/run_ctest/run_test_utils.py +++ b/core/run_ctest/run_test_utils.py @@ -8,7 +8,7 @@ maven_args = p_input["maven_args"] use_surefire = p_input["use_surefire"] ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]') - +project = p_input["project"] class TestResult: def __init__(self, ran_tests_and_time=set(), failed_tests=set()): self.failed_tests = failed_tests @@ -19,7 +19,11 @@ def maven_cmd(test, add_time=False): # surefire:test reuses test build from last compilation # if you modified the test and want to rerun it, you must use `mvn test` test_mode = "surefire:test" if use_surefire else "test" - cmd = ["mvn", test_mode, "-Dtest={}".format(test)] + maven_args + if project == "nifi-commons": + + cmd = ["mvn", test_mode, "-Dtest={}".format(test), "-Dsurefire.failIfNoSpecifiedTests=false"] + maven_args + else: + cmd = ["mvn", test_mode, "-Dtest={}".format(test)] + maven_args if add_time: cmd = ["time"] + cmd print(">>>>[ctest_core] command: " + " ".join(cmd)) diff --git a/core/run_ctest/sample-nifi-commons/ctest-nifi.properties b/core/run_ctest/sample-nifi-commons/ctest-nifi.properties new file mode 100644 index 00000000..e23c5b0d --- /dev/null +++ b/core/run_ctest/sample-nifi-commons/ctest-nifi.properties @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Core Properties # +nifi.flow.configuration.file=./target/flow.xml.gz +nifi.flow.configuration.archive.dir=./target/archive/ +nifi.flowcontroller.autoResumeState=true +nifi.flowcontroller.graceful.shutdown.period=10 sec +nifi.flowservice.writedelay.interval=2 sec +nifi.administrative.yield.duration=30 sec + +nifi.reporting.task.configuration.file=./target/reporting-tasks.xml +nifi.controller.service.configuration.file=./target/controller-services.xml +nifi.templates.directory=./target/templates +nifi.ui.banner.text=UI Banner Text +nifi.ui.autorefresh.interval=30 sec +nifi.nar.library.directory=./target/resources/NiFiProperties/lib/ +nifi.nar.library.directory.alt=./target/resources/NiFiProperties/lib2/ +nifi.nar.working.directory=./target/work/nar/ + +# H2 Settings +nifi.database.directory=./target/database_repository +nifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE + +# FlowFile Repository +nifi.flowfile.repository.directory=./target/test-repo +nifi.flowfile.repository.partitions=1 +nifi.flowfile.repository.checkpoint.interval=2 mins +nifi.queue.swap.threshold=20000 +nifi.swap.storage.directory=./target/test-repo/swap +nifi.swap.in.period=5 sec +nifi.swap.in.threads=1 +nifi.swap.out.period=5 sec +nifi.swap.out.threads=4 + +# Content Repository +nifi.content.claim.max.appendable.size=10 MB +nifi.content.claim.max.flow.files=100 +nifi.content.repository.directory.default=./target/content_repository + +# Provenance Repository Properties +nifi.provenance.repository.storage.directory=./target/provenance_repository +nifi.provenance.repository.max.storage.time=24 hours +nifi.provenance.repository.max.storage.size=1 GB +nifi.provenance.repository.rollover.time=30 secs +nifi.provenance.repository.rollover.size=100 MB + +# Site to Site properties +nifi.remote.input.socket.port=9990 +nifi.remote.input.secure=true + +# web properties # +nifi.web.war.directory=./target/lib +nifi.web.http.host= +nifi.web.http.port=8080 +nifi.web.https.host= +nifi.web.https.port= +nifi.web.jetty.working.directory=./target/work/jetty + +# security properties # +nifi.sensitive.props.key=key +nifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL + +nifi.security.keystore= +nifi.security.keystoreType= +nifi.security.keystorePasswd= +nifi.security.keyPasswd= +nifi.security.truststore= +nifi.security.truststoreType= +nifi.security.truststorePasswd= +nifi.security.user.authorizer= + +# cluster common properties (cluster manager and nodes must have same values) # +nifi.cluster.protocol.heartbeat.interval=5 sec +nifi.cluster.protocol.is.secure=false +nifi.cluster.protocol.socket.timeout=30 sec +nifi.cluster.protocol.connection.handshake.timeout=45 sec +# if multicast is used, then nifi.cluster.protocol.multicast.xxx properties must be configured # +nifi.cluster.protocol.use.multicast=false +nifi.cluster.protocol.multicast.address= +nifi.cluster.protocol.multicast.port= +nifi.cluster.protocol.multicast.service.broadcast.delay=500 ms +nifi.cluster.protocol.multicast.service.locator.attempts=3 +nifi.cluster.protocol.multicast.service.locator.attempts.delay=1 sec + +# cluster node properties (only configure for cluster nodes) # +nifi.cluster.is.node=false +nifi.cluster.node.address= +nifi.cluster.node.protocol.port= +nifi.cluster.node.protocol.threads=2 +# if multicast is not used, nifi.cluster.node.unicast.xxx must have same values as nifi.cluster.manager.xxx # +nifi.cluster.node.unicast.manager.address= +nifi.cluster.node.unicast.manager.protocol.port= +nifi.cluster.node.unicast.manager.authority.provider.port= + +# cluster manager properties (only configure for cluster manager) # +nifi.cluster.is.manager=false +nifi.cluster.manager.address= +nifi.cluster.manager.protocol.port= +nifi.cluster.manager.authority.provider.port= +nifi.cluster.manager.authority.provider.threads=10 +nifi.cluster.manager.node.firewall.file= +nifi.cluster.manager.node.event.history.size=10 +nifi.cluster.manager.node.api.connection.timeout=30 sec +nifi.cluster.manager.node.api.read.timeout=30 sec +nifi.cluster.manager.node.api.request.threads=10 +nifi.cluster.manager.flow.retrieval.delay=5 sec +nifi.cluster.manager.protocol.threads=10 +nifi.cluster.manager.safemode.duration=0 sec + +# analytics properties # +nifi.analytics.predict.interval=3 mins +nifi.analytics.connection.model.implementation=org.apache.nifi.controller.status.analytics.models.OrdinaryLeastSquares \ No newline at end of file diff --git a/data/ctest_mapping/opensource-nifi-commons.json b/data/ctest_mapping/opensource-nifi-commons.json new file mode 100644 index 00000000..9e0ca3ea --- /dev/null +++ b/data/ctest_mapping/opensource-nifi-commons.json @@ -0,0 +1,22 @@ +{ + "nifi.ui.banner.text": [ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testValidateProperties" + ], + "nifi.nar.library.directory": [ + "NiFiPropertiesTest#testProperties", + "NiFiPropertiesTest#testBlankProperties", + "NiFiPropertiesTest#testAdditionalOidcScopesAreTrimmed", + "NiFiPropertiesTest#testMissingProperties", + "NiFiPropertiesTest#testShouldVerifyValidFormatPortValue", + "NiFiPropertiesTest#testShouldVerifyValidPortValue", + "NiFiPropertiesTest#testShouldVerifyExceptionThrownWhenInValidFormatPortValue", + "NiFiPropertiesTest#testValidateProperties" + ] +} \ No newline at end of file