From 0231384f82766619f681888cd3bbb4fff3a85a31 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:39:07 +0200 Subject: [PATCH 1/6] feat: improve default configuration --- rust/operator-binary/src/hdfs_controller.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/hdfs_controller.rs b/rust/operator-binary/src/hdfs_controller.rs index 6394276d..954c3b74 100644 --- a/rust/operator-binary/src/hdfs_controller.rs +++ b/rust/operator-binary/src/hdfs_controller.rs @@ -664,7 +664,15 @@ fn rolegroup_config_map( ) .add("dfs.datanode.registered.hostname", "${env.POD_ADDRESS}") .add("dfs.datanode.registered.port", "${env.DATA_PORT}") - .add("dfs.datanode.registered.ipc.port", "${env.IPC_PORT}"); + .add("dfs.datanode.registered.ipc.port", "${env.IPC_PORT}") + .add("dfs.datanode.sync.behind.writes", "true") + .add("dfs.datanode.synconclose", "true") + .add("io.file.buffer.size", "131072") + .add("dfs.namenode.handler.count", "50") + .add("dfs.datanode.handler.count", "50") + .add("dfs.namenode.replication.max-streams", "4") + .add("dfs.namenode.replication.max-streams-hard-limit", "8") + .add("dfs.datanode.max.transfer.threads", "8192"); if hdfs.has_https_enabled() { hdfs_site.add("dfs.datanode.registered.https.port", "${env.HTTPS_PORT}"); } else { From bfe17176a22b16f5eafb0237dbcefdcbcddc11d9 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:46:20 +0200 Subject: [PATCH 2/6] fix: add missing labels to hdfs-operator-clusterrole-nodes --- deploy/helm/hdfs-operator/templates/roles.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deploy/helm/hdfs-operator/templates/roles.yaml b/deploy/helm/hdfs-operator/templates/roles.yaml index fbc1b8c7..2b846ea5 100644 --- a/deploy/helm/hdfs-operator/templates/roles.yaml +++ b/deploy/helm/hdfs-operator/templates/roles.yaml @@ -186,6 +186,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: {{ include "operator.name" . }}-clusterrole-nodes + labels: + {{- include "operator.labels" . | nindent 4 }} rules: - apiGroups: - "" From 165f335dace1cbdfd07f38fddc97322c67c1371b Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:51:26 +0200 Subject: [PATCH 3/6] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d3d3839..bc376a8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. - The operator now defaults to `AES/CTR/NoPadding` for `dfs.encrypt.data.transfer.cipher.suite` to improve security and performance ([#693]). - The built-in Prometheus servlet is now enabled and metrics are exposed under the `/prom` path of all UI services ([#695]). +- Added several properties to `hdfs-site.xml` that improve general performance and reliability ([#696]) ### Changed @@ -50,6 +51,7 @@ All notable changes to this project will be documented in this file. [#684]: https://github.com/stackabletech/hdfs-operator/pull/684 [#693]: https://github.com/stackabletech/hdfs-operator/pull/693 [#695]: https://github.com/stackabletech/hdfs-operator/pull/695 +[#696]: https://github.com/stackabletech/hdfs-operator/pull/696 ## [25.3.0] - 2025-03-21 From 0baead3aa281d057ee698a819be0a8f563d382fa Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 19 Jun 2025 16:58:54 +0200 Subject: [PATCH 4/6] explain properties --- rust/operator-binary/src/hdfs_controller.rs | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/rust/operator-binary/src/hdfs_controller.rs b/rust/operator-binary/src/hdfs_controller.rs index 954c3b74..c04b3674 100644 --- a/rust/operator-binary/src/hdfs_controller.rs +++ b/rust/operator-binary/src/hdfs_controller.rs @@ -665,13 +665,42 @@ fn rolegroup_config_map( .add("dfs.datanode.registered.hostname", "${env.POD_ADDRESS}") .add("dfs.datanode.registered.port", "${env.DATA_PORT}") .add("dfs.datanode.registered.ipc.port", "${env.IPC_PORT}") + // The following two properties are set to "true" because there is a minor chance that data + // written to HDFS is not synced to disk even if a block has been closed. + // Users in HBase can control this explicitly for the WAL, but for flushes and compactions + // I believe they can't as easily (if at all). + // In theory, HBase should be able to recover from these failures, but that comes at a cost + // and there's always a risk. + // Enabling this behavior causes HDFS to sync to disk as soon as possible. .add("dfs.datanode.sync.behind.writes", "true") .add("dfs.datanode.synconclose", "true") + // The default (4096) hasn't changed since 2009. + // Increase to 128k to allow for faster transfers. .add("io.file.buffer.size", "131072") + // Defaults to 10 since at least 2011. + // This controls the concurrent number of client connections (this includes DataNodes) + // to the NameNode. Ideally, we'd scale this with the number of DataNodes but this would + // lead to restarts of the NameNode. + // This should lead to better performance due to more concurrency. .add("dfs.namenode.handler.count", "50") + // Defaults to 10 since at least 2012. + // This controls the concurrent number of client connections to the DataNodes. + // We have no idea how many clients there may be, so it's hard to assign a good default. + // Increasing to 50 should lead to better performance due to more concurrency, especially + // with use-cases like HBase. .add("dfs.datanode.handler.count", "50") + // The following two properties default to 2 and 4 respectively since around 2013. + // They control the number of maximum replication "jobs" a NameNode assigns to + // a DataNode in a single heartbeat. + // Increasing this number will increase network usage during replication events + // but can lead to faster recovery. .add("dfs.namenode.replication.max-streams", "4") .add("dfs.namenode.replication.max-streams-hard-limit", "8") + // Defaults to 4096 and hasn't changed since at least 2011. + // The number of threads used for actual data transfer, so not very CPU heavy + // but IO bound. This is why the number is relatively high. + // But today's Java and IO should be able to handle more, so bump it to 8192 for + // better performance/concurrency. .add("dfs.datanode.max.transfer.threads", "8192"); if hdfs.has_https_enabled() { hdfs_site.add("dfs.datanode.registered.https.port", "${env.HTTPS_PORT}"); From 05e52b728ed2feeb5a45ce7a85f70e45206fda11 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 19 Jun 2025 17:34:39 +0200 Subject: [PATCH 5/6] move property to core-site.xml --- rust/operator-binary/src/hdfs_controller.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rust/operator-binary/src/hdfs_controller.rs b/rust/operator-binary/src/hdfs_controller.rs index c04b3674..2b388e8c 100644 --- a/rust/operator-binary/src/hdfs_controller.rs +++ b/rust/operator-binary/src/hdfs_controller.rs @@ -674,9 +674,6 @@ fn rolegroup_config_map( // Enabling this behavior causes HDFS to sync to disk as soon as possible. .add("dfs.datanode.sync.behind.writes", "true") .add("dfs.datanode.synconclose", "true") - // The default (4096) hasn't changed since 2009. - // Increase to 128k to allow for faster transfers. - .add("io.file.buffer.size", "131072") // Defaults to 10 since at least 2011. // This controls the concurrent number of client connections (this includes DataNodes) // to the NameNode. Ideally, we'd scale this with the number of DataNodes but this would @@ -720,7 +717,10 @@ fn rolegroup_config_map( .ha_zookeeper_quorum() .security_config(hdfs, cluster_info) .context(BuildSecurityConfigSnafu)? - .enable_prometheus_endpoint(); + .enable_prometheus_endpoint() + // The default (4096) hasn't changed since 2009. + // Increase to 128k to allow for faster transfers. + .add("io.file.buffer.size", "131072"); if let Some(hdfs_opa_config) = hdfs_opa_config { hdfs_opa_config.add_core_site_config(&mut core_site); } From 2930110355ae2b826797df918297c8b7ec53fbc7 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Thu, 19 Jun 2025 17:40:36 +0200 Subject: [PATCH 6/6] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc376a8a..edaec267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ All notable changes to this project will be documented in this file. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. - The operator now defaults to `AES/CTR/NoPadding` for `dfs.encrypt.data.transfer.cipher.suite` to improve security and performance ([#693]). - The built-in Prometheus servlet is now enabled and metrics are exposed under the `/prom` path of all UI services ([#695]). -- Added several properties to `hdfs-site.xml` that improve general performance and reliability ([#696]) +- Added several properties to `hdfs-site.xml` and `core-site.xml` that improve general performance and reliability ([#696]) ### Changed