From 5dc96d3615f437c90fcfe11e910576f97a288a76 Mon Sep 17 00:00:00 2001 From: Peter Van Bouwel Date: Wed, 6 Aug 2025 12:15:38 +0200 Subject: [PATCH 1/5] doc: job-options memory options Clarify the documentation on memory related job-options. If the backend has a default of None then [a limit of 0 bytes is set](https://github.com/Open-EO/openeo-geopyspark-driver/commit/18e512d84c594591a90b2a357becfdc5f00266aa#diff-f606c26975b555ceec5f9fd97aa22cb9372fdd3d508db2eb8ae248f3b559f2eeL134) rather then no enforcement. Also clarify that it is purely a limit and not a reservation. --- openeogeotrellis/job_options.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openeogeotrellis/job_options.py b/openeogeotrellis/job_options.py index 31bab28d7..7419fd19c 100644 --- a/openeogeotrellis/job_options.py +++ b/openeogeotrellis/job_options.py @@ -48,15 +48,17 @@ class JobOptions: default=get_backend_config().default_executor_memoryOverhead, metadata={ "name": "executor-memoryOverhead", - "description": "Memory allocated to the workers in addition to the JVM, for example, to run UDFs. " - "The total available memory of an executor is equal to executor-memory + executor-memoryOverhead [+ python-memory].", + "description": "Memory allocated to the workers in addition to the JVM. " + "The total available memory of an executor is equal to executor-memory + executor-memoryOverhead + python-memory.", }) python_memory: str = field( default=get_backend_config().default_python_memory, metadata={ "description": "Setting to specifically limit the memory used by python on a worker. " + "Leaving this setting empty will enforce the default value of the backend. None will be zero bytes." "Typical processes that use python-memory are UDF's, sar_backscatter or Sentinel 3 data loading. " - "Leaving this setting empty will allow Python to use almost all of the executor-memoryOverhead, but may lead to unclear error messages when the memory limit is reached." + "This memory not a reservation so it can act as executor_memory_overhead but it is enforced as a limit." + "Memory allocation problems with one of aforementioned processes likely warrant an increase of this value." }) executor_cores: int = field( default=get_backend_config().default_executor_cores, From 9ebfdb9763f13016a413b67a304d205b76a50a40 Mon Sep 17 00:00:00 2001 From: pvbouwel <463976+pvbouwel@users.noreply.github.com> Date: Thu, 7 Aug 2025 10:02:12 +0200 Subject: [PATCH 2/5] Update openeogeotrellis/job_options.py Co-authored-by: Stefaan Lippens --- openeogeotrellis/job_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeogeotrellis/job_options.py b/openeogeotrellis/job_options.py index 7419fd19c..fa68c59a3 100644 --- a/openeogeotrellis/job_options.py +++ b/openeogeotrellis/job_options.py @@ -55,7 +55,7 @@ class JobOptions: default=get_backend_config().default_python_memory, metadata={ "description": "Setting to specifically limit the memory used by python on a worker. " - "Leaving this setting empty will enforce the default value of the backend. None will be zero bytes." + "Leaving this setting empty will enforce the default value of the backend. None will be zero bytes. " "Typical processes that use python-memory are UDF's, sar_backscatter or Sentinel 3 data loading. " "This memory not a reservation so it can act as executor_memory_overhead but it is enforced as a limit." "Memory allocation problems with one of aforementioned processes likely warrant an increase of this value." From cd64e0c0d64b655a330bf2ba7b0362c82bf189ad Mon Sep 17 00:00:00 2001 From: Peter Van Bouwel Date: Thu, 7 Aug 2025 10:03:46 +0200 Subject: [PATCH 3/5] doc: typo missing is --- openeogeotrellis/job_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeogeotrellis/job_options.py b/openeogeotrellis/job_options.py index fa68c59a3..72fcd5812 100644 --- a/openeogeotrellis/job_options.py +++ b/openeogeotrellis/job_options.py @@ -57,7 +57,7 @@ class JobOptions: "description": "Setting to specifically limit the memory used by python on a worker. " "Leaving this setting empty will enforce the default value of the backend. None will be zero bytes. " "Typical processes that use python-memory are UDF's, sar_backscatter or Sentinel 3 data loading. " - "This memory not a reservation so it can act as executor_memory_overhead but it is enforced as a limit." + "This memory is not a reservation so it can act as executor_memory_overhead but it is enforced as a limit." "Memory allocation problems with one of aforementioned processes likely warrant an increase of this value." }) executor_cores: int = field( From 8c6f01d106796a75cfc14d2b4786ecc0e8d2bd74 Mon Sep 17 00:00:00 2001 From: Peter Van Bouwel Date: Thu, 7 Aug 2025 10:05:48 +0200 Subject: [PATCH 4/5] doc: typo whitespace --- openeogeotrellis/job_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeogeotrellis/job_options.py b/openeogeotrellis/job_options.py index 72fcd5812..47f45b1ca 100644 --- a/openeogeotrellis/job_options.py +++ b/openeogeotrellis/job_options.py @@ -57,7 +57,7 @@ class JobOptions: "description": "Setting to specifically limit the memory used by python on a worker. " "Leaving this setting empty will enforce the default value of the backend. None will be zero bytes. " "Typical processes that use python-memory are UDF's, sar_backscatter or Sentinel 3 data loading. " - "This memory is not a reservation so it can act as executor_memory_overhead but it is enforced as a limit." + "This memory is not a reservation so it can act as executor_memory_overhead but it is enforced as a limit. " "Memory allocation problems with one of aforementioned processes likely warrant an increase of this value." }) executor_cores: int = field( From 37100bd5fff3a90c78b5b77a2be465e67fc1fb8d Mon Sep 17 00:00:00 2001 From: Peter Van Bouwel Date: Fri, 8 Aug 2025 09:26:11 +0200 Subject: [PATCH 5/5] doc: another attempt on memory options. --- openeogeotrellis/job_options.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openeogeotrellis/job_options.py b/openeogeotrellis/job_options.py index 47f45b1ca..0d706dba8 100644 --- a/openeogeotrellis/job_options.py +++ b/openeogeotrellis/job_options.py @@ -49,16 +49,18 @@ class JobOptions: metadata={ "name": "executor-memoryOverhead", "description": "Memory allocated to the workers in addition to the JVM. " - "The total available memory of an executor is equal to executor-memory + executor-memoryOverhead + python-memory.", + "The total available memory of an executor is equal to executor-memory + executor-memoryOverhead + python-memory. " + "Therefore this option allows to foresee additional memory for processes other than the JVM or the Python runtime.", }) python_memory: str = field( default=get_backend_config().default_python_memory, metadata={ "description": "Setting to specifically limit the memory used by python on a worker. " "Leaving this setting empty will enforce the default value of the backend. None will be zero bytes. " - "Typical processes that use python-memory are UDF's, sar_backscatter or Sentinel 3 data loading. " - "This memory is not a reservation so it can act as executor_memory_overhead but it is enforced as a limit. " - "Memory allocation problems with one of aforementioned processes likely warrant an increase of this value." + "The total available memory of an executor is equal to executor-memory + executor-memoryOverhead + python-memory. " + "Typical processes that use python-memory are UDF's, sar_backscatter or Sentinel 3 data loading " + "for these this python_memory limit should be used rather than executor_memory_overhead. " + "Memory allocation problems for one of the aforementioned processes likely warrant an increase of this value. " }) executor_cores: int = field( default=get_backend_config().default_executor_cores,