From 7fc9be621c45f476598f79c3f354ba5d79c8a4eb Mon Sep 17 00:00:00 2001 From: Ali Date: Tue, 6 May 2025 15:26:44 +0200 Subject: [PATCH 1/3] doc for stashing --- docs/source/topics/calculations/usage.rst | 37 ++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/source/topics/calculations/usage.rst b/docs/source/topics/calculations/usage.rst index 8d3699306a..fe8523f078 100644 --- a/docs/source/topics/calculations/usage.rst +++ b/docs/source/topics/calculations/usage.rst @@ -632,17 +632,46 @@ Using the ``COPY`` mode, the target path defines another location (on the same f } .. note:: + In addition to the ``COPY`` mode, the following modes, these storage efficient modes are also are available: + ``COMPRESS_TAR``, ``COMPRESS_TARBZ2``, ``COMPRESS_TARGZ``, ``COMPRESS_TARXZ``. - In the future, other methods for stashing may be implemented, such as placing all files in a (compressed) tarball or even stash files on tape. +The stashed files and folders are represented by an output node that is attached to the calculation node through the label ``remote_stash``, as a ``RemoteStashFolderData`` node. +Just like the ``remote_folder`` node, this represents a location or files on a remote machine and so is equivalent to a "symbolic link". .. important:: - If the ``stash`` option namespace is defined for a calculation job, the daemon will perform the stashing operations before the files are retrieved. + If the ``stash`` option namespace is defined for a generic calculation job, the daemon will perform the stashing operations before the files are retrieved. This means that the stashing happens before the parsing of the output files (which occurs after the retrieving step), such that that the files will be stashed independent of the final exit status that the parser will assign to the calculation job. This may cause files to be stashed for calculations that will later be considered to have failed. -The stashed files and folders are represented by an output node that is attached to the calculation node through the label ``remote_stash``, as a ``RemoteStashFolderData`` node. -Just like the ``remote_folder`` node, this represents a location or files on a remote machine and so is equivalent to a "symbolic link". +To avoid this scenario, you can instead, stash via a separate calculation job, for example: + +.. code-block:: python + + from aiida.common.datastructures import StashMode + from aiida.orm import load_node + + StashCalculation = CalculationFactory('core.stash') + + calcjob_node = load_node() + remote_folder = calcjob_node.outputs.remote_folder + + inputs = { + 'metadata': { + 'computer': Computer.collection.get(label="localhost"), + 'options': { + 'stash': { + 'source_list': ['aiida.out', 'output.txt'], + 'target_base': '/scratch/', + 'stash_mode': StashMode.COPY.value, + }, + }, + }, + 'source_node': remote_folder, + } + + result = run(StashCalculation, **inputs) + .. important:: From 3f28536fc8892437091b6f9e2ae083b123176fa0 Mon Sep 17 00:00:00 2001 From: Ali Date: Fri, 23 May 2025 16:21:06 +0200 Subject: [PATCH 2/3] removed .value --- docs/source/topics/calculations/usage.rst | 4 ++-- src/aiida/calculations/stash.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/topics/calculations/usage.rst b/docs/source/topics/calculations/usage.rst index fe8523f078..4cecb7f645 100644 --- a/docs/source/topics/calculations/usage.rst +++ b/docs/source/topics/calculations/usage.rst @@ -625,7 +625,7 @@ Using the ``COPY`` mode, the target path defines another location (on the same f 'stash': { 'source_list': ['aiida.out', 'output.txt'], 'target_base': '/storage/project/stash_folder', - 'stash_mode': StashMode.COPY.value, + 'stash_mode': StashMode.COPY, } } } @@ -663,7 +663,7 @@ To avoid this scenario, you can instead, stash via a separate calculation job, f 'stash': { 'source_list': ['aiida.out', 'output.txt'], 'target_base': '/scratch/', - 'stash_mode': StashMode.COPY.value, + 'stash_mode': StashMode.COPY, }, }, }, diff --git a/src/aiida/calculations/stash.py b/src/aiida/calculations/stash.py index 807235170f..089666eb01 100644 --- a/src/aiida/calculations/stash.py +++ b/src/aiida/calculations/stash.py @@ -30,7 +30,7 @@ class StashCalculation(CalcJob): 'options': { 'resources': {'num_machines': 1}, 'stash': { - 'stash_mode': StashMode.COPY.value, + 'stash_mode': StashMode.COPY, 'target_base': '/scratch/my_stashing/', 'source_list': ['aiida.in', '_aiidasubmit.sh'], }, From fb6746ed902d1ccf91f4031128ba98d08d4970c1 Mon Sep 17 00:00:00 2001 From: Ali Date: Thu, 5 Jun 2025 14:20:55 +0200 Subject: [PATCH 3/3] review applied --- docs/source/topics/calculations/usage.rst | 8 ++++---- src/aiida/calculations/stash.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/topics/calculations/usage.rst b/docs/source/topics/calculations/usage.rst index 4cecb7f645..1ecab045d7 100644 --- a/docs/source/topics/calculations/usage.rst +++ b/docs/source/topics/calculations/usage.rst @@ -625,7 +625,7 @@ Using the ``COPY`` mode, the target path defines another location (on the same f 'stash': { 'source_list': ['aiida.out', 'output.txt'], 'target_base': '/storage/project/stash_folder', - 'stash_mode': StashMode.COPY, + 'stash_mode': StashMode.COPY.value, } } } @@ -649,7 +649,7 @@ To avoid this scenario, you can instead, stash via a separate calculation job, f .. code-block:: python from aiida.common.datastructures import StashMode - from aiida.orm import load_node + from aiida.orm import load_node, load_computer StashCalculation = CalculationFactory('core.stash') @@ -658,12 +658,12 @@ To avoid this scenario, you can instead, stash via a separate calculation job, f inputs = { 'metadata': { - 'computer': Computer.collection.get(label="localhost"), + 'computer': load_computer(label="localhost"), 'options': { 'stash': { 'source_list': ['aiida.out', 'output.txt'], 'target_base': '/scratch/', - 'stash_mode': StashMode.COPY, + 'stash_mode': StashMode.COPY.value, }, }, }, diff --git a/src/aiida/calculations/stash.py b/src/aiida/calculations/stash.py index 089666eb01..9be4d4610d 100644 --- a/src/aiida/calculations/stash.py +++ b/src/aiida/calculations/stash.py @@ -26,11 +26,11 @@ class StashCalculation(CalcJob): inputs = { 'metadata': { - 'computer': Computer.collection.get(label="localhost"), + 'computer': load_computer(label="localhost"), 'options': { 'resources': {'num_machines': 1}, 'stash': { - 'stash_mode': StashMode.COPY, + 'stash_mode': StashMode.COPY.value, 'target_base': '/scratch/my_stashing/', 'source_list': ['aiida.in', '_aiidasubmit.sh'], },