Add ParticleDump and FieldDump (openPMD plugin) to PICMI

lib/python/picongpu/picmi/diagnostics/__init__.py

@@ -13,13 +13,20 @@
 from .png import Png
 from .timestepspec import TimeStepSpec
 from .checkpoint import Checkpoint
+from .particle_dump import ParticleDump
+from .field_dump import FieldDump
+from .backend_config import BackendConfig, OpenPMDConfig
 
 __all__ = [
     "Auto",
+    "BackendConfig",
+    "OpenPMDConfig",
     "Binning",
     "PhaseSpace",
     "EnergyHistogram",
     "MacroParticleCount",
+    "ParticleDump",
+    "FieldDump",
     "Png",
     "TimeStepSpec",
     "Checkpoint",

lib/python/picongpu/picmi/diagnostics/backend_config.py

@@ -0,0 +1,18 @@
+"""
+This file is part of PIConGPU.
+Copyright 2025 PIConGPU contributors
+Authors: Julian Lenz
+License: GPLv3+
+"""
+
+from picongpu.pypicongpu.output.openpmd_plugin import OpenPMDConfig as PyPIConGPUOpenPMDConfig
+
+
+class BackendConfig:
+    def result_path(self, prefix_path):
+        raise NotImplementedError()
+
+
+class OpenPMDConfig(PyPIConGPUOpenPMDConfig, BackendConfig):
+    def __init__(self, *args, **kwargs):
+        super(PyPIConGPUOpenPMDConfig, self).__init__(*args, **kwargs)

lib/python/picongpu/picmi/diagnostics/field_dump.py

@@ -0,0 +1,27 @@
+"""
+This file is part of PIConGPU.
+Copyright 2025 PIConGPU contributors
+Authors: Julian Lenz
+License: GPLv3+
+"""
+
+from os import PathLike
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel
+
+from .backend_config import BackendConfig, OpenPMDConfig
+from .timestepspec import TimeStepSpec
+
+
+class FieldDump(BaseModel):
+    fieldname: Literal["E", "B", "J"]
+    period: TimeStepSpec = TimeStepSpec[:]("steps")
+    options: BackendConfig = OpenPMDConfig(file="simData")
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def result_path(self, prefix_path: PathLike):
+        return self.options.result_path(prefix_path=Path(prefix_path) / "simOutput" / "openPMD")

lib/python/picongpu/picmi/diagnostics/particle_dump.py

@@ -0,0 +1,28 @@
+"""
+This file is part of PIConGPU.
+Copyright 2025 PIConGPU contributors
+Authors: Julian Lenz
+License: GPLv3+
+"""
+
+from os import PathLike
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from picongpu.picmi.species import Species
+
+from .backend_config import BackendConfig, OpenPMDConfig
+from .timestepspec import TimeStepSpec
+
+
+class ParticleDump(BaseModel):
+    species: Species
+    period: TimeStepSpec = TimeStepSpec[:]("steps")
+    options: BackendConfig = OpenPMDConfig(file="simData")
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    def result_path(self, prefix_path: PathLike):
+        return self.options.result_path(prefix_path=Path(prefix_path) / "simOutput" / "openPMD")

lib/python/picongpu/picmi/simulation.py

@@ -16,6 +16,8 @@
 import picmistandard
 import typeguard
 
+from picongpu.picmi.diagnostics import ParticleDump, FieldDump
+from picongpu.pypicongpu.output.openpmd_plugin import OpenPMDPlugin, FieldDump as PyPIConGPUFieldDump
 from picongpu.pypicongpu.species.initmanager import InitManager
 
 from .. import pypicongpu
@@ -26,6 +28,15 @@
 from .species import Species
 
 
+def _unique(iterable):
+    # very naive, just for non-hashables that can still be compared
+    result = []
+    for x in iterable:
+        if x not in result:
+            result.append(x)
+    return result
+
+
 def is_iterable(obj):
     try:
         iter(obj)
@@ -67,6 +78,10 @@ def _normalise_template_dir(directory: None | PathLike | typing.Iterable[PathLik
     return directory
 
 
+def handled_via_openpmd(diagnostic):
+    return isinstance(diagnostic, (ParticleDump, FieldDump))
+
+
 # may not use pydantic since inherits from _DocumentedMetaClass
 @typeguard.typechecked
 class Simulation(picmistandard.PICMI_Simulation):
@@ -479,6 +494,37 @@ def step(self, nsteps: int = 1):
             )
         self.picongpu_run()
 
+    def _generate_openpmd_plugins(self, diagnostics, pypicongpu_by_picmi_species, num_steps):
+        diagnostics = list(diagnostics)
+        return [
+            OpenPMDPlugin(
+                sources=[
+                    (
+                        diagnostic.period.get_as_pypicongpu(time_step_size=self.time_step_size, num_steps=num_steps),
+                        pypicongpu_by_picmi_species[diagnostic.species]
+                        if isinstance(diagnostic, ParticleDump)
+                        else PyPIConGPUFieldDump(name=diagnostic.fieldname),
+                    )
+                    for diagnostic in filter(lambda x: x.options == options, diagnostics)
+                ],
+                config=options,
+            )
+            for options in _unique(map(lambda x: x.options, diagnostics))
+        ]
+
+    def _generate_plugins(self, pypicongpu_by_picmi_species, num_steps):
+        return [
+            entry.get_as_pypicongpu(
+                dict_species_picmi_to_pypicongpu=pypicongpu_by_picmi_species,
+                time_step_size=self.time_step_size,
+                num_steps=num_steps,
+            )
+            for entry in self.diagnostics
+            if not handled_via_openpmd(entry)
+        ] + self._generate_openpmd_plugins(
+            filter(handled_via_openpmd, self.diagnostics), pypicongpu_by_picmi_species, num_steps
+        )
+
     def get_as_pypicongpu(self) -> pypicongpu.simulation.Simulation:
         """translate to PyPIConGPU object"""
         s = pypicongpu.simulation.Simulation()
@@ -523,15 +569,7 @@ def get_as_pypicongpu(self) -> pypicongpu.simulation.Simulation:
 
         s.init_manager, pypicongpu_by_picmi_species = self.__get_init_manager()
 
-        s.plugins = [
-            entry.get_as_pypicongpu(
-                dict_species_picmi_to_pypicongpu=pypicongpu_by_picmi_species,
-                time_step_size=self.time_step_size,
-                num_steps=s.time_steps,
-            )
-            for entry in self.diagnostics
-        ]
-
+        s.plugins = self._generate_plugins(pypicongpu_by_picmi_species, s.time_steps)
         # set typical ppc if not set explicitly by user
         if self.picongpu_typical_ppc is None:
             s.typical_ppc = (s.init_manager).get_typical_particle_per_cell()

lib/python/picongpu/pypicongpu/output/__init__.py

@@ -5,9 +5,13 @@
 from .png import Png
 from .timestepspec import TimeStepSpec
 from .checkpoint import Checkpoint
+from .openpmd_plugin import OpenPMDPlugin
+from .plugin import Plugin
 
 __all__ = [
     "Auto",
+    "OpenPMDPlugin",
+    "Plugin",
     "PhaseSpace",
     "EnergyHistogram",
     "MacroParticleCount",

lib/python/picongpu/pypicongpu/output/openpmd_plugin.py

@@ -0,0 +1,119 @@
+"""
+This file is part of PIConGPU.
+Copyright 2025 PIConGPU contributors
+Authors: Julian Lenz
+License: GPLv3+
+"""
+
+from functools import reduce
+from os import PathLike
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Annotated, Iterable, Literal
+from hashlib import sha256
+
+import tomli_w
+from pydantic import AfterValidator, BaseModel
+
+from picongpu.pypicongpu.output.plugin import Plugin
+from picongpu.pypicongpu.output.timestepspec import TimeStepSpec
+from picongpu.pypicongpu.species.species import Species
+
+
+class OpenPMDConfig(BaseModel):
+    file: PathLike | str
+    infix: str = "_%06T"
+    ext: Annotated[str, AfterValidator(lambda s: s.strip("."))] = "h5"
+    backend_config: PathLike | None = None
+    data_preparation_strategy: Literal["mappedMemory", "doubleBuffer"] = "mappedMemory"
+    range: None = None
+
+    def full_filename(self):
+        return f"{self.file}{self.infix}.{self.ext}"
+
+    def result_path(self, prefix_path: PathLike = Path()):
+        filename = self.full_filename()
+        if Path(filename).is_absolute():
+            return filename
+        return (Path(prefix_path) / filename).absolute()
+
+
+def to_string(timestepspec: TimeStepSpec):
+    return ",".join(
+        map(
+            lambda x: "{start}:{stop}:{step}".format(**x),
+            timestepspec.get_rendering_context()["specs"],
+        )
+    )
+
+
+class FieldDump(BaseModel):
+    name: str
+
+    def get_rendering_context(self) -> dict:
+        return self.model_dump(mode="json")
+
+
+class OpenPMDPlugin(Plugin):
+    _name = "openPMD"
+
+    sources: list[tuple[TimeStepSpec, Iterable[Species]]]
+    config: OpenPMDConfig = OpenPMDConfig(file="simData")
+    _setup_dir: Path | None = None
+    # We're using a negation here because now `False` and `None` (evaluating to `False`)
+    # both mean that we can't rely on `setup_dir` being anything permanent:
+    _setup_dir_is_not_temporary: bool | None = None
+
+    def config_filename(self, content, context: Literal["runtime", "setup"]):
+        filename = f"openPMD_config_{sha256(tomli_w.dumps(content).encode()).hexdigest()}.toml"
+        if not self._setup_dir_is_not_temporary or context == "setup":
+            return self.setup_dir / "etc" / filename
+        if context == "runtime":
+            return Path("..") / "input" / "etc" / filename
+        raise ValueError(f"Unknown {context=} upon requesting the openPMD config filename.")
+
+    @property
+    def setup_dir(self):
+        if self._setup_dir_is_not_temporary is None:
+            self._setup_dir_is_not_temporary = self._setup_dir is not None
+
+        if self._setup_dir is None:
+            self._setup_dir = Path(TemporaryDirectory(delete=False).name).absolute()
+
+        return self._setup_dir
+
+    @setup_dir.setter
+    def setup_dir(self, other):
+        self._setup_dir = Path(other)
+
+    def __init__(self, sources, config):
+        self.sources = sources
+        self.config = config
+
+    def _generate_config_file(self):
+        # There's some strange interaction with the custom hashing of TimeStepSpec
+        # that's implemented on RenderedObject
+        # hindering the storage of this data structure.
+        # As a workaround, we're computing this on the fly.
+        # Shouldn't be performance critical but it would be more elegant to normalise early on.
+        sources = reduce(
+            lambda dictionary, key_val: dictionary.setdefault(to_string(key_val[0]), []).append(
+                key_val[1].get_rendering_context()["name"]
+            )
+            or dictionary,
+            self.sources,
+            {},
+        )
+        content = self.config.model_dump(mode="json", exclude_none=True) | {
+            "sink": {"dummy_application_name": {"period": sources}}
+        }
+        with self.config_filename(content, context="setup").open("wb") as file:
+            tomli_w.dump(content, file)
+        return content
+
+    def _get_serialized(self) -> dict | None:
+        content = self._generate_config_file()
+        return {"config_filename": str(self.config_filename(content, context="runtime"))}
+
+    class Config:
+        arbitrary_types_allowed = True

lib/python/picongpu/pypicongpu/runner.py

@@ -5,22 +5,22 @@
 License: GPLv3+
 """
 
-from .simulation import Simulation
-from . import util
-from .rendering import Renderer
-
-from os import path, environ, chdir
+import datetime
+import json
+import logging
+import pathlib
+import re
+import subprocess
+import tempfile
+import typing
+from os import chdir, environ, path
 from pathlib import Path
 
 import typeguard
-import tempfile
-import subprocess
-import logging
-import typing
-import re
-import datetime
-import pathlib
-import json
+
+from . import util
+from .rendering import Renderer
+from .simulation import Simulation
 
 DEFAULT_TEMPLATE_DIRECTORY = (Path(__file__).parents[4] / "share" / "picongpu" / "pypicongpu" / "template").absolute()
 
@@ -289,6 +289,8 @@ def __render_templates(self):
         Delegates work to Renderer(), see there for details.
         """
         logging.info("rendering templates...")
+        # This is kind of a dirty hack:
+        self.sim.spread_directory_information(self.setup_dir)
         # check 1 (implicit): according to schema?
         context = self.sim.get_rendering_context()
         # check 2: structure suitable for renderer?