From 69f9a42674c7b10606a062bb901b9b5ee0c9c788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 7 Jul 2025 20:24:55 +0200 Subject: [PATCH 01/12] Add generalized handling for PETS input using a dedicated factory --- src/instamatic/_collections.py | 29 ++- src/instamatic/_typing.py | 4 + src/instamatic/processing/ImgConversion.py | 117 ++++--------- src/instamatic/processing/ImgConversionDM.py | 2 + src/instamatic/processing/ImgConversionTPX.py | 2 + .../processing/ImgConversionTVIPS.py | 2 + .../processing/PETS_input_factory.py | 165 ++++++++++++++++++ .../processing/PETS_input_keywords.csv | 74 ++++++++ 8 files changed, 309 insertions(+), 86 deletions(-) create mode 100644 src/instamatic/processing/PETS_input_factory.py create mode 100644 src/instamatic/processing/PETS_input_keywords.csv diff --git a/src/instamatic/_collections.py b/src/instamatic/_collections.py index 1af6992a..78d0e4f5 100644 --- a/src/instamatic/_collections.py +++ b/src/instamatic/_collections.py @@ -1,7 +1,8 @@ from __future__ import annotations +import string from collections import UserDict -from typing import Any +from typing import Any, Tuple class NoOverwriteDict(UserDict): @@ -11,3 +12,29 @@ def __setitem__(self, key: Any, value: Any) -> None: if key in self.data: raise KeyError(f'Key "{key}" already exists and cannot be overwritten.') super().__setitem__(key, value) + + +class PartialFormatter(string.Formatter): + """`str.format` alternative, allows for partial replacement of {fields}""" + + def __init__(self, missing: str = '{{{}}}') -> None: + super().__init__() + self.missing: str = missing # used instead of missing values + + def get_field(self, field_name: str, args, kwargs) -> Tuple[Any, str]: + """When field can't be found, return placeholder text instead.""" + try: + obj, used_key = super().get_field(field_name, args, kwargs) + return obj, used_key + except (KeyError, AttributeError, IndexError, TypeError): + return self.missing.format(field_name), field_name + + def format_field(self, value: Any, format_spec: str) -> str: + """If the field was not found, format placeholder as string instead.""" + try: + return super().format_field(value, format_spec) + except (ValueError, TypeError): + return str(value) + + +partial_formatter = PartialFormatter() diff --git a/src/instamatic/_typing.py b/src/instamatic/_typing.py index 28872732..84e8e355 100644 --- a/src/instamatic/_typing.py +++ b/src/instamatic/_typing.py @@ -1,6 +1,10 @@ from __future__ import annotations +import os +from typing import Union + from typing_extensions import Annotated +AnyPath = Union[str, bytes, os.PathLike] int_nm = Annotated[int, 'Length expressed in nanometers'] float_deg = Annotated[float, 'Angle expressed in degrees'] diff --git a/src/instamatic/processing/ImgConversion.py b/src/instamatic/processing/ImgConversion.py index 640a666d..9297555b 100644 --- a/src/instamatic/processing/ImgConversion.py +++ b/src/instamatic/processing/ImgConversion.py @@ -4,13 +4,15 @@ import logging import time from datetime import datetime -from math import cos +from pathlib import Path import numpy as np from instamatic import config +from instamatic._typing import AnyPath from instamatic.formats import read_tiff, write_adsc, write_mrc, write_tiff from instamatic.processing.flatfield import apply_flatfield_correction +from instamatic.processing.PETS_input_factory import PetsInputFactory from instamatic.processing.stretch_correction import affine_transform_ellipse_to_circle from instamatic.tools import ( find_beam_center, @@ -131,6 +133,7 @@ def __init__( rotation_axis: float, # radians, specifies the position of the rotation axis acquisition_time: float, # seconds, acquisition time (exposure time + overhead) flatfield: str = 'flatfield.tiff', + method: str = 'continuous-rotation 3D ED', # or 'stills' or 'precession', used for CIF/documentation ): if flatfield is not None: flatfield, h = read_tiff(flatfield) @@ -188,6 +191,7 @@ def __init__( # self.rotation_speed = get_calibrated_rotation_speed(osc_angle / self.acquisition_time) self.name = 'Instamatic' + self.method = method from .XDS_template import ( XDS_template, # hook XDS_template here, because it is difficult to override as a global @@ -644,93 +648,36 @@ def write_beam_centers(self, path: str) -> None: np.savetxt(path / 'beam_centers.txt', centers, fmt='%10.4f') - def write_pets_inp(self, path: str, tiff_path: str = 'tiff') -> None: - """Write PETS input file `pets.pts` in directory `path`""" - if self.start_angle > self.end_angle: - sign = -1 - else: - sign = 1 - - omega = np.degrees(self.rotation_axis) - - # for pets, 0 <= omega <= 360 - if omega < 0: - omega += 360 - elif omega > 360: - omega -= 360 - - with open(path / 'pets.pts', 'w') as f: - date = str(time.ctime()) - print( - '# PETS input file for Rotation Electron Diffraction generated by `instamatic`', - file=f, - ) - print(f'# {date}', file=f) - print('# For definitions of input parameters, see:', file=f) - print('# http://pets.fzu.cz/ ', file=f) - print('', file=f) - print(f'lambda {self.wavelength}', file=f) - print(f'Aperpixel {self.pixelsize}', file=f) - print(f'phi {float(self.osc_angle) / 2}', file=f) - print(f'omega {omega}', file=f) - print('bin 1', file=f) - print('reflectionsize 20', file=f) - print('noiseparameters 3.5 38', file=f) - print('', file=f) - # print("reconstructions", file=f) - # print("endreconstructions", file=f) - # print("", file=f) - # print("distortions", file=f) - # print("enddistortions", file=f) - # print("", file=f) - print('imagelist', file=f) - for i in self.observed_range: - fn = f'{i:05d}.tiff' - angle = self.start_angle + sign * self.osc_angle * i - print(f'{tiff_path}/{fn} {angle:10.4f} 0.00', file=f) - print('endimagelist', file=f) + def write_pets_inp(self, path: AnyPath, tiff_path: str = 'tiff') -> None: + sign = 1 if self.start_angle < self.end_angle else -1 + omega = np.degrees(self.rotation_axis) % 360 - def write_pets2_inp(self, path: str, tiff_path: str = 'tiff') -> None: - """Write PETS 2 input file `pets.pts2` in directory `path`""" - path.mkdir(exist_ok=True, parents=True) - - if self.start_angle > self.end_angle: - sign = -1 + if 'continuous' in self.method: + geometry = 'continuous' + elif 'precess' in self.method: + geometry = 'precesssion' else: - sign = 1 - - omega = np.degrees(self.rotation_axis) + geometry = 'static' + + p = PetsInputFactory() + p.add('geometry', geometry) + p.add('lambda', self.wavelength) + p.add('Aperpixel', self.pixelsize) + p.add('phi', float(self.osc_angle) / 2) + p.add('omega', omega) + p.add('bin', 1) + p.add('reflectionsize', 20) + p.add('noiseparameters', 3.5, 38) + p.add('') + + s = [] + for i in self.observed_range: + angle = self.start_angle + sign * self.osc_angle * i + s.append(f'{tiff_path}/{i:05d}.tiff {angle:10.4f} 0.00') + p.add('imagelist', *s) - with open(path / 'pets.pts2', 'w') as f: - date = str(time.ctime()) - print( - '# PETS 2 input file for Rotation Electron Diffraction generated by `instamatic`', - file=f, - ) - print(f'# {date}', file=f) - print('# For definitions of input parameters, see:', file=f) - print('# http://pets.fzu.cz/ ', file=f) - print('', file=f) - print('geometry continuous', file=f) - print(f'lambda {self.wavelength}', file=f) - print(f'Aperpixel {self.pixelsize}', file=f) - print(f'phi {float(self.osc_angle) / 2}', file=f) - print(f'omega {omega}', file=f) - print('bin 1', file=f) - print('reflectionsize 15', file=f) - print('noiseparameters 25 10', file=f) - print('i/sigma 5.00 10.00', file=f) - print('', file=f) - print('imagelist', file=f) - tiff_set = {0} - last_img = len(self.observed_range) - tiff_set.update(self.observed_range) - tiff_set.remove(last_img) - for i in tiff_set: - fn = f'{i:04d}.tiff' - angle = self.start_angle + sign * self.osc_angle * i - print(f'{tiff_path}/{fn} {angle:10.4f} 0.00', file=f) - print('endimagelist', file=f) + with open(Path(path) / 'pets.pts2new', 'w') as f: + f.write(str(p.compile(self.__dict__))) def write_REDp_shiftcorrection(self, path: str) -> None: """Write .sc (shift correction) file for REDp in directory `path`""" diff --git a/src/instamatic/processing/ImgConversionDM.py b/src/instamatic/processing/ImgConversionDM.py index c465d666..84f35cdb 100644 --- a/src/instamatic/processing/ImgConversionDM.py +++ b/src/instamatic/processing/ImgConversionDM.py @@ -24,6 +24,7 @@ def __init__( pixelsize: float = None, # p/Angstrom, size of the pixels (overrides camera_length) physical_pixelsize: float = None, # mm, physical size of the pixels (overrides camera length) wavelength: float = None, # Angstrom, relativistic wavelength of the electron beam + method: str = 'continuous-rotation 3D ED', # or 'stills' or 'precession', used for CIF/documentation ): if flatfield is not None: flatfield, h = read_tiff(flatfield) @@ -69,6 +70,7 @@ def __init__( logger.debug(f'Primary beam at: {self.mean_beam_center}') self.name = 'DigitalMicrograph' + self.method = method from .XDS_templateDM import XDS_template diff --git a/src/instamatic/processing/ImgConversionTPX.py b/src/instamatic/processing/ImgConversionTPX.py index b27c8d01..27dfe6ac 100644 --- a/src/instamatic/processing/ImgConversionTPX.py +++ b/src/instamatic/processing/ImgConversionTPX.py @@ -26,6 +26,7 @@ def __init__( wavelength: float = None, # Angstrom, relativistic wavelength of the electron beam stretch_amplitude=0.0, # Stretch correction amplitude, % stretch_azimuth=0.0, # Stretch correction azimuth, degrees + method: str = 'continuous-rotation 3D ED', # or 'stills' or 'precession', used for CIF/documentation ): if flatfield is not None: flatfield, h = read_tiff(flatfield) @@ -81,6 +82,7 @@ def __init__( logger.debug(f'Primary beam at: {self.mean_beam_center}') self.name = 'TimePix_SU' + self.method = method from .XDS_templateTPX import XDS_template diff --git a/src/instamatic/processing/ImgConversionTVIPS.py b/src/instamatic/processing/ImgConversionTVIPS.py index b894a018..c227ba1d 100644 --- a/src/instamatic/processing/ImgConversionTVIPS.py +++ b/src/instamatic/processing/ImgConversionTVIPS.py @@ -24,6 +24,7 @@ def __init__( pixelsize: float = None, # p/Angstrom, size of the pixels (overrides camera_length) physical_pixelsize: float = None, # mm, physical size of the pixels (overrides camera length) wavelength: float = None, # Angstrom, relativistic wavelength of the electron beam + method: str = 'continuous-rotation 3D ED', # or 'stills' or 'precession', used for CIF/documentation ): if flatfield is not None: flatfield, h = read_tiff(flatfield) @@ -71,6 +72,7 @@ def __init__( logger.debug(f'Primary beam at: {self.mean_beam_center}') self.name = 'TVIPS F416' + self.method = method from .XDS_templateTVIPS import XDS_template diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py new file mode 100644 index 00000000..7fe4ef02 --- /dev/null +++ b/src/instamatic/processing/PETS_input_factory.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import dataclasses +import time +from copy import deepcopy +from pathlib import Path +from string import punctuation +from textwrap import dedent +from typing import Any, Iterable, Optional, Union +from warnings import warn + +import numpy as np +import pandas as pd +from typing_extensions import Self + +from instamatic import config +from instamatic._collections import partial_formatter +from instamatic._typing import AnyPath + + +class PetsKeywords: + """Read PETS2 field metadata as read from the PETS2 manual.""" + + def __init__(self, table: pd.DataFrame) -> None: + table.fillna({'end': 'optional'}, inplace=True) + self.table = table + + @classmethod + def from_file(cls, path: AnyPath): + return cls(pd.read_csv(Path(path), index_col='field')) + + @property + def list(self) -> np.ndarray[str]: + return self.table.index.values + + def find(self, text: str) -> set[str]: + first_words = set( + line.strip().split()[0].strip(punctuation).lower() + for line in text.splitlines() + if line.strip() + ) + return first_words.intersection(self.list) + + +pets_keywords = PetsKeywords.from_file( + r'C:\Users\tchon\PycharmProjects\instamatic\src\instamatic\processing\PETS_input_keywords.csv' +) + + +@dataclasses.dataclass +class PetsInputElement: + """Store metadata for a single PETS input element.""" + + keywords: list[str] + values: list[any] + string: Optional[str] = None + + def __str__(self): + return self.string if self.string is not None else self.build_string() + + @classmethod + def from_any(cls, keyword_or_text: str, values: Optional[list[Any]] = None) -> Self: + keywords = pets_keywords.find(keyword_or_text) + if len(keywords) == 1 and values is not None: # a single keyword with some values + return cls([keywords.pop()], values) + else: # any other text block + return cls(keywords=list(keywords), values=[], string=keyword_or_text) + + def has_end(self): + return ( + len(self.keywords) == 1 + and self.keywords[0] in pets_keywords.list + and pets_keywords.table.at[self.keywords[0], 'end'] != 'false' + ) + + def build_string(self): + assert len(self.keywords) == 1 + prefix = [self.keywords[0]] + delimiter = '\n' if self.has_end() else ' ' + suffix = [f'end{self.keywords[0]}'] if self.has_end() else [] + return delimiter.join(str(s) for s in prefix + self.values + suffix) + + +AnyPetsInputElement = Union[PetsInputElement, str] + + +class PetsInputWarning(UserWarning): + pass + + +class PetsInputFactory: + """Compile a PETS / PETS2 input file while preventing duplicates. + + This class is a general replacement for a simple print-to-file mechanism + used previously. Using a list of all PETS2-viable keywords, it parses + input strings and remembers all added commands. In addition to hard-coded + parameters, it includes `config.camera.pets_prefix` (at the beginning) + and `config.camera.pets_suffix` (at the end of the file). When a duplicate + `PetsInputElement` is to be added, it is ignored and a warning is raised. + """ + + def __init__(self, elements: Optional[Iterable[AnyPetsInputElement]] = None) -> None: + """As the input is built as we add, store current string & keywords.""" + self.current_elements: list[PetsInputElement] = [] + self.current_keywords: list[str] = [] + if elements is not None: + for element in elements: + self.add(element) + + def __add__(self, other: Self) -> Self: + new = deepcopy(self) + for element in other.current_elements: + new.add(element) + return new + + def __str__(self) -> str: + return '\n'.join(str(e) for e in self.current_elements) + + def add(self, element: AnyPetsInputElement, *values: Any) -> None: + """Add a PETS kw/values, a '# comment', or a text to be parsed.""" + if values or not isinstance(element, PetsInputElement): + element = PetsInputElement.from_any(element, list(values)) + if self._no_duplicates_in(element.keywords): + self.current_elements.append(element) + + def compile(self, image_converter_attributes: dict) -> Self: + """Build a full version of PETS input with title, prefix, suffix.""" + title = dedent(f""" + # PETS input file for Electron Diffraction generated by `instamatic` + # {str(time.ctime())} + # For definitions of input parameters, see: https://pets.fzu.cz/ + """).strip() + + prefix = getattr(config.camera, 'pets_prefix', None) + if prefix is not None and image_converter_attributes is not None: + prefix = partial_formatter.format(prefix, **image_converter_attributes) + prefix = PetsInputElement.from_any(prefix) + + suffix = getattr(config.camera, 'pets_suffix', None) + if suffix is not None and image_converter_attributes is not None: + suffix = partial_formatter.format(suffix, **image_converter_attributes) + suffix = PetsInputElement.from_any(suffix) + + return self.__class__([title, prefix] + self.current_elements + [suffix]) + + def _no_duplicates_in(self, keywords: Iterable[str]) -> bool: + """Return True & add keywords if there are no duplicates; else warn.""" + no_duplicates = True + for keyword in keywords: + if keyword in self.current_keywords: + warn(f'Duplicate keyword rejected: {keyword}', PetsInputWarning) + no_duplicates = False + if no_duplicates: + self.current_keywords.extend(list(keywords)) + return no_duplicates + + +if __name__ == '__main__': + # test: find which keywords are added from prefix and suffix only + pif = PetsInputFactory().compile({}) + print('PETS DEFAULT INPUT (`config.camera.pets_prefix` + `suffix`:') + print('---') + print(str(pif), end='') + print('---') + print(f'FOUND KEYWORDS: {{{", ".join(pif.current_keywords)}}}') diff --git a/src/instamatic/processing/PETS_input_keywords.csv b/src/instamatic/processing/PETS_input_keywords.csv new file mode 100644 index 00000000..c4921b52 --- /dev/null +++ b/src/instamatic/processing/PETS_input_keywords.csv @@ -0,0 +1,74 @@ +field,end +autotask,true +keepautotasks,false +lambda,false +aperpixel,false +geometry,false +detector,false +noiseparameters,false +phi,false +omega,false +delta,false +pixelsize,false +bin,false +reflectionsize,false +dstarmax,false +dstarmaxps,false +dstarmin,false +centerradius,false +beamstop,optional +badpixels,true +avoidicerings,false +icerings,true +peaksearchmode,false +center,false +centermode,false +i/sigma,false +mask,true +moreaveprofiles,false +peakprofileparams,false +peakprofilesectors,false +background,false +backgroundmethod,false +peakanalysis,false +minclusterpoints,false +indexingmode,false +indexingparameters,false +maxindex,false +cellrefinemode,false +cellrefineparameters,false +referencecell,false +intensitymethod,false +adjustreflbox,false +resshellfraction,false +saturationlimit,false +skipsaturated,false +minrotaxisdist,false +minreflpartiality,false +rcshape,false +integrationmode,false +integrationparams,false +intkinematical,false +intdynamical,false +dynamicalscales,false +dynamicalerrormodel,false +errormodel,false +outliers,false +refinecamelparams,false +orientationparams,false +simulationpower,false +interpolationparams,false +distcenterasoffset,false +distortunits,false +distortions,true +distortionskeys,true +mapformat,false +reconstruction,true +reconstructionparams,false +removebackground,false +serialed,false +virtualframes,false +cifentries,true +imagelist,true +celllist,true +cellitem,true From ced02313236ea55fdc54a89872c46c3206602d62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 7 Jul 2025 20:54:20 +0200 Subject: [PATCH 02/12] Since we need to support python 3.7 and 3.8, move csv file inside py --- .../processing/PETS_input_factory.py | 88 ++++++++++++++++++- .../processing/PETS_input_keywords.csv | 74 ---------------- 2 files changed, 84 insertions(+), 78 deletions(-) delete mode 100644 src/instamatic/processing/PETS_input_keywords.csv diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py index 7fe4ef02..256f8518 100644 --- a/src/instamatic/processing/PETS_input_factory.py +++ b/src/instamatic/processing/PETS_input_factory.py @@ -3,6 +3,7 @@ import dataclasses import time from copy import deepcopy +from io import StringIO from pathlib import Path from string import punctuation from textwrap import dedent @@ -17,6 +18,83 @@ from instamatic._collections import partial_formatter from instamatic._typing import AnyPath +pets_input_keywords_csv = """ +field,end +autotask,true +keepautotasks,false +lambda,false +aperpixel,false +geometry,false +detector,false +noiseparameters,false +phi,false +omega,false +delta,false +pixelsize,false +bin,false +reflectionsize,false +dstarmax,false +dstarmaxps,false +dstarmin,false +centerradius,false +beamstop,optional +badpixels,true +avoidicerings,false +icerings,true +peaksearchmode,false +center,false +centermode,false +i/sigma,false +mask,true +moreaveprofiles,false +peakprofileparams,false +peakprofilesectors,false +background,false +backgroundmethod,false +peakanalysis,false +minclusterpoints,false +indexingmode,false +indexingparameters,false +maxindex,false +cellrefinemode,false +cellrefineparameters,false +referencecell,false +intensitymethod,false +adjustreflbox,false +resshellfraction,false +saturationlimit,false +skipsaturated,false +minrotaxisdist,false +minreflpartiality,false +rcshape,false +integrationmode,false +integrationparams,false +intkinematical,false +intdynamical,false +dynamicalscales,false +dynamicalerrormodel,false +errormodel,false +outliers,false +refinecamelparams,false +orientationparams,false +simulationpower,false +interpolationparams,false +distcenterasoffset,false +distortunits,false +distortions,true +distortionskeys,true +mapformat,false +reconstruction,true +reconstructionparams,false +removebackground,false +serialed,false +virtualframes,false +cifentries,true +imagelist,true +celllist,true +cellitem,true +""" + class PetsKeywords: """Read PETS2 field metadata as read from the PETS2 manual.""" @@ -26,7 +104,11 @@ def __init__(self, table: pd.DataFrame) -> None: self.table = table @classmethod - def from_file(cls, path: AnyPath): + def from_string(cls, string: str) -> Self: + return cls(pd.read_csv(StringIO(string), index_col='field')) + + @classmethod + def from_file(cls, path: AnyPath) -> Self: return cls(pd.read_csv(Path(path), index_col='field')) @property @@ -42,9 +124,7 @@ def find(self, text: str) -> set[str]: return first_words.intersection(self.list) -pets_keywords = PetsKeywords.from_file( - r'C:\Users\tchon\PycharmProjects\instamatic\src\instamatic\processing\PETS_input_keywords.csv' -) +pets_keywords = PetsKeywords.from_string(pets_input_keywords_csv) @dataclasses.dataclass diff --git a/src/instamatic/processing/PETS_input_keywords.csv b/src/instamatic/processing/PETS_input_keywords.csv deleted file mode 100644 index c4921b52..00000000 --- a/src/instamatic/processing/PETS_input_keywords.csv +++ /dev/null @@ -1,74 +0,0 @@ -field,end -autotask,true -keepautotasks,false -lambda,false -aperpixel,false -geometry,false -detector,false -noiseparameters,false -phi,false -omega,false -delta,false -pixelsize,false -bin,false -reflectionsize,false -dstarmax,false -dstarmaxps,false -dstarmin,false -centerradius,false -beamstop,optional -badpixels,true -avoidicerings,false -icerings,true -peaksearchmode,false -center,false -centermode,false -i/sigma,false -mask,true -moreaveprofiles,false -peakprofileparams,false -peakprofilesectors,false -background,false -backgroundmethod,false -peakanalysis,false -minclusterpoints,false -indexingmode,false -indexingparameters,false -maxindex,false -cellrefinemode,false -cellrefineparameters,false -referencecell,false -intensitymethod,false -adjustreflbox,false -resshellfraction,false -saturationlimit,false -skipsaturated,false -minrotaxisdist,false -minreflpartiality,false -rcshape,false -integrationmode,false -integrationparams,false -intkinematical,false -intdynamical,false -dynamicalscales,false -dynamicalerrormodel,false -errormodel,false -outliers,false -refinecamelparams,false -orientationparams,false -simulationpower,false -interpolationparams,false -distcenterasoffset,false -distortunits,false -distortions,true -distortionskeys,true -mapformat,false -reconstruction,true -reconstructionparams,false -removebackground,false -serialed,false -virtualframes,false -cifentries,true -imagelist,true -celllist,true -cellitem,true From 2c3af427e0eef2c6931a849b528621f762315efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 7 Jul 2025 21:03:49 +0200 Subject: [PATCH 03/12] Fix name of produced pets file to `pets.pts` --- src/instamatic/processing/ImgConversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instamatic/processing/ImgConversion.py b/src/instamatic/processing/ImgConversion.py index 9297555b..1c32530e 100644 --- a/src/instamatic/processing/ImgConversion.py +++ b/src/instamatic/processing/ImgConversion.py @@ -676,7 +676,7 @@ def write_pets_inp(self, path: AnyPath, tiff_path: str = 'tiff') -> None: s.append(f'{tiff_path}/{i:05d}.tiff {angle:10.4f} 0.00') p.add('imagelist', *s) - with open(Path(path) / 'pets.pts2new', 'w') as f: + with open(Path(path) / 'pets.pts', 'w') as f: f.write(str(p.compile(self.__dict__))) def write_REDp_shiftcorrection(self, path: str) -> None: From d05f4dac0b71e5668bb2d82d69409407cafbaa86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 7 Jul 2025 21:45:00 +0200 Subject: [PATCH 04/12] Fix creating `pets.pts` if prefix or suffix are not defined --- .../processing/PETS_input_factory.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py index 256f8518..9cfc2e78 100644 --- a/src/instamatic/processing/PETS_input_factory.py +++ b/src/instamatic/processing/PETS_input_factory.py @@ -210,18 +210,23 @@ def compile(self, image_converter_attributes: dict) -> Self: # {str(time.ctime())} # For definitions of input parameters, see: https://pets.fzu.cz/ """).strip() + pets_element_list = [title] prefix = getattr(config.camera, 'pets_prefix', None) - if prefix is not None and image_converter_attributes is not None: - prefix = partial_formatter.format(prefix, **image_converter_attributes) - prefix = PetsInputElement.from_any(prefix) + if prefix is not None: + if image_converter_attributes is not None: + prefix = partial_formatter.format(prefix, **image_converter_attributes) + pets_element_list.append(PetsInputElement.from_any(prefix)) + + pets_element_list.extend(self.current_elements) suffix = getattr(config.camera, 'pets_suffix', None) - if suffix is not None and image_converter_attributes is not None: - suffix = partial_formatter.format(suffix, **image_converter_attributes) - suffix = PetsInputElement.from_any(suffix) + if suffix is not None: + if image_converter_attributes is not None: + suffix = partial_formatter.format(suffix, **image_converter_attributes) + pets_element_list.append(PetsInputElement.from_any(suffix)) - return self.__class__([title, prefix] + self.current_elements + [suffix]) + return self.__class__(pets_element_list) def _no_duplicates_in(self, keywords: Iterable[str]) -> bool: """Return True & add keywords if there are no duplicates; else warn.""" From f8fe078922159512ebbb3a3bc3c999128a23ca57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 8 Jul 2025 14:12:28 +0200 Subject: [PATCH 05/12] Split PETS affix to blocks pre-adding: rejects only duplicate lines, not all affix --- .../processing/PETS_input_factory.py | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py index 9cfc2e78..93a68b55 100644 --- a/src/instamatic/processing/PETS_input_factory.py +++ b/src/instamatic/processing/PETS_input_factory.py @@ -1,22 +1,18 @@ from __future__ import annotations -import dataclasses -import time from copy import deepcopy +from dataclasses import dataclass from io import StringIO -from pathlib import Path -from string import punctuation from textwrap import dedent +from time import ctime from typing import Any, Iterable, Optional, Union from warnings import warn -import numpy as np import pandas as pd from typing_extensions import Self from instamatic import config from instamatic._collections import partial_formatter -from instamatic._typing import AnyPath pets_input_keywords_csv = """ field,end @@ -107,27 +103,19 @@ def __init__(self, table: pd.DataFrame) -> None: def from_string(cls, string: str) -> Self: return cls(pd.read_csv(StringIO(string), index_col='field')) - @classmethod - def from_file(cls, path: AnyPath) -> Self: - return cls(pd.read_csv(Path(path), index_col='field')) - @property - def list(self) -> np.ndarray[str]: - return self.table.index.values + def list(self) -> list[str]: + return self.table.index.values.tolist() def find(self, text: str) -> set[str]: - first_words = set( - line.strip().split()[0].strip(punctuation).lower() - for line in text.splitlines() - if line.strip() - ) - return first_words.intersection(self.list) + firsts = {t.strip().split()[0].lower() for t in text.splitlines() if t.strip()} + return firsts.intersection(self.list) pets_keywords = PetsKeywords.from_string(pets_input_keywords_csv) -@dataclasses.dataclass +@dataclass class PetsInputElement: """Store metadata for a single PETS input element.""" @@ -146,6 +134,29 @@ def from_any(cls, keyword_or_text: str, values: Optional[list[Any]] = None) -> S else: # any other text block return cls(keywords=list(keywords), values=[], string=keyword_or_text) + @classmethod + def list_from_text(cls, text: str) -> list[Self]: + """Split a text with many commands it into `list[PetsInputElement]`""" + lines = text.splitlines() + firsts = [t.strip().split()[0].lower() if t.strip() else '' for t in lines] + + def split2blocks(start: int) -> list[str]: + if start >= len(lines): + return [] + if firsts[start] not in pets_keywords.list: + return [lines[start]] + split2blocks(start + 1) + if pets_keywords.table.at[firsts[start], 'end'] == 'false': + return [lines[start]] + split2blocks(start + 1) + try: + end = firsts.index('end' + firsts[start], start) + except ValueError: + if pets_keywords.table.at[firsts[start], 'end'] == 'optional': + return [lines[start]] + split2blocks(start + 1) + end = len(lines) + return ['\n'.join(lines[start : end + 1])] + split2blocks(end + 1) + + return [PetsInputElement.from_any(block) for block in split2blocks(0)] + def has_end(self): return ( len(self.keywords) == 1 @@ -154,7 +165,8 @@ def has_end(self): ) def build_string(self): - assert len(self.keywords) == 1 + if len(self.keywords) > 1: + warn(f'Building `str(PetsInputElement)` with >1 keywords! {self.keywords}') prefix = [self.keywords[0]] delimiter = '\n' if self.has_end() else ' ' suffix = [f'end{self.keywords[0]}'] if self.has_end() else [] @@ -207,7 +219,7 @@ def compile(self, image_converter_attributes: dict) -> Self: """Build a full version of PETS input with title, prefix, suffix.""" title = dedent(f""" # PETS input file for Electron Diffraction generated by `instamatic` - # {str(time.ctime())} + # {str(ctime())} # For definitions of input parameters, see: https://pets.fzu.cz/ """).strip() pets_element_list = [title] @@ -216,7 +228,7 @@ def compile(self, image_converter_attributes: dict) -> Self: if prefix is not None: if image_converter_attributes is not None: prefix = partial_formatter.format(prefix, **image_converter_attributes) - pets_element_list.append(PetsInputElement.from_any(prefix)) + pets_element_list.extend(PetsInputElement.list_from_text(prefix)) pets_element_list.extend(self.current_elements) @@ -224,7 +236,7 @@ def compile(self, image_converter_attributes: dict) -> Self: if suffix is not None: if image_converter_attributes is not None: suffix = partial_formatter.format(suffix, **image_converter_attributes) - pets_element_list.append(PetsInputElement.from_any(suffix)) + pets_element_list.extend(PetsInputElement.list_from_text(suffix)) return self.__class__(pets_element_list) @@ -245,6 +257,6 @@ def _no_duplicates_in(self, keywords: Iterable[str]) -> bool: pif = PetsInputFactory().compile({}) print('PETS DEFAULT INPUT (`config.camera.pets_prefix` + `suffix`:') print('---') - print(str(pif), end='') + print(str(pif)) print('---') print(f'FOUND KEYWORDS: {{{", ".join(pif.current_keywords)}}}') From 9e9c0e42df8e5a73bcee346a88ca6dfeec7f0d31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 8 Jul 2025 20:53:32 +0200 Subject: [PATCH 06/12] Add doc, tests, move csv to resources, bump min Python version to 3.9 --- .github/workflows/test.yml | 2 +- MANIFEST.in | 1 + docs/config.md | 36 +++++ environment.yml | 2 +- pyproject.toml | 18 ++- readme.md | 2 +- .../processing/PETS_input_factory.py | 131 +++++------------- .../processing/PETS_input_keywords.csv | 74 ++++++++++ tests/test_pets_input.py | 69 +++++++++ 9 files changed, 229 insertions(+), 106 deletions(-) create mode 100644 src/instamatic/processing/PETS_input_keywords.csv create mode 100644 tests/test_pets_input.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b5abadd7..90ba7741 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.x'] steps: - uses: actions/checkout@v3 diff --git a/MANIFEST.in b/MANIFEST.in index 9f4fd525..08885fa0 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -27,3 +27,4 @@ include src/instamatic/config/camera/*.yaml include src/instamatic/config/microscope/*.yaml include src/instamatic/config/scripts/*.md include src/instamatic/neural_network/*.p +include src/instamatic/processing/PETS_input_keywords.csv diff --git a/docs/config.md b/docs/config.md index 262afddb..1d6bb857 100644 --- a/docs/config.md +++ b/docs/config.md @@ -249,6 +249,42 @@ This file holds the specifications of the camera. This file is must be located t DiffShift: {gridsize: 5, stepsize: 300} ``` +**pets_prefix** +: Arbitrary information to be added at the beginning of the `.pts` file created after an experiment. The prefix can include any [valid PETS2 input lines](http://pets.fzu.cz/download/pets2_manual.pdf). In the case of duplicate commands, prefix lines take precedence over hard-coded and suffix commands, and prevent the latter ones from being added. Additionally, this field can contain new python-style [replacement fields](https://pyformat.info/) which, if present among the `ImgConversion` instance attributes, will be filled automatically after each experiment (see the `pets_suffix` example). A typical `pets_prefix`, capable of overwriting the default detector specification output can look like this: +```yaml +pets_prefix: "noiseparameters 4.2 0\nreflectionsize 8\ndetector asi" +``` + +**pets_suffix** +: Arbitrary information to be added at the end of the `.pts` file created after an experiment. Similarly to the `pets_prefix`, the suffix can include any [valid PETS2 input lines](http://pets.fzu.cz/download/pets2_manual.pdf) as well as new python-style [replacement fields](https://pyformat.info/). In contrast to prefix, any duplicate commands added to suffix will be ignored. This field can be useful to add backup or meta information about the experiment: +```yaml +pets_suffix: | + cifentries + _exptl_special_details + ; + {method} data collected using Instamatic. + Tilt step: {osc_angle:.3f} deg + Exposure: {headers[0][ImageExposureTime]:.6f} s per frame + ; + _diffrn_ambient_temperature ? + _diffrn_source 'Lanthanum hexaboride cathode' + _diffrn_source_voltage 200 + _diffrn_radiation_type electron + _diffrn_radiation_wavelength 0.0251 + _diffrn_measurement_device 'Transmission electron microscope' + _diffrn_measurement_device_type 'FEI Tecnai G2 20' + _diffrn_detector 'ASI Cheetah' + _diffrn_measurement_method '{method}' + _diffrn_measurement_specimen_support 'Cu grid with amorphous carbon foil' + _diffrn_standards_number 0 + endcifentries + + badpixels + 359 32 + 279 513 + endbadpixels +``` + ## microscope.yaml This file holds all the specifications of the microscope as necessary. It is important to set up the camera lengths, magnifications, and magnification modes. This file is must be located the `microscope/camera` directory, and can have any name as defined in `settings.yaml`. diff --git a/environment.yml b/environment.yml index 045ed1fa..15c48193 100644 --- a/environment.yml +++ b/environment.yml @@ -4,4 +4,4 @@ channels: - conda-forge - defaults dependencies: - - python==3.7 + - python==3.9 diff --git a/pyproject.toml b/pyproject.toml index 9f9b6906..59488d17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ name = "instamatic" version = "2.1.1" description = "Python program for automated electron diffraction data collection" readme = "readme.md" -requires-python = ">=3.7" +requires-python = ">=3.9" authors = [ {name = "Stef Smeets", email = "s.smeets@esciencecenter.nl"}, ] @@ -24,11 +24,11 @@ keywords = [ ] license = {text = "BSD License"} classifiers = [ - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "License :: OSI Approved :: BSD License", @@ -123,8 +123,18 @@ publishing = [ # setup "instamatic.autoconfig" = "instamatic.config.autoconfig:main" +[tool.setuptools] +packages = ["instamatic"] +package-dir = {"" = "src"} +include-package-data = true + +[tool.setuptools.package-data] +instamatic = [ + "processing/PETS_input_keywords.csv" +] + [tool.ruff] -target-version = 'py37' +target-version = 'py39' line-length = 96 [tool.ruff.lint] diff --git a/readme.md b/readme.md index 4958c3f8..96316367 100644 --- a/readme.md +++ b/readme.md @@ -39,7 +39,7 @@ conda create -n instamatic python=3.11 conda activate instamatic ``` -Install using pip, works with python versions 3.7 or newer: +Install using pip, works with python versions 3.9 or newer: ```bash pip install instamatic diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py index 93a68b55..e3e92c4f 100644 --- a/src/instamatic/processing/PETS_input_factory.py +++ b/src/instamatic/processing/PETS_input_factory.py @@ -2,7 +2,7 @@ from copy import deepcopy from dataclasses import dataclass -from io import StringIO +from importlib import resources from textwrap import dedent from time import ctime from typing import Any, Iterable, Optional, Union @@ -13,83 +13,7 @@ from instamatic import config from instamatic._collections import partial_formatter - -pets_input_keywords_csv = """ -field,end -autotask,true -keepautotasks,false -lambda,false -aperpixel,false -geometry,false -detector,false -noiseparameters,false -phi,false -omega,false -delta,false -pixelsize,false -bin,false -reflectionsize,false -dstarmax,false -dstarmaxps,false -dstarmin,false -centerradius,false -beamstop,optional -badpixels,true -avoidicerings,false -icerings,true -peaksearchmode,false -center,false -centermode,false -i/sigma,false -mask,true -moreaveprofiles,false -peakprofileparams,false -peakprofilesectors,false -background,false -backgroundmethod,false -peakanalysis,false -minclusterpoints,false -indexingmode,false -indexingparameters,false -maxindex,false -cellrefinemode,false -cellrefineparameters,false -referencecell,false -intensitymethod,false -adjustreflbox,false -resshellfraction,false -saturationlimit,false -skipsaturated,false -minrotaxisdist,false -minreflpartiality,false -rcshape,false -integrationmode,false -integrationparams,false -intkinematical,false -intdynamical,false -dynamicalscales,false -dynamicalerrormodel,false -errormodel,false -outliers,false -refinecamelparams,false -orientationparams,false -simulationpower,false -interpolationparams,false -distcenterasoffset,false -distortunits,false -distortions,true -distortionskeys,true -mapformat,false -reconstruction,true -reconstructionparams,false -removebackground,false -serialed,false -virtualframes,false -cifentries,true -imagelist,true -celllist,true -cellitem,true -""" +from instamatic._typing import AnyPath class PetsKeywords: @@ -100,8 +24,8 @@ def __init__(self, table: pd.DataFrame) -> None: self.table = table @classmethod - def from_string(cls, string: str) -> Self: - return cls(pd.read_csv(StringIO(string), index_col='field')) + def from_file(cls, path: AnyPath) -> Self: + return cls(pd.read_csv(path, index_col='field')) @property def list(self) -> list[str]: @@ -112,7 +36,8 @@ def find(self, text: str) -> set[str]: return firsts.intersection(self.list) -pets_keywords = PetsKeywords.from_string(pets_input_keywords_csv) +with resources.files('instamatic.processing').joinpath('PETS_input_keywords.csv').open() as f: + pets_keywords = PetsKeywords.from_file(f) # noqa: default keywords read at import @dataclass @@ -120,7 +45,7 @@ class PetsInputElement: """Store metadata for a single PETS input element.""" keywords: list[str] - values: list[any] + values: list[Any] string: Optional[str] = None def __str__(self): @@ -129,16 +54,15 @@ def __str__(self): @classmethod def from_any(cls, keyword_or_text: str, values: Optional[list[Any]] = None) -> Self: keywords = pets_keywords.find(keyword_or_text) - if len(keywords) == 1 and values is not None: # a single keyword with some values + if len(keywords) == 1 and values is not None: return cls([keywords.pop()], values) - else: # any other text block - return cls(keywords=list(keywords), values=[], string=keyword_or_text) + return cls(keywords=list(keywords), values=[], string=keyword_or_text) @classmethod def list_from_text(cls, text: str) -> list[Self]: """Split a text with many commands it into `list[PetsInputElement]`""" lines = text.splitlines() - firsts = [t.strip().split()[0].lower() if t.strip() else '' for t in lines] + firsts = [ts.split()[0].lower() if (ts := t.strip()) else '' for t in lines] def split2blocks(start: int) -> list[str]: if start >= len(lines): @@ -168,8 +92,8 @@ def build_string(self): if len(self.keywords) > 1: warn(f'Building `str(PetsInputElement)` with >1 keywords! {self.keywords}') prefix = [self.keywords[0]] - delimiter = '\n' if self.has_end() else ' ' - suffix = [f'end{self.keywords[0]}'] if self.has_end() else [] + delimiter = '\n' if (has_end := self.has_end()) else ' ' + suffix = [f'end{self.keywords[0]}'] if has_end else [] return delimiter.join(str(s) for s in prefix + self.values + suffix) @@ -191,6 +115,22 @@ class PetsInputFactory: `PetsInputElement` is to be added, it is ignored and a warning is raised. """ + @classmethod + def get_prefix(cls) -> Union[str, None]: + return getattr(config.camera, 'pets_prefix', None) + + @classmethod + def get_suffix(cls) -> Union[str, None]: + return getattr(config.camera, 'pets_suffix', None) + + @classmethod + def get_title(cls) -> str: + return dedent(f""" + # PETS input file for Electron Diffraction generated by `instamatic` + # {str(ctime())} + # For definitions of input parameters, see: https://pets.fzu.cz/ + """).strip() + def __init__(self, elements: Optional[Iterable[AnyPetsInputElement]] = None) -> None: """As the input is built as we add, store current string & keywords.""" self.current_elements: list[PetsInputElement] = [] @@ -217,23 +157,16 @@ def add(self, element: AnyPetsInputElement, *values: Any) -> None: def compile(self, image_converter_attributes: dict) -> Self: """Build a full version of PETS input with title, prefix, suffix.""" - title = dedent(f""" - # PETS input file for Electron Diffraction generated by `instamatic` - # {str(ctime())} - # For definitions of input parameters, see: https://pets.fzu.cz/ - """).strip() - pets_element_list = [title] + pets_element_list = [self.get_title()] - prefix = getattr(config.camera, 'pets_prefix', None) - if prefix is not None: + if (prefix := self.get_prefix()) is not None: if image_converter_attributes is not None: prefix = partial_formatter.format(prefix, **image_converter_attributes) pets_element_list.extend(PetsInputElement.list_from_text(prefix)) pets_element_list.extend(self.current_elements) - suffix = getattr(config.camera, 'pets_suffix', None) - if suffix is not None: + if (suffix := self.get_suffix()) is not None: if image_converter_attributes is not None: suffix = partial_formatter.format(suffix, **image_converter_attributes) pets_element_list.extend(PetsInputElement.list_from_text(suffix)) @@ -253,7 +186,7 @@ def _no_duplicates_in(self, keywords: Iterable[str]) -> bool: if __name__ == '__main__': - # test: find which keywords are added from prefix and suffix only + # check pets input added from config.camera.pets_prefix and _suffix only pif = PetsInputFactory().compile({}) print('PETS DEFAULT INPUT (`config.camera.pets_prefix` + `suffix`:') print('---') diff --git a/src/instamatic/processing/PETS_input_keywords.csv b/src/instamatic/processing/PETS_input_keywords.csv new file mode 100644 index 00000000..c4921b52 --- /dev/null +++ b/src/instamatic/processing/PETS_input_keywords.csv @@ -0,0 +1,74 @@ +field,end +autotask,true +keepautotasks,false +lambda,false +aperpixel,false +geometry,false +detector,false +noiseparameters,false +phi,false +omega,false +delta,false +pixelsize,false +bin,false +reflectionsize,false +dstarmax,false +dstarmaxps,false +dstarmin,false +centerradius,false +beamstop,optional +badpixels,true +avoidicerings,false +icerings,true +peaksearchmode,false +center,false +centermode,false +i/sigma,false +mask,true +moreaveprofiles,false +peakprofileparams,false +peakprofilesectors,false +background,false +backgroundmethod,false +peakanalysis,false +minclusterpoints,false +indexingmode,false +indexingparameters,false +maxindex,false +cellrefinemode,false +cellrefineparameters,false +referencecell,false +intensitymethod,false +adjustreflbox,false +resshellfraction,false +saturationlimit,false +skipsaturated,false +minrotaxisdist,false +minreflpartiality,false +rcshape,false +integrationmode,false +integrationparams,false +intkinematical,false +intdynamical,false +dynamicalscales,false +dynamicalerrormodel,false +errormodel,false +outliers,false +refinecamelparams,false +orientationparams,false +simulationpower,false +interpolationparams,false +distcenterasoffset,false +distortunits,false +distortions,true +distortionskeys,true +mapformat,false +reconstruction,true +reconstructionparams,false +removebackground,false +serialed,false +virtualframes,false +cifentries,true +imagelist,true +celllist,true +cellitem,true diff --git a/tests/test_pets_input.py b/tests/test_pets_input.py new file mode 100644 index 00000000..30543df4 --- /dev/null +++ b/tests/test_pets_input.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +from contextlib import nullcontext +from dataclasses import dataclass +from typing import Optional + +import pytest + +from instamatic.processing.PETS_input_factory import PetsInputFactory, PetsInputWarning + + +@dataclass +class PetsInputTestInfixCase: + prefix: Optional[str] + suffix: Optional[str] + result: str + warnings: tuple[type[UserWarning]] = () + + +case0 = PetsInputTestInfixCase( # tests if factory works in general + prefix=None, + suffix=None, + result='# Title\ndetector asi\nreflectionsize 20', +) + +case1 = PetsInputTestInfixCase( # tests if prefix overwrites commands overwrites suffix + prefix='detector default', + suffix='reflectionsize 8', + result='# Title\ndetector default\nreflectionsize 20', + warnings=(PetsInputWarning,), +) + +case2 = PetsInputTestInfixCase( # tests if {format} fields are partially substituted + prefix='detector {detector}\nreflectionsize {reflectionsize}', + suffix=None, + result='# Title\ndetector {detector}\nreflectionsize 9', + warnings=(PetsInputWarning,), +) + +case3 = PetsInputTestInfixCase( # tests if partially-duplicate suffix is partially removed + prefix=None, + suffix='reflectionsize 8\nnoiseparameters 3.5 38', + result='# Title\ndetector asi\nreflectionsize 20\nnoiseparameters 3.5 38', + warnings=(PetsInputWarning,), +) + +case4 = PetsInputTestInfixCase( # tests the consistency of comment and empty line behavior + prefix='# Prefix1\n\n# Prefix3\n\n', # trailing \n cut + suffix='', # empty suffix ignored, so \n is not added + result='# Title\n# Prefix1\n\n# Prefix3\n\ndetector asi\nreflectionsize 20', +) + + +@pytest.mark.parametrize('infix_case', [case0, case1, case2, case3, case4]) +def test_pets_input(infix_case): + pif = PetsInputFactory() + + # monkey patch PetsInputFactory class methods for the purpose of testing + pif.get_title = lambda: '# Title' + if (infix_case_prefix := infix_case.prefix) is not None: + pif.get_prefix = lambda: infix_case_prefix + if (infix_case_suffix := infix_case.suffix) is not None: + pif.get_suffix = lambda: infix_case_suffix + + pif.add('detector', 'asi') + pif.add('reflectionsize', 20) + with pytest.warns(w) if (w := infix_case.warnings) else nullcontext(): + pif_compiled = pif.compile(dict(reflectionsize=9)) + assert str(pif_compiled) == infix_case.result From 5e2ff6c23903620ef9430dcb202597f0c6658f10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 8 Jul 2025 21:05:32 +0200 Subject: [PATCH 07/12] GitHub tests fail for Python 3.13, may be not supported by some libraries yet --- .github/workflows/test.yml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90ba7741..8c6428d8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.x'] + python-version: ['3.9', '3.10', '3.11', '3.12', ] steps: - uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index 59488d17..458c93b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,6 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "License :: OSI Approved :: BSD License", From be9fab41939821c53e7f41a15f07837091880ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 15 Jul 2025 15:56:41 +0200 Subject: [PATCH 08/12] Remove `[tool.setuptools.package-data]` from `pyproject.toml` Co-authored-by: Stef Smeets --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 458c93b7..94789eed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,11 +127,6 @@ packages = ["instamatic"] package-dir = {"" = "src"} include-package-data = true -[tool.setuptools.package-data] -instamatic = [ - "processing/PETS_input_keywords.csv" -] - [tool.ruff] target-version = 'py39' line-length = 96 From c12ac7a0575e333ede4d58d371903994816e170e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 15 Jul 2025 15:59:46 +0200 Subject: [PATCH 09/12] Fix typo in src/instamatic/processing/ImgConversion.py Co-authored-by: Stef Smeets --- src/instamatic/processing/ImgConversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/instamatic/processing/ImgConversion.py b/src/instamatic/processing/ImgConversion.py index 1c32530e..c1c33ef6 100644 --- a/src/instamatic/processing/ImgConversion.py +++ b/src/instamatic/processing/ImgConversion.py @@ -655,7 +655,7 @@ def write_pets_inp(self, path: AnyPath, tiff_path: str = 'tiff') -> None: if 'continuous' in self.method: geometry = 'continuous' elif 'precess' in self.method: - geometry = 'precesssion' + geometry = 'precession' else: geometry = 'static' From 024124d946f74220e54ad404eb3c74b914b281e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 15 Jul 2025 16:04:57 +0200 Subject: [PATCH 10/12] Move `PetsInputWarning` to the top of the `PETS_input_factory.py` file --- src/instamatic/processing/PETS_input_factory.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py index e3e92c4f..bf236e3c 100644 --- a/src/instamatic/processing/PETS_input_factory.py +++ b/src/instamatic/processing/PETS_input_factory.py @@ -16,6 +16,10 @@ from instamatic._typing import AnyPath +class PetsInputWarning(UserWarning): + pass + + class PetsKeywords: """Read PETS2 field metadata as read from the PETS2 manual.""" @@ -100,10 +104,6 @@ def build_string(self): AnyPetsInputElement = Union[PetsInputElement, str] -class PetsInputWarning(UserWarning): - pass - - class PetsInputFactory: """Compile a PETS / PETS2 input file while preventing duplicates. From 413b5022a9ed00fec82ce6850109d2611e8f428f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 15 Jul 2025 16:15:45 +0200 Subject: [PATCH 11/12] Make `PETS_KEYWORDS` uppercase, fix `PetsKeywords.from_file` type hint --- .../processing/PETS_input_factory.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/instamatic/processing/PETS_input_factory.py b/src/instamatic/processing/PETS_input_factory.py index bf236e3c..90ea728e 100644 --- a/src/instamatic/processing/PETS_input_factory.py +++ b/src/instamatic/processing/PETS_input_factory.py @@ -5,7 +5,7 @@ from importlib import resources from textwrap import dedent from time import ctime -from typing import Any, Iterable, Optional, Union +from typing import IO, Any, Iterable, Optional, Union from warnings import warn import pandas as pd @@ -28,8 +28,8 @@ def __init__(self, table: pd.DataFrame) -> None: self.table = table @classmethod - def from_file(cls, path: AnyPath) -> Self: - return cls(pd.read_csv(path, index_col='field')) + def from_file(cls, path_or_buffer: Union[AnyPath, IO[str]]) -> Self: + return cls(pd.read_csv(path_or_buffer, index_col='field')) @property def list(self) -> list[str]: @@ -41,7 +41,7 @@ def find(self, text: str) -> set[str]: with resources.files('instamatic.processing').joinpath('PETS_input_keywords.csv').open() as f: - pets_keywords = PetsKeywords.from_file(f) # noqa: default keywords read at import + PETS_KEYWORDS = PetsKeywords.from_file(f) @dataclass @@ -57,7 +57,7 @@ def __str__(self): @classmethod def from_any(cls, keyword_or_text: str, values: Optional[list[Any]] = None) -> Self: - keywords = pets_keywords.find(keyword_or_text) + keywords = PETS_KEYWORDS.find(keyword_or_text) if len(keywords) == 1 and values is not None: return cls([keywords.pop()], values) return cls(keywords=list(keywords), values=[], string=keyword_or_text) @@ -71,14 +71,14 @@ def list_from_text(cls, text: str) -> list[Self]: def split2blocks(start: int) -> list[str]: if start >= len(lines): return [] - if firsts[start] not in pets_keywords.list: + if firsts[start] not in PETS_KEYWORDS.list: return [lines[start]] + split2blocks(start + 1) - if pets_keywords.table.at[firsts[start], 'end'] == 'false': + if PETS_KEYWORDS.table.at[firsts[start], 'end'] == 'false': return [lines[start]] + split2blocks(start + 1) try: end = firsts.index('end' + firsts[start], start) except ValueError: - if pets_keywords.table.at[firsts[start], 'end'] == 'optional': + if PETS_KEYWORDS.table.at[firsts[start], 'end'] == 'optional': return [lines[start]] + split2blocks(start + 1) end = len(lines) return ['\n'.join(lines[start : end + 1])] + split2blocks(end + 1) @@ -88,8 +88,8 @@ def split2blocks(start: int) -> list[str]: def has_end(self): return ( len(self.keywords) == 1 - and self.keywords[0] in pets_keywords.list - and pets_keywords.table.at[self.keywords[0], 'end'] != 'false' + and self.keywords[0] in PETS_KEYWORDS.list + and PETS_KEYWORDS.table.at[self.keywords[0], 'end'] != 'false' ) def build_string(self): From 24e243427e4b3f2e2de944f5e3c6afd819700213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 16 Jul 2025 13:53:51 +0200 Subject: [PATCH 12/12] Bump the suggested conda-forge Python version to 3.12 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 15c48193..d97d7cb5 100644 --- a/environment.yml +++ b/environment.yml @@ -4,4 +4,4 @@ channels: - conda-forge - defaults dependencies: - - python==3.9 + - python==3.12