Fi feature detect misalignment

.github/workflows/buildPlugin.yml

@@ -6,6 +6,9 @@ on:
   pull_request:
     branches: [devel, master]
 
+  workflow_dispatch:
+
+
 # Define the job that should be run
 jobs:
   build:
@@ -101,12 +104,10 @@ jobs:
       working-directory: ${{ github.workspace }}
       run: ../scipion/scipion3 installp -p . --devel
 
-    # Currently, test script is being downloaded from scipion-chem, but that should be temporary
-    # Ideally, the script might be included in scipion core to be able to import it,
-    # or maybe host it in a separate repository common with useful scripts for all scipion plugins
     - name: Run tests
       working-directory: ${{ github.workspace }}/${{ vars.FOLDER_WITH_VERSION }}
       run: |
-        wget https://raw.githubusercontent.com/scipion-chem/scipion-chem/devel/pwchem/runTests.py
         eval "$(${{ github.workspace }}/../miniconda/bin/conda shell.bash hook)"
-        python runTests.py ${{ github.workspace }}/../scipion/scipion3 ${{ vars.FOLDER_WITH_VERSION }} -noGPU -testData=testData.json -j=3
+        pip install --user scipion-testrunner
+        ${{ github.workspace }}/../scipion/scipion3 run conda install -c conda-forge libstdcxx-ng
+        scipion_testrunner ${{ github.workspace }}/../scipion/scipion3 ${{ vars.FOLDER_WITH_VERSION }} --noGpu --testData=testData.json

requirements.txt

@@ -2,3 +2,5 @@ scipion-em-xmipp
 scipion-em-tomo
 scipion-em >= 3.0.8
 scipion-em-xmippBase
+pandas <= 1.3.5
+statsmodels

xmipptomo/protocols.conf

@@ -7,7 +7,11 @@ Tomography = [
 	    {"tag": "protocol_group", "text": "Tilt-series preprocess", "openItem": "False", "children": [
                 {"tag": "protocol", "value": "XmippProtResizeTiltSeries", "text": "default"},
                 {"tag": "protocol", "value": "XmippProtApplyTransformationMatrixTS", "text": "default"},
+                {"tag": "protocol", "value": "XmippProtDetectLandmarkTS", "text": "default"},
                 {"tag": "protocol", "value": "XmippProtSplitTiltSeries", "text": "default"}
+        ]},
+	    {"tag": "protocol_group", "text": "Alignment", "openItem": "False", "children": [
+	        {"tag": "protocol", "value": "XmippProtDetectMisalignmentTiltSeries", "text": "default"}
             ]}
 	]},
 	{"tag": "section", "text": "Tomograms", "children": [

xmipptomo/protocols/__init__.py

@@ -37,6 +37,8 @@
 from .protocol_crop_tomograms import XmippProtCropTomograms
 from .protocol_extract_subtomos import XmippProtExtractSubtomos
 from .protocol_flexalign import XmippProtTsFlexAlign
+from .protocol_detect_misalignment_TS import XmippProtDetectMisalignmentTiltSeries
+from .protocol_detect_landmarks_TS import XmippProtDetectLandmarkTS
 from .protocol_peak_high_contrast import XmippProtPeakHighContrast
 from .protocol_phantom_subtomo import XmippProtPhantomSubtomo
 from .protocol_phantom_tomo import XmippProtPhantomTomo

xmipptomo/protocols/protocol_applyAlignmentTS.py

@@ -1,6 +1,6 @@
 # **************************************************************************
 # *
-# * Authors:     Federico P. de Isidro Gomez (fp.deisidro@cnb.csi.es) [1]
+# * Authors:     Federico P. de Isidro Gomez (fp.deisidro@cnb.csic.es) [1]
 # *
 # * [1] Centro Nacional de Biotecnologia, CSIC, Spain
 # *

xmipptomo/protocols/protocol_detect_landmarks_TS.py

@@ -0,0 +1,240 @@
+# **************************************************************************
+# *
+# * Authors:     Federico P. de Isidro Gomez (fp.deisidro@cnb.csi.es) [1]
+# *
+# * [1] Centro Nacional de Biotecnologia, CSIC, Spain
+# *
+# * This program is free software; you can redistribute it and/or modify
+# * it under the terms of the GNU General Public License as published by
+# * the Free Software Foundation; either version 2 of the License, or
+# * (at your option) any later version.
+# *
+# * This program is distributed in the hope that it will be useful,
+# * but WITHOUT ANY WARRANTY; without even the implied warranty of
+# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# * GNU General Public License for more details.
+# *
+# * You should have received a copy of the GNU General Public License
+# * along with this program; if not, write to the Free Software
+# * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# * 02111-1307  USA
+# *
+# *  All comments concerning this program package may be sent to the
+# *  e-mail address 'scipion@cnb.csic.es'
+# *
+# **************************************************************************
+
+import os
+from pyworkflow import BETA
+import pyworkflow.protocol.params as params
+import pyworkflow.utils.path as path
+from pyworkflow.object import Set, Boolean
+from pyworkflow.protocol.constants import STEPS_PARALLEL
+from pwem.protocols import EMProtocol
+import tomo.objects as tomoObj
+from tomo.protocols import ProtTomoBase
+import xmipptomo.utils as utils
+
+
+OUTPUT_COORDS_FILENAME = "outputLandmarkCoordinates.xmd"
+
+
+class XmippProtDetectLandmarkTS(EMProtocol, ProtTomoBase):
+    """
+    Scipion protocol for xmipp_tomo_detect_landmarks. This protocol detect fiducial landmarks in a tilt series. It will
+    not produce a correspondence of landmark between images so chains will not be constructed only single landmark per
+    tilt image.
+    """
+
+    _label = 'detect landmarks TS'
+    _devStatus = BETA
+
+    def __init__(self, **kwargs):
+        EMProtocol.__init__(self, **kwargs)
+        ProtTomoBase.__init__(self)
+        self.stepsExecutionMode = STEPS_PARALLEL
+
+    # -------------------------- DEFINE param functions -----------------------
+    def _defineParams(self, form):
+        form.addSection('Input')
+
+        form.addParam('inputSetOfTiltSeries',
+                      params.PointerParam,
+                      pointerClass='SetOfTiltSeries',
+                      important=True,
+                      label='Input set of tilt-series')
+
+        form.addParam('fiducialSize',
+                      params.FloatParam,
+                      important=True,
+                      label='Landmark size (nm)',
+                      help='Landmark size in nanometers (nm).')
+
+        # Advanced params
+        form.addParam('thrSD',
+                      params.FloatParam,
+                      expertLevel=params.LEVEL_ADVANCED,
+                      default=5,
+                      label='Coordinate value SD threshold',
+                      help='Number of SD a coordinate value must be over the mean to consider that it belongs to a '
+                           'high contrast feature.')
+
+        form.addParam('targetLMsize',
+                      params.FloatParam,
+                      expertLevel=params.LEVEL_ADVANCED,
+                      default=8,
+                      label='Target landmark size (px)',
+                      help='Target landmark size to adjust down sampling before filtering. Default value is 8 px.')
+
+        form.addParam('numberFTdirOfDirections',
+                      params.IntParam,
+                      expertLevel=params.LEVEL_ADVANCED,
+                      default=16,
+                      label='Fourier filter directions',
+                      help='Number of directions to analyze in the Fourier directional filter.')
+
+    # -------------------------- INSERT steps functions ---------------------
+    def _insertAllSteps(self):
+        allcosId = []
+
+        for ts in self.inputSetOfTiltSeries.get():
+            tsObjId = ts.getObjId()
+            cisID = self._insertFunctionStep(self.convertInputStep,
+                                             tsObjId,
+                                             prerequisites=[])
+            dlsID = self._insertFunctionStep(self.detectLandmarksStep,
+                                             tsObjId,
+                                             prerequisites=[cisID])
+            cosID = self._insertFunctionStep(self.createOutputStep,
+                                             tsObjId,
+                                             prerequisites=[dlsID])
+            allcosId.append(cosID)
+
+        self._insertFunctionStep(self.closeOutputSetsStep,
+                                 prerequisites=allcosId)
+
+    # --------------------------- STEPS functions ----------------------------
+    def convertInputStep(self, tsObjId):
+        ts = self.inputSetOfTiltSeries.get()[tsObjId]
+        tsId = ts.getTsId()
+
+        extraPrefix = self._getExtraPath(tsId)
+        tmpPrefix = self._getTmpPath(tsId)
+
+        path.makePath(tmpPrefix)
+        path.makePath(extraPrefix)
+
+        firstItem = ts.getFirstItem()
+
+        """Apply the transformation form the input tilt-series"""
+        # Use Xmipp interpolation via Scipion
+        if firstItem.hasTransform():
+            avgRotAngle = utils.calculateAverageRotationAngleFromTM(ts)
+            swap = True if (avgRotAngle > 45 or avgRotAngle < -45) else False
+
+            outputTsFileName = os.path.join(tmpPrefix, firstItem.parseFileName())
+            ts.applyTransform(outputTsFileName, swapXY=swap)
+
+        else:
+            outputTsFileName = os.path.join(tmpPrefix, firstItem.parseFileName())
+            ts.applyTransform(outputTsFileName)
+
+    def detectLandmarksStep(self, tsObjId):
+        ts = self.inputSetOfTiltSeries.get()[tsObjId]
+        tsId = ts.getTsId()
+
+        extraPrefix = self._getExtraPath(tsId)
+        tmpPrefix = self._getTmpPath(tsId)
+
+        firstItem = ts.getFirstItem()
+
+        params = {
+            'i': os.path.join(tmpPrefix, firstItem.parseFileName() + ":mrcs"),
+            'o': os.path.join(extraPrefix, OUTPUT_COORDS_FILENAME),
+            'thrSD': self.thrSD.get(),
+            'samplingRate': self.inputSetOfTiltSeries.get().getSamplingRate(),
+            'fiducialSize': self.fiducialSize.get() * 10,
+            'targetLMsize': self.targetLMsize.get(),
+            'numberFTdirOfDirections': self.numberFTdirOfDirections.get()
+        }
+
+        args = "-i %(i)s " \
+               "-o %(o)s " \
+               "--thrSD %(thrSD).2f " \
+               "--samplingRate %(samplingRate).2f " \
+               "--fiducialSize %(fiducialSize).2f " \
+               "--targetLMsize %(targetLMsize).4f " \
+               "--numberFTdirOfDirections %(numberFTdirOfDirections)d "
+
+        self.runJob('xmipp_tomo_detect_landmarks', args % params)
+
+    def createOutputStep(self, tsObjId):
+        ts = self.inputSetOfTiltSeries.get()[tsObjId]
+        tsId = ts.getTsId()
+
+        extraPrefix = self._getExtraPath(tsId)
+
+        firstItem = ts.getFirstItem()
+
+        self.getOutputSetOfLandmarkModels()
+
+        lmFileName = os.path.join(extraPrefix, firstItem.parseFileName(suffix='_lm', extension='.sfid'))
+        lm = tomoObj.LandmarkModel(tsId=tsId,
+                                   fileName=lmFileName,
+                                   modelName=None,
+                                   size=self.fiducialSize.get() * 10,
+                                   applyTSTransformation=Boolean(False))
+        lm.setTiltSeries(ts)
+
+        lmList = utils.parseLandmarkCoordinatesFile(os.path.join(extraPrefix, OUTPUT_COORDS_FILENAME))
+
+        if not lmList:
+            print("WARNING: no landmark detected in tilt-series")
+            return  # Do not create output if no detected landmarks in tilt-series
+
+        for i, lmInfo in enumerate(lmList):
+            lm.addLandmark(xCoor=lmInfo[0],
+                           yCoor=lmInfo[1],
+                           tiltIm=lmInfo[2]+1,
+                           chainId=i+1,
+                           xResid=0.0,
+                           yResid=0.0)
+
+        self.outputSetOfLandmarkModels.append(lm)
+        self._store()
+
+    def closeOutputSetsStep(self):
+        if hasattr(self, "outputSetOfLandmarkModels"):
+            self.outputSetOfLandmarkModels.setStreamState(Set.STREAM_CLOSED)
+            self.outputSetOfLandmarkModels.write()
+
+        self._store()
+
+    # --------------------------- UTILS functions ----------------------------
+    def getOutputSetOfLandmarkModels(self):
+        if hasattr(self, "outputSetOfLandmarkModels"):
+            self.outputSetOfLandmarkModels.setStreamState(Set.STREAM_OPEN)
+            self.outputSetOfLandmarkModels.enableAppend()
+
+        else:
+            outputSetOfLandmarkModels = self._createSetOfLandmarkModels()
+            outputSetOfLandmarkModels.setSetOfTiltSeries(self.inputSetOfTiltSeries)
+
+            outputSetOfLandmarkModels.copyInfo(self.inputSetOfTiltSeries.get())
+
+            outputSetOfLandmarkModels.setStreamState(Set.STREAM_OPEN)
+
+            self._defineOutputs(outputSetOfLandmarkModels=outputSetOfLandmarkModels)
+            self._defineSourceRelation(self.inputSetOfTiltSeries, outputSetOfLandmarkModels)
+
+        return self.outputSetOfLandmarkModels
+
+    # --------------------------- INFO functions ----------------------------
+    def _summary(self):
+        summary = []
+        return summary
+
+    def _methods(self):
+        methods = []
+
+        return methods