Mb/pymetised

init.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 export PYTHONPATH=.
-export SOF_DATA=~/astar/METIS_Simulations/ESO/output
+export SOF=~/astar/METIS_Pipeline_Test_Data/metis_sim_small_1/sof
+export SOF_DATA=~/astar/METIS_Pipeline_Test_Data/metis_sim_small_1/data
 export PYESOREX_PLUGIN_DIR=~/astar/METIS_Pipeline/metisp/pyrecipes/
+export PYESOREX_OUTPUT_DIR=~/astar/METIS_Pipeline_Test_Data/metis_sim_small_1/data

metisp/pymetis/src/pymetis/base/__init__.py

@@ -0,0 +1,3 @@
+from .recipe import MetisRecipe
+from .impl import MetisRecipeImpl
+from .product import PipelineProduct

metisp/pymetis/src/pymetis/base/impl.py

@@ -5,7 +5,7 @@
 from cpl.core import Msg
 
 from pymetis.base.product import PipelineProduct
-from pymetis.inputs import PipelineInputSet
+from pymetis.inputs.inputset import PipelineInputSet
 
 
 class MetisRecipeImpl(ABC):
@@ -99,35 +99,3 @@ def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui
             product_frames.append(product.as_frame())
 
         return product_frames
-
-
-class MetisRecipe(cpl.ui.PyRecipe):
-    """
-        The abstract base class for all METIS recipes.
-        In an ideal world it would also be abstract (derived from ABC, or metaclass=abc.ABCMeta),
-        but `pyesorex` wants to instantiate all recipes it finds
-        and would crash with an abstract class.
-        The underscored _fields must be present but should be overwritten
-        by every child class (`pyesorex` actually checks for their presence).
-    """
-    _name = "metis_abstract_base"
-    _version = "0.0.1"
-    _author = "Martin Baláž"
-    _email = "[email protected]"
-    _copyright = "CPL-3.0-or-later"
-    _synopsis = "Abstract-like base class for METIS recipes"
-    _description = "This class serves as the base class for all METIS recipes."
-
-    parameters = cpl.ui.ParameterList([])   # By default, classes do not have any parameters
-    implementation_class = str              # Dummy class, this is instantiated but not used, `str` does not hurt.
-
-    def __init__(self):
-        super().__init__()
-        self.implementation = self.implementation_class(self)
-
-    def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet:
-        """
-            The main method, as required by PyCPL.
-            It just calls the same method in the decoupled implementation.
-        """
-        return self.implementation.run(frameset, settings)

metisp/pymetis/src/pymetis/base/product.py

@@ -18,11 +18,11 @@ class PipelineProduct(ABC):
     frame_type: cpl.ui.Frame.FrameType = None
 
     def __init__(self,
-                 recipe: 'MetisRecipeImpl',
+                 recipe_impl: 'MetisRecipeImpl',
                  header: cpl.core.PropertyList,
                  image: cpl.core.Image,
                  **kwargs):
-        self.recipe: 'MetisRecipeImpl' = recipe
+        self.recipe: 'MetisRecipeImpl' = recipe_impl
         self.header: cpl.core.PropertyList = header
         self.image: cpl.core.Image = image
         self.properties = cpl.core.PropertyList()
@@ -87,9 +87,9 @@ def save(self):
         )
 
     @property
-    @abstractmethod
     def category(self) -> str:
         """ Every product must define ESO PRO CATG """
+        return self.tag
 
     @property
     def output_file_name(self) -> str:

metisp/pymetis/src/pymetis/base/recipe.py

@@ -0,0 +1,35 @@
+from typing import Dict, Any
+
+import cpl
+
+
+class MetisRecipe(cpl.ui.PyRecipe):
+    """
+        The abstract base class for all METIS recipes.
+        In an ideal world it would also be abstract (derived from ABC, or metaclass=abc.ABCMeta),
+        but `pyesorex` wants to instantiate all recipes it finds
+        and would crash with an abstract class.
+        The underscored _fields must be present but should be overwritten
+        by every child class (`pyesorex` actually checks for their presence).
+    """
+    _name = "metis_abstract_base"
+    _version = "0.0.1"
+    _author = "METIS PIP team"
+    _email = "[email protected]" # ToDo some sensible default maybe?
+    _copyright = "CPL-3.0-or-later"
+    _synopsis = "Abstract-like base class for METIS recipes"
+    _description = "This class serves as the base class for all METIS recipes."
+
+    parameters = cpl.ui.ParameterList([])          # By default, classes do not have any parameters
+    implementation_class: type = str               # Dummy class, this must be overridden in the derived classes anyway
+
+    def __init__(self):
+        super().__init__()
+        self.implementation = self.implementation_class(self)
+
+    def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet:
+        """
+            The main method, as required by PyCPL.
+            It just calls the same method in the decoupled implementation.
+        """
+        return self.implementation.run(frameset, settings)

metisp/pymetis/src/pymetis/inputs/__init__.py

@@ -2,4 +2,10 @@
 
 from .base import PipelineInput, SinglePipelineInput, MultiplePipelineInput
 
-from .common import RawInput, MasterDarkInput, MasterFlatInput, LinearityInput, PersistenceMapInput, GainMapInput
+from .common import (RawInput,
+                     MasterDarkInput,
+                     MasterFlatInput,
+                     LinearityInput,
+                     BadpixMapInput,
+                     PersistenceMapInput,
+                     GainMapInput)

metisp/pymetis/src/pymetis/inputs/base.py

@@ -1,16 +1,15 @@
 from abc import abstractmethod
-from xml.sax.handler import property_dom_node
 
 import cpl
 
 from cpl.core import Msg
 
 
 class PipelineInput:
-    _title: str = None                      # No univerrsal title makes sense
+    _title: str = None                      # No universal title makes sense
     _required: bool = True                  # By default, inputs are required to be present
     _tags: [str] = None                     # No universal tags are provided
-    _group: str = None
+    _group: str = None                      # No sensible default, must be provided explicitly
 
     @property
     def title(self):
@@ -61,7 +60,10 @@ def __init__(self,
 
         # ...and that they are a list of strings (not a single string -- this leads to nasty errors)
         if not isinstance(self.tags, list):
-            raise TypeError(f"Tags must be a list of template strings, got '{self.tags}'")
+            raise TypeError(f"Tags must be a list of string templates, got '{self.tags}'")
+        for tag in self.tags:
+            if not isinstance(tag, str):
+                raise TypeError(f"Tags must be a list of string templates, got '{type(tag)}'")
 
         # Override `required` if requested
         if required is not None:
@@ -75,7 +77,7 @@ def __init__(self,
     @abstractmethod
     def verify(self) -> None:
         """
-        Verify that the input has all the required inputs
+        Verify that the input has all the required frames. There is no default logic.
         """
 
     def print_debug(self, *, offset: int = 0):
@@ -185,21 +187,29 @@ def _verify_same_detector(self) -> None:
             None on success
         """
         detectors = []
-        return
         for frame in self.frameset:
             header = cpl.core.PropertyList.load(frame.file, 0)
-            det = header['ESO DPR TECH'].value
             try:
-                detectors.append({
-                                     'IMAGE,LM': '2RG',
-                                     'IMAGE,N': 'GEO',
-                                     'IFU': 'IFU',
-                                 }[det])
+                det = header['ESO DPR TECH'].value
+                try:
+                    detectors.append({
+                                         'IMAGE,LM': '2RG',
+                                         'IMAGE,N': 'GEO',
+                                         'IFU': 'IFU',
+                                     }[det])
+                except KeyError as e:
+                    raise KeyError(f"Invalid detector name! In {frame.file}, ESO DPR TECH is '{det}'") from e
             except KeyError as e:
-                raise KeyError(f"Invalid detector name! In {frame.file}, ESO DPR TECH is '{det}'") from e
+                Msg.warning(self.__class__.__qualname__, f"No detector (ESO DPR TECH) set!")
+
 
         # Check if all the raws have the same detector, if not, we have a problem
         if len(unique := list(set(detectors))) == 1:
             self._detector_name = unique[0]
+        elif len(unique) == 0:
+            Msg.warning(self.__class__.__qualname__,
+                        f"No detectors specified (this is probably fine in skeleton stage)")
         else:
-            raise ValueError(f"Darks from more than one detector found: {set(detectors)}!")
+            # raise ValueError(f"Darks from more than one detector found: {set(detectors)}!")
+            Msg.warning(self.__class__.__qualname__,
+                        f"Darks from more than one detector found: {set(detectors)}!")

metisp/pymetis/src/pymetis/inputs/common.py

@@ -13,15 +13,15 @@
 
 You should override class attributes:
 
- -  `_title`,
+ -  `_title`
     the short description of an Input class (this is used for log output)
- -  `_group`,
+ -  `_group`
     the `cpl.ui.Frame.FrameGroup` property that is required by PyEsoRex and also by CPL
- -  `_tags`,
+ -  `_tags`
     the list of tags that are accepted by this input. Note that in some classes it is not defined yet, but definition
     is deferred to further children (but has to be defined *somewhere*). Tags can contain format template placeholders,
     which will be filled from **kwargs in the __init__ method, to allow various detectors or bands.
- -  `_required`,
+ -  `_required`
     a boolean telling the recipe if this input is required or not. Default is True, so it is enough to say
     `_required = False` for optional inputs.
 """
@@ -30,34 +30,37 @@
 class RawInput(MultiplePipelineInput):
     _title: str = "raw"
     _group = cpl.ui.Frame.FrameGroup.RAW
-    _required: bool = True
 
 
 class MasterDarkInput(SinglePipelineInput):
     _title: str = "master dark"
     _tags: [str] = ["MASTER_DARK_{det}"]
     _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
-    _required: bool = True
 
 
 class MasterFlatInput(SinglePipelineInput):
     _title: str = "master flat"
-    _tags: [str] = ["MASTER_FLAT_{det}"]
+    _tags: [str] = ["MASTER_IMG_FLAT_LAMP_{det}"]
     _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
-    _required: bool = True
 
 
 class LinearityInput(SinglePipelineInput):
     _title: str = "linearity map"
     _tags: [str] = ["LINEARITY_{det}"]
     _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
-    _required: bool = True
+
+
+class BadpixMapInput(SinglePipelineInput):
+    _title: str = "bad pixel map"
+    _tags: [str] = ["BADPIX_MAP_{det}"]
+    _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
 
 
 class PersistenceMapInput(SinglePipelineInput):
     _title: str = "persistence map"
     _tags: [str] = ["PERSISTENCE_MAP"]
     _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.CALIB
+    _required: bool = False     # Persistence maps are usually optional (can be overridden)
 
 
 class GainMapInput(SinglePipelineInput):

metisp/pymetis/src/pymetis/inputs/inputset.py

@@ -1,4 +1,4 @@
-from abc import ABCMeta, abstractmethod
+from abc import ABCMeta
 
 import cpl
 from cpl.core import Msg

metisp/pymetis/src/pymetis/inputs/mixins.py

@@ -0,0 +1,29 @@
+import cpl.ui
+
+from pymetis.inputs import PersistenceMapInput, PipelineInputSet
+
+
+class Detector2rgMixin:
+    detector: str = '2RG'
+    band: str = 'LM'
+
+class DetectorGeoMixin:
+    detector: str = 'GEO'
+    band: str = 'N'
+
+class DetectorIfuMixin:
+    detector: str = 'IFU'
+
+class TargetSciMixin:
+    target: str = 'SCI'
+
+class TargetStdMixin:
+    target: str = 'STD'
+
+
+
+class PersistenceInputSetMixin(PipelineInputSet):
+    def __init__(self, frameset: cpl.ui.FrameSet):
+        self.persistence_map = PersistenceMapInput(frameset)
+        super().__init__(frameset)
+        self.inputs += [self.persistence_map]

metisp/pymetis/src/pymetis/prefabricates/darkimage.py → metisp/pymetis/src/pymetis/prefab/darkimage.py

@@ -2,8 +2,7 @@
 
 import cpl.ui
 
-from pymetis.inputs.common import MasterDarkInput
-from pymetis.prefabricates.rawimage import RawImageProcessor
+from pymetis.prefab.rawimage import RawImageProcessor
 
 
 class DarkImageProcessor(RawImageProcessor, ABC):

metisp/pymetis/src/pymetis/prefabricates/flat.py → metisp/pymetis/src/pymetis/prefab/flat.py

@@ -6,9 +6,9 @@
 
 from pymetis.inputs import PipelineInputSet
 from pymetis.inputs.common import RawInput, MasterDarkInput
-from pymetis.base.product import PipelineProduct
 
-from pymetis.prefabricates.darkimage import DarkImageProcessor
+from .darkimage import DarkImageProcessor
+from ..base.product import PipelineProduct
 
 
 class MetisBaseImgFlatImpl(DarkImageProcessor, ABC):

metisp/pymetis/src/pymetis/prefabricates/rawimage.py → metisp/pymetis/src/pymetis/prefab/rawimage.py

@@ -4,10 +4,8 @@
 import cpl
 from cpl.core import Msg
 
-from pymetis.base.impl import MetisRecipeImpl
-from pymetis.base.input import RecipeInput
+from pymetis.base import MetisRecipeImpl
 from pymetis.inputs import PipelineInputSet
-from pymetis.inputs.common import RawInput
 
 
 class RawImageProcessor(MetisRecipeImpl, ABC):