mb/tidy-up

.github/workflows/run_edps.yaml

@@ -54,18 +54,36 @@ jobs:
           export SOF_DATA="$(pwd)/METIS_Pipeline_Test_Data/small202402/outputSmall/"
           export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/small202402/sofFiles/"
           export PYESOREX_OUTPUT_DIR="$SOF_DATA"
+
+          # LIST RECIPES
           pyesorex --recipes
 
           # DET RECIPES
           pyesorex metis_det_lingain "${SOF_DIR}/metis_det_lingain.lm.sof"
           pyesorex metis_det_dark "${SOF_DIR}/metis_det_dark.lm.sof"
 
           # IMG LM RECIPES
+          pyesorex metis_lm_img_distortion  "${SOF_DIR}/metis_lm_img_distortion.sof"
           pyesorex metis_lm_img_flat  "${SOF_DIR}/metis_lm_img_flat.lamp.sof"
           pyesorex metis_lm_img_basic_reduce "${SOF_DIR}/metis_lm_img_basic_reduce.std.sof"
+          pyesorex metis_lm_img_basic_reduce "${SOF_DIR}/metis_lm_img_basic_reduce.sci.sof"
+          pyesorex metis_lm_img_basic_reduce "${SOF_DIR}/metis_lm_img_basic_reduce.sky.sof"
+          pyesorex metis_lm_img_background "${SOF_DIR}/metis_lm_img_background.std.sof"
+          pyesorex metis_lm_img_background "${SOF_DIR}/metis_lm_img_background.sci.sof"
+          pyesorex metis_lm_img_std_process "${SOF_DIR}/metis_lm_img_std_process.sof"
+          pyesorex metis_lm_img_calibrate "${SOF_DIR}/metis_lm_img_calibrate.sof"
+          pyesorex metis_lm_img_sci_postprocess "${SOF_DIR}/metis_lm_img_sci_postprocess.sof"
 
           # IFU RECIPES
+          pyesorex metis_ifu_distortion "${SOF_DIR}/metis_ifu_distortion.sof"
+          pyesorex metis_ifu_wavecal "${SOF_DIR}/metis_ifu_wavecal.sof"
           pyesorex metis_ifu_rsrf "${SOF_DIR}/metis_ifu_rsrf.sof"
+          pyesorex metis_ifu_reduce "${SOF_DIR}/metis_ifu_reduce.std.sof"
+          pyesorex metis_ifu_reduce "${SOF_DIR}/metis_ifu_reduce.sci.sof"
+          pyesorex metis_ifu_telluric "${SOF_DIR}/metis_ifu_telluric.std.sof"
+          pyesorex metis_ifu_telluric "${SOF_DIR}/metis_ifu_telluric.sci.sof"
+          pyesorex metis_ifu_calibrate "${SOF_DIR}/metis_ifu_calibrate.sof"
+          pyesorex metis_ifu_postprocess "${SOF_DIR}/metis_ifu_postprocess.sof"
 
           # CAL RECIPES
           pyesorex metis_cal_chophome "${SOF_DIR}/metis_cal_chophome.sof"
@@ -84,5 +102,5 @@ jobs:
           edps -lw
           edps -w metis.metis_wkf -i $SOF_DATA -c
           edps -w metis.metis_wkf -i $SOF_DATA -lt
-          edps -w metis.metis_wkf -i $SOF_DATA -m all| tee edps.stdout.txt
-          ! grep "'FAILED'" edps.stdout.txt
+          edps -w metis.metis_wkf -i $SOF_DATA -m all | tee edps.stdout.txt
+          ! grep "'FAILED'" edps.stdout.txt

metisp/pymetis/src/pymetis/base/impl.py

@@ -34,23 +34,26 @@ class MetisRecipeImpl(ABC):
     by particular pipeline recipe implementations.
     """
     InputSet: type[PipelineInputSet] = None
-    Product: type[PipelineProduct] = None
 
     # Available parameters are a class variable. This must be present, even if empty.
     parameters = cpl.ui.ParameterList([])
 
     def __init__(self, recipe: 'MetisRecipe') -> None:
+        """
+        Initializes the recipe implementation object with parameters from the recipe
+        and sets internal attributes to None / empty as needed.
+        """
         super().__init__()
         self.name = recipe.name
         self.version = recipe.version
         self.parameters = recipe.parameters
 
-        self.inputset: PipelineInputSet = None
+        self.inputset: PipelineInputSet | None = None
         self.frameset: cpl.ui.FrameSet | None = None
         self.header: cpl.core.PropertyList | None = None
-        self.products: Dict[str, PipelineProduct] = {}
+        self.products: [PipelineProduct] = {}
         self.product_frames = cpl.ui.FrameSet()
-        
+
     def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet:
         """
             The main function of the recipe implementation. It mirrors the signature of `Recipe.run`
@@ -64,14 +67,14 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
 
         try:
             self.frameset = frameset
-            self.import_settings(settings)                # Import and process the provided settings dict
-            self.inputset = self.InputSet(frameset)       # Create an appropriate InputSet object
+            self.import_settings(settings)                  # Import and process the provided settings dict
+            self.inputset = self.InputSet(frameset)         # Create an appropriate InputSet object
             self.inputset.print_debug()
-            self.inputset.validate()                      # Verify that they are valid (maybe with `schema` too?)
-            products = self.process_images()              # Do all the actual processing
-            self.save_products(products)                  # Save the output products
+            self.inputset.validate()                        # Verify that they are valid (maybe with `schema` too?)
+            self.products = self.process_images()           # Do all the actual processing
+            self._save_products()                           # Save the output products
 
-            return self.build_product_frameset(products)  # Return the output as a pycpl FrameSet
+            return self.build_product_frameset()            # Return the output as a pycpl FrameSet
         except cpl.core.DataNotFoundError as e:
             Msg.error(self.__class__.__qualname__, f"Data not found error: {e.message}")
             raise e
@@ -84,20 +87,20 @@ def import_settings(self, settings: Dict[str, Any]) -> None:
                 self.parameters[key].value = value
             except KeyError:
                 Msg.warning(self.__class__.__qualname__,
-                            f"Settings includes '{key}':{value} "
+                            f"Settings include '{key}' = {value} "
                             f"but class {self.__class__.__qualname__} "
                             f"has no parameter named {key}.")
 
     @abstractmethod
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         """
         The core method of the recipe implementation. It should contain all the processing logic.
         At its entry point the Input class must be already loaded and validated.
 
         All pixel manipulation should happen inside this function (or something it calls from within).
         Put explicitly,
-            - no pixel manipulation before entering `process_images`,
-            - and no pixel manipulation after exiting `process_images`.
+            - no pixel manipulation *before* entering `process_images`,
+            - and no pixel manipulation *after* exiting `process_images`.
 
         The basic workflow inside this function should be as follows:
 
@@ -118,22 +121,24 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         """
         return {}
 
-    def save_products(self, products: Dict[str, PipelineProduct]) -> None:
+    def _save_products(self) -> None:
         """
         Save and register the created products.
         """
-        for name, product in products.items():
-            Msg.debug(self.__class__.__qualname__,
-                      f"Saving product {name}")
+        assert self.products is not None, "Products have not been created yet!"
+
+        Msg.debug(self.__class__.__qualname__,
+                  f"Saving {len(self.products)} products: {self.products}")
+        for product in self.products:
             product.save()
 
-    def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui.FrameSet:
+    def build_product_frameset(self) -> cpl.ui.FrameSet:
         """
         Gather all the products and build a FrameSet from their frames so that it can be returned from `run`.
         """
         Msg.debug(self.__class__.__qualname__,
                   f"Building the product frameset")
-        return cpl.ui.FrameSet([product.as_frame() for product in products.values()])
+        return cpl.ui.FrameSet([product.as_frame() for product in self.products])
 
     def as_dict(self) -> dict[str, Any]:
         """
@@ -147,7 +152,7 @@ def as_dict(self) -> dict[str, Any]:
             'title': self.name,
             'inputset': self.inputset.as_dict(),
             'products': {
-                product.tag: product.as_dict() for product in self.products.values()
+                str(product.category): product.as_dict() for product in self.products
             }
         }
 

metisp/pymetis/src/pymetis/base/product.py

@@ -17,7 +17,7 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 """
 
-from abc import ABC, abstractmethod
+from abc import ABC
 
 import cpl
 from cpl.core import Msg
@@ -31,10 +31,10 @@ class PipelineProduct(ABC):
         one FITS file with associated headers and a frame
     """
 
-    tag: str = None
-    group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.PRODUCT        # ToDo: Is this a sensible default?
-    level: cpl.ui.Frame.FrameLevel = None
-    frame_type: cpl.ui.Frame.FrameType = None
+    _tag: str = None
+    _group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.PRODUCT        # ToDo: Is this a sensible default?
+    _level: cpl.ui.Frame.FrameLevel = None
+    _frame_type: cpl.ui.Frame.FrameType = None
 
     def __init__(self,
                  recipe_impl: 'MetisRecipeImpl',
@@ -120,6 +120,22 @@ def save(self):
             header=self.header,
         )
 
+    @property
+    def tag(self):
+        return self._tag
+
+    @property
+    def group(self):
+        return self._group
+
+    @property
+    def level(self):
+        return self._level
+
+    @property
+    def frame_type(self):
+        return self._frame_type
+
     @property
     def category(self) -> str:
         """
@@ -181,3 +197,33 @@ def __init__(self,
                                       f"{self.__class__.__qualname__} does not")
 
         super().__init__(recipe, header, image, **kwargs)
+
+
+class BandSpecificProduct(PipelineProduct, ABC):
+    """
+    Product specific to one band. Probably should be merged with all other similar classes.
+    """
+    band = None
+
+    def __init__(self,
+                 recipe: 'MetisRecipe',
+                 header: cpl.core.PropertyList,
+                 image: cpl.core.Image,
+                 *,
+                 band: str = None,
+                 **kwargs):
+
+        if band is not None:
+            self.band = band
+
+        """
+            At the moment of instantiation, the `band` attribute must already be set *somehow*. Either
+            -   as a class attribute (if it is constant)
+            -   from the constructor (if it is determined from the data)
+            -   or as a provided property (if it has to be computed dynamically)
+        """
+        if self.band is None:
+            raise NotImplementedError(f"Products specific to a target must define 'band', but "
+                                      f"{self.__class__.__qualname__} does not")
+
+        super().__init__(recipe, header, image, **kwargs)

metisp/pymetis/src/pymetis/base/recipe.py

@@ -42,6 +42,8 @@ class MetisRecipe(cpl.ui.PyRecipe):
                     "Bonus points if it is not visible from pyesorex.")
 
     parameters = cpl.ui.ParameterList([])           # By default, a recipe does not have any parameters.
+                                                    # ToDo: There is a pyesorex bug that prevents
+                                                    # this from being actually used.
     implementation_class: type["MetisRecipeImpl"]   # Dummy class, this must be overridden in the derived classes anyway
 
     def __init__(self):
@@ -61,4 +63,4 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
             It just calls the same method in the decoupled implementation.
         """
         self.implementation = self.dispatch_implementation_class(frameset)(self)
-        return self.implementation.run(frameset, settings)
+        return self.implementation.run(frameset, settings)

metisp/pymetis/src/pymetis/inputs/multiple.py

@@ -54,19 +54,19 @@ def __init__(self,
         self.extract_tag_parameters(tag_matches)
 
     @classmethod
-    def get_target_name(cls, frameset: cpl.ui.FrameSet):  
+    def get_target_name(cls, frameset: cpl.ui.FrameSet):
         """
         Extracts the 'target' name from the input string based on the '_tags' regex.
-        
+
         :param inputString: The string to be matched against the pattern.
         :return: The target name if a match is found, otherwise None.
         """
         for frame in frameset:
             if match := cls._tags.match(frame.tag):
                 return match.group("target")
-            else: 
+            else:
                 return None
-    
+
     def extract_tag_parameters(self, matches: [dict[str, str]]):
         if len(matches) == 0:
             return

metisp/pymetis/src/pymetis/prefab/flat.py

@@ -46,15 +46,15 @@ class RawInput(RawInput):
 
         def __init__(self, frameset: cpl.ui.FrameSet):
             super().__init__(frameset)
-            #self.persistence = PersistenceMapInput(frameset)
-            #self.linearity = LinearityInput(frameset)
-            #self.gain_map = GainMapInput(frameset)
-            #self.inputs |= {self.persistence, self.linearity, self.gain_map}
+            self.persistence = PersistenceMapInput(frameset)
+            self.linearity = LinearityInput(frameset)
+            self.gain_map = GainMapInput(frameset)
+            self.inputs |= {self.persistence, self.linearity, self.gain_map}
 
     class Product(PipelineProduct):
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _group = cpl.ui.Frame.FrameGroup.PRODUCT
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
         band: str = None
 
         @property
@@ -69,7 +69,7 @@ def output_file_name(self) -> str:
         def tag(self) -> str:
             return self.category
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         """
         Do the actual processing of the images.
         Here, it means loading the input images and a master dark,
@@ -94,7 +94,6 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
         combined_image = self.combine_images(self.inputset.load_raw_images(), method)
 
-        self.products = {
-            self.name.upper(): self.Product(self, header, combined_image),
-        }
-        return self.products
+        product = self.Product(self, header, combined_image)
+
+        return [product]

metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py

@@ -62,9 +62,9 @@ class ProductCombined(PipelineProduct):
         """
         Final product: combined, background-subtracted images of the WCU source
         """
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
-        level = cpl.ui.Frame.FrameLevel.FINAL
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _group = cpl.ui.Frame.FrameGroup.PRODUCT
+        _level = cpl.ui.Frame.FrameLevel.FINAL
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
         @property
         def category(self) -> str:
@@ -83,9 +83,9 @@ class ProductBackground(PipelineProduct):
         """
         Intermediate product: the instrumental background (WCU OFF)
         """
-        group = cpl.ui.Frame.FrameGroup.PRODUCT
-        level = cpl.ui.Frame.FrameLevel.INTERMEDIATE
-        frame_type = cpl.ui.Frame.FrameType.IMAGE
+        _group = cpl.ui.Frame.FrameGroup.PRODUCT
+        _level = cpl.ui.Frame.FrameLevel.INTERMEDIATE
+        _frame_type = cpl.ui.Frame.FrameType.IMAGE
 
         @property
         def category(self) -> str:
@@ -100,7 +100,7 @@ def tag(self) -> str:
             return rf"{self.category}"
 
 
-    def process_images(self) -> Dict[str, PipelineProduct]:
+    def process_images(self) -> [PipelineProduct]:
         """do something"""
 
         background_hdr = cpl.core.PropertyList()
@@ -113,13 +113,10 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         raw_images.subtract_image(background_img)
         combined_img = self.combine_images(raw_images, "median")
 
-        self.products = {
-            rf"{self.target}_COMBINED":
+        return [
             self.ProductCombined(self, combined_hdr, combined_img),
-            rf"{self.target}_BACKGROUND":
             self.ProductBackground(self, background_hdr, background_img),
-        }
-        return self.products
+        ]
 
     def load_images(self, frameset: cpl.ui.FrameSet) -> cpl.core.ImageList:
         """Load an imagelist from a FrameSet