Mb/testament

metisp/pymetis/src/pymetis/base/impl.py

@@ -33,8 +33,8 @@ class MetisRecipeImpl(ABC):
     Contains central data flow control and also provides abstract methods to be overridden
     by particular pipeline recipe implementations.
     """
-    InputSet: PipelineInputSet = None
-    Product: PipelineProduct = None
+    InputSet: type[PipelineInputSet] = None
+    Product: type[PipelineProduct] = None
 
     # Available parameters are a class variable. This must be present, even if empty.
     parameters = cpl.ui.ParameterList([])
@@ -45,7 +45,7 @@ def __init__(self, recipe: 'MetisRecipe') -> None:
         self.version = recipe.version
         self.parameters = recipe.parameters
 
-        self.inputset = None
+        self.inputset: PipelineInputSet | None = None
         self.frameset = None
         self.header = None
         self.products: Dict[str, PipelineProduct] = {}
@@ -67,7 +67,7 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
             self.import_settings(settings)                # Import and process the provided settings dict
             self.inputset = self.InputSet(frameset)       # Create an appropriate InputSet object
             self.inputset.print_debug()
-            self.inputset.validate()                        # Verify that they are valid (maybe with `schema` too?)
+            self.inputset.validate()                      # Verify that they are valid (maybe with `schema` too?)
             products = self.process_images()              # Do all the actual processing
             self.save_products(products)                  # Save the output products
 
@@ -124,14 +124,15 @@ def save_products(self, products: Dict[str, PipelineProduct]) -> None:
         """
         for name, product in products.items():
             Msg.debug(self.__class__.__qualname__,
-                      f"Saving {name}")
+                      f"Saving product {name}")
             product.save()
 
     def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui.FrameSet:
         """
         Gather all the products and build a FrameSet from their frames so that it can be returned from `run`.
         """
-        Msg.debug(self.__class__.__qualname__, f"Building the product frameset")
+        Msg.debug(self.__class__.__qualname__,
+                  f"Building the product frameset")
         return cpl.ui.FrameSet([product.as_frame() for product in products.values()])
 
     def as_dict(self) -> dict[str, Any]:

metisp/pymetis/src/pymetis/prefab/flat.py

@@ -32,29 +32,18 @@
 
 
 class MetisBaseImgFlatImpl(DarkImageProcessor, ABC):
-    class InputSet(PipelineInputSet):
+    class InputSet(DarkImageProcessor.InputSet):
         """
         Base class for Inputs which create flats. Requires a set of raw frames and a master dark.
         """
-        class RawFlatInput(RawInput):
+        MasterDarkInput = MasterDarkInput
+
+        class RawInput(RawInput):
             """
             A subclass of RawInput that is handling the flat image raws.
             """
             _tags = re.compile(r"(?P<band>(LM|N))_FLAT_(?P<target>LAMP|TWILIGHT)_RAW")
 
-        class DarkFlatInput(MasterDarkInput):
-            """
-            Just a plain MasterDarkInput.
-            """
-            pass
-
-        def __init__(self, frameset):
-            super().__init__(frameset)
-            self.raw = self.RawFlatInput(frameset)
-            self.master_dark = MasterDarkInput(frameset)
-            self.inputs = [self.raw, self.master_dark]
-
-
     class Product(PipelineProduct):
         group = cpl.ui.Frame.FrameGroup.PRODUCT
         level = cpl.ui.Frame.FrameLevel.FINAL
@@ -83,7 +72,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         # TODO: Detect detector
         # TODO: Twilight
 
-        raw_images = self.load_raw_images()
+        raw_images = self.inputset.load_raw_images()
         master_dark = cpl.core.Image.load(self.inputset.master_dark.frame.file, extension=0)
 
         for raw_image in raw_images:
@@ -96,7 +85,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         # TODO: preprocessing steps like persistence correction / nonlinearity (or not) should come here
 
         header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
-        combined_image = self.combine_images(self.load_raw_images(), method)
+        combined_image = self.combine_images(self.inputset.load_raw_images(), method)
 
         self.products = {
             self.name.upper(): self.Product(self, header, combined_image),

metisp/pymetis/src/pymetis/prefab/rawimage.py

@@ -42,32 +42,31 @@ def __init__(self, frameset: cpl.ui.FrameSet):
             self.raw = self.RawInput(frameset)
             self.inputs += [self.raw]
 
-    def load_raw_images(self) -> cpl.core.ImageList:
-        """
-        Always load a set of raw images, as determined by the tags.
-        Chi-Hung has warned Martin that this is unnecessary and fills the memory quickly,
-        but if we are to use CPL functions, Martin does not see a way around it.
-        """
-        output = cpl.core.ImageList()
-
-        for idx, frame in enumerate(self.inputset.raw.frameset):
-            Msg.info(self.__class__.__qualname__, f"Processing input frame #{idx}: {frame.file!r}...")
+        def load_raw_images(self) -> cpl.core.ImageList:
+            """
+            Always load a set of raw images, as determined by the tags.
+            Chi-Hung has warned Martin that this is unnecessary and fills the memory quickly,
+            but if we are to use CPL functions, Martin does not see a way around it.
+            """
+            output = cpl.core.ImageList()
 
-            # Append the loaded image to an image list
-            Msg.debug(self.__class__.__qualname__, f"Loading input image {frame.file}")
-            output.append(cpl.core.Image.load(frame.file, extension=1))
+            for idx, frame in enumerate(self.raw.frameset):
+                Msg.info(self.__class__.__qualname__,
+                         f"Processing input frame #{idx}: {frame.file!r}...")
+                output.append(cpl.core.Image.load(frame.file, extension=1))
 
-        return output
+            return output
 
     @classmethod
     def combine_images(cls,
                        images: cpl.core.ImageList,
-                       method: Literal['add'] | Literal['average'] | Literal['median']):
+                       method: Literal['add'] | Literal['average'] | Literal['median']) -> cpl.core.Image:
         """
         Basic helper method to combine images using one of `add`, `average` or `median`.
         Probably not a universal panacea, but it recurs often enough to warrant being here.
         """
-        Msg.info(cls.__qualname__, f"Combining images using method {method!r}")
+        Msg.info(cls.__qualname__,
+                 f"Combining images using method {method!r}")
         combined_image = None
         match method:
             case "add":

metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_reduce.py

@@ -104,7 +104,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         self.target = self.inputset.tag_parameters["target"]
 
         header = cpl.core.PropertyList()
-        images = self.load_raw_images()
+        images = self.inputset.load_raw_images()
         image = self.combine_images(images, "add")
 
         self.products = {

metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_rsrf.py

@@ -77,7 +77,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         # self.apply_fluxcal()
 
         header = cpl.core.PropertyList()
-        images = self.load_raw_images()
+        images = self.inputset.load_raw_images()
         image = self.combine_images(images, "add")
 
         self.products = {

metisp/pymetis/src/pymetis/recipes/ifu/metis_ifu_wavecal.py

@@ -53,7 +53,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         # self.apply_fluxcal()
 
         header = cpl.core.PropertyList()
-        images = self.load_raw_images()
+        images = self.inputset.load_raw_images()
         image = self.combine_images(images, "add")
 
         self.products = {

metisp/pymetis/src/pymetis/recipes/instrument/metis_pupil_imaging.py

@@ -24,6 +24,8 @@
 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 """
 
+import re
+
 from typing import Dict
 
 import cpl
@@ -64,26 +66,24 @@ class InputSet(DarkImageProcessor.InputSet):
         # RawImageProcessor.InputSet. It already knows that it wants a RawInput and MasterDarkInput class,
         # but does not know about the tags yet. So here we define tags for the raw input
         class Raw(RawInput):
-            _tags = ["LM_PUPIL_RAW"]
+            _tags = re.compile("LM_PUPIL_RAW")
 
         # Also one master flat is required. We use a prefabricated class
         class MasterFlat(MasterFlatInput):
-            _tags = ["MASTER_IMG_FLAT_LAMP_LM"]
+            _tags = re.compile("MASTER_IMG_FLAT_LAMP_LM")
 
         # We could define the master dark explicitly too, but we can use a prefabricated class instead.
         # That already has its tags defined (for master darks it's always "MASTER_DARK_{det}"), so we just define
         # the detector and band. Those are now available for all Input classes here.
         # Of course, we could be more explicit and define them directly.
 
-        detector: str = '2RG'
-        band: str = 'LM'
         RawInput = Raw
         MasterDarkInput = MasterDarkInput
 
         def __init__(self, frameset: cpl.ui.FrameSet):
             super().__init__(frameset)
             self.master_flat = self.MasterFlat(frameset,
-                                               tags=["MASTER_IMG_FLAT_LAMP_{band}", "MASTER_IMG_FLAT_TWILIGHT_{band}"],
+                                               tags=re.compile("MASTER_IMG_FLAT_(?P<target>LAMP|TWILIGHT)_(?P<band>LM|N)"),
                                                band="LM", det=self.detector)
             self.linearity = LinearityInput(frameset, det=self.detector)
             self.persistence = PersistenceMapInput(frameset, required=False)
@@ -203,19 +203,12 @@ class MetisPupilImaging(MetisRecipe):
 
     parameters = cpl.ui.ParameterList([
         cpl.ui.ParameterEnum(
-            name="pupil_imaging.stacking.method",
-            context="pupil_imaging",
+            name="metis_pupil_imaging.stacking.method",
+            context="metis_pupil_imaging",
             description="Name of the method used to combine the input images",
             default="add",
             alternatives=("add", "average", "median"),
         ),
-        cpl.ui.ParameterEnum(
-            name="pupil_imaging.band",
-            context="pupil_imaging",
-            description="band to run",
-            default="lm",
-            alternatives=("lm", "n",),
-        )
     ])
 
     implementation_class = MetisPupilImagingImpl

metisp/pymetis/src/pymetis/recipes/metis_det_dark.py

@@ -80,7 +80,7 @@ def process_images(self) -> Dict[str, PipelineProduct]:
         Msg.info(self.__class__.__qualname__, f"Combining images using method {method!r}")
 
         # TODO: preprocessing steps like persistence correction / nonlinearity (or not)
-        raw_images = self.load_raw_images()
+        raw_images = self.inputset.load_raw_images()
         combined_image = self.combine_images(raw_images, method)
         header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
 

metisp/pymetis/src/pymetis/recipes/metis_det_lingain.py

@@ -60,7 +60,7 @@ def tag(self) -> str:
             return f"BADPIX_MAP_{self.detector:s}"
 
     def process_images(self) -> Dict[str, PipelineProduct]:
-        raw_images = self.load_raw_images()
+        raw_images = self.inputset.load_raw_images()
         combined_image = self.combine_images(raw_images,
                                              method=self.parameters["metis_det_lingain.stacking.method"].value)
 

metisp/pymetis/src/pymetis/tests/generic.py

@@ -26,7 +26,7 @@
 
 import cpl
 
-from pymetis.inputs import PipelineInputSet
+from pymetis.inputs import PipelineInputSet, MultiplePipelineInput
 from pymetis.base.product import PipelineProduct
 
 
@@ -75,12 +75,19 @@ def test_all_inputs(self, load_frameset, sof):
 
 
 class RawInputSetTest(BaseInputSetTest):
-    def test_raw_input_count(self, load_frameset, sof):
+    def test_is_raw_input_count_correct(self, load_frameset, sof):
         instance = self.impl.InputSet(load_frameset(sof))
         assert len(instance.raw.frameset) == self.count
 
+    def test_inputset_has_raw(self, load_frameset, sof):
+        instance = self.impl.InputSet(load_frameset(sof))
+        assert isinstance(instance.raw, MultiplePipelineInput)
+
 
 class BaseRecipeTest(ABC):
+    """
+    Integration / regression tests for verifying that the recipe can be run
+    """
     _recipe = None
 
     @classmethod
@@ -96,7 +103,7 @@ def test_recipe_can_be_run_directly(self, load_frameset, sof):
         instance = self._recipe()
         frameset = cpl.ui.FrameSet(load_frameset(sof))
         instance.run(frameset, {})
-        #pprint.pprint(instance.implementation.as_dict())
+        # pprint.pprint(instance.implementation.as_dict(), width=200)
 
     def test_recipe_can_be_run_with_pyesorex(self, name, create_pyesorex):
         pyesorex = create_pyesorex(self._recipe)

metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_rsrf.py

@@ -0,0 +1,43 @@
+"""
+This file is part of the METIS Pipeline.
+Copyright (C) 2024 European Southern Observatory
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+"""
+
+import pytest
+
+from pymetis.recipes.ifu.metis_ifu_rsrf import (MetisIfuRsrf as Recipe, MetisIfuRsrfImpl as Impl)
+from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest
+
+
+@pytest.fixture
+def name():
+    return 'metis_ifu_rsrf'
+
+
+@pytest.fixture
+def sof(name):
+    return f'{name}.sof'
+
+
+class TestRecipe(BaseRecipeTest):
+    """ A bunch of extremely simple and stupid test cases... just to see if it does something """
+    _recipe = Recipe
+
+
+class TestInputSet(BaseInputSetTest):
+    impl = Impl
+    count = 1

metisp/pymetis/src/pymetis/tests/ifu/test_metis_ifu_telluric.py

@@ -37,6 +37,10 @@ class TestRecipe(BaseRecipeTest):
     """ A bunch of extremely simple and stupid test cases... just to see if it does something """
     _recipe = Recipe
 
+    @pytest.mark.parametrize("sof", ["metis_ifu_telluric.std.sof", "metis_ifu_telluric.sci.sof"])
+    def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex):
+        super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex)
+
 
 class TestInputSet(BaseInputSetTest):
     impl = Impl

metisp/pymetis/src/pymetis/tests/img/test_metis_lm_img_flat.py

@@ -19,8 +19,9 @@
 
 import pytest
 
-from pymetis.recipes.img.metis_lm_img_flat import MetisLmImgFlat as Recipe, MetisLmImgFlatImpl as Impl
-from generic import BaseInputSetTest, BaseRecipeTest, BaseProductTest
+from pymetis.tests.generic import BaseRecipeTest, BaseInputSetTest, BaseProductTest
+from pymetis.recipes.img.metis_lm_img_flat import (MetisLmImgFlat as Recipe,
+                                                   MetisLmImgFlatImpl as Impl)
 
 
 @pytest.fixture
@@ -30,13 +31,16 @@ def name():
 
 @pytest.fixture
 def sof(name):
-    return f"{name}.lamp.sof"
+    return f'{name}.lamp.sof'
 
 
 class TestRecipe(BaseRecipeTest):
-    """ A bunch of extremely simple test cases... just to see if it does something """
     _recipe = Recipe
 
+    @pytest.mark.parametrize("sof", [f"metis_lm_img_flat.{target}.sof" for target in ['lamp', 'twilight']])
+    def test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(self, name, sof, create_pyesorex):
+        super().test_pyesorex_runs_with_zero_exit_code_and_empty_stderr(name, sof, create_pyesorex)
+
 
 class TestInputSet(BaseInputSetTest):
     impl = Impl