Skip to content

Commit ae97870

Browse files
authored
Merge pull request #98 from AstarVienna/mb/tidy-up
Everything knotted together
2 parents 28cd3b8 + 3059864 commit ae97870

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+850
-545
lines changed

.github/workflows/run_edps.yaml

+20-2
Original file line numberDiff line numberDiff line change
@@ -54,18 +54,36 @@ jobs:
5454
export SOF_DATA="$(pwd)/METIS_Pipeline_Test_Data/small202402/outputSmall/"
5555
export SOF_DIR="$(pwd)/METIS_Pipeline_Test_Data/small202402/sofFiles/"
5656
export PYESOREX_OUTPUT_DIR="$SOF_DATA"
57+
58+
# LIST RECIPES
5759
pyesorex --recipes
5860
5961
# DET RECIPES
6062
pyesorex metis_det_lingain "${SOF_DIR}/metis_det_lingain.lm.sof"
6163
pyesorex metis_det_dark "${SOF_DIR}/metis_det_dark.lm.sof"
6264
6365
# IMG LM RECIPES
66+
pyesorex metis_lm_img_distortion "${SOF_DIR}/metis_lm_img_distortion.sof"
6467
pyesorex metis_lm_img_flat "${SOF_DIR}/metis_lm_img_flat.lamp.sof"
6568
pyesorex metis_lm_img_basic_reduce "${SOF_DIR}/metis_lm_img_basic_reduce.std.sof"
69+
pyesorex metis_lm_img_basic_reduce "${SOF_DIR}/metis_lm_img_basic_reduce.sci.sof"
70+
pyesorex metis_lm_img_basic_reduce "${SOF_DIR}/metis_lm_img_basic_reduce.sky.sof"
71+
pyesorex metis_lm_img_background "${SOF_DIR}/metis_lm_img_background.std.sof"
72+
pyesorex metis_lm_img_background "${SOF_DIR}/metis_lm_img_background.sci.sof"
73+
pyesorex metis_lm_img_std_process "${SOF_DIR}/metis_lm_img_std_process.sof"
74+
pyesorex metis_lm_img_calibrate "${SOF_DIR}/metis_lm_img_calibrate.sof"
75+
pyesorex metis_lm_img_sci_postprocess "${SOF_DIR}/metis_lm_img_sci_postprocess.sof"
6676
6777
# IFU RECIPES
78+
pyesorex metis_ifu_distortion "${SOF_DIR}/metis_ifu_distortion.sof"
79+
pyesorex metis_ifu_wavecal "${SOF_DIR}/metis_ifu_wavecal.sof"
6880
pyesorex metis_ifu_rsrf "${SOF_DIR}/metis_ifu_rsrf.sof"
81+
pyesorex metis_ifu_reduce "${SOF_DIR}/metis_ifu_reduce.std.sof"
82+
pyesorex metis_ifu_reduce "${SOF_DIR}/metis_ifu_reduce.sci.sof"
83+
pyesorex metis_ifu_telluric "${SOF_DIR}/metis_ifu_telluric.std.sof"
84+
pyesorex metis_ifu_telluric "${SOF_DIR}/metis_ifu_telluric.sci.sof"
85+
pyesorex metis_ifu_calibrate "${SOF_DIR}/metis_ifu_calibrate.sof"
86+
pyesorex metis_ifu_postprocess "${SOF_DIR}/metis_ifu_postprocess.sof"
6987
7088
# CAL RECIPES
7189
pyesorex metis_cal_chophome "${SOF_DIR}/metis_cal_chophome.sof"
@@ -84,5 +102,5 @@ jobs:
84102
edps -lw
85103
edps -w metis.metis_wkf -i $SOF_DATA -c
86104
edps -w metis.metis_wkf -i $SOF_DATA -lt
87-
edps -w metis.metis_wkf -i $SOF_DATA -m all| tee edps.stdout.txt
88-
! grep "'FAILED'" edps.stdout.txt
105+
edps -w metis.metis_wkf -i $SOF_DATA -m all | tee edps.stdout.txt
106+
! grep "'FAILED'" edps.stdout.txt

metisp/pymetis/src/pymetis/base/impl.py

+26-21
Original file line numberDiff line numberDiff line change
@@ -34,23 +34,26 @@ class MetisRecipeImpl(ABC):
3434
by particular pipeline recipe implementations.
3535
"""
3636
InputSet: type[PipelineInputSet] = None
37-
Product: type[PipelineProduct] = None
3837

3938
# Available parameters are a class variable. This must be present, even if empty.
4039
parameters = cpl.ui.ParameterList([])
4140

4241
def __init__(self, recipe: 'MetisRecipe') -> None:
42+
"""
43+
Initializes the recipe implementation object with parameters from the recipe
44+
and sets internal attributes to None / empty as needed.
45+
"""
4346
super().__init__()
4447
self.name = recipe.name
4548
self.version = recipe.version
4649
self.parameters = recipe.parameters
4750

48-
self.inputset: PipelineInputSet = None
51+
self.inputset: PipelineInputSet | None = None
4952
self.frameset: cpl.ui.FrameSet | None = None
5053
self.header: cpl.core.PropertyList | None = None
51-
self.products: Dict[str, PipelineProduct] = {}
54+
self.products: [PipelineProduct] = {}
5255
self.product_frames = cpl.ui.FrameSet()
53-
56+
5457
def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.FrameSet:
5558
"""
5659
The main function of the recipe implementation. It mirrors the signature of `Recipe.run`
@@ -64,14 +67,14 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
6467

6568
try:
6669
self.frameset = frameset
67-
self.import_settings(settings) # Import and process the provided settings dict
68-
self.inputset = self.InputSet(frameset) # Create an appropriate InputSet object
70+
self.import_settings(settings) # Import and process the provided settings dict
71+
self.inputset = self.InputSet(frameset) # Create an appropriate InputSet object
6972
self.inputset.print_debug()
70-
self.inputset.validate() # Verify that they are valid (maybe with `schema` too?)
71-
products = self.process_images() # Do all the actual processing
72-
self.save_products(products) # Save the output products
73+
self.inputset.validate() # Verify that they are valid (maybe with `schema` too?)
74+
self.products = self.process_images() # Do all the actual processing
75+
self._save_products() # Save the output products
7376

74-
return self.build_product_frameset(products) # Return the output as a pycpl FrameSet
77+
return self.build_product_frameset() # Return the output as a pycpl FrameSet
7578
except cpl.core.DataNotFoundError as e:
7679
Msg.error(self.__class__.__qualname__, f"Data not found error: {e.message}")
7780
raise e
@@ -84,20 +87,20 @@ def import_settings(self, settings: Dict[str, Any]) -> None:
8487
self.parameters[key].value = value
8588
except KeyError:
8689
Msg.warning(self.__class__.__qualname__,
87-
f"Settings includes '{key}':{value} "
90+
f"Settings include '{key}' = {value} "
8891
f"but class {self.__class__.__qualname__} "
8992
f"has no parameter named {key}.")
9093

9194
@abstractmethod
92-
def process_images(self) -> Dict[str, PipelineProduct]:
95+
def process_images(self) -> [PipelineProduct]:
9396
"""
9497
The core method of the recipe implementation. It should contain all the processing logic.
9598
At its entry point the Input class must be already loaded and validated.
9699
97100
All pixel manipulation should happen inside this function (or something it calls from within).
98101
Put explicitly,
99-
- no pixel manipulation before entering `process_images`,
100-
- and no pixel manipulation after exiting `process_images`.
102+
- no pixel manipulation *before* entering `process_images`,
103+
- and no pixel manipulation *after* exiting `process_images`.
101104
102105
The basic workflow inside this function should be as follows:
103106
@@ -118,22 +121,24 @@ def process_images(self) -> Dict[str, PipelineProduct]:
118121
"""
119122
return {}
120123

121-
def save_products(self, products: Dict[str, PipelineProduct]) -> None:
124+
def _save_products(self) -> None:
122125
"""
123126
Save and register the created products.
124127
"""
125-
for name, product in products.items():
126-
Msg.debug(self.__class__.__qualname__,
127-
f"Saving product {name}")
128+
assert self.products is not None, "Products have not been created yet!"
129+
130+
Msg.debug(self.__class__.__qualname__,
131+
f"Saving {len(self.products)} products: {self.products}")
132+
for product in self.products:
128133
product.save()
129134

130-
def build_product_frameset(self, products: Dict[str, PipelineProduct]) -> cpl.ui.FrameSet:
135+
def build_product_frameset(self) -> cpl.ui.FrameSet:
131136
"""
132137
Gather all the products and build a FrameSet from their frames so that it can be returned from `run`.
133138
"""
134139
Msg.debug(self.__class__.__qualname__,
135140
f"Building the product frameset")
136-
return cpl.ui.FrameSet([product.as_frame() for product in products.values()])
141+
return cpl.ui.FrameSet([product.as_frame() for product in self.products])
137142

138143
def as_dict(self) -> dict[str, Any]:
139144
"""
@@ -147,7 +152,7 @@ def as_dict(self) -> dict[str, Any]:
147152
'title': self.name,
148153
'inputset': self.inputset.as_dict(),
149154
'products': {
150-
product.tag: product.as_dict() for product in self.products.values()
155+
str(product.category): product.as_dict() for product in self.products
151156
}
152157
}
153158

metisp/pymetis/src/pymetis/base/product.py

+51-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
1818
"""
1919

20-
from abc import ABC, abstractmethod
20+
from abc import ABC
2121

2222
import cpl
2323
from cpl.core import Msg
@@ -31,10 +31,10 @@ class PipelineProduct(ABC):
3131
one FITS file with associated headers and a frame
3232
"""
3333

34-
tag: str = None
35-
group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.PRODUCT # ToDo: Is this a sensible default?
36-
level: cpl.ui.Frame.FrameLevel = None
37-
frame_type: cpl.ui.Frame.FrameType = None
34+
_tag: str = None
35+
_group: cpl.ui.Frame.FrameGroup = cpl.ui.Frame.FrameGroup.PRODUCT # ToDo: Is this a sensible default?
36+
_level: cpl.ui.Frame.FrameLevel = None
37+
_frame_type: cpl.ui.Frame.FrameType = None
3838

3939
def __init__(self,
4040
recipe_impl: 'MetisRecipeImpl',
@@ -120,6 +120,22 @@ def save(self):
120120
header=self.header,
121121
)
122122

123+
@property
124+
def tag(self):
125+
return self._tag
126+
127+
@property
128+
def group(self):
129+
return self._group
130+
131+
@property
132+
def level(self):
133+
return self._level
134+
135+
@property
136+
def frame_type(self):
137+
return self._frame_type
138+
123139
@property
124140
def category(self) -> str:
125141
"""
@@ -181,3 +197,33 @@ def __init__(self,
181197
f"{self.__class__.__qualname__} does not")
182198

183199
super().__init__(recipe, header, image, **kwargs)
200+
201+
202+
class BandSpecificProduct(PipelineProduct, ABC):
203+
"""
204+
Product specific to one band. Probably should be merged with all other similar classes.
205+
"""
206+
band = None
207+
208+
def __init__(self,
209+
recipe: 'MetisRecipe',
210+
header: cpl.core.PropertyList,
211+
image: cpl.core.Image,
212+
*,
213+
band: str = None,
214+
**kwargs):
215+
216+
if band is not None:
217+
self.band = band
218+
219+
"""
220+
At the moment of instantiation, the `band` attribute must already be set *somehow*. Either
221+
- as a class attribute (if it is constant)
222+
- from the constructor (if it is determined from the data)
223+
- or as a provided property (if it has to be computed dynamically)
224+
"""
225+
if self.band is None:
226+
raise NotImplementedError(f"Products specific to a target must define 'band', but "
227+
f"{self.__class__.__qualname__} does not")
228+
229+
super().__init__(recipe, header, image, **kwargs)

metisp/pymetis/src/pymetis/base/recipe.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ class MetisRecipe(cpl.ui.PyRecipe):
4242
"Bonus points if it is not visible from pyesorex.")
4343

4444
parameters = cpl.ui.ParameterList([]) # By default, a recipe does not have any parameters.
45+
# ToDo: There is a pyesorex bug that prevents
46+
# this from being actually used.
4547
implementation_class: type["MetisRecipeImpl"] # Dummy class, this must be overridden in the derived classes anyway
4648

4749
def __init__(self):
@@ -61,4 +63,4 @@ def run(self, frameset: cpl.ui.FrameSet, settings: Dict[str, Any]) -> cpl.ui.Fra
6163
It just calls the same method in the decoupled implementation.
6264
"""
6365
self.implementation = self.dispatch_implementation_class(frameset)(self)
64-
return self.implementation.run(frameset, settings)
66+
return self.implementation.run(frameset, settings)

metisp/pymetis/src/pymetis/inputs/multiple.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -54,19 +54,19 @@ def __init__(self,
5454
self.extract_tag_parameters(tag_matches)
5555

5656
@classmethod
57-
def get_target_name(cls, frameset: cpl.ui.FrameSet):
57+
def get_target_name(cls, frameset: cpl.ui.FrameSet):
5858
"""
5959
Extracts the 'target' name from the input string based on the '_tags' regex.
60-
60+
6161
:param inputString: The string to be matched against the pattern.
6262
:return: The target name if a match is found, otherwise None.
6363
"""
6464
for frame in frameset:
6565
if match := cls._tags.match(frame.tag):
6666
return match.group("target")
67-
else:
67+
else:
6868
return None
69-
69+
7070
def extract_tag_parameters(self, matches: [dict[str, str]]):
7171
if len(matches) == 0:
7272
return

metisp/pymetis/src/pymetis/prefab/flat.py

+11-12
Original file line numberDiff line numberDiff line change
@@ -46,15 +46,15 @@ class RawInput(RawInput):
4646

4747
def __init__(self, frameset: cpl.ui.FrameSet):
4848
super().__init__(frameset)
49-
#self.persistence = PersistenceMapInput(frameset)
50-
#self.linearity = LinearityInput(frameset)
51-
#self.gain_map = GainMapInput(frameset)
52-
#self.inputs |= {self.persistence, self.linearity, self.gain_map}
49+
self.persistence = PersistenceMapInput(frameset)
50+
self.linearity = LinearityInput(frameset)
51+
self.gain_map = GainMapInput(frameset)
52+
self.inputs |= {self.persistence, self.linearity, self.gain_map}
5353

5454
class Product(PipelineProduct):
55-
group = cpl.ui.Frame.FrameGroup.PRODUCT
56-
level = cpl.ui.Frame.FrameLevel.FINAL
57-
frame_type = cpl.ui.Frame.FrameType.IMAGE
55+
_group = cpl.ui.Frame.FrameGroup.PRODUCT
56+
_level = cpl.ui.Frame.FrameLevel.FINAL
57+
_frame_type = cpl.ui.Frame.FrameType.IMAGE
5858
band: str = None
5959

6060
@property
@@ -69,7 +69,7 @@ def output_file_name(self) -> str:
6969
def tag(self) -> str:
7070
return self.category
7171

72-
def process_images(self) -> Dict[str, PipelineProduct]:
72+
def process_images(self) -> [PipelineProduct]:
7373
"""
7474
Do the actual processing of the images.
7575
Here, it means loading the input images and a master dark,
@@ -94,7 +94,6 @@ def process_images(self) -> Dict[str, PipelineProduct]:
9494
header = cpl.core.PropertyList.load(self.inputset.raw.frameset[0].file, 0)
9595
combined_image = self.combine_images(self.inputset.load_raw_images(), method)
9696

97-
self.products = {
98-
self.name.upper(): self.Product(self, header, combined_image),
99-
}
100-
return self.products
97+
product = self.Product(self, header, combined_image)
98+
99+
return [product]

metisp/pymetis/src/pymetis/recipes/cal/metis_cal_chophome.py

+9-12
Original file line numberDiff line numberDiff line change
@@ -62,9 +62,9 @@ class ProductCombined(PipelineProduct):
6262
"""
6363
Final product: combined, background-subtracted images of the WCU source
6464
"""
65-
group = cpl.ui.Frame.FrameGroup.PRODUCT
66-
level = cpl.ui.Frame.FrameLevel.FINAL
67-
frame_type = cpl.ui.Frame.FrameType.IMAGE
65+
_group = cpl.ui.Frame.FrameGroup.PRODUCT
66+
_level = cpl.ui.Frame.FrameLevel.FINAL
67+
_frame_type = cpl.ui.Frame.FrameType.IMAGE
6868

6969
@property
7070
def category(self) -> str:
@@ -83,9 +83,9 @@ class ProductBackground(PipelineProduct):
8383
"""
8484
Intermediate product: the instrumental background (WCU OFF)
8585
"""
86-
group = cpl.ui.Frame.FrameGroup.PRODUCT
87-
level = cpl.ui.Frame.FrameLevel.INTERMEDIATE
88-
frame_type = cpl.ui.Frame.FrameType.IMAGE
86+
_group = cpl.ui.Frame.FrameGroup.PRODUCT
87+
_level = cpl.ui.Frame.FrameLevel.INTERMEDIATE
88+
_frame_type = cpl.ui.Frame.FrameType.IMAGE
8989

9090
@property
9191
def category(self) -> str:
@@ -100,7 +100,7 @@ def tag(self) -> str:
100100
return rf"{self.category}"
101101

102102

103-
def process_images(self) -> Dict[str, PipelineProduct]:
103+
def process_images(self) -> [PipelineProduct]:
104104
"""do something"""
105105

106106
background_hdr = cpl.core.PropertyList()
@@ -113,13 +113,10 @@ def process_images(self) -> Dict[str, PipelineProduct]:
113113
raw_images.subtract_image(background_img)
114114
combined_img = self.combine_images(raw_images, "median")
115115

116-
self.products = {
117-
rf"{self.target}_COMBINED":
116+
return [
118117
self.ProductCombined(self, combined_hdr, combined_img),
119-
rf"{self.target}_BACKGROUND":
120118
self.ProductBackground(self, background_hdr, background_img),
121-
}
122-
return self.products
119+
]
123120

124121
def load_images(self, frameset: cpl.ui.FrameSet) -> cpl.core.ImageList:
125122
"""Load an imagelist from a FrameSet

0 commit comments

Comments
 (0)