handle empty set properly

Author: guidopetri

src/feature_engineering.py

@@ -81,6 +81,9 @@ def explode_moves(player: str,
                                          data_date=data_date,
                                          )
     df = pd.read_parquet(io_dir / f'{prefix}_cleaned_df.parquet')
+    if df.empty:
+        df.to_parquet(io_dir / f'{prefix}_exploded_moves.parquet')
+        return
     df = df[['game_link', 'moves']]
 
     df = df.explode('moves')
@@ -101,6 +104,9 @@ def explode_clocks(player: str,
                                          data_date=data_date,
                                          )
     df = pd.read_parquet(io_dir / f'{prefix}_cleaned_df.parquet')
+    if df.empty:
+        df.to_parquet(io_dir / f'{prefix}_exploded_clocks.parquet')
+        return
     df = df[['game_link', 'clocks']]
 
     df = df.explode('clocks')
@@ -122,6 +128,9 @@ def explode_positions(player: str,
                                          data_date=data_date,
                                          )
     df = pd.read_parquet(io_dir / f'{prefix}_cleaned_df.parquet')
+    if df.empty:
+        df.to_parquet(io_dir / f'{prefix}_exploded_positions.parquet')
+        return
     df = df[['game_link', 'positions']]
 
     df = df.explode('positions')
@@ -144,6 +153,9 @@ def explode_materials(player: str,
                                          data_date=data_date,
                                          )
     df = pd.read_parquet(io_dir / f'{prefix}_cleaned_df.parquet')
+    if df.empty:
+        df.to_parquet(io_dir / f'{prefix}_exploded_materials.parquet')
+        return
     df = df[['game_link', 'material_by_move']]
 
     df = df.explode('material_by_move')

src/inference.py

@@ -24,6 +24,10 @@ def estimate_win_probabilities(player: str,
     positions = pd.read_parquet(io_dir / f'{prefix}_exploded_positions.parquet')  # noqa
     game_clocks = pd.read_parquet(io_dir / f'{prefix}_exploded_clocks.parquet')
 
+    if all(df.empty for df in [game_infos, evals, positions, game_clocks]):
+        game_infos.to_parquet(io_dir / f'{prefix}_win_probabilities.parquet')
+        return
+
     game_infos['has_increment'] = (game_infos['increment'] > 0).astype(int)
 
     game_infos_cols = ['game_link',

src/pipeline_import/postgres_templates.py

@@ -173,6 +173,9 @@ def _load_to_table(table_name: str,
                      )
 
     reader = pq.ParquetFile(io_dir / f'{parquet_filename}.parquet')
+    if not reader.scan_contents():
+        print('did not find any rows to load, exiting')
+        return
 
     temp_table_name = f'temp_{table_name}'
 

src/pipeline_import/transforms.py

@@ -190,6 +190,9 @@ def transform_game_data(player: str,
                                          data_date=data_date,
                                          )
     df = pd.read_parquet(io_dir / f'{prefix}_cleaned_df.parquet')
+    if df.empty:
+        df.to_parquet(io_dir / f'{prefix}_game_infos.parquet')
+        return
     df['player'] = player
 
     if 'black_rating_diff' not in df.columns:

src/vendors/stockfish.py

@@ -21,6 +21,9 @@ def get_evals(player: str,
                                          data_date=data_date,
                                          )
     df = pd.read_parquet(io_dir / f'{prefix}_cleaned_df.parquet')
+    if df.empty:
+        df.to_parquet(io_dir / f'{prefix}_evals.parquet')
+        return
 
     sf_params = get_cfg('stockfish_cfg')