Enhance storage fixture management and add dynamic library fixture; streamline cleanup and improve test reliability

Author: G-D-Petrov

python/arcticdb/storage_fixtures/api.py

@@ -132,7 +132,14 @@ def slow_cleanup(self, failure_consequence=""):
         self.libs_from_factory.clear()
 
         arctic = self.create_arctic()
-        for lib in self.libs_names_from_arctic[:]:
+        libs = set(self.libs_names_from_arctic[:])
+        with handle_cleanup_exception(self, arctic, consequence=failure_consequence):
+            # There are some tests that add libraries without using the factory directly (e.g. test_move_storage)
+            # so make sure that we capture all of the libraries
+            for lib in arctic.list_libraries():
+                libs.add(lib)
+
+        for lib in libs:
             with handle_cleanup_exception(self, lib, consequence=failure_consequence):
                 arctic.delete_library(lib)
 
@@ -155,7 +162,7 @@ def replace_uri_field(cls, uri: str, field: ArcticUriFields, replacement: str, s
         regex = cls._FIELD_REGEX[field]
         match = regex.search(uri)
         assert match, f"{uri} does not have {field}"
-        return f"{uri[:match.start(start)]}{replacement}{uri[match.end(end):]}"
+        return f"{uri[: match.start(start)]}{replacement}{uri[match.end(end) :]}"
 
 
 class StorageFixtureFactory(_SaferContextManager):
@@ -188,5 +195,4 @@ def enforcing_permissions_context(self, set_to=True):
             self.enforcing_permissions = saved
 
     @abstractmethod
-    def create_fixture(self) -> StorageFixture:
-        ...
+    def create_fixture(self) -> StorageFixture: ...

python/tests/conftest.py

@@ -212,6 +212,12 @@ def s3_clean_bucket(s3_storage_factory) -> Generator[S3Bucket, None, None]:
         yield f
 
 
+@pytest.fixture(scope="function")
+def azurite_clean_bucket(azurite_storage_factory) -> Generator[S3Bucket, None, None]:
+    with azurite_storage_factory.create_fixture() as f:
+        yield f
+
+
 @pytest.fixture(scope="function")
 def nfs_clean_bucket(nfs_backed_s3_storage_factory) -> Generator[NfsS3Bucket, None, None]:
     with nfs_backed_s3_storage_factory.create_fixture() as f:
@@ -479,6 +485,13 @@ def arctic_library(arctic_client, lib_name) -> Library:
     arctic_client.delete_library(lib_name)
 
 
+@pytest.fixture
+def arctic_library_dynamic(arctic_client, lib_name) -> Library:
+    lib_opts = LibraryOptions(dynamic_schema=True)
+    yield arctic_client.create_library(lib_name, library_options=lib_opts)
+    arctic_client.delete_library(lib_name)
+
+
 @pytest.fixture
 def arctic_library_v1(arctic_client_v1, lib_name) -> Library:
     yield arctic_client_v1.create_library(lib_name)

python/tests/integration/arcticdb/test_arctic.py

@@ -1157,12 +1157,10 @@ def test_segment_slicing(arctic_client, lib_name):
     assert num_data_segments == math.ceil(rows / rows_per_segment) * math.ceil(columns / columns_per_segment)
 
 
-# skip this test for now
-@pytest.mark.skip(reason="There is a strange problem with this one TODO")
 @pytest.mark.parametrize("fixture", ["s3_storage", pytest.param("azurite_storage", marks=AZURE_TESTS_MARK)])
-def test_reload_symbol_list(fixture, request, lib_name):
+def test_reload_symbol_list(fixture, request):
     storage_fixture: StorageFixture = request.getfixturevalue(fixture)
-    lib_name = f"{lib_name}_reload_symbol_list"
+    lib_name = "test_reload_symbol_list"
 
     def get_symbol_list_keys(lib_name):
         keys = storage_fixture.iter_underlying_object_names()

python/tests/integration/arcticdb/test_arctic_move_storage.py

@@ -42,9 +42,6 @@ def test_move_storage(storage_type, host_attr, request):
             # When - we copy the underlying objects without using the Arctic API
             source_storage.copy_underlying_objects_to(dest_storage)
 
-            # TODO: FIX THIS
-            ac.delete_library("lib")
-
         # When - we leave the source_storage context, the storage should have been cleaned up
         assert not source_storage.create_arctic().list_libraries()
 
@@ -55,7 +52,6 @@ def test_move_storage(storage_type, host_attr, request):
         assert lib.list_symbols() == ["sym"]
         assert_frame_equal(df, lib.read("sym").data)
         assert dest_host in lib.read("sym").host
-        new_ac.delete_library("lib")
 
 
 @pytest.mark.skipif(sys.platform == "darwin", reason="Test doesn't raise an exception on MacOS ARM")

python/tests/integration/arcticdb/version_store/test_basic_version_store.py

@@ -2490,7 +2490,7 @@ def test_missing_first_version_key_batch(basic_store):
 
         write_times.append(pd.Timestamp(vit.timestamp))
         expected.append(df1)
-        # time.sleep(1)
+
         df2 = pd.DataFrame(
             {"d": range(x + 1, l + x + 1), "e": range(x + 2, l + x + 2), "f": range(x + 3, l + x + 3)}, index=idx
         )

python/tests/stress/arcticdb/version_store/test_large_df.py

@@ -13,6 +13,8 @@
 import numpy as np
 import pandas as pd
 
+from tests.util.mark import SLOW_TESTS_MARK
+
 from arcticdb.util.test import (
     assert_frame_equal,
     dataframe_for_date,
@@ -63,7 +65,7 @@ def test_write_and_update_large_df_in_chunks(lmdb_version_store_very_big_map):
     assert_frame_equal(lib.head(symbol).data, expected_head)
 
 
-@pytest.mark.skip(reason="Uses too much storage and fails on GitHub runners")
+@SLOW_TESTS_MARK
 def test_write_large_df_in_chunks(lmdb_version_store_big_map):
     symbol = "symbol"
     lib = lmdb_version_store_big_map

python/tests/stress/arcticdb/version_store/test_mem_leaks.py

@@ -325,11 +325,10 @@ def gen_random_date(start: pd.Timestamp, end: pd.Timestamp):
 
 @SLOW_TESTS_MARK
 @SKIP_CONDA_MARK  # Conda CI runner doesn't have enough storage to perform these stress tests
-# @pytest.mark.skipif(
-#     WINDOWS, reason="Not enough storage on Windows runners, due to large Win OS footprint and less free mem"
-# )
-# @pytest.mark.skipif(MACOS, reason="Problem on MacOs most probably similar to WINDOWS")
-@pytest.mark.skip(reason="This test is not conclusive for memory leaks and is very flaky")
+@pytest.mark.skipif(
+    WINDOWS, reason="Not enough storage on Windows runners, due to large Win OS footprint and less free mem"
+)
+@pytest.mark.skipif(MACOS, reason="Problem on MacOs most probably similar to WINDOWS")
 def test_mem_leak_read_all_arctic_lib(arctic_library_lmdb_100gb):
     lib: adb.Library = arctic_library_lmdb_100gb
 

python/tests/stress/arcticdb/version_store/test_stress_delete.py

@@ -50,8 +50,9 @@ def test_stress_delete(object_store_factory):
     print("Delete took {}".format(datetime.now() - start_time))
 
     # Make sure that the symbols are deleted
-    with pytest.raises(NoDataFoundException) as e:
-        lib1.batch_read(syms)
+    for x in range(num_tests):
+        with pytest.raises(NoDataFoundException) as e:
+            lib1.read(f"symbol_{x}")
 
     res = lib2.batch_read(syms)
     for i, sym in enumerate(syms):
@@ -63,5 +64,6 @@ def test_stress_delete(object_store_factory):
     check_no_keys(lib2)
 
     # Make sure that the symbols are deleted
-    with pytest.raises(NoDataFoundException) as e:
-        lib2.batch_read(syms)
+    for x in range(num_tests):
+        with pytest.raises(NoDataFoundException) as e:
+            lib2.read(f"symbol_{x}")

python/tests/unit/arcticdb/version_store/test_sort.py

@@ -41,13 +41,8 @@ def test_stage_finalize(arctic_library):
     pd.testing.assert_frame_equal(result, expected)
 
 
-def create_lib_dynamic(ac, lib_name):
-    lib_opts = adb.LibraryOptions(dynamic_schema=True)
-    return ac.get_library(lib_name, create_if_missing=True, library_options=lib_opts)
-
-
-def test_stage_finalize_dynamic(arctic_client, lib_name):
-    arctic_library = create_lib_dynamic(arctic_client, lib_name)
+def test_stage_finalize_dynamic(arctic_library_dynamic):
+    arctic_library = arctic_library_dynamic
     symbol = "AAPL"
     sort_cols = ["timestamp", "col1"]
 
@@ -79,7 +74,6 @@ def test_stage_finalize_dynamic(arctic_client, lib_name):
 
     expected = pd.concat([df1, df2]).sort_values(sort_cols)
     pd.testing.assert_frame_equal(result, expected)
-    arctic_client.delete_library(lib_name)
 
 
 def random_strings(count, max_length):
@@ -124,8 +118,8 @@ def test_stage_finalize_strings(arctic_library):
     pd.testing.assert_frame_equal(result, expected)
 
 
-def test_stage_finalize_strings_dynamic(arctic_client, lib_name):
-    arctic_library = create_lib_dynamic(arctic_client, lib_name)
+def test_stage_finalize_strings_dynamic(arctic_library_dynamic):
+    arctic_library = arctic_library_dynamic
     symbol = "AAPL"
     sort_cols = ["timestamp", "col1"]