From 6d83933d5c207071ea94114d7688d144282162f3 Mon Sep 17 00:00:00 2001 From: Xbc-TOP <2765024335@qq.com> Date: Fri, 27 Sep 2024 15:19:34 +0800 Subject: [PATCH 1/2] update pytest --- openbox/core/ea/adaptive_ea_advisor.py | 2 +- .../test_random_configuration_chooser.py | 17 +- test/acquisition_function/test_acquisition.py | 115 ++++++++++- test/core/ea/test_adaptive_ea_advisor.py | 28 ++- test/core/ea/test_differential_ea_advisor.py | 35 +++- test/core/ea/test_nsga2_ea_advisor.py | 18 +- test/core/ea/test_saea_advisor.py | 72 ++++++- test/core/test_async_batch_advisor.py | 13 +- test/core/test_base.py | 9 +- test/core/test_ea_advisor.py | 19 ++ test/core/test_generic_advisor.py | 59 +++++- test/core/test_mc_advisor.py | 20 +- test/core/test_mf_batch_advisor.py | 18 +- test/core/test_random_advisor.py | 18 +- test/core/test_sync_batch_advisor.py | 17 +- test/core/test_tpe_advisor.py | 26 ++- .../test_examples_ask_and_tell_interface.py | 5 +- ...les_evaluate_sync_parallel_optimization.py | 81 +++++++- test/examples/test_examples_quick_example.py | 3 +- test/fixtures/acqusition_funcs.py | 186 +++++++++++++++++- test/fixtures/configspace.py | 63 +++++- test/fixtures/history.py | 63 ++++++ test/fixtures/surrogate_models.py | 11 ++ test/optimizer/test_parallel_smbo.py | 4 +- test/surrogate/base/test_base_model.py | 20 +- test/surrogate/base/test_gp_base_prior.py | 5 +- test/utils/test_early_stop.py | 2 +- 27 files changed, 839 insertions(+), 90 deletions(-) diff --git a/openbox/core/ea/adaptive_ea_advisor.py b/openbox/core/ea/adaptive_ea_advisor.py index 15f72052..99fbfa51 100755 --- a/openbox/core/ea/adaptive_ea_advisor.py +++ b/openbox/core/ea/adaptive_ea_advisor.py @@ -51,7 +51,7 @@ def __init__(self, self.last_observations = [] def get_suggestion(self): - if not self.last_suggestions: + while not self.last_suggestions: self.update_observations(self.last_observations) self.last_observations = [] self.last_suggestions = self.get_suggestions() diff --git a/test/acq_optimizer/test_random_configuration_chooser.py b/test/acq_optimizer/test_random_configuration_chooser.py index aa2d40d5..a9514c6c 100644 --- a/test/acq_optimizer/test_random_configuration_chooser.py +++ b/test/acq_optimizer/test_random_configuration_chooser.py @@ -13,26 +13,41 @@ def test_no_cool_down_chooser_returns_expected_result(self): self.assertTrue(chooser.check(2)) self.assertFalse(chooser.check(3)) + chooser2 = ChooserNoCoolDown(1.0) + self.assertTrue(chooser2.modulus == 1.0) + def test_linear_cool_down_chooser_returns_expected_result(self): chooser = ChooserLinearCoolDown(2.0, 0.3, np.inf) self.assertTrue(chooser.check(2)) self.assertFalse(chooser.check(3)) + chooser.next_smbo_iteration() + self.assertTrue(chooser.check(3)) + def test_prob_chooser_returns_expected_result(self): chooser = ChooserProb(0.5, self.rng) self.assertTrue(chooser.check(1)) self.assertFalse(chooser.check(2)) def test_prob_cool_down_chooser_returns_expected_result(self): - chooser = ChooserProbCoolDown(0.5, 0.1, self.rng) + chooser = ChooserProbCoolDown(0.5, 2, self.rng) self.assertTrue(chooser.check(1)) self.assertFalse(chooser.check(2)) + chooser.next_smbo_iteration() + self.assertTrue(chooser.check(3)) + def test_cosine_annealing_chooser_returns_expected_result(self): chooser = ChooserCosineAnnealing(0.5, 0.1, 2, self.rng) self.assertTrue(chooser.check(1)) self.assertFalse(chooser.check(2)) + chooser.next_smbo_iteration() + self.assertTrue(chooser.iteration == 1) + chooser.next_smbo_iteration() + chooser.next_smbo_iteration() + self.assertTrue(chooser.iteration == 0) + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/test/acquisition_function/test_acquisition.py b/test/acquisition_function/test_acquisition.py index 740cec22..a5279c49 100644 --- a/test/acquisition_function/test_acquisition.py +++ b/test/acquisition_function/test_acquisition.py @@ -12,7 +12,15 @@ LCB, Uncertainty, ) - +''' +加入其他函数的测试 +''' +from openbox.acquisition_function.multi_objective_acquisition import ( + MaxvalueEntropySearch, + MESMO, + MESMOC, + MESMOC2, +) class ConfigurationMock: def __init__(self, values=None): self.values = values @@ -22,6 +30,13 @@ def get_array(self): return np.array(self.values, dtype=np.float64) class MockModel: + ''' + 其实axis的重点在于方向，而不是行和列。具体到各种用法而言也是如此。 + 当axis=1时，如果是求平均，那么是从左到右横向求平均；如果是拼接，那么也是左右横向拼接；如果是drop，那么也是横向发生变化，体现为列的减少。 + mean_col_vector = np.mean(X, axis=1).reshape(-1, 1) +return mean_col_vector, mean_col_vector + 这里代码重复了？ + ''' def predict_marginalized_over_instances(self, X): return np.array([np.mean(X, axis=1).reshape((1, -1))]).reshape((-1, 1)), np.array( [np.mean(X, axis=1).reshape((1, -1))] @@ -45,11 +60,30 @@ def __init__(self, num_targets=1): self.num_targets = num_targets def predict_marginalized_over_instances(self, X): + ''' + 这个乘法真的没有问题？不是引用？ + np.full(X.shape[0], 0.0)：创建一个与 X 的行数相同的全 0 数组。 + >>> b=[a] + >>> b + [array([1, 2])] + >>> b*2 + [array([1, 2]), array([1, 2])] + >>> c=b*2 + >>> c + [array([1, 2]), array([1, 2])] + >>> c[0][0]=2 + >>> c + [array([2, 2]), array([2, 2])] + >>> + ''' return np.array([np.mean(X, axis=1).reshape((1, -1))] * self.num_targets).reshape((-1, 2)), np.array( [np.full(X.shape[0], 0.0).reshape((1, -1))] * self.num_targets ).reshape((-1, 2)) class MockConstraintModel: + ''' + np.full(m.shape, 0.1)：创建一个与 m 形状相同的数组，数组中的每个元素都为 0.1 + ''' def predict_marginalized_over_instances(self, X): m = -np.abs(np.mean(X, axis=1)).reshape((-1, 1)) v = np.full(m.shape, 0.1) @@ -60,6 +94,7 @@ def predict_marginalized_over_instances(self, X): return np.full(X.shape[0], np.nan).reshape((-1, 1)), np.full(X.shape[0], np.nan).reshape((-1, 1)) class MockModelWrongVShape: + #错的v shape？ def predict_marginalized_over_instances(self, X): m = np.mean(X, axis=1).reshape((-1, 2)) v = np.var(X, axis=1).reshape((-1, 1)) @@ -150,6 +185,12 @@ def test_ei_eta_fail(model, acquisition_function): ei.update(model=model) configurations = [ConfigurationMock([1.0])] with pytest.raises(ValueError): + ''' + 1. with pytest.raises(ValueError): +pytest.raises 是 pytest 提供的一个上下文管理器，用于检查某段代码是否抛出特定类型的异常。 +ValueError 是期望的异常类型。 +这段代码的含义是：在 with 语句的块内，pytest 期望执行的代码会引发 ValueError。如果代码没有抛出 ValueError，测试将会失败。 + ''' ei(configurations) def test_ei_1x1(model, acquisition_function): @@ -426,7 +467,7 @@ def test_pi_1xD(model, acq_pi): def test_pi_NxD(model, acq_pi): pi = acq_pi pi.update(model=model, eta=1.0) - configurations = [ + configurations = [ ConfigurationMock([0.0001, 0.0001, 0.0001]), ConfigurationMock([0.1, 0.1, 0.1]), ConfigurationMock([1.0, 1.0, 1.0]), @@ -549,4 +590,72 @@ def test_uncertainty_with_nan(model_with_nan, acq_uncer): configurations = [ConfigurationMock([0.5])] acq = uncertainty(configurations) - assert np.all(acq == 0) \ No newline at end of file + assert np.all(acq == 0) + + + + +''' +import numpy as np + +# 创建一个形状为 (2, 3, 4) 的三维数组 +data = np.array([ + [[1, 2, 3, 4], # 实验 1 + [5, 6, 7, 8], + [9, 10, 11, 12]], + + [[13, 14, 15, 16], # 实验 2 + [17, 18, 19, 20], + [21, 22, 23, 24]] +]) + +print("原始数组:") +print(data) +mean_axis_0 = np.mean(data, axis=0) +print("沿 axis=0 计算的均值:") +print(mean_axis_0) +# 输出: +# [[ 7. 8. 9. 10.] +# [11. 12. 13. 14.] +# [15. 16. 17. 18.]] +mean_axis_1 = np.mean(data, axis=1) + +print("沿 axis=1 计算的均值:") +print(mean_axis_1) +# 输出: +# [[ 5. 6. 7. 8.] +# [19. 20. 21. 22.]] + +mean_axis_neg1 = np.mean(data, axis=-1) +print("沿 axis=-1 计算的均值:") +print(mean_axis_neg1) +# 输出: +# [[ 2.5 6.5 10.5] +# [14.5 18.5 22.5]] +总结 +''' + +''' +使用 np.expand_dims +当我们调用： +expanded = np.expand_dims(self.cell_lower_bounds, axis=(1, 2)) +axis=(1, 2) 指定在第 1 和第 2 维的位置插入新维度。 +变化步骤 +第一次扩展 (axis=1): + +原始数组形状为 (m,)，在第 1 维插入一个新维度后，形状变为 (1, m)： +python +复制代码 +[[1, 2, 3]] # 形状为 (1, 3) +第二次扩展 (axis=2): + +在新数组的第 2 维插入新维度，形状变为 (1, m, 1)： +python +复制代码 +[[[1], + [2], + [3]]] # 形状为 (1, 3, 1) +总结 +因此，经过两次扩展后，self.cell_lower_bounds 的形状最终变为 (1, m, 1)，表示有一个批次（第一维为 1），m 个目标（第二维为 m）， +以及每个目标的单个值（第三维为 1）。这种形状使得该数组可以与其他三维数组（如 Y_samples）进行有效的广播和运算。 +''' diff --git a/test/core/ea/test_adaptive_ea_advisor.py b/test/core/ea/test_adaptive_ea_advisor.py index 1c6c1359..a8c1cdd4 100644 --- a/test/core/ea/test_adaptive_ea_advisor.py +++ b/test/core/ea/test_adaptive_ea_advisor.py @@ -7,7 +7,12 @@ def test_adaptive_ea_advisor(configspace_tiny): config_space = configspace_tiny - advisor = AdaptiveEAAdvisor(config_space, population_size=4, subset_size=2, num_objectives=2, pc=1, pm=1) + advisor = AdaptiveEAAdvisor(config_space, population_size=8, subset_size=2, num_objectives=2, pc=1, pm=1) + #换一个oldest策略再来一次? + ''' + 目前看来这个只支持单目标优化，这里的多目标似乎是假的 + ''' + advisor_oldest = AdaptiveEAAdvisor(config_space, strategy='oldest', population_size=8, subset_size=2, num_objectives=2, pc=1, pm=1) assert advisor.config_space == config_space assert advisor.subset_size == 2 assert advisor.epsilon == 0.2 @@ -20,13 +25,23 @@ def test_adaptive_ea_advisor(configspace_tiny): assert advisor.k4 == 0.3 assert advisor.last_suggestions == [] assert advisor.last_observations == [] - - perfs = [[2, 1], [1, 2], [2, 2], [1, 1], [1, 2]] - for i in range(4): + ''' + 多assert + ''' + perfs = [[2, 1], [1, 2], [2, 2], [1, 1], [1, 2], [1, 1], [1, 1], [1, 1], [1, 1], [2, 3], [4, 5], [6, 7], [3, 6]] + #修改为10次 ref_point多给点 + for i in range(13): suggestion1 = advisor.get_suggestion() assert isinstance(suggestion1, Configuration) observation1 = Observation(suggestion1, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) advisor.update_observation(observation1) + ''' + oldest strategy + ''' + suggestion1 = advisor_oldest.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_oldest.update_observation(observation1) config_a = config_space.sample_configuration() config_b = config_space.sample_configuration() @@ -35,4 +50,7 @@ def test_adaptive_ea_advisor(configspace_tiny): config = config_space.sample_configuration() next_config = advisor.mutation(config) - assert isinstance(next_config, Configuration) \ No newline at end of file + ## 测试None + # config = None + # next_config = advisor.mutation(config) + # assert isinstance(next_config, Configuration) \ No newline at end of file diff --git a/test/core/ea/test_differential_ea_advisor.py b/test/core/ea/test_differential_ea_advisor.py index 6e6390de..109d9b65 100644 --- a/test/core/ea/test_differential_ea_advisor.py +++ b/test/core/ea/test_differential_ea_advisor.py @@ -6,9 +6,19 @@ from openbox.utils.constants import MAXINT, SUCCESS -def test_cmaes_ea_advisor_initialization(configspace_tiny): +def test_cmaes_ea_advisor_initialization(configspace_tiny, configspace_cat): config_space = configspace_tiny advisor = DifferentialEAAdvisor(config_space, num_objectives=2, population_size=4) + config_cat = configspace_cat + #添加变化的f以及cr + + ''' + 在单目标优化中，允许使用动态的 f 和 cr 来提升算法的表现。 + 在多目标优化中，为了保持多个目标的平衡，限制动态调整 f 和 cr，因此在这种情况下需要固定这些参数。 + 断言的目的是避免算法在不合适的情况下使用动态参数，确保差分进化算法在不同优化问题上的一致性和正确性。 + ''' + advisor_vary = DifferentialEAAdvisor(config_space, f=(0,1), cr=(0,1), num_objectives=1, population_size=4) + advisor_cat = DifferentialEAAdvisor(config_cat, num_objectives=1, population_size=4) assert advisor.config_space == config_space assert advisor.num_objectives == 2 assert advisor.num_constraints == 0 @@ -24,13 +34,23 @@ def test_cmaes_ea_advisor_initialization(configspace_tiny): assert advisor.filter_gen_population is None assert advisor.keep_unexpected_population is True assert advisor.save_cached_configuration is True - - perfs = [[2, 1], [1, 2], [2, 2], [1, 1], [1, 2]] - for i in range(5): + perfs = [[2, 1], [1, 2], [2, 2], [1, 1], [1, 2], [1, 1], [1, 1], [-1, 1]] + for i in range(8): suggestion1 = advisor.get_suggestion() assert isinstance(suggestion1, Configuration) observation1 = Observation(suggestion1, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + #这个update内有next_population的更新，会把suggestion1也就是gen的config加入sub，让sel能使用sub advisor.update_observation(observation1) + + suggestion2 = advisor_vary.get_suggestion() + assert isinstance(suggestion2, Configuration) + observation2 = Observation(suggestion2, [perfs[i][0]], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_vary.update_observation(observation2) + + # suggestion3 = advisor_cat.get_suggestion() + # assert isinstance(suggestion3, Configuration) + # observation3 = Observation(suggestion3, [1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + # advisor_cat.update_observation(observation3) config_a = config_space.sample_configuration() config_b = config_space.sample_configuration() @@ -40,3 +60,10 @@ def test_cmaes_ea_advisor_initialization(configspace_tiny): config = advisor.cross_over(config_a, config_b, 0.5) assert isinstance(config, Configuration) + + + config_a = config_cat.sample_configuration() + config_b = config_cat.sample_configuration() + config_c = config_cat.sample_configuration() + config = advisor_cat.mutate(config_a, config_b, config_c, 0.5) + #这里的prefs对应ref_point吧？传到Observation中对应objectives属性 \ No newline at end of file diff --git a/test/core/ea/test_nsga2_ea_advisor.py b/test/core/ea/test_nsga2_ea_advisor.py index 14b454de..7ceee1a5 100644 --- a/test/core/ea/test_nsga2_ea_advisor.py +++ b/test/core/ea/test_nsga2_ea_advisor.py @@ -6,7 +6,7 @@ from openbox.utils.constants import MAXINT, SUCCESS -def test_nsga2_ea_advisor_initialization(configspace_tiny): +def test_nsga2_ea_advisor_initialization(configspace_tiny, configspace_huge): config_space = configspace_tiny advisor = NSGA2EAdvisor(config_space, num_objectives=2, subset_size=2, population_size=4) assert advisor.config_space == config_space @@ -25,4 +25,18 @@ def test_nsga2_ea_advisor_initialization(configspace_tiny): suggestion1 = advisor.get_suggestion() assert isinstance(suggestion1, Configuration) observation1 = Observation(suggestion1, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) - advisor.update_observation(observation1) \ No newline at end of file + advisor.update_observation(observation1) + + obs = list() + for i in range(5): + configs = advisor.get_suggestions() + for config in configs: + obs.append(Observation(config, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={})) + advisor.update_observations(obs) + + + #test cross_over + advisor_2 = NSGA2EAdvisor(config_space=configspace_huge) + for i in range(5): + config = advisor_2.cross_over(configspace_huge.sample_configuration(), configspace_huge.sample_configuration()) + assert config.configuration_space == configspace_huge \ No newline at end of file diff --git a/test/core/ea/test_saea_advisor.py b/test/core/ea/test_saea_advisor.py index 6cfdf91a..3cc4a853 100644 --- a/test/core/ea/test_saea_advisor.py +++ b/test/core/ea/test_saea_advisor.py @@ -8,9 +8,31 @@ from openbox.utils.constants import MAXINT, SUCCESS +from openbox.core.ea.regularized_ea_advisor import RegularizedEAAdvisor + + +''' +urrogate-assisted evolutionary algorithm, SAEA +''' + def test_saea_advisor(configspace_tiny): config_space = configspace_tiny - advisor = SAEAAdvisor(config_space, population_size=4, num_objectives=1) + advisor = SAEAAdvisor(config_space, population_size=4, num_objectives=1, ea=RegularizedEAAdvisor(config_space, subset_size=2, population_size=4, num_objectives=1)) + advisor_eic = SAEAAdvisor(config_space, population_size=4, num_constraints=1, ea=RegularizedEAAdvisor(config_space, subset_size=2, population_size=4, num_constraints=1)) + + advisor_ehvic = SAEAAdvisor(config_space, population_size=4, num_objectives=2, num_constraints=1, ref_point=[1,1], ea=RegularizedEAAdvisor(config_space, subset_size=2, population_size=4, num_objectives=2, num_constraints=1)) + + advisor_ehvi = SAEAAdvisor(config_space, population_size=4, num_objectives=2, ref_point=[1,1], ea=RegularizedEAAdvisor(config_space, subset_size=2, population_size=4, num_objectives=2)) + + ''' + mesmo不需要ref_point + ''' + advisor_mesmo = SAEAAdvisor(config_space, population_size=4, num_objectives=2, ea=RegularizedEAAdvisor(config_space, subset_size=2, population_size=4, num_objectives=2)) + + advisor_mesmoc = SAEAAdvisor(config_space, population_size=4, num_objectives=2, num_constraints=1, ea=RegularizedEAAdvisor(config_space, subset_size=2, num_objectives=2, population_size=4, num_constraints=1)) + + advisor_parego = SAEAAdvisor(config_space, population_size=4, acq='parego', num_objectives=2, num_constraints=1, ea=RegularizedEAAdvisor(config_space, subset_size=2, num_objectives=2, population_size=4, num_constraints=1)) + assert advisor.config_space == config_space assert advisor.gen_multiplier == 50 assert advisor.is_models_trained is False @@ -21,3 +43,51 @@ def test_saea_advisor(configspace_tiny): assert isinstance(suggestion1, Configuration) observation1 = Observation(suggestion1, perfs[i][:1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) advisor.update_observation(observation1) + if i == 0: + advisor.sel() + #第二轮清空一下cache，不然会无法循环了 + if i == 1: + advisor.cached_config = [] + assert advisor.is_models_trained is True + + suggestion1 = advisor_eic.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i][:1], constraints=[1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_eic.update_observation(observation1) + if i == 0: + advisor_eic.sel() + + suggestion1 = advisor_ehvic.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i], constraints=[1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_ehvic.update_observation(observation1) + if i == 0: + advisor_ehvic.sel() + + suggestion1 = advisor_ehvi.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_ehvi.update_observation(observation1) + if i == 0: + advisor_ehvi.sel() + + suggestion1 = advisor_mesmoc.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i], constraints=[1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_mesmoc.update_observation(observation1) + # if i == 0: + # advisor_mesmoc.sel() + + suggestion1 = advisor_mesmo.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_mesmo.update_observation(observation1) + # if i == 0: + # advisor_mesmo.sel() + + suggestion1 = advisor_parego.get_suggestion() + assert isinstance(suggestion1, Configuration) + observation1 = Observation(suggestion1, perfs[i], constraints=[1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_parego.update_observation(observation1) + # if i == 0: + # advisor_parego.sel() diff --git a/test/core/test_async_batch_advisor.py b/test/core/test_async_batch_advisor.py index 1bcb8264..ecfb91e3 100644 --- a/test/core/test_async_batch_advisor.py +++ b/test/core/test_async_batch_advisor.py @@ -1,12 +1,9 @@ import pytest from unittest.mock import MagicMock, patch - -from openbox.utils.early_stop import EarlyStopException from openbox.core.async_batch_advisor import AsyncBatchAdvisor from openbox.utils.config_space import ConfigurationSpace from openbox.utils.history import History, Observation from openbox.utils.constants import MAXINT, SUCCESS -from openbox import logger def test_async_batch_advisor_initialization(configspace_tiny, history_single_obs): @@ -33,4 +30,12 @@ def test_async_batch_advisor_initialization(configspace_tiny, history_single_obs observation = Observation(suggestion, [0.1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) advisor.update_observation(observation) assert len(advisor.history) == 1 - +# + +# def test_async_batch_advisor_update_observation(): +# config_space = mock_config_space() +# advisor = AsyncBatchAdvisor(config_space) +# observation = mock_observation() +# advisor.running_configs.append(observation.config) +# advisor.update_observation(observation) +# assert observation.config not in advisor.running_configs \ No newline at end of file diff --git a/test/core/test_base.py b/test/core/test_base.py index 95bfcb67..a8d1e5fa 100644 --- a/test/core/test_base.py +++ b/test/core/test_base.py @@ -11,13 +11,13 @@ from openbox.surrogate.tlbo.rgpe import RGPE from openbox.surrogate.tlbo.stacking_gpr import SGPR from openbox.surrogate.tlbo.topo_variant3 import TOPO_V3 - +from openbox.surrogate.mo.parego import ParEGOSurrogate def test_build_acq_func(configspace_tiny, surrogate_model_abs, surrogate_model_gp): config_space = configspace_tiny model_abs = surrogate_model_abs model_gp = surrogate_model_gp - + #"?"未识别的情况 for acq_func in ['ei', 'eips', 'logei', 'pi', 'lcb', 'lpei', 'mcei', 'parego']: result = build_acq_func(acq_func, model_abs) assert isinstance(result, AbstractAcquisitionFunction) @@ -50,7 +50,6 @@ def test_build_acq_func(configspace_tiny, surrogate_model_abs, surrogate_model_g result = build_acq_func(acq_func, model_abs, [model_gp, model_gp]) assert isinstance(result, AbstractAcquisitionFunction) - def test_build_surrogate(configspace_tiny, transfer_learning_history_single): config_space = configspace_tiny rng = np.random.RandomState(0) @@ -80,6 +79,10 @@ def test_build_surrogate(configspace_tiny, transfer_learning_history_single): surrogate = build_surrogate('tlbo_topov3_gp', config_space, rng, transfer_learning_history) assert isinstance(surrogate, TOPO_V3) + #start with parego + surrogate = build_surrogate('parego_gp', config_space, rng, transfer_learning_history) + assert isinstance(surrogate, ParEGOSurrogate) + with pytest.raises(ValueError, match='.* for tlbo surrogate!'): build_surrogate('tlbo_invalid', config_space, rng, transfer_learning_history) diff --git a/test/core/test_ea_advisor.py b/test/core/test_ea_advisor.py index 823a3a10..7c302bbb 100644 --- a/test/core/test_ea_advisor.py +++ b/test/core/test_ea_advisor.py @@ -1,4 +1,5 @@ import pytest +import random from unittest.mock import MagicMock, patch from openbox.core.ea_advisor import EA_Advisor from openbox.utils.config_space import ConfigurationSpace @@ -29,6 +30,8 @@ def test_ea_advisor_initialization(configspace_tiny): assert advisor.population == [] assert advisor.history is not None + + config = advisor.get_suggestion() assert config in advisor.all_configs assert config in advisor.running_configs @@ -53,3 +56,19 @@ def test_ea_advisor_initialization(configspace_tiny): # test get_history() history = advisor.get_history() assert history == advisor.history + + # test update_observation() + configs = advisor.get_suggestions(batch_size=30) + objectives = [(i)/10 for i in range(0,30)] + for (config, objective) in zip(configs, objectives): + obesrvation = Observation(config, objective, trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor.update_observation(obesrvation) + + assert len(advisor.population) == 30 + assert dict(config=configs[0], age=2, pref=0.0) not in advisor.population + + # test select a parent + configs = advisor.get_suggestions(batch_size=10) + for config in configs: + assert config in advisor.all_configs + assert config in advisor.running_configs diff --git a/test/core/test_generic_advisor.py b/test/core/test_generic_advisor.py index 228cd2e4..038884cf 100644 --- a/test/core/test_generic_advisor.py +++ b/test/core/test_generic_advisor.py @@ -5,12 +5,59 @@ from openbox.utils.util_funcs import check_random_state from openbox.utils.history import Observation from openbox.utils.constants import MAXINT, SUCCESS -from openbox import logger -def test_generic_advisor(configspace_tiny, history_single_obs): +def test_generic_advisor(configspace_tiny,configspace_cat,configspace_big,configspace_mid,history_single_obs): config_space = configspace_tiny - advisor = Advisor(config_space) + #设置其他类型的config_space,需要修改fixtures下面的 + config_space_2 = configspace_cat + #传入其他类型的参数,非连续和cat我不太知道，毕竟space.const的语法是啥？我看不了二进制，可能得去MIT源码看 + config_space_3 = configspace_big + config_space_4 = configspace_mid + + advisor_cat = Advisor(config_space_2,early_stop=True,early_stop_kwargs={"min_iter":1}) + config=advisor_cat.get_suggestion() + observation = Observation(config, [0.1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) + advisor_cat.update_observation(observation) + + advisor_cat.get_suggestion() + #设置早停机制 + advisor_initial = Advisor(config_space) + #传入early_stop_kwards可以提前终止，这个kwards中的min_iter是控制何时终止的，大于了就终止 + advisor = Advisor(config_space,early_stop=True) + advisor_2 = Advisor(config_space,ref_point=[0, 0],num_objectives=2,num_constraints=1) + advisor_3 = Advisor(config_space_3,num_objectives=2,ref_point=[0, 0],init_strategy='halton') + advisor_3.get_suggestion() + #advisor_3是ehvic + advisor_4 = Advisor(config_space_4,num_objectives=5,ref_point=[0, 0, 0, 0, 0],init_strategy='random') + advisor_4.get_suggestion() + #我们直接修改advisor_4的acq_type，让那段代码能运行 + advisor_4.acq_type='ehvi' + advisor_4.check_setup() + advisor_4.acq_type='mesmo';advisor_4.surrogate_type='gp' + advisor_4.check_setup() + + + advisor_5 = Advisor(config_space_4,num_objectives=5,num_constraints=1,ref_point=[0, 0, 0, 0, 0],init_strategy='sobol') + advisor_5.get_suggestion() + #拿advisor_5撞set_up + advisor_5.acq_type='ehvic' + advisor_5.check_setup() + advisor_5.acq_type='mesmoc';advisor_5.surrogate_type='gp' + advisor_5.check_setup() + advisor_5.acq_type='mesmoc';advisor_5.constraint_surrogate_type='gp' + advisor_5.check_setup() + advisor_6 = Advisor(config_space_4,num_constraints=1,init_strategy='latin_hypercube') + advisor_6.get_suggestion() + #这里建立usemo和parego + # advisor_6.acq_type = 'parego' + # advisor_6.setup_bo_basics() + # advisor_6.acq_type = 'usemo' + # advisor_6.setup_bo_basics() + ''' + 进行一些assert看是否与预期相同(to do) + ''' + assert advisor.config_space == config_space assert advisor.num_objectives == 1 assert advisor.num_constraints == 0 @@ -24,7 +71,6 @@ def test_generic_advisor(configspace_tiny, history_single_obs): assert advisor.output_dir == 'logs' assert advisor.task_id == 'OpenBox' assert isinstance(advisor.rng, np.random.RandomState) - config = advisor.get_suggestion(history=history_single_obs) assert config is not None @@ -38,6 +84,11 @@ def test_generic_advisor(configspace_tiny, history_single_obs): history = advisor.get_history() assert history == advisor.history + ''' + 让initial configuration有值,注意传参不能跳过默认参数，要么就用关键字形式赋值 + ''' + initial_configurations = advisor.create_initial_design(advisor.init_strategy) + new_advisor = Advisor(config_space,initial_configurations=initial_configurations) advisor.save_json("test/datas/test.json") advisor.load_json("test/datas/test.json") diff --git a/test/core/test_mc_advisor.py b/test/core/test_mc_advisor.py index 140f3543..1691ae14 100644 --- a/test/core/test_mc_advisor.py +++ b/test/core/test_mc_advisor.py @@ -55,4 +55,22 @@ def test_mc_advisor_early_stop(configspace_tiny): advisor.update_observation(observation) with pytest.raises(EarlyStopException): - advisor.get_suggestion() \ No newline at end of file + advisor.get_suggestion() + +def test_mc_different_constraints_and_objectives(configspace_tiny, multi_start_history_double_obs): + config_space = configspace_tiny + + Ob_array = [1, 2, 2, 5, 5] + Co_array = [1, 0, 1, 0, 1] + Ty_array = ['mceic', 'mcehvi', 'mcehvic', 'mcparego', 'mcparegoc'] + + for (o,c,t) in zip(Ob_array, Co_array, Ty_array): + ref_point = None + if 'ehvi' in t: + ref_point = [0.1,0.2] + ad = MCAdvisor(config_space, o, c, ref_point=ref_point) + if c == 0: + ad.get_suggestion(multi_start_history_double_obs) + else: + ad = MCAdvisor(config_space, o, c, ref_point=ref_point) + assert ad.acq_type == t \ No newline at end of file diff --git a/test/core/test_mf_batch_advisor.py b/test/core/test_mf_batch_advisor.py index 4ddcd2ae..88a1d374 100644 --- a/test/core/test_mf_batch_advisor.py +++ b/test/core/test_mf_batch_advisor.py @@ -1,6 +1,7 @@ import pytest +from unittest.mock import MagicMock, patch from openbox.core.mf_batch_advisor import MFBatchAdvisor -from openbox.utils.early_stop import EarlyStopException +from openbox.utils.config_space import ConfigurationSpace from openbox.utils.history import Observation from openbox.utils.constants import MAXINT, SUCCESS @@ -43,18 +44,3 @@ def test_mf_batch_advisor(configspace_tiny, history_single_obs): assert len(advisor.history_list) == 4 assert len(advisor.resource_identifiers) == 4 - - - -def test_mf_advisor_early_stop(configspace_tiny): - config_space = configspace_tiny - advisor = MFBatchAdvisor(config_space, early_stop=True, early_stop_kwargs={'min_iter': 3, 'min_improvement_percentage': 1e8}) - - for i in range(3): - suggestion = advisor.get_suggestions(1)[0] - observation = Observation(suggestion, [10-i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) - advisor.update_observation(observation, 0.8) - advisor.update_observation(observation, 1) - - with pytest.raises(EarlyStopException): - advisor.get_suggestions(1)[0] diff --git a/test/core/test_random_advisor.py b/test/core/test_random_advisor.py index 97cc6d99..c58c40d0 100644 --- a/test/core/test_random_advisor.py +++ b/test/core/test_random_advisor.py @@ -4,7 +4,6 @@ from openbox.utils.config_space import ConfigurationSpace from openbox.utils.history import History, Observation from openbox.utils.constants import MAXINT, SUCCESS -from openbox.utils.early_stop import EarlyStopException def test_random_advisor_initialization(configspace_tiny, history_single_obs): config_space = configspace_tiny @@ -17,6 +16,7 @@ def test_random_advisor_initialization(configspace_tiny, history_single_obs): assert advisor.task_id == 'OpenBox' assert advisor.rng is not None + advisor = RandomAdvisor(config_space) config = advisor.get_suggestion(history=history_single_obs) assert config is not None @@ -26,19 +26,3 @@ def test_random_advisor_initialization(configspace_tiny, history_single_obs): configs = advisor.sample_random_configs(config_space, 5) assert len(configs) == 5 - - -def test_random_advisor_early_stop(configspace_tiny, history_single_obs): - config_space = configspace_tiny - advisor = RandomAdvisor(config_space, early_stop=True, early_stop_kwargs={'max_no_improvement_rounds': 10}) - - for i in range(13): - if i < 12: - config = advisor.get_suggestion() - observation = Observation(config, [0.1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) - advisor.update_observation(observation) - assert len(advisor.history) == i + 1 - else: - with pytest.raises(EarlyStopException): - config = advisor.get_suggestion() - diff --git a/test/core/test_sync_batch_advisor.py b/test/core/test_sync_batch_advisor.py index b52bedbd..ff74b29a 100644 --- a/test/core/test_sync_batch_advisor.py +++ b/test/core/test_sync_batch_advisor.py @@ -1,7 +1,8 @@ import pytest +from unittest.mock import MagicMock, patch from openbox.core.sync_batch_advisor import SyncBatchAdvisor -from openbox.utils.early_stop import EarlyStopException +from openbox.utils.config_space import ConfigurationSpace from openbox.utils.history import History, Observation from openbox.utils.constants import MAXINT, SUCCESS @@ -50,17 +51,3 @@ def test_sync_batch_advisora(configspace_tiny, history_single_obs): observation = Observation(suggestions[0], [0.1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) advisor.update_observation(observation) assert len(advisor.history) == 1 - - - -def test_sync_batch_advisor_early_stop(configspace_tiny): - config_space = configspace_tiny - advisor = SyncBatchAdvisor(config_space, early_stop=True, early_stop_kwargs={'min_iter': 3, 'min_improvement_percentage': 1e8}) - - for i in range(3): - suggestion = advisor.get_suggestion() - observation = Observation(suggestion, [10-i], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) - advisor.update_observation(observation) - - with pytest.raises(EarlyStopException): - advisor.get_suggestion() \ No newline at end of file diff --git a/test/core/test_tpe_advisor.py b/test/core/test_tpe_advisor.py index 0b8e1500..22d825dd 100644 --- a/test/core/test_tpe_advisor.py +++ b/test/core/test_tpe_advisor.py @@ -5,9 +5,10 @@ from openbox.utils.config_space import ConfigurationSpace from openbox.utils.history import History, Observation from openbox.utils.constants import MAXINT, SUCCESS +from openbox import space as sp -def test_tpe_advisor_initialization(configspace_tiny, history_single_obs): +def test_tpe_advisor_initialization(configspace_tiny, configspace_2, history_single_obs, history_single_obs_2): config_space = configspace_tiny advisor = TPE_Advisor(config_space) assert advisor.config_space == config_space @@ -19,6 +20,7 @@ def test_tpe_advisor_initialization(configspace_tiny, history_single_obs): assert advisor.task_id == 'OpenBox' assert advisor.output_dir == 'logs' assert advisor.rng is not None + assert advisor.kde_vartypes == 'cc' config = advisor.get_suggestion(history=history_single_obs) observation = Observation(config, [0.1], trial_state=SUCCESS, elapsed_time=2.0, extra_info={}) @@ -29,6 +31,10 @@ def test_tpe_advisor_initialization(configspace_tiny, history_single_obs): array = np.array([[1, 2, np.nan], [4, np.nan, 6], [3, 8, 9]]) imputed_array = advisor.impute_conditional_data(array) assert not np.isnan(imputed_array).any() + + array_2 = np.array([[1, np.nan]]) + imputed_array = advisor.impute_conditional_data(array_2) + assert not np.isnan(imputed_array).any() advisor.fit_kde_models(history_single_obs) assert 'good' in advisor.kde_models @@ -39,4 +45,20 @@ def test_tpe_advisor_initialization(configspace_tiny, history_single_obs): history = advisor.get_history() assert history == advisor.history - \ No newline at end of file + + + # test when min_points_in_model < len(Parameters)+1 + + advisor_2 = TPE_Advisor(configspace_2, min_points_in_model=1) + assert advisor_2.min_points_in_model == 3 + assert advisor_2.kde_vartypes == 'cu' + + config = advisor_2.get_suggestion(history_single_obs_2) + + + config = advisor_2.get_suggestion() + assert len(advisor_2.history) == 0 + assert config == configspace_2.get_default_configuration() + + imputed_array = advisor_2.impute_conditional_data(array_2) + assert not np.isnan(imputed_array).any() \ No newline at end of file diff --git a/test/examples/test_examples_ask_and_tell_interface.py b/test/examples/test_examples_ask_and_tell_interface.py index f2e8630a..e2d20b01 100644 --- a/test/examples/test_examples_ask_and_tell_interface.py +++ b/test/examples/test_examples_ask_and_tell_interface.py @@ -49,7 +49,10 @@ def test_examples_ask_and_tell_interface(): plt.close() # install pyrfr to use get_importance() - print(history.get_importance()) + + #我修改了method + + print(history.get_importance(method='shap')) # Have a try on the new HTML visualization feature! # You can also call visualize_html() after optimization. diff --git a/test/examples/test_examples_evaluate_sync_parallel_optimization.py b/test/examples/test_examples_evaluate_sync_parallel_optimization.py index 90357392..64cac45f 100644 --- a/test/examples/test_examples_evaluate_sync_parallel_optimization.py +++ b/test/examples/test_examples_evaluate_sync_parallel_optimization.py @@ -3,8 +3,13 @@ import matplotlib.pyplot as plt from openbox import ParallelOptimizer, space as sp from openbox.utils.constants import SUCCESS - - +from unittest.mock import Mock +from openbox.core.computation.parallel_process import ParallelEvaluation +from openbox.optimizer.parallel_smbo import wrapper +from openbox.core.computation.nondaemonic_processpool import ProcessPool +''' +这个也用来测试pSMBO 测试并没有发现wrapper被执行，暂时不知道原因 +''' # Define Objective Function def branin(config): x1, x2 = config['x1'], config['x2'] @@ -12,7 +17,25 @@ def branin(config): + 10 * (1 - 1 / (8 * np.pi)) * np.cos(x1) + 10 return {'objectives': [y]} +''' +尝试，但是wrapper依然没有执行 + # 使用 Mock 作为回调函数，检查是否调用 + mock_callback = Mock() + proc = ProcessPool(4) + _config = space.sample_configuration() + _param = [branin, _config, None, [np.inf]] + result = proc.apply_async(wrapper, (_param,), callback=mock_callback) + + # 等待任务完成 + result.wait() + + # 检查回调是否被调用 + mock_callback.assert_called_once() +''' +''' +直接调用wrapper +''' def test_examples_evaluate_sync_parallel_optimization(): max_runs = 20 @@ -21,6 +44,8 @@ def test_examples_evaluate_sync_parallel_optimization(): x1 = sp.Real("x1", -5, 10, default_value=0) x2 = sp.Real("x2", 0, 15, default_value=0) space.add_variables([x1, x2]) + _config = space.sample_configuration() + wrapper( (branin, _config, None, [np.inf]) ) # Parallel Evaluation on Local Machine opt = ParallelOptimizer( @@ -38,6 +63,58 @@ def test_examples_evaluate_sync_parallel_optimization(): logging_dir='logs/pytest/', ) history = opt.run() + # 测试更多并行方法 + opt_syn_ea = ParallelOptimizer( + branin, + space, + parallel_strategy='sync', + batch_size=4, + batch_strategy='default', + sample_strategy='ea', + num_objectives=1, + num_constraints=0, + max_runs=max_runs, + # surrogate_type='gp', + surrogate_type='auto', + task_id='parallel_sync', + logging_dir='logs/pytest/', +) + history = opt_syn_ea.run() + + opt_asyn_ea = ParallelOptimizer( + branin, + space, + parallel_strategy='async', + batch_size=4, + batch_strategy='default', + sample_strategy='ea', + num_objectives=1, + num_constraints=0, + max_runs=max_runs, + # surrogate_type='gp', + surrogate_type='auto', + task_id='parallel_sync', + logging_dir='logs/pytest/', +) + history = opt_asyn_ea.run() + + + opt_asyn_default = ParallelOptimizer( + branin, + space, + parallel_strategy='async', + batch_size=4, + batch_strategy='default', + num_objectives=1, + num_constraints=0, + max_runs=max_runs, + # surrogate_type='gp', + surrogate_type='auto', + task_id='parallel_sync', + logging_dir='logs/pytest/', +) + history = opt_asyn_default.run() + opt_asyn_default.async_iterate() print(history) diff --git a/test/examples/test_examples_quick_example.py b/test/examples/test_examples_quick_example.py index 6a106792..403fd960 100644 --- a/test/examples/test_examples_quick_example.py +++ b/test/examples/test_examples_quick_example.py @@ -45,7 +45,8 @@ def test_examples_quick_example(): plt.close() # install pyrfr to use get_importance() - print(history.get_importance()) + #method改成shap + print(history.get_importance(method='shap')) # Have a try on the new HTML visualization feature! # You can also call visualize_html() after optimization. diff --git a/test/fixtures/acqusition_funcs.py b/test/fixtures/acqusition_funcs.py index 0d91cfa9..c8767162 100644 --- a/test/fixtures/acqusition_funcs.py +++ b/test/fixtures/acqusition_funcs.py @@ -1,7 +1,10 @@ import pytest from openbox.acquisition_function.acquisition import EI -from openbox.acquisition_function.multi_objective_acquisition import USeMO +from openbox.acquisition_function.multi_objective_acquisition import USeMO,MESMO,MESMOC,MESMOC2 +from openbox.acquisition_function.mc_multi_objective_acquisition import MCEHVI,MCParEGO,MCParEGOC,MCEHVIC +from openbox.utils.multi_objective import NondominatedPartitioning +from openbox.acquisition_function.mc_acquisition import MCEI,MCEIC @pytest.fixture @@ -22,6 +25,46 @@ def acq_func_ei(surrogate_model_gp, history_single_obs): return ei +@pytest.fixture +def acq_func_mcei(configspace_tiny, surrogate_model_gp, history_single_obs): + surrogate_model = surrogate_model_gp + mcei = MCEI(surrogate_model) + + X = history_single_obs.get_config_array(transform='scale') + Y = history_single_obs.get_objectives(transform='infeasible') + + surrogate_model.train(X, Y) + + mcei.update(model=surrogate_model, + constraint_models=None, + eta=history_single_obs.get_incumbent_value(), + num_data=len(history_single_obs) + ) + mcei([configspace_tiny.sample_configuration()]) + return mcei + + +@pytest.fixture +def acq_func_mceic(configspace_tiny, surrogate_model_gp, history_single_obs): + surrogate_model = surrogate_model_gp + constriant_model = surrogate_model_gp + mceic = MCEIC(surrogate_model, [constriant_model]) + + X = history_single_obs.get_config_array(transform='scale') + Y = history_single_obs.get_objectives(transform='infeasible') + cY = history_single_obs.get_constraints(transform='bilog') + + surrogate_model.train(X, Y) + + mceic.update(model=surrogate_model, + constraint_models=[constriant_model], + constraint_perfs=cY, + eta=history_single_obs.get_incumbent_value(), + num_data=len(history_single_obs) + ) + mceic([configspace_tiny.sample_configuration()]) + return mceic + @pytest.fixture def acq_func_usemo(configspace_tiny, surrogate_model_gp, history_double_obs): @@ -42,5 +85,146 @@ def acq_func_usemo(configspace_tiny, surrogate_model_gp, history_double_obs): eta=history_double_obs.get_mo_incumbent_values(), num_data=len(history_double_obs), X=X, Y=Y) + #usemo([configspace_tiny.sample_configuration()]) return usemo + +@pytest.fixture +def acq_func_mesmo(configspace_tiny, surrogate_model_gp_rbf, history_double_obs): + surrogate_model1 = surrogate_model_gp_rbf + surrogate_model2 = surrogate_model_gp_rbf + memso = MESMO([surrogate_model1, surrogate_model2], config_space=configspace_tiny) + + X = history_double_obs.get_config_array(transform='scale') + Y = history_double_obs.get_objectives(transform='infeasible') + cY = history_double_obs.get_constraints(transform='bilog') + + surrogate_model1.train(X, Y[:, 0]) + surrogate_model1.train(X, Y[:, 1]) + + memso.update(model=[surrogate_model1, surrogate_model2], + constraint_models=None, + constraint_perfs=cY, # for MESMOC + eta=history_double_obs.get_mo_incumbent_values(), + num_data=len(history_double_obs), + X=X, Y=Y) + memso([configspace_tiny.sample_configuration()]) + return memso + + +@pytest.fixture +def acq_func_mesmoc(configspace_tiny, surrogate_model_gp_rbf, history_double_cons_obs): + surrogate_model1 = surrogate_model_gp_rbf + surrogate_model2 = surrogate_model_gp_rbf + surrogate_model3 = surrogate_model_gp_rbf + surrogate_model4 = surrogate_model_gp_rbf + memsoc = MESMOC([surrogate_model1, surrogate_model2], [surrogate_model3, surrogate_model4], config_space=configspace_tiny) + + X = history_double_cons_obs.get_config_array(transform='scale') + Y = history_double_cons_obs.get_objectives(transform='infeasible') + cY = history_double_cons_obs.get_constraints(transform='bilog') + + surrogate_model1.train(X, Y[:, 0]) + surrogate_model1.train(X, Y[:, 1]) + + memsoc.update(model=[surrogate_model1, surrogate_model2], + constraint_models= [surrogate_model3, surrogate_model4], + constraint_perfs=cY, # for MESMOC + eta=history_double_cons_obs.get_mo_incumbent_values(), + num_data=len(history_double_cons_obs), + X=X, Y=Y) + memsoc([configspace_tiny.sample_configuration()]) + return memsoc + + +@pytest.fixture +def acq_func_mesmoc2(configspace_tiny, surrogate_model_gp_rbf, history_double_cons_obs): + surrogate_model1 = surrogate_model_gp_rbf + surrogate_model2 = surrogate_model_gp_rbf + surrogate_model3 = surrogate_model_gp_rbf + surrogate_model4 = surrogate_model_gp_rbf + memsoc2 = MESMOC2([surrogate_model1, surrogate_model2], [surrogate_model3, surrogate_model4], config_space=configspace_tiny) + + X = history_double_cons_obs.get_config_array(transform='scale') + Y = history_double_cons_obs.get_objectives(transform='infeasible') + cY = history_double_cons_obs.get_constraints(transform='bilog') + + surrogate_model1.train(X, Y[:, 0]) + surrogate_model1.train(X, Y[:, 1]) + + memsoc2.update(model=[surrogate_model1, surrogate_model2], + constraint_models= [surrogate_model3, surrogate_model4], + constraint_perfs=cY, # for MESMOC + eta=history_double_cons_obs.get_mo_incumbent_values(), + num_data=len(history_double_cons_obs), + X=X, Y=Y) + #计算这个点的mesmoc值,但是这里只是为了测试，严格来说要勇敢X中的config，X含有4个config，最后后一次性计算他们的acq + memsoc2([configspace_tiny.sample_configuration()]) + return memsoc2 +''' +cell_bounds = partitioning.get_hypercell_bounds(ref_point=self.ref_point) +self.acquisition_function.update(model=self.surrogate_model, + constraint_models=self.constraint_models, + cell_lower_bounds=cell_bounds[0], + cell_upper_bounds=cell_bounds[1]) +要主动传入cell_bound，update，并且update这些属性 +''' +@pytest.fixture +def acq_func_mc_ehvic(configspace_tiny, surrogate_model_gp_rbf, history_double_cons_obs): + surrogate_model1 = surrogate_model_gp_rbf + surrogate_model2 = surrogate_model_gp_rbf + surrogate_model3 = surrogate_model_gp_rbf + surrogate_model4 = surrogate_model_gp_rbf + mc_ehvic = MCEHVIC([surrogate_model1, surrogate_model2], [surrogate_model3, surrogate_model4], config_space=configspace_tiny, ref_point=[1,1]) + + X = history_double_cons_obs.get_config_array(transform='scale') + Y = history_double_cons_obs.get_objectives(transform='infeasible') + cY = history_double_cons_obs.get_constraints(transform='bilog') + partitioning = NondominatedPartitioning(2, Y) + surrogate_model1.train(X, Y[:, 0]) + surrogate_model1.train(X, Y[:, 1]) + cell_bounds = partitioning.get_hypercell_bounds(ref_point=[1,1]) + mc_ehvic.update(model=[surrogate_model1, surrogate_model2], + constraint_models= [surrogate_model3, surrogate_model4], + constraint_perfs=cY, # for MESMOC + eta=history_double_cons_obs.get_mo_incumbent_values(), + num_data=len(history_double_cons_obs), + X=X, Y=Y, + cell_lower_bounds=cell_bounds[0],cell_upper_bounds=cell_bounds[1]) + #计算这个点的mesmoc值,但是这里只是为了测试，严格来说要勇敢X中的config，X含有4个config，最后后一次性计算他们的acq + mc_ehvic([configspace_tiny.sample_configuration()]) + return mc_ehvic + + + + +@pytest.fixture +def acq_func_mc_parEGOC(configspace_tiny, surrogate_model_gp_rbf, history_double_cons_obs): + surrogate_model1 = surrogate_model_gp_rbf + surrogate_model2 = surrogate_model_gp_rbf + surrogate_model3 = surrogate_model_gp_rbf + surrogate_model4 = surrogate_model_gp_rbf + mc_parEGOC = MCParEGOC([surrogate_model1, surrogate_model2], [surrogate_model3, surrogate_model4], config_space=configspace_tiny, ref_point=[1,1]) + + X = history_double_cons_obs.get_config_array(transform='scale') + Y = history_double_cons_obs.get_objectives(transform='infeasible') + cY = history_double_cons_obs.get_constraints(transform='bilog') + partitioning = NondominatedPartitioning(2, Y) + surrogate_model1.train(X, Y[:, 0]) + surrogate_model1.train(X, Y[:, 1]) + cell_bounds = partitioning.get_hypercell_bounds(ref_point=[1,1]) + mc_parEGOC.update(model=[surrogate_model1, surrogate_model2], + constraint_models= [surrogate_model3, surrogate_model4], + constraint_perfs=cY, # for MESMOC + eta=history_double_cons_obs.get_mo_incumbent_values(), + num_data=len(history_double_cons_obs), + X=X, Y=Y, + cell_lower_bounds=cell_bounds[0],cell_upper_bounds=cell_bounds[1]) + #计算这个点的mesmoc值,但是这里只是为了测试，严格来说要勇敢X中的config，X含有4个config，最后后一次性计算他们的acq + mc_parEGOC([configspace_tiny.sample_configuration()]) + return mc_parEGOC + + + + + diff --git a/test/fixtures/configspace.py b/test/fixtures/configspace.py index 24739c78..ba4b545b 100644 --- a/test/fixtures/configspace.py +++ b/test/fixtures/configspace.py @@ -1,6 +1,8 @@ import pytest from openbox import space as sp - +''' +记得换下种子，不然在一个space内了 +''' @pytest.fixture def configspace_tiny() -> sp.Space: cs = sp.Space(seed=0) @@ -8,4 +10,63 @@ def configspace_tiny() -> sp.Space: x2 = sp.Real("x2", 0, 15, default_value=0) cs.add_variables([x1, x2]) + return cs +''' +添加其他不同的夹具 +''' +@pytest.fixture +def configspace_cat()-> sp.Space: + cs_1 = sp.Space(seed=1) + name = 'cat_hp' + choices = ['red', 'green', 'blue'] + default_value = 'red' + meta = {'description': 'Test Meta Data'} + weights = [0.2, 0.3, 0.5] + + # Create an instance of the Categorical variable + categorical_var = sp.Categorical(name=name, choices=choices, default_value=default_value, meta=meta, + weights=weights) + #注意这里是add_variable不能是variables + cs_1.add_variable(categorical_var) + return cs_1 + +@pytest.fixture +def configspace_big()-> sp.Space: + #100个连续案例 + cs = sp.Space(seed=2) + for i in range(100): + x1 = sp.Real("x{}".format(i), -i, 10, default_value=0) + x2 = sp.Real("x{}".format(i+100), 0, 15, default_value=0) + cs.add_variables([x1, x2]) + return cs + +@pytest.fixture +def configspace_mid()-> sp.Space: + #15个连续案例 + cs = sp.Space(seed=3) + for i in range(15): + x2 = sp.Real("x{}".format(i-200), 0, 15, default_value=0) + x1 = sp.Real("x{}".format(i-100), -i, 10, default_value=0) + cs.add_variables([x1, x2]) + return cs + + +@pytest.fixture +def configspace_2() -> sp.Space: + cs = sp.Space(seed=0) + x1 = sp.Real(name='x1', lower=0, upper=10, default_value=5) + x2 = sp.Categorical(name='x2', choices=["car", "train", "plane"], default_value="train") + cs.add_variables([x1, x2]) + + return cs + +@pytest.fixture +def configspace_huge() -> sp.Space: + cs = sp.Space(seed=0) + x1 = sp.Real("x1", -5, 10, default_value=0) + x2 = sp.Real("x2", 0, 15, default_value=0) + x3 = sp.Categorical(name='x3', choices=["car", "train", "plane"], default_value="train") + x4 = sp.Int("x4", -5, 15, default_value=0) + cs.add_variables([x1, x2, x3, x4]) + return cs \ No newline at end of file diff --git a/test/fixtures/history.py b/test/fixtures/history.py index c07c3f69..2311f38c 100644 --- a/test/fixtures/history.py +++ b/test/fixtures/history.py @@ -75,6 +75,44 @@ def history_single_obs(configspace_tiny): return history +@pytest.fixture +def history_single_obs_2(configspace_2): + config_space = configspace_2 + # Create a History instance + config1 = config_space.sample_configuration() + config2 = config_space.sample_configuration() + config3 = config_space.sample_configuration() + config4 = config_space.sample_configuration() + config5 = config_space.sample_configuration() + config6 = config_space.sample_configuration() + config7 = config_space.sample_configuration() + config8 = config_space.sample_configuration() + config9 = config_space.sample_configuration() + config10 = config_space.sample_configuration() + + # Set up observations with different objective values + observation1 = Observation(config=config1, objectives=[0.1], trial_state=SUCCESS) + observation2 = Observation(config=config2, objectives=[0.3], trial_state=SUCCESS) + observation3 = Observation(config=config3, objectives=[0.5], trial_state=SUCCESS) + observation4 = Observation(config=config4, objectives=[2.0], trial_state=FAILED) + observation5 = Observation(config=config5, objectives=[3.0], trial_state=SUCCESS) + observation6 = Observation(config=config6, objectives=[10.5], trial_state=SUCCESS) + observation7 = Observation(config=config7, objectives=[30.1], trial_state=SUCCESS) + observation8 = Observation(config=config8, objectives=[100.5], trial_state=SUCCESS) + observation9 = Observation(config=config9, objectives=[33.0], trial_state=SUCCESS) + observation10 = Observation(config=config10, objectives=[12.5], trial_state=SUCCESS) + + history = History( + num_objectives=1, + config_space=configspace_2 + ) + + # Add observations to history + history.update_observations([observation1, observation2, observation3, observation4, observation5, + observation6, observation7, observation8, observation9, observation10]) + + return history + @pytest.fixture def history_double(configspace_tiny): @@ -177,6 +215,31 @@ def multi_start_history_single_obs(configspace_tiny): return history +@pytest.fixture +def multi_start_history_double_obs(configspace_tiny): + config_space = configspace_tiny + # Create a History instance + config1 = config_space.sample_configuration() + config2 = config_space.sample_configuration() + config3 = config_space.sample_configuration() + config4 = config_space.sample_configuration() + + # Set up observations with different objective values + observation1 = Observation(config=config1, objectives=[0.1,0.2], trial_state=SUCCESS) + observation2 = Observation(config=config2, objectives=[0.3,0.4], trial_state=SUCCESS) + observation3 = Observation(config=config3, objectives=[0.5,0.6], trial_state=SUCCESS) + observation4 = Observation(config=config4, objectives=[2.0,0.3], trial_state=FAILED) + + history = MultiStartHistory( + num_objectives=2, + config_space=configspace_tiny + ) + + # Add observations to history + history.update_observations([observation1, observation2, observation3, observation4]) + + return history + @pytest.fixture def transfer_learning_history_single(history_single_obs): diff --git a/test/fixtures/surrogate_models.py b/test/fixtures/surrogate_models.py index 74094363..b8adb714 100644 --- a/test/fixtures/surrogate_models.py +++ b/test/fixtures/surrogate_models.py @@ -24,6 +24,17 @@ def surrogate_model_gp(configspace_tiny): bounds=bounds, rng=rng) +@pytest.fixture +def surrogate_model_gp_rbf(configspace_tiny): + config_space = configspace_tiny + rng = np.random.RandomState(0) + types, bounds = get_types(config_space) + return create_gp_model(model_type='gp_rbf', + config_space=config_space, + types=types, + bounds=bounds, + rng=rng) + @pytest.fixture def cons_kernel(): diff --git a/test/optimizer/test_parallel_smbo.py b/test/optimizer/test_parallel_smbo.py index b7aa0eaa..fb928d37 100644 --- a/test/optimizer/test_parallel_smbo.py +++ b/test/optimizer/test_parallel_smbo.py @@ -56,7 +56,9 @@ def test_psmbo_async(configspace_tiny): psmbo.async_run() history = psmbo.run() assert len(history) > 0 - + ''' + 看起来这个async_iterate一般不会被调用？他是异步生成n个observation + ''' observations = psmbo.async_iterate(n=2) assert len(observations) == 2 diff --git a/test/surrogate/base/test_base_model.py b/test/surrogate/base/test_base_model.py index f4ae1aa6..c717a333 100644 --- a/test/surrogate/base/test_base_model.py +++ b/test/surrogate/base/test_base_model.py @@ -8,10 +8,14 @@ class AbstractModelTests(unittest.TestCase): def setUp(self): self.types = np.array([0, 0]) self.bounds = [(0, 1), (0, 1)] - # self.instance_features = np.array([[0, 1], [1, 0]]) self.instance_features = None - self.pca_components = 2 + self.pca_components = 1 self.model = AbstractModel(self.types, self.bounds, self.instance_features, self.pca_components) + + self.types2 = np.array([0, 0, 0, 0]) + self.instance_features2 = np.array([[0, 1], [1, 0], [1, 1], [2, 1], [4, 2]]) + self.bounds2 = [(0, 1), (0, 1), (0, 1), (0, 1)] + self.model2 = AbstractModel(self.types2,self.bounds2, self.instance_features2, self.pca_components) def test_model_train_raises_not_implemented(self): @@ -48,6 +52,18 @@ def test_model_predict_marginalized_over_instances(self): with self.assertRaises(NotImplementedError): self.model.predict_marginalized_over_instances(X) + def test_model_with_instances(self): + X = np.array([[0, 1], [3, 4], [0, 1], [1, 0]]) + Y = np.array([[0], [1], [0], [1]]) + + for X_, Y_ in zip(X, Y): + X_ = np.hstack((np.tile(X_, (5,1)), self.instance_features2)) + Y_ = np.tile(Y_, (5, 1)) + with self.assertRaises(NotImplementedError): + self.model2.train(X_, Y_) + + with self.assertRaises(NotImplementedError): + self.model2.predict_marginalized_over_instances(X) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/test/surrogate/base/test_gp_base_prior.py b/test/surrogate/base/test_gp_base_prior.py index c4b619e1..ee3e5c97 100644 --- a/test/surrogate/base/test_gp_base_prior.py +++ b/test/surrogate/base/test_gp_base_prior.py @@ -30,6 +30,7 @@ def test_horseshoe_prior_sample_from_prior(self): samples = prior.sample_from_prior(5) self.assertEqual(len(samples), 5) self.assertFalse(all(s > 0 for s in samples)) + prior.gradient(1) def test_lognormal_prior_lnprob(self): prior = LognormalPrior(1, self.rng) @@ -53,6 +54,8 @@ def test_soft_tophat_prior_sample_from_prior(self): samples = prior.sample_from_prior(5) self.assertEqual(len(samples), 5) self.assertFalse(all(1 <= s <= 10 for s in samples)) + prior.gradient(-2) + prior.gradient(10) def test_gamma_prior_lnprob(self): prior = GammaPrior(1, 1, 0, self.rng) @@ -64,7 +67,7 @@ def test_gamma_prior_sample_from_prior(self): samples = prior.sample_from_prior(5) self.assertEqual(len(samples), 5) self.assertFalse(all(s > 0 for s in samples)) - + prior.gradient(2) if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/test/utils/test_early_stop.py b/test/utils/test_early_stop.py index 25a28d86..a66b03b8 100644 --- a/test/utils/test_early_stop.py +++ b/test/utils/test_early_stop.py @@ -81,7 +81,7 @@ def test_es_decide_early_stop_before_suggest(): def test_es_decide_early_stop_after_suggest(): # check not reach min history = HistoryMock() - early_stop = EarlyStopAlgorithm(min_iter=5, min_improvement_percentage=0.05, max_no_improvement_rounds=0) + early_stop = EarlyStopAlgorithm(min_iter=5, max_no_improvement_rounds=0) history.observations = [[1], [1], [1]] assert not early_stop.decide_early_stop_after_suggest(history) From d1b13f9e3bf6db68ae79ee7989830eb993e00f5d Mon Sep 17 00:00:00 2001 From: xbc bw Date: Fri, 27 Sep 2024 15:58:57 +0800 Subject: [PATCH 2/2] add codecov token --- .github/workflows/codecov.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 6a9ce265..0e147fb4 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -40,3 +40,5 @@ jobs: - name: Upload coverage reports to Codecov with GitHub Action # if: ${{ runner.os == 'Linux' && matrix.python-version == 3.8 }} uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}