Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "iohinspector"
version = "0.1.0"
version = "0.1.1"
authors = [
{ name="Diederick Vermetten", email="d.vermetten@gmail.com" },
{ name="Jacob de Nobel", email="jacobdenobel@gmail.com" },
Expand Down
13 changes: 8 additions & 5 deletions src/iohinspector/align.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ def align_data(
y_col: str = "raw_y",
output: str = "long",
maximization: bool = False,
silence_warning: bool = False
) -> pl.DataFrame:
"""Align data based on function evaluation counts

Expand All @@ -24,12 +25,14 @@ def align_data(
y_col (str, optional): function value column. Defaults to 'raw_y'.
output (str, optional): whether to return a long or wide dataframe as output. Defaults to 'long'.
maximization (bool, optional): whether the data comes from maximization or minimization. Defaults to False (minimization).
silence_warning (bool, optional): whether to silence the deprication warning

Returns:
pl.DataFrame: Alligned DataFrame
"""
if not silence_warning:
warnings.warn( "turbo_align is favoured over this function", DeprecationWarning)

warnings.warn( "Turbo align is favoured over this function", DeprecationWarning)
evals_df = pl.DataFrame({x_col: evals})

def merge_asof_group(group):
Expand Down Expand Up @@ -60,7 +63,7 @@ def merge_asof_group(group):
if output == "long":
return result_df

pivot_df = result_df.pivot(index=x_col, columns=group_cols, values=y_col)
pivot_df = result_df.pivot(index=x_col, on=group_cols, values=y_col)
return pivot_df


Expand Down Expand Up @@ -103,16 +106,16 @@ def turbo_align(

if x_col != "evaluations" and maximization:
result_df = x_vals.join_asof(
df, by="data_id", on=x_col, strategy="forward"
df, by="data_id", on=x_col, strategy="forward", check_sortedness=False,
)
else:
result_df = x_vals.join_asof(
df, by="data_id", on=x_col, strategy="backward"
df, by="data_id", on=x_col, strategy="backward", check_sortedness=False,
)


if output == "long":
return result_df

pivot_df = result_df.pivot(index=x_col, columns=("data_id",), values=y_col)
pivot_df = result_df.pivot(index=x_col, on=("data_id",), values=y_col)
return pivot_df
10 changes: 5 additions & 5 deletions src/iohinspector/metrics/eaf.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

from iohinspector.align import align_data
from iohinspector.align import align_data, turbo_align
from iohinspector.metrics import transform_fval, get_sequence
import numpy as np
import pandas as pd
Expand Down Expand Up @@ -52,13 +52,13 @@ def get_discritized_eaf_single_objective(
eval_min, eval_max, eval_targets, scale_log=scale_eval_log, cast_to_int=True
)

dt_aligned = align_data(
dt_aligned = turbo_align(
data,
eval_values,
x_col=eval_var,
y_col=fval_var,
output="long"
)
)
dt_aligned = transform_fval(
dt_aligned,
lb=f_min,
Expand Down Expand Up @@ -105,8 +105,8 @@ def get_eaf_data(
if eval_max is None:
eval_max = data[eval_var].max()

evals = get_sequence(eval_min, eval_max, 50, scale_eval_log, True)
long = align_data(data, np.array(evals, "uint64"), ["data_id"], output="long")
evals = get_sequence(eval_min, eval_max, 50, scale_eval_log, True).astype("uint64")
long = turbo_align(data, evals, output='long')

if return_as_pandas:
return long.to_pandas()
Expand Down
1 change: 1 addition & 0 deletions src/iohinspector/metrics/ecdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def get_data_ecdf(
x_col=eval_var,
y_col=fval_var,
maximization=maximization,
silence_warning=True
)
dt_ecdf = (
transform_fval(
Expand Down
1 change: 1 addition & 0 deletions src/iohinspector/metrics/fixed_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ def aggregate_convergence(
x_col=eval_var,
y_col=fval_var,
maximization=maximization,
silence_warning=True
)
aggregations = [
pl.mean(fval_var).alias("mean"),
Expand Down
1 change: 1 addition & 0 deletions src/iohinspector/metrics/fixed_target.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ def aggregate_running_time(
x_col=fval_var,
y_col=eval_var,
maximization=maximization,
silence_warning=True
)

if eval_max is None:
Expand Down
2 changes: 1 addition & 1 deletion src/iohinspector/metrics/ranking.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def get_tournament_ratings(
fids = data[fid_vars].unique()
aligned_comps = data.pivot(
index=alg_vars,
columns=fid_vars,
on=fid_vars,
values=fval_var,
aggregate_function=pl.element(),
)
Expand Down
2 changes: 2 additions & 0 deletions src/iohinspector/metrics/trajectory.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,15 @@ def get_trajectory(data: pl.DataFrame,
else:
max_fevals = traj_length + min_fevals
x_values = np.arange(min_fevals, max_fevals + 1)

data_aligned = align_data(
data.cast({evaluation_variable: pl.Int64}),
x_values,
group_cols=["data_id"] + free_variables,
x_col=evaluation_variable,
y_col=fval_variable,
maximization=maximization,
silence_warning=True
)
if return_as_pandas:
data_aligned = data_aligned.to_pandas()
Expand Down
21 changes: 14 additions & 7 deletions src/iohinspector/plots/attractor_network.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import warnings
from dataclasses import dataclass
from typing import Iterable

import numpy as np
import pandas as pd
import polars as pl
from typing import Iterable, Tuple
import matplotlib
import matplotlib.pyplot as plt

from iohinspector.metrics import get_attractor_network
from iohinspector.plots.utils import BasePlotArgs, _create_plot_args, _save_fig

Expand Down Expand Up @@ -134,11 +137,16 @@ def plot_attractor_network(
)
network.remove_edges_from(nx.selfloop_edges(network))

decision_matrix = [network.nodes[node]["decision"] for node in network.nodes()]
mds = MDS(n_components=1, random_state=0)
x_positions = mds.fit_transform(
decision_matrix
).flatten() # Flatten to get 1D array for x-axis
D = [network.nodes[node]["decision"] for node in network.nodes()]
mds = MDS(n_components=1, random_state=0, n_init=4)
if len(D[0]) == len(D):
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
x_positions = mds.fit_transform(D)
else:
x_positions = mds.fit_transform(D)

x_positions = x_positions.flatten() # Flatten to get 1D array for x-axis
y_positions = [network.nodes[node]["fitness"] for node in network.nodes()]
pos = {
node: (x, y) for node, x, y in zip(network.nodes(), x_positions, y_positions)
Expand Down Expand Up @@ -200,5 +208,4 @@ def plot_attractor_network(
plot_args.apply(ax)

_save_fig(fig, file_name, plot_args)

return ax, nodes, edges
10 changes: 4 additions & 6 deletions src/iohinspector/plots/ranking.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
from typing import Iterable, Optional
from iohinspector.metrics.ranking import get_robustrank_changes, get_robustrank_over_time
from iohinspector.plots.utils import BasePlotArgs, _create_plot_args, _save_fig

import polars as pl
import numpy as np
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sbs

from iohinspector.metrics.ranking import get_robustrank_changes, get_robustrank_over_time
from iohinspector.plots.utils import BasePlotArgs, _create_plot_args, _save_fig
from iohinspector.metrics import get_tournament_ratings
from iohinspector.indicators import add_indicator


def plot_tournament_ranking(
Expand Down Expand Up @@ -215,7 +214,6 @@ def plot_robustrank_changes(
fig, ax = plt.subplots(1, 1, figsize=plot_args.figsize)
else:
fig = None

plot_line_ranks(comparisons, ax=ax)

plot_args.apply(ax)
Expand Down
16 changes: 7 additions & 9 deletions tests/test_align.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import unittest
import numpy as np
import polars as pl
from iohinspector.align import align_data
from iohinspector.align import turbo_align
from iohinspector.align import align_data, turbo_align


class TestAlignData(unittest.TestCase):
Expand All @@ -15,7 +13,7 @@ def test_align_data_minimization_long(self):
})

evals = [1, 2, 3, 4, 5]
result = align_data(df, evals, group_cols=("data_id",), x_col="evaluations", y_col="raw_y", output="long", maximization=False)
result = align_data(df, evals, group_cols=("data_id",), x_col="evaluations", y_col="raw_y", output="long", maximization=False, silence_warning=True)
expected = pl.DataFrame({
"evaluations": [1, 2, 3, 4, 5, 1, 2, 3, 4, 5],
"raw_y": [10, 8, 8, 8, 6, 20, 20, 20, 18, 16],
Expand All @@ -32,7 +30,7 @@ def test_align_data_maximization_long(self):
"raw_y": [5, 7, 6]
})
evals = [1, 2, 3]
result = align_data(df, evals, group_cols=("data_id",), x_col="evaluations", y_col="raw_y", output="long", maximization=True)
result = align_data(df, evals, group_cols=("data_id",), x_col="evaluations", y_col="raw_y", output="long", maximization=True, silence_warning=True)
expected = pl.DataFrame({
"evaluations": [1, 2, 3],
"raw_y": [5, 7, 7],
Expand All @@ -50,7 +48,7 @@ def test_align_data_wide_output(self):
})
evals = [1, 2, 3, 4, 5]

result = align_data(df, evals, group_cols=("data_id",), x_col="evaluations", y_col="raw_y", output="wide", maximization=False)
result = align_data(df, evals, group_cols=("data_id",), x_col="evaluations", y_col="raw_y", output="wide", maximization=False, silence_warning=True)
# Should pivot to wide format
self.assertIn("1", result.columns)
self.assertIn("2", result.columns)
Expand All @@ -65,7 +63,7 @@ def test_align_data_custom_group_col(self):
"raw_y": [5, 3, 7, 6]
})
evals = [1, 2]
result = align_data(df, evals, group_cols=("exp_id",), x_col="evaluations", y_col="raw_y", output="long", maximization=False)
result = align_data(df, evals, group_cols=("exp_id",), x_col="evaluations", y_col="raw_y", output="long", maximization=False, silence_warning=True)
self.assertTrue(set(result["exp_id"].to_list()) == {1, 2})

def test_align_data_non_default_x_col(self):
Expand All @@ -75,7 +73,7 @@ def test_align_data_non_default_x_col(self):
"score": [100, 90, 80]
})
evals = [10, 20, 30]
result = align_data(df, evals, group_cols=("data_id",), x_col="steps", y_col="score", output="long", maximization=False)
result = align_data(df, evals, group_cols=("data_id",), x_col="steps", y_col="score", output="long", maximization=False, silence_warning=True)
self.assertTrue(result["steps"].to_list() == [10, 20, 30])

class TestTurboAlignData(unittest.TestCase):
Expand Down Expand Up @@ -134,7 +132,7 @@ def test_turbo_align_non_default_x_col(self):
"score": [100, 90, 80]
})
evals = [10, 20, 30]
result = align_data(df, evals, group_cols=("data_id",), x_col="steps", y_col="score", output="long", maximization=False)
result = turbo_align(df, evals, x_col="steps", y_col="score", output="long", maximization=False)
self.assertTrue(result["steps"].to_list() == [10, 20, 30])

if __name__ == "__main__":
Expand Down
6 changes: 3 additions & 3 deletions tests/test_metrics/test_eaf.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def test_basic_single_data_id(self):
self.assertTrue(len(result.columns) == 10) # default x_targets
self.assertEqual(result.shape[0], 101) # default y_targets
# Assert all values are 1 or 0
self.assertTrue(result[self.data["evaluations"].to_list()].applymap(lambda x: x in [1, 0]).all().all())
self.assertTrue(result[self.data["evaluations"].to_list()].map(lambda x: x in [1, 0]).all().all())
self.assertEqual(result[1].tolist()[-1], 0)
self.assertEqual(result[1000].tolist()[0], 1)

Expand All @@ -46,7 +46,7 @@ def test_basic_multi_data_id(self):
self.assertTrue(len(result.columns) == 10) # default x_targets
self.assertEqual(result.shape[0], 101) # default y_targets
# Assert all values are 1, 0.5 or 0
self.assertTrue(result[self.multi_data["evaluations"].to_list()].applymap(lambda x: x in [1, 0.5, 0]).all().all())
self.assertTrue(result[self.multi_data["evaluations"].to_list()].map(lambda x: x in [1, 0.5, 0]).all().all())
self.assertEqual(result[1].tolist()[-1], 0)
self.assertEqual(result[1000].tolist()[0], 1)

Expand All @@ -60,7 +60,7 @@ def test_custom_eval_min_max(self):
self.assertTrue(all(x in result.columns for x in [2, 4]))

def test_custom_f_min_max_targets(self):
result = get_discritized_eaf_single_objective(self.data, f_min=0.0, f_max=1.0, f_targets=5)
result = get_discritized_eaf_single_objective(self.data, f_min=1e-12, f_max=1.0, f_targets=5)
self.assertEqual(result.shape[0], 5)
self.assertAlmostEqual(result.index.min(), 0.0)
self.assertAlmostEqual(result.index.max(), 1.0)
Expand Down
7 changes: 4 additions & 3 deletions tests/test_plots/test_attractor_network.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ class TestPlotAttractorNetwork(unittest.TestCase):
def setUp(self):
self.data = pl.DataFrame({
"x1": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
"x2": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
"x2": [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 3, 2],
"x3": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13],
"raw_y": [35, 33, 31, 29, 27, 23, 18, 16, 14, 12, 10, 9, 6],
"evaluations": [1,42, 81,121,161,201,241,281,321,361,401,442,481],
"data_id": [1]*13
Expand All @@ -21,10 +22,10 @@ def setUp(self):
def test_basic_call_returns_axes_and_data(self):
ax, nodes, edges = plot_attractor_network(
self.data,
coord_vars=["x1", "x2"],
coord_vars=["x1", "x2", "x3"],
fval_var="raw_y",
eval_var="evaluations",
)
)

self.assertIsNotNone(ax)
self.assertIsNotNone(nodes)
Expand Down
35 changes: 20 additions & 15 deletions tests/test_plots/test_ranking.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
import unittest
import warnings

import polars as pl
import matplotlib
from iohinspector.plots import plot_robustrank_over_time,plot_tournament_ranking, plot_robustrank_changes
from iohinspector.indicators import HyperVolume

matplotlib.use("Agg") # Use non-interactive backend for tests
import matplotlib.pyplot as plt

from iohinspector.plots import plot_robustrank_over_time,plot_tournament_ranking, plot_robustrank_changes
from iohinspector.indicators import HyperVolume

class TestPlotTournamentRanking(unittest.TestCase):
def setUp(self):
Expand Down Expand Up @@ -72,12 +73,14 @@ def setUp(self):

def test_basic_call_returns_axes_and_data(self):
evals = [1, 10, 100]
axs, comparison, benchmark = plot_robustrank_over_time(
self.data,
obj_vars=["f1", "f2", "f3"],
evals=evals,
indicator=HyperVolume(reference_point=[5.0, 5.0, 5.0]),
)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
axs, comparison, benchmark = plot_robustrank_over_time(
self.data,
obj_vars=["f1", "f2", "f3"],
evals=evals,
indicator=HyperVolume(reference_point=[5.0, 5.0, 5.0]),
)
self.assertIsNotNone(axs)
self.assertIsNotNone(comparison)
self.assertIsNotNone(benchmark)
Expand Down Expand Up @@ -135,12 +138,14 @@ def setUp(self):

def test_basic_call_returns_axes_and_data(self):
evals = [1, 10, 100]
ax, dt = plot_robustrank_changes(
self.data,
obj_vars=["f1","f2", "f3"],
evals=evals,
indicator=HyperVolume(reference_point=[5.0, 5.0, 5.0]),
)
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=UserWarning)
ax, dt = plot_robustrank_changes(
self.data,
obj_vars=["f1","f2", "f3"],
evals=evals,
indicator=HyperVolume(reference_point=[5.0, 5.0, 5.0]),
)
self.assertIsNotNone(ax)
self.assertIsNotNone(dt)

Expand Down