From f6ce700d5ba0e5a703f35d145ec314ebb390162a Mon Sep 17 00:00:00 2001 From: James Guana Date: Fri, 6 Jun 2025 00:22:56 +0800 Subject: [PATCH 1/6] [ENH] Add timeseries integration test for various models --- .../test_data/test_timeseries_integration.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 tests/test_data/test_timeseries_integration.py diff --git a/tests/test_data/test_timeseries_integration.py b/tests/test_data/test_timeseries_integration.py new file mode 100644 index 000000000..1142a881f --- /dev/null +++ b/tests/test_data/test_timeseries_integration.py @@ -0,0 +1,95 @@ +import pickle +import shutil + +import lightning.pytorch as pl +from lightning.pytorch.callbacks import EarlyStopping +from lightning.pytorch.loggers import TensorBoardLogger +import numpy as np +import pandas as pd +import pytest + +from pytorch_forecasting.data.timeseries import TimeSeriesDataSet +from pytorch_forecasting.metrics import MQF2DistributionLoss, QuantileLoss +from pytorch_forecasting.metrics.distributions import ( + ImplicitQuantileNetworkDistributionLoss, +) +from pytorch_forecasting.models import ( + Baseline, + DeepAR, + DecoderMLP, + NBeats, + NHiTS, + GRU, + LSTM, + MultiEmbedding, + get_rnn, + RecurrentNetwork, + TemporalFusionTransformer, + TiDEModel, + TimeXer, +) + +from pytorch_forecasting.utils._dependencies import _get_installed_packages + + +MODELS = [ + Baseline, + DeepAR, + DecoderMLP, + NBeats, + NHiTS, + GRU, + LSTM, + MultiEmbedding, + get_rnn, + RecurrentNetwork, + TemporalFusionTransformer, + TiDEModel, + TimeXer, +] + + +@pytest.mark.parametrize("model", MODELS) +def test_integration(model): + n_timeseries = 10 + time_points = 10 + data = pd.DataFrame( + data={ + "target": np.random.rand(time_points * n_timeseries), + "time_varying_known_real_1": np.random.rand(time_points * n_timeseries), + "time_idx": np.tile(np.arange(time_points), n_timeseries), + "group_id": np.repeat(np.arange(n_timeseries), time_points), + } + ) + training_dataset = TimeSeriesDataSet( + data=data, + time_idx="time_idx", + target="target", + group_ids=["group_id"], + time_varying_unknown_reals=["target"], + time_varying_known_reals=(["time_varying_known_real_1"]), + max_prediction_length=max_prediction_length, + max_encoder_length=3, + ) + training_data_loader = training_dataset.to_dataloader(train=True) + forecaster = model.from_dataset(training_dataset, log_val_interval=1) + trainer = pl.Trainer( + accelerator="cpu", + max_epochs=3, + min_epochs=2, + limit_train_batches=10, + ) + trainer.fit( + forecaster, + train_dataloaders=training_data_loader, + ) + validation_dataset = TimeSeriesDataSet.from_dataset( + training_dataset, data, stop_randomization=True, predict=True + ) + validation_data_loader = validation_dataset.to_dataloader(train=False) + forecaster.predict( + validation_data_loader, + fast_dev_run=True, + return_index=True, + return_decoder_lengths=True, + ) From 0760ccd731e09edcfc54b406ddb04be8edd6bdda Mon Sep 17 00:00:00 2001 From: James Guana Date: Fri, 6 Jun 2025 01:37:46 +0800 Subject: [PATCH 2/6] Move the timeseries integration to test_all_estimators --- .../tests/test_all_estimators.py | 55 +++++++++++ .../test_data/test_timeseries_integration.py | 95 ------------------- 2 files changed, 55 insertions(+), 95 deletions(-) delete mode 100644 tests/test_data/test_timeseries_integration.py diff --git a/pytorch_forecasting/tests/test_all_estimators.py b/pytorch_forecasting/tests/test_all_estimators.py index 45d4772d5..2f5a95c1e 100644 --- a/pytorch_forecasting/tests/test_all_estimators.py +++ b/pytorch_forecasting/tests/test_all_estimators.py @@ -6,9 +6,12 @@ import lightning.pytorch as pl from lightning.pytorch.callbacks import EarlyStopping from lightning.pytorch.loggers import TensorBoardLogger +import numpy as np +import pandas as pd from skbase.testing import BaseFixtureGenerator as _BaseFixtureGenerator from pytorch_forecasting._registry import all_objects +from pytorch_forecasting.data.timeseries import TimeSeriesDataSet from pytorch_forecasting.tests._config import EXCLUDE_ESTIMATORS, EXCLUDED_TESTS # whether to test only estimators from modules that are changed w.r.t. main @@ -242,6 +245,53 @@ def _integration( ) +def _timeseries_integration(model, name): + print(f"Testing {name} integration with timeseries") + n_timeseries = 10 + time_points = 10 + max_prediction_length = 2 + data = pd.DataFrame( + data={ + "target": np.random.rand(time_points * n_timeseries), + "time_varying_known_real_1": np.random.rand(time_points * n_timeseries), + "time_idx": np.tile(np.arange(time_points), n_timeseries), + "group_id": np.repeat(np.arange(n_timeseries), time_points), + } + ) + training_dataset = TimeSeriesDataSet( + data=data, + time_idx="time_idx", + target="target", + group_ids=["group_id"], + time_varying_unknown_reals=["target"], + time_varying_known_reals=(["time_varying_known_real_1"]), + max_prediction_length=max_prediction_length, + max_encoder_length=3, + ) + training_data_loader = training_dataset.to_dataloader(train=True) + forecaster = model.from_dataset(training_dataset, log_val_interval=1) + trainer = pl.Trainer( + accelerator="cpu", + max_epochs=3, + min_epochs=2, + limit_train_batches=10, + ) + trainer.fit( + forecaster, + train_dataloaders=training_data_loader, + ) + validation_dataset = TimeSeriesDataSet.from_dataset( + training_dataset, data, stop_randomization=True, predict=True + ) + validation_data_loader = validation_dataset.to_dataloader(train=False) + forecaster.predict( + validation_data_loader, + fast_dev_run=True, + return_index=True, + return_decoder_lengths=True, + ) + + class TestAllPtForecasters(PackageConfig, BaseFixtureGenerator): """Generic tests for all objects in the mini package.""" @@ -251,6 +301,11 @@ def test_doctest_examples(self, object_class): run_doctest(object_class, name=f"class {object_class.__name__}") + def test_timeseries_itengration(self, object_class): + """Runs timeseries integration for estimator class.""" + + _timeseries_integration(object_class, name=f"class {object_class.__name__}") + def test_integration( self, object_metadata, diff --git a/tests/test_data/test_timeseries_integration.py b/tests/test_data/test_timeseries_integration.py deleted file mode 100644 index 1142a881f..000000000 --- a/tests/test_data/test_timeseries_integration.py +++ /dev/null @@ -1,95 +0,0 @@ -import pickle -import shutil - -import lightning.pytorch as pl -from lightning.pytorch.callbacks import EarlyStopping -from lightning.pytorch.loggers import TensorBoardLogger -import numpy as np -import pandas as pd -import pytest - -from pytorch_forecasting.data.timeseries import TimeSeriesDataSet -from pytorch_forecasting.metrics import MQF2DistributionLoss, QuantileLoss -from pytorch_forecasting.metrics.distributions import ( - ImplicitQuantileNetworkDistributionLoss, -) -from pytorch_forecasting.models import ( - Baseline, - DeepAR, - DecoderMLP, - NBeats, - NHiTS, - GRU, - LSTM, - MultiEmbedding, - get_rnn, - RecurrentNetwork, - TemporalFusionTransformer, - TiDEModel, - TimeXer, -) - -from pytorch_forecasting.utils._dependencies import _get_installed_packages - - -MODELS = [ - Baseline, - DeepAR, - DecoderMLP, - NBeats, - NHiTS, - GRU, - LSTM, - MultiEmbedding, - get_rnn, - RecurrentNetwork, - TemporalFusionTransformer, - TiDEModel, - TimeXer, -] - - -@pytest.mark.parametrize("model", MODELS) -def test_integration(model): - n_timeseries = 10 - time_points = 10 - data = pd.DataFrame( - data={ - "target": np.random.rand(time_points * n_timeseries), - "time_varying_known_real_1": np.random.rand(time_points * n_timeseries), - "time_idx": np.tile(np.arange(time_points), n_timeseries), - "group_id": np.repeat(np.arange(n_timeseries), time_points), - } - ) - training_dataset = TimeSeriesDataSet( - data=data, - time_idx="time_idx", - target="target", - group_ids=["group_id"], - time_varying_unknown_reals=["target"], - time_varying_known_reals=(["time_varying_known_real_1"]), - max_prediction_length=max_prediction_length, - max_encoder_length=3, - ) - training_data_loader = training_dataset.to_dataloader(train=True) - forecaster = model.from_dataset(training_dataset, log_val_interval=1) - trainer = pl.Trainer( - accelerator="cpu", - max_epochs=3, - min_epochs=2, - limit_train_batches=10, - ) - trainer.fit( - forecaster, - train_dataloaders=training_data_loader, - ) - validation_dataset = TimeSeriesDataSet.from_dataset( - training_dataset, data, stop_randomization=True, predict=True - ) - validation_data_loader = validation_dataset.to_dataloader(train=False) - forecaster.predict( - validation_data_loader, - fast_dev_run=True, - return_index=True, - return_decoder_lengths=True, - ) From ba404aa5fbe32e4144a6463156d6541cd044d82b Mon Sep 17 00:00:00 2001 From: jobs-git Date: Fri, 6 Jun 2025 02:00:44 +0800 Subject: [PATCH 3/6] Fix typo --- pytorch_forecasting/tests/test_all_estimators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_forecasting/tests/test_all_estimators.py b/pytorch_forecasting/tests/test_all_estimators.py index 2f5a95c1e..6f3dd06ac 100644 --- a/pytorch_forecasting/tests/test_all_estimators.py +++ b/pytorch_forecasting/tests/test_all_estimators.py @@ -301,7 +301,7 @@ def test_doctest_examples(self, object_class): run_doctest(object_class, name=f"class {object_class.__name__}") - def test_timeseries_itengration(self, object_class): + def test_timeseries_integration(self, object_class): """Runs timeseries integration for estimator class.""" _timeseries_integration(object_class, name=f"class {object_class.__name__}") From 070ffc2ffe134e20bb26e23e80bda85fdbf292cb Mon Sep 17 00:00:00 2001 From: jobs-git Date: Fri, 6 Jun 2025 23:20:25 +0800 Subject: [PATCH 4/6] Limit the data to target only --- pytorch_forecasting/tests/test_all_estimators.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pytorch_forecasting/tests/test_all_estimators.py b/pytorch_forecasting/tests/test_all_estimators.py index 6f3dd06ac..0e5ba44ae 100644 --- a/pytorch_forecasting/tests/test_all_estimators.py +++ b/pytorch_forecasting/tests/test_all_estimators.py @@ -253,9 +253,6 @@ def _timeseries_integration(model, name): data = pd.DataFrame( data={ "target": np.random.rand(time_points * n_timeseries), - "time_varying_known_real_1": np.random.rand(time_points * n_timeseries), - "time_idx": np.tile(np.arange(time_points), n_timeseries), - "group_id": np.repeat(np.arange(n_timeseries), time_points), } ) training_dataset = TimeSeriesDataSet( From 4afd88afc270f8f024ae59e37a473333dc204008 Mon Sep 17 00:00:00 2001 From: jobs-git Date: Fri, 6 Jun 2025 23:24:03 +0800 Subject: [PATCH 5/6] Limit the data to target only --- pytorch_forecasting/tests/test_all_estimators.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pytorch_forecasting/tests/test_all_estimators.py b/pytorch_forecasting/tests/test_all_estimators.py index 0e5ba44ae..e7ccba284 100644 --- a/pytorch_forecasting/tests/test_all_estimators.py +++ b/pytorch_forecasting/tests/test_all_estimators.py @@ -257,11 +257,7 @@ def _timeseries_integration(model, name): ) training_dataset = TimeSeriesDataSet( data=data, - time_idx="time_idx", target="target", - group_ids=["group_id"], - time_varying_unknown_reals=["target"], - time_varying_known_reals=(["time_varying_known_real_1"]), max_prediction_length=max_prediction_length, max_encoder_length=3, ) From 9ce89ff75038d9cfe7a9596bb575ea6fcec619c1 Mon Sep 17 00:00:00 2001 From: jobs-git Date: Fri, 6 Jun 2025 23:47:26 +0800 Subject: [PATCH 6/6] Fix check failure on other models --- pytorch_forecasting/tests/test_all_estimators.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pytorch_forecasting/tests/test_all_estimators.py b/pytorch_forecasting/tests/test_all_estimators.py index e7ccba284..56226985a 100644 --- a/pytorch_forecasting/tests/test_all_estimators.py +++ b/pytorch_forecasting/tests/test_all_estimators.py @@ -253,11 +253,16 @@ def _timeseries_integration(model, name): data = pd.DataFrame( data={ "target": np.random.rand(time_points * n_timeseries), + "time_idx": np.tile(np.arange(time_points), n_timeseries), + "group_id": np.repeat(np.arange(n_timeseries), time_points), } ) training_dataset = TimeSeriesDataSet( data=data, + time_idx="time_idx", target="target", + group_ids=["group_id"], + time_varying_unknown_reals=["target"], max_prediction_length=max_prediction_length, max_encoder_length=3, )