Problem loading pickle model

[HadiKutabi]

Hallo,

Im having a problem in loading the the models "rf_complete.pkl" & "sgd_complete.pkl" when initializing dswizard.core.master.Master.

Im running python 3.8, joblib 1.1.1, sklearn 1.0.2 just as it says in the requirement.txt.

for me it looks like a problem with the version of joblib or pickle. maybe the model was exported with an older version.

This is my code and the resulting error:

import logging
import os

from dswizard.core.master import Master
from dswizard.core.model import Dataset
from dswizard.optimizers.bandit_learners.pseudo import PseudoBandit
from dswizard.util import util
from dswizard.optimizers.config_generators import Hyperopt
from dswizard.optimizers.structure_generators.mcts import MCTS, TransferLearning



wallclock_limit = 700
cutoff = 120
log_dir = "run/"
fold = 2
task = 3913


util.setup_logging(os.path.join(log_dir, str(task), 'log.txt'))
logger = logging.getLogger()

# Load dataset
# Tasks: 18, 53, 9983, 146822, 168912
logger.info(f'Processing task {task}')
ds, ds_test = Dataset.from_openml(task, fold, 'roc_auc')

master = Master(
    ds=ds,
    working_directory=os.path.join(log_dir, str(task)),
    n_workers=1,
    model='/home/hadi/Desktop/dswizard-main/dswizard/assets/rf_complete.pkl',
    wallclock_limit=wallclock_limit,
    cutoff=cutoff,
    pre_sample=False,
    config_generator_class=Hyperopt,
    structure_generator_class=MCTS,
    structure_generator_kwargs={'policy': TransferLearning},
    # structure_generator_class=FixedStructure,
    # structure_generator_kwargs={'steps': steps},

    bandit_learner_class=PseudoBandit
)


if __name__ == "__main__":

    print("Starting Optimizing")
    pipeline, run_history, ensemble = master.optimize() 

Error:

/home/hadi/Desktop/dswizard-main/dswizard/optimizers/config_generators/__init__.py:9: UserWarning: SMAC not installed
  warnings.warn("SMAC not installed")
2023-01-30 08:28:56,986 INFO     root            MainThread Processing task 3913
2023-01-30 08:28:57,017 INFO     openml.config   MainThread No config file found at /root/.config/openml/config, using default configuration.
2023-01-30 08:28:57,029 INFO     openml.datasets.dataset MainThread pickle load data kc2
2023-01-30 08:28:57,031 DEBUG    Meta-Features   MainThread Calculating MF
2023-01-30 08:28:57,166 DEBUG    Meta-Features   MainThread Calculating MF
Traceback (most recent call last):
  File "/home/hadi/Desktop/dswizard-main/1_optimize.py", line 41, in <module>
    master = Master(
  File "/home/hadi/Desktop/dswizard-main/dswizard/core/master.py", line 134, in __init__
    self.cfg_cache: ConfigCache = self.mgr.ConfigCache(
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 740, in temp
    token, exp = self._create(typeid, *args, **kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 625, in _create
    id, exposed = dispatch(conn, None, 'create', (typeid,)+args, kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 91, in dispatch
    raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError: 
---------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 210, in handle_request
    result = func(c, *args, **kwds)
  File "/usr/lib/python3.8/multiprocessing/managers.py", line 403, in create
    obj = callable(*args, **kwds)
  File "/home/hadi/Desktop/dswizard-main/dswizard/core/config_cache.py", line 47, in __init__
    self.model, _ = joblib.load(f)
  File "/home/hadi/Desktop/dswizard-main/venv/lib/python3.8/site-packages/joblib/numpy_pickle.py", line 577, in load
    obj = _unpickle(fobj)
  File "/home/hadi/Desktop/dswizard-main/venv/lib/python3.8/site-packages/joblib/numpy_pickle.py", line 506, in _unpickle
    obj = unpickler.load()
  File "/usr/lib/python3.8/pickle.py", line 1212, in load
    dispatch[key[0]](self)
KeyError: 118
---------------------------------------------------------------------------

Process finished with exit code 1