when running examples/tensorflow_v2_example/ train_and_serve.py, an error occurred

[leepand]

Traceback (most recent call last):
File "train_and_serve.py", line 58, in
main()
File "train_and_serve.py", line 50, in main
model = ebonite.create_model(tf_keras_model, test_data, 'tf_v2_model')
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/client/helpers.py", line 19, in create_model
return Model.create(model_object, input_data, model_name, params, description, **kwargs)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/core/objects/core.py", line 1037, in create
wrapper: ModelWrapper = custom_wrapper or ModelAnalyzer.analyze(model_object, input_data=input_data)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/core/analyzer/base.py", line 159, in analyze
return cls._find_hook(obj).process(obj, **kwargs)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/core/analyzer/model.py", line 30, in process
return self._wrapper_factory().bind_model(obj, **kwargs)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/core/objects/wrapper.py", line 104, in bind_model
self.methods, self.requirements = self._prepare_methods_and_requirements(input_data)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/core/objects/wrapper.py", line 109, in _prepare_methods_and_requirements
requirements += self._model_requirements()
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/core/objects/wrapper.py", line 184, in _model_requirements
return get_object_requirements(self)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 466, in get_object_requirements
a.dump(obj)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 445, in dump
StockPickler.dump(self, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 437, in dump
self.save(obj)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 819, in save_list
self._batch_appends(obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 846, in _batch_appends
save(tmp[0])
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 662, in save_reduce
save(state)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/dill/_dill.py", line 912, in save_module_dict
StockPickler.save_dict(pickler, obj)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 859, in save_dict
self._batch_setitems(obj.items())
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 549, in save
self.save_reduce(obj=obj, *rv)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 659, in save_reduce
self._batch_setitems(dictitems)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 885, in _batch_setitems
save(v)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 504, in save
f(self, obj) # Call unbound method with explicit self
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 774, in save_tuple
save(element)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/ebonite/utils/module.py", line 446, in save
return super(EbonitePickler, self).save(obj, save_persistent_id)
File "/Users/leepand/miniconda3/lib/python3.7/pickle.py", line 524, in save
rv = reduce(self.proto)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1000, in reduce
return convert_to_tensor, (self._numpy(),)
File "/Users/leepand/miniconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1039, in _numpy
six.raise_from(core._status_to_exception(e.code, e.message), None) # pylint: disable=protected-access
File "", line 3, in raise_from
tensorflow.python.framework.errors_impl.InvalidArgumentError: Cannot convert a Tensor of dtype resource to a NumPy array.

[xsa-dev]

i have the same error

My OS

Distributor ID:	Ubuntu
Description:	Ubuntu 20.04.3 LTS
Release:	20.04
Codename:	focal

Python

Python 3.7.11 (default, Jul 27 2021, 14:32:16) 
[GCC 7.5.0] :: Anaconda, Inc. on linux

Error

---------------------------------------------------------------------------
InvalidArgumentError                      Traceback (most recent call last)
/tmp/ipykernel_11593/2993359119.py in <module>
     12 
     13 if __name__ == '__main__':
---> 14     main()

/tmp/ipykernel_11593/2993359119.py in main()
      4 
      5     #  create model 'tf_v2_model' from TF Keras model and test data
----> 6     model = ebonite.create_model(tf_keras_model, test_data, 'tf_v2_model')
      7 
      8     # run flask service with this model

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/client/helpers.py in create_model(model_object, input_data, model_name, params, description, **kwargs)
     17     :return: :class:`~ebonite.core.objects.core.Model` instance
     18     """
---> 19     return Model.create(model_object, input_data, model_name, params, description, **kwargs)

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/core/objects/core.py in create(cls, model_object, input_data, model_name, params, description, additional_artifacts, additional_requirements, custom_wrapper, custom_artifact, custom_requirements)
   1035         :returns: :py:class:`Model`
   1036         """
-> 1037         wrapper: ModelWrapper = custom_wrapper or ModelAnalyzer.analyze(model_object, input_data=input_data)
   1038         name = model_name or _generate_model_name(wrapper)
   1039 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/core/analyzer/base.py in analyze(cls, obj, **kwargs)
    157             :return: Instance of {return_type.__name__}
    158             """
--> 159             return cls._find_hook(obj).process(obj, **kwargs)
    160 
    161         @classmethod

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/core/analyzer/model.py in process(self, obj, **kwargs)
     28 
     29     def process(self, obj, **kwargs) -> ModelWrapper:
---> 30         return self._wrapper_factory().bind_model(obj, **kwargs)
     31 
     32     @abstractmethod

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/core/objects/wrapper.py in bind_model(self, model, input_data, **kwargs)
    102 
    103         self.model = model
--> 104         self.methods, self.requirements = self._prepare_methods_and_requirements(input_data)
    105         return self
    106 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/core/objects/wrapper.py in _prepare_methods_and_requirements(self, input_data)
    107     def _prepare_methods_and_requirements(self, input_data):
    108         requirements = Requirements()
--> 109         requirements += self._model_requirements()
    110 
    111         arg_type = DatasetAnalyzer.analyze(input_data)

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/core/objects/wrapper.py in _model_requirements(self)
    182         :return: :class:`.Requirements` object representing runtime requirements of bound module object
    183         """
--> 184         return get_object_requirements(self)
    185 
    186     @abstractmethod

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in get_object_requirements(obj)
    464     """
    465     a = _EboniteRequirementAnalyzer(recurse=True)
--> 466     a.dump(obj)
    467     return a.to_requirements()

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in dump(self, obj)
    443             raise PicklingError(msg)
    444         else:
--> 445             StockPickler.dump(self, obj)
    446         stack.clear()  # clear record of 'recursion-sensitive' pickled objects
    447         return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in dump(self, obj)
    485         if self.proto >= 4:
    486             self.framer.start_framing()
--> 487         self.save(obj)
    488         self.write(STOP)
    489         self.framer.end_framing()

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    715         if state is not None:
    716             if state_setter is None:
--> 717                 save(state)
    718                 write(BUILD)
    719             else:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    910             # we only care about session the first pass thru
    911             pickler._session = False
--> 912         StockPickler.save_dict(pickler, obj)
    913         log.info("# D2")
    914     return

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_dict(self, obj)
    969 
    970         self.memoize(obj)
--> 971         self._batch_setitems(obj.items())
    972 
    973     dispatch[dict] = save_dict

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    601 
    602         # Save the reduce() output and finally memoize the object
--> 603         self.save_reduce(obj=obj, *rv)
    604 
    605     def persistent_id(self, obj):

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, state_setter, obj)
    711 
    712         if dictitems is not None:
--> 713             self._batch_setitems(dictitems)
    714 
    715         if state is not None:

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in _batch_setitems(self, items)
    995                 for k, v in tmp:
    996                     save(k)
--> 997                     save(v)
    998                 write(SETITEMS)
    999             elif n:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    558             f = self.dispatch.get(t)
    559             if f is not None:
--> 560                 f(self, obj)  # Call unbound method with explicit self
    561                 return
    562 

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save_tuple(self, obj)
    884         if n <= 3 and self.proto >= 2:
    885             for element in obj:
--> 886                 save(element)
    887             # Subtle.  Same as in the big comment below.
    888             if id(obj) in memo:

~/anaconda3/envs/RL_Snake/lib/python3.8/site-packages/ebonite/utils/module.py in save(self, obj, save_persistent_id)
    444         self._add_requirement(obj)
    445         try:
--> 446             return super(EbonitePickler, self).save(obj, save_persistent_id)
    447         except (ValueError, TypeError, PickleError) as e:
    448             # if object cannot be serialized, it's probably a C object and we don't need to go deeper

~/anaconda3/envs/RL_Snake/lib/python3.8/pickle.py in save(self, obj, save_persistent_id)
    576                 reduce = getattr(obj, "__reduce_ex__", None)
    577                 if reduce is not None:
--> 578                     rv = reduce(self.proto)
    579                 else:
    580                     reduce = getattr(obj, "__reduce__", None)

~/.local/lib/python3.8/site-packages/tensorflow/python/framework/ops.py in __reduce__(self)
   1013 
   1014   def __reduce__(self):
-> 1015     return convert_to_tensor, (self._numpy(),)
   1016 
   1017   def __copy__(self):

~/.local/lib/python3.8/site-packages/tensorflow/python/framework/ops.py in _numpy(self)
   1060       return self._numpy_internal()
   1061     except core._NotOkStatusException as e:  # pylint: disable=protected-access
-> 1062       six.raise_from(core._status_to_exception(e.code, e.message), None)  # pylint: disable=protected-access
   1063 
   1064   @property

~/.local/lib/python3.8/site-packages/six.py in raise_from(value, from_value)

InvalidArgumentError: Cannot convert a Tensor of dtype resource to a NumPy array.