From d7ae02681f244d888de054156c7763adceac978d Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Fri, 13 May 2022 16:11:40 -0400 Subject: [PATCH 1/8] Upgrade to the newest version of dill. 0.3.4 --- sdks/python/apache_beam/internal/dill_pickler.py | 11 ----------- .../python/container/py37/base_image_requirements.txt | 4 ++-- .../python/container/py38/base_image_requirements.txt | 4 ++-- .../python/container/py39/base_image_requirements.txt | 4 ++-- sdks/python/setup.py | 2 +- 5 files changed, 7 insertions(+), 18 deletions(-) diff --git a/sdks/python/apache_beam/internal/dill_pickler.py b/sdks/python/apache_beam/internal/dill_pickler.py index f99bf2af59eb..9a20d8888ad7 100644 --- a/sdks/python/apache_beam/internal/dill_pickler.py +++ b/sdks/python/apache_beam/internal/dill_pickler.py @@ -58,17 +58,6 @@ def __exit__(self, *unused_exc_info): # Pickling, especially unpickling, causes broken module imports on Python 3 # if executed concurrently, see: BEAM-8651, http://bugs.python.org/issue38884. _pickle_lock = threading.RLock() -# Dill 0.28.0 renamed dill.dill to dill._dill: -# https://github.com/uqfoundation/dill/commit/f0972ecc7a41d0b8acada6042d557068cac69baa -# TODO: Remove this once Beam depends on dill >= 0.2.8 -if not getattr(dill, 'dill', None): - dill.dill = dill._dill - sys.modules['dill.dill'] = dill._dill - -# TODO: Remove once Dataflow has containers with a preinstalled dill >= 0.2.8 -if not getattr(dill, '_dill', None): - dill._dill = dill.dill - sys.modules['dill._dill'] = dill.dill def _is_nested_class(cls): diff --git a/sdks/python/container/py37/base_image_requirements.txt b/sdks/python/container/py37/base_image_requirements.txt index c721f4e89186..7bf53995b438 100644 --- a/sdks/python/container/py37/base_image_requirements.txt +++ b/sdks/python/container/py37/base_image_requirements.txt @@ -17,7 +17,7 @@ # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. -# You will need Python intepreters for all versions supported by Beam, see: +# You will need Python interpreters for all versions supported by Beam, see: # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. @@ -38,7 +38,7 @@ crcmod==1.7 cryptography==37.0.2 Cython==0.29.28 deprecation==2.1.0 -dill==0.3.1.1 +dill==0.3.4 docker==5.0.3 docopt==0.6.2 execnet==1.9.0 diff --git a/sdks/python/container/py38/base_image_requirements.txt b/sdks/python/container/py38/base_image_requirements.txt index 37d194cd9535..77d42128c952 100644 --- a/sdks/python/container/py38/base_image_requirements.txt +++ b/sdks/python/container/py38/base_image_requirements.txt @@ -17,7 +17,7 @@ # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. -# You will need Python intepreters for all versions supported by Beam, see: +# You will need Python interpreters for all versions supported by Beam, see: # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. @@ -37,7 +37,7 @@ crcmod==1.7 cryptography==37.0.2 Cython==0.29.28 deprecation==2.1.0 -dill==0.3.1.1 +dill==0.3.4 docker==5.0.3 docopt==0.6.2 execnet==1.9.0 diff --git a/sdks/python/container/py39/base_image_requirements.txt b/sdks/python/container/py39/base_image_requirements.txt index 5daabd22d611..21e545653e70 100644 --- a/sdks/python/container/py39/base_image_requirements.txt +++ b/sdks/python/container/py39/base_image_requirements.txt @@ -17,7 +17,7 @@ # Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update. # Do not edit manually, adjust ../base_image_requirements_manual.txt or # Apache Beam's setup.py instead, and regenerate the list. -# You will need Python intepreters for all versions supported by Beam, see: +# You will need Python interpreters for all versions supported by Beam, see: # https://s.apache.org/beam-python-dev-wiki # Reach out to a committer if you need help. @@ -37,7 +37,7 @@ crcmod==1.7 cryptography==37.0.2 Cython==0.29.28 deprecation==2.1.0 -dill==0.3.1.1 +dill==0.3.4 docker==5.0.3 docopt==0.6.2 execnet==1.9.0 diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a02f9f9c339c..165f2f7ad977 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -133,7 +133,7 @@ def get_version(): # version of dill. It is best to use the same version of dill on client and # server, therefore list of allowed versions is very narrow. # See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.1.1,<0.3.2', + 'dill>=0.3.4.0,<0.3.5', 'cloudpickle>=2.0.0,<3', 'fastavro>=0.23.6,<2', 'grpcio>=1.29.0,<2', From 57c829505737ced548e4505ab083847a8fb1459b Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Fri, 13 May 2022 16:27:12 -0400 Subject: [PATCH 2/8] added back in sys modules code, this can be fixed later --- sdks/python/apache_beam/internal/dill_pickler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/sdks/python/apache_beam/internal/dill_pickler.py b/sdks/python/apache_beam/internal/dill_pickler.py index 9a20d8888ad7..f99bf2af59eb 100644 --- a/sdks/python/apache_beam/internal/dill_pickler.py +++ b/sdks/python/apache_beam/internal/dill_pickler.py @@ -58,6 +58,17 @@ def __exit__(self, *unused_exc_info): # Pickling, especially unpickling, causes broken module imports on Python 3 # if executed concurrently, see: BEAM-8651, http://bugs.python.org/issue38884. _pickle_lock = threading.RLock() +# Dill 0.28.0 renamed dill.dill to dill._dill: +# https://github.com/uqfoundation/dill/commit/f0972ecc7a41d0b8acada6042d557068cac69baa +# TODO: Remove this once Beam depends on dill >= 0.2.8 +if not getattr(dill, 'dill', None): + dill.dill = dill._dill + sys.modules['dill.dill'] = dill._dill + +# TODO: Remove once Dataflow has containers with a preinstalled dill >= 0.2.8 +if not getattr(dill, '_dill', None): + dill._dill = dill.dill + sys.modules['dill._dill'] = dill.dill def _is_nested_class(cls): From 19e347ee0bdf6c22df1996735dfa8edbba754177 Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Wed, 18 May 2022 14:05:06 -0400 Subject: [PATCH 3/8] fixed typo --- sdks/python/container/run_generate_requirements.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/container/run_generate_requirements.sh b/sdks/python/container/run_generate_requirements.sh index 9fb39795b7d2..a0d3ee6021b6 100755 --- a/sdks/python/container/run_generate_requirements.sh +++ b/sdks/python/container/run_generate_requirements.sh @@ -27,7 +27,7 @@ # ./gradlew :sdks:python:container:generatePythonRequirementsAll # ./gradlew :sdks:python:container:py38:generatePythonRequirements -# You will need Python intepreters for all versions supported by Beam, see: +# You will need Python interpreters for all versions supported by Beam, see: # https://s.apache.org/beam-python-dev-wiki if [[ $# != 2 ]]; then From eade96678f37db937fb290e94991860b94c6b478 Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Wed, 18 May 2022 15:17:12 -0400 Subject: [PATCH 4/8] dill for python --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 65610a9a7fcd..02ade55e6129 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -65,6 +65,7 @@ * X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). * Go SDK users can now use generic registration functions to optimize their DoFn execution. ([BEAM-14347](https://issues.apache.org/jira/browse/BEAM-14347)) * Go SDK users may now write self-checkpointing Splittable DoFns to read from streaming sources. ([BEAM-11104](https://issues.apache.org/jira/browse/BEAM-11104)) +* The dill library for python is updated to version 0.3.4 ([BEAM-11167](https://issues.apache.org/jira/browse/BEAM-11167)) ## Breaking Changes From fec1d6cf62899267a9a5cecd916db9beebf42c77 Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Thu, 19 May 2022 09:39:18 -0400 Subject: [PATCH 5/8] Update CHANGES.md Co-authored-by: tvalentyn --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 02ade55e6129..dc0ba4a1090e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -65,7 +65,7 @@ * X feature added (Java/Python) ([BEAM-X](https://issues.apache.org/jira/browse/BEAM-X)). * Go SDK users can now use generic registration functions to optimize their DoFn execution. ([BEAM-14347](https://issues.apache.org/jira/browse/BEAM-14347)) * Go SDK users may now write self-checkpointing Splittable DoFns to read from streaming sources. ([BEAM-11104](https://issues.apache.org/jira/browse/BEAM-11104)) -* The dill library for python is updated to version 0.3.4 ([BEAM-11167](https://issues.apache.org/jira/browse/BEAM-11167)) +* The serialization (pickling) library for Python is updated to dill==0.3.4 ([BEAM-11167](https://issues.apache.org/jira/browse/BEAM-11167)) ## Breaking Changes From 252d4c80dba2bf4a11ec65bf34503c6ae93b62bb Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Thu, 2 Jun 2022 10:54:43 -0400 Subject: [PATCH 6/8] update dill version in setup --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 038016146b54..770486dc09a6 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -211,7 +211,7 @@ def get_portability_package_data(): # using older version of dill. It is best to use the same version of # dill on client and server, therefore list of allowed versions is very # narrow. See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.4,<0.3.5', + 'dill>=0.3.5,<0.3.6', 'cloudpickle>=2.0.0,<3', 'fastavro>=0.23.6,<2', 'grpcio>=1.29.0,<2', From a943fcd46f2e657635c49376bdfaecfbf48e9edd Mon Sep 17 00:00:00 2001 From: Ryan Thompson Date: Thu, 2 Jun 2022 12:49:30 -0400 Subject: [PATCH 7/8] Update CHANGES.md Co-authored-by: tvalentyn --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 5abd68f679a3..108b6bf31188 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -66,7 +66,7 @@ * Go SDK users can now use generic registration functions to optimize their DoFn execution. ([BEAM-14347](https://issues.apache.org/jira/browse/BEAM-14347)) * Go SDK users may now write self-checkpointing Splittable DoFns to read from streaming sources. ([BEAM-11104](https://issues.apache.org/jira/browse/BEAM-11104)) * Go SDK textio Reads have been moved to Splittable DoFns exclusively. ([BEAM-14489](https://issues.apache.org/jira/browse/BEAM-14489)) -* The serialization (pickling) library for Python is updated to dill==0.3.4 ([BEAM-11167](https://issues.apache.org/jira/browse/BEAM-11167)) +* The serialization (pickling) library for Python is updated to dill==0.3.5.1 ([BEAM-11167](https://issues.apache.org/jira/browse/BEAM-11167)) ## Breaking Changes From fee88f2b2dfd3a1db40117a963a515216c5a3aef Mon Sep 17 00:00:00 2001 From: tvalentyn Date: Thu, 2 Jun 2022 18:50:01 +0200 Subject: [PATCH 8/8] Update sdks/python/setup.py --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 770486dc09a6..2797574a09f3 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -211,7 +211,7 @@ def get_portability_package_data(): # using older version of dill. It is best to use the same version of # dill on client and server, therefore list of allowed versions is very # narrow. See: https://github.com/uqfoundation/dill/issues/341. - 'dill>=0.3.5,<0.3.6', + 'dill>=0.3.5.1,<0.3.6', 'cloudpickle>=2.0.0,<3', 'fastavro>=0.23.6,<2', 'grpcio>=1.29.0,<2',