From 68c56ab13f806e7f864cbdb7943ef8a49435f7e6 Mon Sep 17 00:00:00 2001 From: Salim Hbeiliny Date: Thu, 2 Jul 2020 10:45:26 +0200 Subject: [PATCH 1/4] Build based on Blender 2.83.1 Add script to create an AWS Lambda layer --- .gitignore | 3 +- README.md | 3 + build.sh | 2 +- docker/Dockerfile | 141 +++++++++++++++++----------------------------- lambda_layer.sh | 5 ++ shim/__init__.py | 33 +++++++---- test.sh | 2 +- 7 files changed, 87 insertions(+), 102 deletions(-) create mode 100755 lambda_layer.sh diff --git a/.gitignore b/.gitignore index 04c66a2..5d3664d 100644 --- a/.gitignore +++ b/.gitignore @@ -101,4 +101,5 @@ ENV/ .mypy_cache/ # Don't include build files in source tree -bpy_lambda/ \ No newline at end of file +bpy_lambda/ +bpy_lambda_layer.zip diff --git a/README.md b/README.md index d63a616..cdad011 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,9 @@ except ImportError: from bpy_lambda import bpy ``` +### AWS Lambda Layer + +AWS Lambda has limits on package size. It is better to package this module as a layer for AWS Lambda. The script `./lamda_layer.sh` will create `bpy_lambda_layer.zip`, which can be used to create a new AWS Lambda Layer. ## Contribution / Building from Source diff --git a/build.sh b/build.sh index 53b9a6d..6892edf 100755 --- a/build.sh +++ b/build.sh @@ -3,7 +3,7 @@ docker build -t bpy_lambda ./docker echo "*** COPYING bpy_lambda ARTIFACTS ***" docker run -d -it --name bpy_lambda bpy_lambda /bin/bash -docker cp bpy_lambda:/bpy_lambda . +docker cp bpy_lambda:/root/bpy_lambda . cp shim/__init__.py ./bpy_lambda echo "*** CLEANING UP ***" diff --git a/docker/Dockerfile b/docker/Dockerfile index 9b69ca7..7c1b668 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,24 +1,29 @@ # Docker file for building Blender as a Python module for AWS Lambda # Compiles to ~/blender-git/blender_build/bin FROM amazonlinux:2018.03 -RUN yum install git cmake -y RUN yum groupinstall 'Development Tools' -y +RUN yum install gcc72 gcc72-c++ git subversion make libX11-devel libXi-devel libXcursor-devel libXrandr-devel libXinerama-devel libstdc++-static \ + libtiff-devel libpng-devel libjpeg-devel openjpeg openjpeg-devel freetype-devel \ + wget -y -RUN yum install gcc-c++ \ - libXi-devel openexr-devel SDL-devel fftw-devel libtiff-devel \ - freetype-devel libogg-devel libjpeg-devel openjpeg openjpeg-devel \ - libpng-devel vim libGLU-devel wget -y +RUN wget https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz && \ + tar -xvzf cmake-3.10.0.tar.gz && \ + cd cmake-3.10.0 && \ + ./bootstrap && \ + make && \ + make install -# Install Python3.6 -ENV LD_LIBRARY_PATH /usr/local/lib -RUN cd ~ && \ - wget https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz && \ - tar xzf Python-3.6.6.tgz && \ - cd Python-3.6.6 && \ - ./configure --enable-shared && \ - make install && \ - ldconfig && python3 --version +# Install Python3.7 +#ENV LD_LIBRARY_PATH /usr/local/lib +#RUN cd ~ && \ +# wget https://www.python.org/ftp/python/3.7.8/Python-3.7.8.tgz && \ +# tar xzf Python-3.7.8.tgz && \ +# cd Python-3.7.8 && \ +# ./configure --enable-shared && \ +# make install && \ +# ldconfig && python3 --version +RUN yum install libXmu-devel libXi-devel libGL-devel -y RUN git clone https://github.com/nigels-com/glew ~/glew && \ cd ~/glew && \ make extensions && \ @@ -29,91 +34,51 @@ RUN mkdir ~/blender-git && \ cd ~/blender-git && \ git clone --progress --no-checkout https://git.blender.org/blender.git && \ cd blender && \ - git checkout tags/v2.79 && \ + git checkout tags/v2.83.1 && \ git submodule update --init --recursive && \ git submodule foreach git checkout master && \ git submodule foreach git pull --rebase origin master -RUN ~/blender-git/blender/build_files/build_environment/install_deps.sh \ +RUN yum install bzip2 sudo libffi libffi-devel freeglut-devel fftw-devel -y +RUN cd ~/blender-git && \ + ./blender/build_files/build_environment/install_deps.sh \ --no-sudo \ --no-confirm \ - --build-boost \ - --build-openexr \ - --build-oiio \ - --skip-python \ --skip-numpy \ - --skip-ocio \ - --skip-osl \ - --skip-osd \ --skip-openvdb \ - --skip-llvm \ - --skip-alembic \ - --skip-opencollada \ - --skip-ffmpeg - -ENV BOOST_ROOT ~/src/blender-deps/boost-1.60.0 + --skip-ffmpeg \ + --skip-usd -# Maintainer's note: calculate DPYTHON_INCLUDE_DIR with the following command: -# ``` -# python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())" -# ``` -RUN mkdir ~/blender-git/blender_build && \ - cd ~/blender-git/blender_build && \ - cmake ../blender \ - -DPYTHON_INCLUDE_DIR=/usr/local/include/python3.6m \ - -DPYTHON_LIBRARY=/usr/local/lib/libpython3.6m.so \ - -DPYTHON_LIBPATH=/usr/local/lib \ - -DWITH_INTERNATIONAL=OFF \ - -DWITH_CYCLES=ON \ - -DWITH_AUDASPACE=OFF \ - -DWITH_BOOST=ON \ - -DWITH_BULLET=OFF \ - -DWITH_CODEC_AVI=OFF \ - -DWITH_COMPOSITOR=OFF \ - -DWITH_GAMEENGINE=OFF \ - -DWITH_GAMEENGINE_DECKLINK=OFF \ - -DWITH_HEADLESS=ON \ - -DWITH_IMAGE_DDS=OFF \ - -DWITH_IMAGE_CINEON=OFF \ - -DWITH_IMAGE_OPENEXR=OFF \ - -DWITH_IMAGE_OPENJPEG=OFF \ - -DWITH_IMAGE_TIFF=OFF \ - -DWITH_INPUT_NDOF=OFF \ - -DWITH_LIBMV=OFF \ - -DWITH_MOD_REMESH=ON \ - -DWITH_MOD_SMOKE=OFF \ - -DWITH_OPENAL=OFF \ - -DWITH_OPENIMAGEIO=OFF \ - -DWITH_PYTHON_INSTALL_NUMPY=OFF \ - -DWITH_INSTALL_PORTABLE=ON \ - -DWITH_PYTHON_INSTALL_REQUESTS=OFF \ - -DWITH_PYTHON_MODULE=ON \ - -DWITH_PYTHON_INSTALL=OFF \ - -DWITH_STATIC_LIBS=ON \ - -DWITH_SYSTEM_GLEW=OFF \ - -DWITH_SYSTEM_GLES=OFF \ - -DWITH_FREESTYLE=ON - -RUN cd ~/blender-git/blender_build && \ - make && \ - make install +RUN cd ~/blender-git/blender && \ + make -j8 BUILD_CMAKE_ARGS="-U *SNDFILE* -U PYTHON* -U *BOOST* -U *Boost* -U *OPENCOLORIO* -U *OPENEXR* -U *OPENIMAGEIO* -U *LLVM* -U *CYCLES* -U *OPENSUBDIV* -U *OPENVDB* -U *COLLADA* -U *FFMPEG* -U *ALEMBIC* -U *USD* -D WITH_CODEC_SNDFILE=ON -D PYTHON_VERSION=3.7 -D PYTHON_ROOT_DIR=/opt/lib/python-3.7 -D BOOST_ROOT=/opt/lib/boost -D Boost_NO_SYSTEM_PATHS=ON -D WITH_OPENCOLORIO=ON -D OPENCOLORIO_ROOT_DIR=/opt/lib/ocio -D OPENEXR_ROOT_DIR=/opt/lib/openexr -D WITH_OPENIMAGEIO=ON -D OPENIMAGEIO_ROOT_DIR=/opt/lib/oiio -D WITH_CYCLES_OSL=ON -D WITH_LLVM=ON -D LLVM_VERSION=9.0.1 -D OSL_ROOT_DIR=/opt/lib/osl -D LLVM_ROOT_DIR=/opt/lib/llvm -D LLVM_STATIC=ON -D WITH_OPENSUBDIV=ON -D OPENSUBDIV_ROOT_DIR=/opt/lib/osd -D WITH_ALEMBIC=ON -D ALEMBIC_ROOT_DIR=/opt/lib/alembic -D WITH_XR_OPENXR=ON -D XR_OPENXR_SDK_ROOT_DIR=/opt/lib/xr-openxr-sdk -D WITH_TBB=OFF -D WITH_CODEC_FFMPEG=OFF" bpy -RUN mkdir /bpy_lambda && \ - cd /bpy_lambda && \ - cp -R ~/blender-git/blender_build/bin/2.79 . && \ - cp -R ~/blender-git/blender_build/bin/bpy.so . && \ - cp -L /opt/lib/openexr-2.2.0/lib/libHalf.so.12 . && \ - cp -L /opt/lib/openexr-2.2.0/lib/libIex.so.12 . && \ - cp -L /opt/lib/openexr-2.2.0/lib/libIlmImf.so.22 . && \ - cp -L /opt/lib/openexr-2.2.0/lib/libIlmThread.so.12 . && \ - cp -L /opt/lib/openexr-2.2.0/lib/libImath.so.12 . && \ - cp -L /opt/lib/oiio-1.7.15/lib/libOpenImageIO.so.1.7 . && \ +RUN mkdir ~/bpy_lambda && \ + cd ~/bpy_lambda && \ + cp -R /opt/lib/python-3.7/lib/python3.7/site-packages/2.83 . && \ + cp -R ~/blender-git/build_linux_bpy/bin/bpy.so . && \ + cp -L /opt/lib/openexr-2.4.0/lib64/libHalf.so . && \ + cp -L /opt/lib/openexr-2.4.0/lib64/libIex.so . && \ + cp -L /opt/lib/openexr-2.4.0/lib64/libIlmImf.so . && \ + cp -L /opt/lib/openexr-2.4.0/lib64/libIlmThread.so . && \ + cp -L /opt/lib/openexr-2.4.0/lib64/libImath.so . && \ + cp -L /opt/lib/openexr-2.4.0/lib64/libIexMath-2_4.so.24 . && \ + cp -L /opt/lib/oiio-1.8.13/lib64/libOpenImageIO.so . && \ + cp -L /opt/lib/osd-3.4.0_RC2/lib/libosdGPU.so.3.4.0 . && \ + cp -L /opt/lib/osd-3.4.0_RC2/lib/libosdCPU.so.3.4.0 . && \ + cp -L /opt/lib/osl-1.10.9/lib64/liboslcomp.so.1.10 . && \ + cp -L /opt/lib/osl-1.10.9/lib64/liboslexec.so.1.10 . && \ + cp -L /opt/lib/osl-1.10.9/lib64/liboslquery.so.1.10 . && \ + cp -L /usr/lib64/libfftw3.so.3 . && \ cp -L /usr/lib64/libopenjpeg.so.2 . && \ - cp -L /opt/lib/boost-1.60.0/lib/libboost_filesystem.so.1.60.0 . && \ - cp -L /opt/lib/boost-1.60.0/lib/libboost_regex.so.1.60.0 . && \ - cp -L /opt/lib/boost/lib/libboost_system.so.1.60.0 . && \ - cp -L /opt/lib/boost/lib/libboost_thread.so.1.60.0 . && \ - cp -L /usr/local/lib/libpython3.6m.so.1.0 . && \ + cp -L /usr/lib64/libGLEW.so.2.2 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_chrono.so.1.70.0 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_date_time.so.1.70.0 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_filesystem.so.1.70.0 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_regex.so.1.70.0 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_system.so.1.70.0 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_thread.so.1.70.0 . && \ + cp -L /opt/lib/boost-1.70.0/lib/libboost_wave.so.1.70.0 . && \ + cp -L /opt/lib/python-3.7.4/lib/libpython3.7m.so.1.0 . && \ cp -L /usr/lib64/libGLU.so.1 . # Build artifact stored in /bpy_lambda diff --git a/lambda_layer.sh b/lambda_layer.sh new file mode 100755 index 0000000..ff820f7 --- /dev/null +++ b/lambda_layer.sh @@ -0,0 +1,5 @@ +rm bpy_lambda_layer.zip +mkdir package && mkdir package/python && mv bpy_lambda package/python/ +cd package && zip -r9 ../bpy_lambda_layer.zip . +cd .. && mv ./package/python/bpy_lambda . +rmdir package/python && rmdir package diff --git a/shim/__init__.py b/shim/__init__.py index 4dadef0..8557c83 100644 --- a/shim/__init__.py +++ b/shim/__init__.py @@ -4,19 +4,30 @@ lib_dir = os.path.dirname(os.path.realpath(__file__)) LIBRARIES = [ - 'libHalf.so.12', - 'libIex.so.12', - 'libImath.so.12', - 'libIlmThread.so.12', - 'libIlmImf.so.22', - 'libboost_system.so.1.60.0', - 'libboost_filesystem.so.1.60.0', - 'libboost_regex.so.1.60.0', - 'libboost_thread.so.1.60.0', - 'libpython3.6m.so.1.0', + 'libHalf.so', + 'libIex.so', + 'libIexMath-2_4.so.24', + 'libImath.so', + 'libIlmThread.so', + 'libIlmImf.so', + 'libboost_chrono.so.1.70.0', + 'libboost_date_time.so.1.70.0', + 'libboost_system.so.1.70.0', + 'libboost_filesystem.so.1.70.0', + 'libboost_regex.so.1.70.0', + 'libboost_thread.so.1.70.0', + 'libboost_wave.so.1.70.0', + 'libpython3.7m.so.1.0', 'libGLU.so.1', + 'libfftw3.so.3', 'libopenjpeg.so.2', - 'libOpenImageIO.so.1.7', + 'libOpenImageIO.so', + 'libGLEW.so.2.2', + 'liboslcomp.so.1.10', + 'liboslexec.so.1.10', + 'liboslquery.so.1.10', + 'libosdCPU.so.3.4.0', + 'libosdGPU.so.3.4.0', ] for lib in LIBRARIES: diff --git a/test.sh b/test.sh index 2ecd29a..b1a303a 100755 --- a/test.sh +++ b/test.sh @@ -1 +1 @@ -docker run --rm -u root -v "$PWD":/var/task:ro,delegated lambci/lambda:python3.6 test.handler +docker run --rm -u root -v "$PWD":/var/task:ro,delegated lambci/lambda:python3.7 test.handler From 30de256e935c883a44cbdc671fe92f4c0e427c6c Mon Sep 17 00:00:00 2001 From: salimhb Date: Tue, 7 Jul 2020 09:19:11 +0200 Subject: [PATCH 2/4] Update README.md Co-authored-by: Ben Congdon --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cdad011..1c51e01 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ except ImportError: ### AWS Lambda Layer -AWS Lambda has limits on package size. It is better to package this module as a layer for AWS Lambda. The script `./lamda_layer.sh` will create `bpy_lambda_layer.zip`, which can be used to create a new AWS Lambda Layer. +AWS Lambda has limits on package size. It is better to package this module as a layer for AWS Lambda. The script `./lambda_layer.sh` will create `bpy_lambda_layer.zip`, which can be used to create a new AWS Lambda Layer. ## Contribution / Building from Source From e7e3caf8e78e810a778fb19dc35ba0431f1792cf Mon Sep 17 00:00:00 2001 From: Salim Hbeiliny Date: Tue, 7 Jul 2020 15:37:39 +0200 Subject: [PATCH 3/4] merge package installation steps --- docker/Dockerfile | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7c1b668..c78f1eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,9 +2,11 @@ # Compiles to ~/blender-git/blender_build/bin FROM amazonlinux:2018.03 RUN yum groupinstall 'Development Tools' -y -RUN yum install gcc72 gcc72-c++ git subversion make libX11-devel libXi-devel libXcursor-devel libXrandr-devel libXinerama-devel libstdc++-static \ - libtiff-devel libpng-devel libjpeg-devel openjpeg openjpeg-devel freetype-devel \ - wget -y +RUN yum install gcc72 gcc72-c++ git subversion make libX11-devel libXi-devel \ + libXcursor-devel libXrandr-devel libXinerama-devel libstdc++-static \ + libtiff-devel libpng-devel libjpeg-devel openjpeg openjpeg-devel \ + freetype-devel wget libXmu-devel libXi-devel libGL-devel bzip2 sudo libffi \ + libffi-devel freeglut-devel fftw-devel -y RUN wget https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz && \ tar -xvzf cmake-3.10.0.tar.gz && \ @@ -13,17 +15,6 @@ RUN wget https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz && \ make && \ make install -# Install Python3.7 -#ENV LD_LIBRARY_PATH /usr/local/lib -#RUN cd ~ && \ -# wget https://www.python.org/ftp/python/3.7.8/Python-3.7.8.tgz && \ -# tar xzf Python-3.7.8.tgz && \ -# cd Python-3.7.8 && \ -# ./configure --enable-shared && \ -# make install && \ -# ldconfig && python3 --version - -RUN yum install libXmu-devel libXi-devel libGL-devel -y RUN git clone https://github.com/nigels-com/glew ~/glew && \ cd ~/glew && \ make extensions && \ @@ -39,7 +30,6 @@ RUN mkdir ~/blender-git && \ git submodule foreach git checkout master && \ git submodule foreach git pull --rebase origin master -RUN yum install bzip2 sudo libffi libffi-devel freeglut-devel fftw-devel -y RUN cd ~/blender-git && \ ./blender/build_files/build_environment/install_deps.sh \ --no-sudo \ From f0c09fbe076215e82f6f010a7dbaae4e4801a845 Mon Sep 17 00:00:00 2001 From: Salim Hbeiliny Date: Tue, 7 Jul 2020 15:38:04 +0200 Subject: [PATCH 4/4] clarify usage of lambda_layer.sh --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1c51e01..ff40b70 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ except ImportError: ### AWS Lambda Layer -AWS Lambda has limits on package size. It is better to package this module as a layer for AWS Lambda. The script `./lambda_layer.sh` will create `bpy_lambda_layer.zip`, which can be used to create a new AWS Lambda Layer. +AWS Lambda has limits on package size. It is better to package this module as a layer for AWS Lambda. After having run `./build.sh`, the script `./lambda_layer.sh` can be run to create `bpy_lambda_layer.zip`, which can be used to create a new AWS Lambda Layer. ## Contribution / Building from Source