bcongdon · salimhb · Jul 2, 2020 · Jul 7, 2020 · Jul 7, 2020 · Jul 7, 2020
diff --git a/.gitignore b/.gitignore
@@ -101,4 +101,5 @@ ENV/
 .mypy_cache/
 
 # Don't include build files in source tree
-bpy_lambda/
+bpy_lambda/
+bpy_lambda_layer.zip
diff --git a/README.md b/README.md
@@ -38,6 +38,9 @@ except ImportError:
     from bpy_lambda import bpy
 ```
 
+### AWS Lambda Layer
+
+AWS Lambda has limits on package size. It is better to package this module as a layer for AWS Lambda. After having run `./build.sh`, the script `./lambda_layer.sh` can be run to create `bpy_lambda_layer.zip`, which can be used to create a new AWS Lambda Layer.
 
 ## Contribution / Building from Source
 

diff --git a/build.sh b/build.sh
@@ -3,7 +3,7 @@ docker build -t bpy_lambda ./docker
 
 echo "*** COPYING bpy_lambda ARTIFACTS ***"
 docker run -d -it --name bpy_lambda bpy_lambda /bin/bash
-docker cp bpy_lambda:/bpy_lambda .
+docker cp bpy_lambda:/root/bpy_lambda .
 cp shim/__init__.py ./bpy_lambda
 
 echo "*** CLEANING UP ***"

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,23 +1,19 @@
 # Docker file for building Blender as a Python module for AWS Lambda
 # Compiles to ~/blender-git/blender_build/bin
 FROM amazonlinux:2018.03
-RUN yum install git cmake -y
 RUN yum groupinstall 'Development Tools' -y
+RUN yum install gcc72 gcc72-c++ git subversion make libX11-devel libXi-devel \
+    libXcursor-devel libXrandr-devel libXinerama-devel libstdc++-static \
+    libtiff-devel libpng-devel libjpeg-devel openjpeg openjpeg-devel \
+    freetype-devel wget libXmu-devel libXi-devel libGL-devel bzip2 sudo libffi \
+    libffi-devel freeglut-devel fftw-devel -y
 
-RUN yum install gcc-c++ \
-    libXi-devel openexr-devel SDL-devel fftw-devel libtiff-devel \
-    freetype-devel libogg-devel libjpeg-devel openjpeg openjpeg-devel \
-    libpng-devel vim libGLU-devel wget -y
-
-# Install Python3.6
-ENV LD_LIBRARY_PATH /usr/local/lib
-RUN cd ~ && \
-    wget https://www.python.org/ftp/python/3.6.6/Python-3.6.6.tgz && \
-    tar xzf Python-3.6.6.tgz && \
-    cd Python-3.6.6 && \
-    ./configure --enable-shared && \
-    make install && \
-    ldconfig && python3 --version
+RUN wget https://cmake.org/files/v3.10/cmake-3.10.0.tar.gz && \
+    tar -xvzf cmake-3.10.0.tar.gz && \
+    cd cmake-3.10.0 && \
+    ./bootstrap && \
+    make && \
+    make install
 
 RUN git clone https://github.com/nigels-com/glew ~/glew && \
     cd ~/glew && \
@@ -29,91 +25,50 @@ RUN mkdir ~/blender-git && \
     cd ~/blender-git && \
     git clone --progress --no-checkout https://git.blender.org/blender.git && \
     cd blender && \
-    git checkout tags/v2.79 && \
+    git checkout tags/v2.83.1 && \
     git submodule update --init --recursive && \
     git submodule foreach git checkout master && \
     git submodule foreach git pull --rebase origin master
 
-RUN ~/blender-git/blender/build_files/build_environment/install_deps.sh \
+RUN cd ~/blender-git && \
+    ./blender/build_files/build_environment/install_deps.sh \
     --no-sudo \
     --no-confirm \
-    --build-boost \
-    --build-openexr \
-    --build-oiio \
-    --skip-python \
     --skip-numpy \
-    --skip-ocio \
-    --skip-osl \
-    --skip-osd \
     --skip-openvdb \
-    --skip-llvm \
-    --skip-alembic \
-    --skip-opencollada \
-    --skip-ffmpeg
-
-ENV BOOST_ROOT ~/src/blender-deps/boost-1.60.0
+    --skip-ffmpeg \
+    --skip-usd
 
-# Maintainer's note: calculate DPYTHON_INCLUDE_DIR with the following command:
-# ```
-# python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())"
-# ```
-RUN mkdir ~/blender-git/blender_build && \
-    cd ~/blender-git/blender_build && \
-    cmake ../blender \
-    -DPYTHON_INCLUDE_DIR=/usr/local/include/python3.6m  \
-    -DPYTHON_LIBRARY=/usr/local/lib/libpython3.6m.so \
-    -DPYTHON_LIBPATH=/usr/local/lib \
-    -DWITH_INTERNATIONAL=OFF \
-    -DWITH_CYCLES=ON \
-    -DWITH_AUDASPACE=OFF \
-    -DWITH_BOOST=ON \
-    -DWITH_BULLET=OFF \
-    -DWITH_CODEC_AVI=OFF \
-    -DWITH_COMPOSITOR=OFF \
-    -DWITH_GAMEENGINE=OFF \
-    -DWITH_GAMEENGINE_DECKLINK=OFF \
-    -DWITH_HEADLESS=ON \
-    -DWITH_IMAGE_DDS=OFF \
-    -DWITH_IMAGE_CINEON=OFF \
-    -DWITH_IMAGE_OPENEXR=OFF \
-    -DWITH_IMAGE_OPENJPEG=OFF \
-    -DWITH_IMAGE_TIFF=OFF \
-    -DWITH_INPUT_NDOF=OFF \
-    -DWITH_LIBMV=OFF \
-    -DWITH_MOD_REMESH=ON \
-    -DWITH_MOD_SMOKE=OFF \
-    -DWITH_OPENAL=OFF \
-    -DWITH_OPENIMAGEIO=OFF \
-    -DWITH_PYTHON_INSTALL_NUMPY=OFF \
-    -DWITH_INSTALL_PORTABLE=ON \
-    -DWITH_PYTHON_INSTALL_REQUESTS=OFF \
-    -DWITH_PYTHON_MODULE=ON \
-    -DWITH_PYTHON_INSTALL=OFF \
-    -DWITH_STATIC_LIBS=ON \
-    -DWITH_SYSTEM_GLEW=OFF \
-    -DWITH_SYSTEM_GLES=OFF \
-    -DWITH_FREESTYLE=ON
-
-RUN cd ~/blender-git/blender_build && \
-    make && \
-    make install
+RUN cd ~/blender-git/blender && \
+    make -j8 BUILD_CMAKE_ARGS="-U *SNDFILE* -U PYTHON* -U *BOOST* -U *Boost* -U *OPENCOLORIO* -U *OPENEXR* -U *OPENIMAGEIO* -U *LLVM* -U *CYCLES* -U *OPENSUBDIV* -U *OPENVDB* -U *COLLADA* -U *FFMPEG* -U *ALEMBIC* -U *USD* -D WITH_CODEC_SNDFILE=ON -D PYTHON_VERSION=3.7 -D PYTHON_ROOT_DIR=/opt/lib/python-3.7 -D BOOST_ROOT=/opt/lib/boost -D Boost_NO_SYSTEM_PATHS=ON -D WITH_OPENCOLORIO=ON -D OPENCOLORIO_ROOT_DIR=/opt/lib/ocio -D OPENEXR_ROOT_DIR=/opt/lib/openexr -D WITH_OPENIMAGEIO=ON -D OPENIMAGEIO_ROOT_DIR=/opt/lib/oiio -D WITH_CYCLES_OSL=ON -D WITH_LLVM=ON -D LLVM_VERSION=9.0.1 -D OSL_ROOT_DIR=/opt/lib/osl -D LLVM_ROOT_DIR=/opt/lib/llvm -D LLVM_STATIC=ON -D WITH_OPENSUBDIV=ON -D OPENSUBDIV_ROOT_DIR=/opt/lib/osd -D WITH_ALEMBIC=ON -D ALEMBIC_ROOT_DIR=/opt/lib/alembic -D WITH_XR_OPENXR=ON -D XR_OPENXR_SDK_ROOT_DIR=/opt/lib/xr-openxr-sdk -D WITH_TBB=OFF -D WITH_CODEC_FFMPEG=OFF" bpy
 
-RUN mkdir /bpy_lambda && \
-    cd /bpy_lambda && \
-    cp -R ~/blender-git/blender_build/bin/2.79 . && \
-    cp -R ~/blender-git/blender_build/bin/bpy.so . && \
-    cp -L /opt/lib/openexr-2.2.0/lib/libHalf.so.12 . && \
-    cp -L /opt/lib/openexr-2.2.0/lib/libIex.so.12 . && \
-    cp -L /opt/lib/openexr-2.2.0/lib/libIlmImf.so.22 . && \
-    cp -L /opt/lib/openexr-2.2.0/lib/libIlmThread.so.12 . && \
-    cp -L /opt/lib/openexr-2.2.0/lib/libImath.so.12 . && \
-    cp -L /opt/lib/oiio-1.7.15/lib/libOpenImageIO.so.1.7 . && \
+RUN mkdir ~/bpy_lambda && \
+    cd ~/bpy_lambda && \
+    cp -R /opt/lib/python-3.7/lib/python3.7/site-packages/2.83 . && \
+    cp -R ~/blender-git/build_linux_bpy/bin/bpy.so . && \
+    cp -L /opt/lib/openexr-2.4.0/lib64/libHalf.so . && \
+    cp -L /opt/lib/openexr-2.4.0/lib64/libIex.so . && \
+    cp -L /opt/lib/openexr-2.4.0/lib64/libIlmImf.so . && \
+    cp -L /opt/lib/openexr-2.4.0/lib64/libIlmThread.so . && \
+    cp -L /opt/lib/openexr-2.4.0/lib64/libImath.so . && \
+    cp -L /opt/lib/openexr-2.4.0/lib64/libIexMath-2_4.so.24 . && \
+    cp -L /opt/lib/oiio-1.8.13/lib64/libOpenImageIO.so . && \
+    cp -L /opt/lib/osd-3.4.0_RC2/lib/libosdGPU.so.3.4.0 . && \
+    cp -L /opt/lib/osd-3.4.0_RC2/lib/libosdCPU.so.3.4.0 . && \
+    cp -L /opt/lib/osl-1.10.9/lib64/liboslcomp.so.1.10 . && \
+    cp -L /opt/lib/osl-1.10.9/lib64/liboslexec.so.1.10 . && \
+    cp -L /opt/lib/osl-1.10.9/lib64/liboslquery.so.1.10 . && \
+    cp -L /usr/lib64/libfftw3.so.3 . && \
     cp -L /usr/lib64/libopenjpeg.so.2 . && \
-    cp -L /opt/lib/boost-1.60.0/lib/libboost_filesystem.so.1.60.0 . && \
-    cp -L /opt/lib/boost-1.60.0/lib/libboost_regex.so.1.60.0 . && \
-    cp -L /opt/lib/boost/lib/libboost_system.so.1.60.0 . && \
-    cp -L /opt/lib/boost/lib/libboost_thread.so.1.60.0 . && \
-    cp -L /usr/local/lib/libpython3.6m.so.1.0 . && \
+    cp -L /usr/lib64/libGLEW.so.2.2 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_chrono.so.1.70.0 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_date_time.so.1.70.0 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_filesystem.so.1.70.0 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_regex.so.1.70.0 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_system.so.1.70.0 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_thread.so.1.70.0 . && \
+    cp -L /opt/lib/boost-1.70.0/lib/libboost_wave.so.1.70.0 . && \
+    cp -L /opt/lib/python-3.7.4/lib/libpython3.7m.so.1.0 . && \
     cp -L /usr/lib64/libGLU.so.1 .
 
 # Build artifact stored in /bpy_lambda
diff --git a/lambda_layer.sh b/lambda_layer.sh
@@ -0,0 +1,5 @@
+rm bpy_lambda_layer.zip
+mkdir package && mkdir package/python && mv bpy_lambda package/python/
+cd package && zip -r9 ../bpy_lambda_layer.zip .
+cd .. && mv ./package/python/bpy_lambda .
+rmdir package/python && rmdir package
diff --git a/shim/__init__.py b/shim/__init__.py
@@ -4,19 +4,30 @@
 lib_dir = os.path.dirname(os.path.realpath(__file__))
 
 LIBRARIES = [
-    'libHalf.so.12',
-    'libIex.so.12',
-    'libImath.so.12',
-    'libIlmThread.so.12',
-    'libIlmImf.so.22',
-    'libboost_system.so.1.60.0',
-    'libboost_filesystem.so.1.60.0',
-    'libboost_regex.so.1.60.0',
-    'libboost_thread.so.1.60.0',
-    'libpython3.6m.so.1.0',
+    'libHalf.so',
+    'libIex.so',
+    'libIexMath-2_4.so.24',
+    'libImath.so',
+    'libIlmThread.so',
+    'libIlmImf.so',
+    'libboost_chrono.so.1.70.0',
+    'libboost_date_time.so.1.70.0',
+    'libboost_system.so.1.70.0',
+    'libboost_filesystem.so.1.70.0',
+    'libboost_regex.so.1.70.0',
+    'libboost_thread.so.1.70.0',
+    'libboost_wave.so.1.70.0',
+    'libpython3.7m.so.1.0',
     'libGLU.so.1',
+    'libfftw3.so.3',
     'libopenjpeg.so.2',
-    'libOpenImageIO.so.1.7',
+    'libOpenImageIO.so',
+    'libGLEW.so.2.2',
+    'liboslcomp.so.1.10',
+    'liboslexec.so.1.10',
+    'liboslquery.so.1.10',
+    'libosdCPU.so.3.4.0',
+    'libosdGPU.so.3.4.0',
 ]
 
 for lib in LIBRARIES:

diff --git a/test.sh b/test.sh
@@ -1 +1 @@
-docker run --rm -u root -v "$PWD":/var/task:ro,delegated lambci/lambda:python3.6 test.handler
+docker run --rm -u root -v "$PWD":/var/task:ro,delegated lambci/lambda:python3.7 test.handler
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		docker run --rm -u root -v "$PWD":/var/task:ro,delegated lambci/lambda:python3.6 test.handler
		docker run --rm -u root -v "$PWD":/var/task:ro,delegated lambci/lambda:python3.7 test.handler
Copy link Owner bcongdon Jul 7, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Using 3.7 hangs indefinitely for me, but 3.6 runs correctly. Are you able to get a successful test run using 3.7? Copy link Author salimhb Jul 7, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. 3.7 hangs for me too. I wasn't sure about reverting to 3.6 or figuring out why `bpy.ops.object.modifier_add(type='REMESH')` causes this issue with Python 3.7 On a real AWS Lambda, it actually causes a segmentation fault: `START RequestId: ... Version: $LATEST Info: Deleted 3 object(s) Info: Deleted 3 object(s) END RequestId: 458a32a9-7db8-4ae4-93a8-74d88ba75159 REPORT RequestId: 458a32a9-7db8-4ae4-93a8-74d88ba75159 Duration: 228.16 ms Billed Duration: 300 ms Memory Size: 1024 MB Max Memory Used: 235 MB Init Duration: 1426.11 ms RequestId: ... Error: Runtime exited with error: signal: segmentation fault (core dumped) Runtime.ExitError` I also tested on an Amazon Linux based Light Sail instance and got the segmentation fault with both Python 3.6 and 3.7 What do you think could be the issue behind this? Copy link markhohmann Jul 7, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Not a lot of help to add, but I was able to run 3.7 with a different test.py script that I configured to test specific to my project. Does not have "remesh" so perhaps that's the culprit as salimhb identifies. Copy link Owner bcongdon Jul 8, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. Hmm... Yeah, the only thing I can think of is that the `WITH_MOD_REMESH` is not currently being set in the make options. However, when I enabled that flag I still wasn't able to get the test to pass on 3.7. I'd be fine with changing the test -- but it is unfortunate to have this regression. @markhohmann: Do you have ideas for a test handler that'd exercise basic blender functionality without calling the remesh operation? Copy link Author salimhb Jul 10, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. @markhohmann can you try this patch and check if it resolves the error "Unable to open display". I added some X11 and GL libs: https://gist.github.com/salimhb/92be32bd629b15b03e94bdc0f8e8b8f5 Copy link markhohmann Jul 20, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. @bcongdon For me I think the basic functionality anyone would need would be: 1) create an object 2) add a material and 3) render to output. Meaning, would people typically use blender on the server side without doing those things? I'm sure there are a ton of use cases that fall outside, but if we test for these major ones, hopefully we can create a foundational test others could extend. Let me know if you have thoughts on essential use cases beyond the three above. I'll submit a test handler. Copy link Author salimhb Sep 3, 2020 Choose a reason for hiding this comment The reason will be displayed to describe this comment to others. Learn more. @markhohmann I think these steps sound good. Did you try my last patch to fix the error "Unable to open display"? I've added some X11libs that might fix it. If yes, then I can update this PR with this commit railslove@`df514ef`