From b4b892b30b6c70cc3030f30d3fad1f99d6491297 Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 14:13:15 -0600
Subject: [PATCH 1/9] Cache site-packages and set links

---
 bin/boxfile                |  11 ++--
 lib/deps.sh                |  62 ++++++++++++++++++++++
 lib/python.sh              | 106 +++++++++++++++++--------------------
 templates/profile.d/pip.sh |  20 +++++++
 4 files changed, 134 insertions(+), 65 deletions(-)
 create mode 100644 lib/deps.sh
 create mode 100755 templates/profile.d/pip.sh

diff --git a/bin/boxfile b/bin/boxfile
index 8645ff8..53cb2f3 100755
--- a/bin/boxfile
+++ b/bin/boxfile
@@ -5,13 +5,10 @@
 cat <<-END
 run.config:
   cache_dirs:
-    - .nanobox/cache/.pip
+    - .nanobox/pip_cache
+
+  build_triggers:
+    - requirements.txt
 END
-if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then
-	cat <<-END
-	  build_triggers:
-	    - requirements.txt
-	END
-fi
 
 exit 0
diff --git a/lib/deps.sh b/lib/deps.sh
new file mode 100644
index 0000000..22f6cc2
--- /dev/null
+++ b/lib/deps.sh
@@ -0,0 +1,62 @@
+# compiles a list of dependencies that will need to be installed
+query_dependencies() {
+  
+  # break early if there isn't a requirements.txt
+  if [[ ! -f $(nos_code_dir)/requirements.txt ]]; then
+    echo ""
+    return
+  fi
+  
+  deps=()
+
+  # mssql
+  if [[ `grep -i 'pymssql' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(freetds)
+  fi
+  # mysql
+  if [[ `grep -i 'MySQLdb\|mysqlclient\|MySQL-python' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(mysql-client)
+  fi
+  # memcache
+  if [[ `grep -i 'memcache\|libmc' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libmemcached)
+  fi
+  # postgres
+  if [[ `grep -i 'psycopg2' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(postgresql94-client)
+  fi
+  # redis
+  if [[ `grep -i 'redis' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(redis)
+  fi
+  # curl
+  if [[ `grep -i 'pycurl' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(curl)
+  fi
+  # pillow
+  if [[ `grep -i 'pillow' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libjpeg-turbo tiff zlib freetype2 lcms2 libwebp tcl tk)
+  fi
+  # boto3
+  if [[ `grep -i 'boto3' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=("$(condensed_runtime)-cElementTree")
+  fi
+  # xmlsec
+  if [[ `grep -i 'xmlsec' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libxml2 libxslt xmlsec1 pkgconf)
+  fi
+  # python3-saml
+  if [[ `grep -i 'python3-saml' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libxml2 libxslt xmlsec1 pkgconf)
+  fi
+  # pygraphviz
+  if [[ `grep -i 'pygraphviz' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libxshmfence libva libvdpau libLLVM-3.8 graphviz)
+  fi
+  # scipy
+  if [[ `grep -i 'scipy' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(blas lapack)
+  fi
+
+  echo "${deps[@]}"
+}
diff --git a/lib/python.sh b/lib/python.sh
index 7e40e24..c81baf2 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -1,6 +1,9 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+# source the deps functions
+. ${engine_lib_dir}/deps.sh
+
 # Copy the code into the live directory which will be used to run the app
 publish_release() {
   nos_print_bullet "Moving build into live app directory..."
@@ -30,7 +33,9 @@ condensed_runtime() {
 
 # Install the python runtime along with any dependencies.
 install_runtime_packages() {
-  pkgs=("$(runtime)" "$(condensed_runtime)-setuptools" "$(condensed_runtime)-pip")
+  pkgs=("$(runtime)" \
+        "$(condensed_runtime)-setuptools" \
+        "$(condensed_runtime)-pip")
   
   # add packages that are usually part of the stdlib
   pkgs+=(\
@@ -56,68 +61,53 @@ uninstall_build_packages() {
   fi
 }
 
-# compiles a list of dependencies that will need to be installed
-query_dependencies() {
-  deps=()
-
-  # mssql
-  if [[ `grep -i 'pymssql' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(freetds)
-  fi
-  # mysql
-  if [[ `grep -i 'MySQLdb\|mysqlclient\|MySQL-python' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(mysql-client)
-  fi
-  # memcache
-  if [[ `grep -i 'memcache\|libmc' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libmemcached)
-  fi
-  # postgres
-  if [[ `grep -i 'psycopg2' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(postgresql94-client)
-  fi
-  # redis
-  if [[ `grep -i 'redis' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(redis)
-  fi
-  # curl
-  if [[ `grep -i 'pycurl' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(curl)
-  fi
-  # pillow
-  if [[ `grep -i 'pillow' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libjpeg-turbo tiff zlib freetype2 lcms2 libwebp tcl tk)
-  fi
-  # boto3
-  if [[ `grep -i 'boto3' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=("$(condensed_runtime)-cElementTree")
-  fi
-  # xmlsec
-  if [[ `grep -i 'xmlsec' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libxml2 libxslt xmlsec1 pkgconf)
-  fi
-  # python3-saml
-  if [[ `grep -i 'python3-saml' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libxml2 libxslt xmlsec1 pkgconf)
-  fi
-  # pygraphviz
-  if [[ `grep -i 'pygraphviz' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libxshmfence libva libvdpau libLLVM-3.8 graphviz)
-  fi
-  # scipy
-  if [[ `grep -i 'scipy' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(blas lapack)
-  fi
-
-  echo "${deps[@]}"
-}
-
-# set any necessary python environment variables
+# Here we get to be a bit clever. Python will store it's cache (deps, etc)
+# in the site-packages dir. Changing this is not easily configurable, so we'll
+# essentially setup a cache dir in ~/.nanobox/pip_cache, and symlink
+# the site-packages to the cached location. Also, we'll copy anything
+# into the cache on the first run.
+# 
+# Additionally, we'll set some environment variables specifically for python
 setup_python_env() {
   # ensure python doesn't buffer even when not attached to a pty
   nos_template_file \
     "env.d/PYTHONUNBUFFERED" \
     "$(nos_etc_dir)/env.d/PYTHONUNBUFFERED"
+    
+  # Ensure the cache destination exists for site-packages
+  if [[ ! -d "$(nos_code_dir)/.nanobox/pip_cache/site-packages" ]]; then
+    mkdir -p "$(nos_code_dir)/.nanobox/pip_cache/site-packages"
+  fi
+  
+  # If anything exists before we symlink, copy it into the cache
+  if [[ -d "$(nos_data_dir)/lib/$(runtime)/site-packages" ]]; then
+    mv \
+      "$(nos_data_dir)/lib/$(runtime)/site-packages/*" \
+      "$(nos_code_dir)/.nanobox/pip_cache/site-packages"
+  fi
+  
+  # set the profile script that correctly sets up the links
+  set_python_profile_script
+}
+
+# Generate the payload to render the python profile template
+python_profile_payload() {
+  cat <<-END
+{
+  "code_dir": "$(nos_code_dir)",
+  "data_dir": "$(nos_data_dir)",
+  "runtime": "$(runtime)"
+}
+END
+}
+
+# Profile script to ensure symlinks for pip
+set_python_profile_script() {
+  mkdir -p "$(nos_etc_dir)/profile.d"
+  nos_template \
+    "profile.d/pip.sh" \
+    "$(nos_etc_dir)/profile.d/pip.sh" \
+    "$(python_profile_payload)"
 }
 
 # fetch the user-specified pip install command or use a default
diff --git a/templates/profile.d/pip.sh b/templates/profile.d/pip.sh
new file mode 100755
index 0000000..5080983
--- /dev/null
+++ b/templates/profile.d/pip.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Here we get to be a bit clever. pip will store it's cache (deps, etc)
+# in the site-packages dir. Changing this is not easily configurable, so we'll
+# essentially setup a cache dir in ~/.nanobox/pip_cache, and symlink
+# the site-packages to the cached location.
+
+site_packages="{{data_dir}}/lib/{{runtime}}/site-packages"
+cache_dir="{{code_dir}}/.nanobox/pip_cache/site-packages"
+
+# remove site-packages if it's a directory
+if [[ -d ${site_packages} ]]; then
+  rm -rf ${site_packages}
+fi
+
+# if site-packages isn't a symlink, create it
+if [[ ! -s ${site_packages} ]]; then
+  mkdir -p ${cache_dir}
+  ln -s ${cache_dir} ${site_packages}
+fi

From 000f591a0dd7769d802115044b33ef6aa5691d1e Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 14:19:25 -0600
Subject: [PATCH 2/9] Condense the lib runtime

---
 lib/python.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/lib/python.sh b/lib/python.sh
index c81baf2..21979b3 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -31,6 +31,13 @@ condensed_runtime() {
   echo "${version//[.thon-]/}"
 }
 
+# In the data directory, the dir will look like python2.7 so
+# we need to fetch to lib runtime whenever we're messing with libs
+lib_runtime() {
+  version=$(runtime)
+  echo "${version//[-]/}"
+}
+
 # Install the python runtime along with any dependencies.
 install_runtime_packages() {
   pkgs=("$(runtime)" \
@@ -80,9 +87,9 @@ setup_python_env() {
   fi
   
   # If anything exists before we symlink, copy it into the cache
-  if [[ -d "$(nos_data_dir)/lib/$(runtime)/site-packages" ]]; then
+  if [[ -d "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" ]]; then
     mv \
-      "$(nos_data_dir)/lib/$(runtime)/site-packages/*" \
+      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages/*" \
       "$(nos_code_dir)/.nanobox/pip_cache/site-packages"
   fi
   
@@ -96,7 +103,7 @@ python_profile_payload() {
 {
   "code_dir": "$(nos_code_dir)",
   "data_dir": "$(nos_data_dir)",
-  "runtime": "$(runtime)"
+  "runtime": "$(lib_runtime)"
 }
 END
 }

From d3b8e8102930e878c988afa347d7aa52509dc976 Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 14:34:55 -0600
Subject: [PATCH 3/9] Exit on error

---
 bin/boxfile | 2 ++
 bin/build   | 2 ++
 bin/cleanup | 4 +++-
 bin/release | 2 ++
 4 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/bin/boxfile b/bin/boxfile
index 53cb2f3..dc14c69 100755
--- a/bin/boxfile
+++ b/bin/boxfile
@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+set -e
+
 cat <<-END
 run.config:
   cache_dirs:
diff --git a/bin/build b/bin/build
index 383012a..4f684d2 100755
--- a/bin/build
+++ b/bin/build
@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+set -e
+
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
 
diff --git a/bin/cleanup b/bin/cleanup
index 7781c57..650e31c 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+set -e
+
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
 
@@ -14,4 +16,4 @@ nos_init "$@"
 # Remove build dependencies to ensure the build env is small
 uninstall_build_packages
 
-exit 0
\ No newline at end of file
+exit 0
diff --git a/bin/release b/bin/release
index 0d173b7..4066a46 100755
--- a/bin/release
+++ b/bin/release
@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+set -e
+
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
 

From a0066762709d9817e26f7a074d9052f00a3036f7 Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 14:38:43 -0600
Subject: [PATCH 4/9] Simplify mv command

---
 lib/python.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/python.sh b/lib/python.sh
index 21979b3..1cb684d 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -89,8 +89,8 @@ setup_python_env() {
   # If anything exists before we symlink, copy it into the cache
   if [[ -d "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" ]]; then
     mv \
-      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages/*" \
-      "$(nos_code_dir)/.nanobox/pip_cache/site-packages"
+      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" \
+      "$(nos_code_dir)/.nanobox/pip_cache/"
   fi
   
   # set the profile script that correctly sets up the links

From 0f103f4f525b5603a8f85d04cd3f66a89a8ac901 Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 14:53:19 -0600
Subject: [PATCH 5/9] Add output to trace the copy

---
 lib/python.sh | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/lib/python.sh b/lib/python.sh
index 1cb684d..bdbb952 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -88,9 +88,15 @@ setup_python_env() {
   
   # If anything exists before we symlink, copy it into the cache
   if [[ -d "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" ]]; then
-    mv \
-      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" \
-      "$(nos_code_dir)/.nanobox/pip_cache/"
+    echo "COPY!"
+    ls -lah "$(nos_data_dir)/lib/$(lib_runtime)/site-packages"
+    cp -a \
+      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages/*" \
+      "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
+    
+    ls -lah "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
+  else
+    echo "DONT COPY"
   fi
   
   # set the profile script that correctly sets up the links

From a2d7b5c85f163208591315a420233f7f86488df9 Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 15:00:33 -0600
Subject: [PATCH 6/9] Use cp correctly

---
 lib/python.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/python.sh b/lib/python.sh
index bdbb952..bd50d13 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -91,8 +91,8 @@ setup_python_env() {
     echo "COPY!"
     ls -lah "$(nos_data_dir)/lib/$(lib_runtime)/site-packages"
     cp -a \
-      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages/*" \
-      "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
+      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" \
+      "$(nos_code_dir)/.nanobox/pip_cache/"
     
     ls -lah "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
   else

From ced85a2c8ba9870f00f22703e789e5b8a9a644ab Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 15:12:19 -0600
Subject: [PATCH 7/9] Add additional tracing to debug the cp

---
 lib/python.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/python.sh b/lib/python.sh
index bd50d13..5fbbc82 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -89,6 +89,7 @@ setup_python_env() {
   # If anything exists before we symlink, copy it into the cache
   if [[ -d "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" ]]; then
     echo "COPY!"
+    ls -lah "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
     ls -lah "$(nos_data_dir)/lib/$(lib_runtime)/site-packages"
     cp -a \
       "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" \

From 46ed37f6db4dff96a3cc4728b50edf4926a0e00d Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 15:18:17 -0600
Subject: [PATCH 8/9] Remove traces and ensure pip isn't removed

---
 lib/python.sh | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/lib/python.sh b/lib/python.sh
index 5fbbc82..9895686 100644
--- a/lib/python.sh
+++ b/lib/python.sh
@@ -60,7 +60,7 @@ install_runtime_packages() {
 
 # Uninstall build dependencies
 uninstall_build_packages() {
-  pkgs=("$(condensed_runtime)-pip")
+  pkgs=()
 
   # if pkgs isn't empty, let's uninstall what we don't need
   if [[ ${#pkgs[@]} -gt 0 ]]; then
@@ -88,16 +88,9 @@ setup_python_env() {
   
   # If anything exists before we symlink, copy it into the cache
   if [[ -d "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" ]]; then
-    echo "COPY!"
-    ls -lah "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
-    ls -lah "$(nos_data_dir)/lib/$(lib_runtime)/site-packages"
     cp -a \
       "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" \
       "$(nos_code_dir)/.nanobox/pip_cache/"
-    
-    ls -lah "$(nos_code_dir)/.nanobox/pip_cache/site-packages/"
-  else
-    echo "DONT COPY"
   fi
   
   # set the profile script that correctly sets up the links

From 2a63c382055713a777ac0c615898ce32f5681a18 Mon Sep 17 00:00:00 2001
From: Tyler Flint <tylerflint@gmail.com>
Date: Wed, 18 Oct 2017 20:14:41 -0600
Subject: [PATCH 9/9] Be less strict about failures

---
 bin/boxfile | 2 +-
 bin/build   | 2 +-
 bin/cleanup | 2 +-
 bin/release | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bin/boxfile b/bin/boxfile
index dc14c69..f575314 100755
--- a/bin/boxfile
+++ b/bin/boxfile
@@ -2,7 +2,7 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
-set -e
+# set -e
 
 cat <<-END
 run.config:
diff --git a/bin/build b/bin/build
index 4f684d2..d082562 100755
--- a/bin/build
+++ b/bin/build
@@ -2,7 +2,7 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
-set -e
+# set -e
 
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
diff --git a/bin/cleanup b/bin/cleanup
index 650e31c..d296f31 100755
--- a/bin/cleanup
+++ b/bin/cleanup
@@ -2,7 +2,7 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
-set -e
+# set -e
 
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
diff --git a/bin/release b/bin/release
index 4066a46..52cc3c2 100755
--- a/bin/release
+++ b/bin/release
@@ -2,7 +2,7 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
-set -e
+# set -e
 
 # source the Nos framework
 . /opt/nanobox/nos/common.sh