Feature/cache site packages

bin/boxfile

@@ -2,16 +2,15 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+# set -e
+
 cat <<-END
 run.config:
   cache_dirs:
-    - .nanobox/cache/.pip
+    - .nanobox/pip_cache
+
+  build_triggers:
+    - requirements.txt
 END
-if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then
-	cat <<-END
-	  build_triggers:
-	    - requirements.txt
-	END
-fi
 
 exit 0

bin/build

@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+# set -e
+
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
 

bin/cleanup

@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+# set -e
+
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
 
@@ -14,4 +16,4 @@ nos_init "$@"
 # Remove build dependencies to ensure the build env is small
 uninstall_build_packages
 
-exit 0
+exit 0

bin/release

@@ -2,6 +2,8 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+# set -e
+
 # source the Nos framework
 . /opt/nanobox/nos/common.sh
 

lib/deps.sh

@@ -0,0 +1,62 @@
+# compiles a list of dependencies that will need to be installed
+query_dependencies() {
+
+  # break early if there isn't a requirements.txt
+  if [[ ! -f $(nos_code_dir)/requirements.txt ]]; then
+    echo ""
+    return
+  fi
+
+  deps=()
+
+  # mssql
+  if [[ `grep -i 'pymssql' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(freetds)
+  fi
+  # mysql
+  if [[ `grep -i 'MySQLdb\|mysqlclient\|MySQL-python' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(mysql-client)
+  fi
+  # memcache
+  if [[ `grep -i 'memcache\|libmc' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libmemcached)
+  fi
+  # postgres
+  if [[ `grep -i 'psycopg2' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(postgresql94-client)
+  fi
+  # redis
+  if [[ `grep -i 'redis' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(redis)
+  fi
+  # curl
+  if [[ `grep -i 'pycurl' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(curl)
+  fi
+  # pillow
+  if [[ `grep -i 'pillow' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libjpeg-turbo tiff zlib freetype2 lcms2 libwebp tcl tk)
+  fi
+  # boto3
+  if [[ `grep -i 'boto3' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=("$(condensed_runtime)-cElementTree")
+  fi
+  # xmlsec
+  if [[ `grep -i 'xmlsec' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libxml2 libxslt xmlsec1 pkgconf)
+  fi
+  # python3-saml
+  if [[ `grep -i 'python3-saml' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libxml2 libxslt xmlsec1 pkgconf)
+  fi
+  # pygraphviz
+  if [[ `grep -i 'pygraphviz' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(libxshmfence libva libvdpau libLLVM-3.8 graphviz)
+  fi
+  # scipy
+  if [[ `grep -i 'scipy' $(nos_code_dir)/requirements.txt` ]]; then
+    deps+=(blas lapack)
+  fi
+
+  echo "${deps[@]}"
+}

lib/python.sh

@@ -1,6 +1,9 @@
 # -*- mode: bash; tab-width: 2; -*-
 # vim: ts=2 sw=2 ft=bash noet
 
+# source the deps functions
+. ${engine_lib_dir}/deps.sh
+
 # Copy the code into the live directory which will be used to run the app
 publish_release() {
   nos_print_bullet "Moving build into live app directory..."
@@ -28,9 +31,18 @@ condensed_runtime() {
   echo "${version//[.thon-]/}"
 }
 
+# In the data directory, the dir will look like python2.7 so
+# we need to fetch to lib runtime whenever we're messing with libs
+lib_runtime() {
+  version=$(runtime)
+  echo "${version//[-]/}"
+}
+
 # Install the python runtime along with any dependencies.
 install_runtime_packages() {
-  pkgs=("$(runtime)" "$(condensed_runtime)-setuptools" "$(condensed_runtime)-pip")
+  pkgs=("$(runtime)" \
+        "$(condensed_runtime)-setuptools" \
+        "$(condensed_runtime)-pip")
 
   # add packages that are usually part of the stdlib
   pkgs+=(\
@@ -48,76 +60,61 @@ install_runtime_packages() {
 
 # Uninstall build dependencies
 uninstall_build_packages() {
-  pkgs=("$(condensed_runtime)-pip")
+  pkgs=()
 
   # if pkgs isn't empty, let's uninstall what we don't need
   if [[ ${#pkgs[@]} -gt 0 ]]; then
     nos_uninstall ${pkgs[@]}
   fi
 }
 
-# compiles a list of dependencies that will need to be installed
-query_dependencies() {
-  deps=()
-
-  # mssql
-  if [[ `grep -i 'pymssql' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(freetds)
-  fi
-  # mysql
-  if [[ `grep -i 'MySQLdb\|mysqlclient\|MySQL-python' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(mysql-client)
-  fi
-  # memcache
-  if [[ `grep -i 'memcache\|libmc' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libmemcached)
-  fi
-  # postgres
-  if [[ `grep -i 'psycopg2' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(postgresql94-client)
-  fi
-  # redis
-  if [[ `grep -i 'redis' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(redis)
-  fi
-  # curl
-  if [[ `grep -i 'pycurl' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(curl)
-  fi
-  # pillow
-  if [[ `grep -i 'pillow' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libjpeg-turbo tiff zlib freetype2 lcms2 libwebp tcl tk)
-  fi
-  # boto3
-  if [[ `grep -i 'boto3' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=("$(condensed_runtime)-cElementTree")
-  fi
-  # xmlsec
-  if [[ `grep -i 'xmlsec' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libxml2 libxslt xmlsec1 pkgconf)
-  fi
-  # python3-saml
-  if [[ `grep -i 'python3-saml' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libxml2 libxslt xmlsec1 pkgconf)
-  fi
-  # pygraphviz
-  if [[ `grep -i 'pygraphviz' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(libxshmfence libva libvdpau libLLVM-3.8 graphviz)
-  fi
-  # scipy
-  if [[ `grep -i 'scipy' $(nos_code_dir)/requirements.txt` ]]; then
-    deps+=(blas lapack)
-  fi
-
-  echo "${deps[@]}"
-}
-
-# set any necessary python environment variables
+# Here we get to be a bit clever. Python will store it's cache (deps, etc)
+# in the site-packages dir. Changing this is not easily configurable, so we'll
+# essentially setup a cache dir in ~/.nanobox/pip_cache, and symlink
+# the site-packages to the cached location. Also, we'll copy anything
+# into the cache on the first run.
+# 
+# Additionally, we'll set some environment variables specifically for python
 setup_python_env() {
   # ensure python doesn't buffer even when not attached to a pty
   nos_template_file \
     "env.d/PYTHONUNBUFFERED" \
     "$(nos_etc_dir)/env.d/PYTHONUNBUFFERED"
+
+  # Ensure the cache destination exists for site-packages
+  if [[ ! -d "$(nos_code_dir)/.nanobox/pip_cache/site-packages" ]]; then
+    mkdir -p "$(nos_code_dir)/.nanobox/pip_cache/site-packages"
+  fi
+
+  # If anything exists before we symlink, copy it into the cache
+  if [[ -d "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" ]]; then
+    cp -a \
+      "$(nos_data_dir)/lib/$(lib_runtime)/site-packages" \
+      "$(nos_code_dir)/.nanobox/pip_cache/"
+  fi
+
+  # set the profile script that correctly sets up the links
+  set_python_profile_script
+}
+
+# Generate the payload to render the python profile template
+python_profile_payload() {
+  cat <<-END
+{
+  "code_dir": "$(nos_code_dir)",
+  "data_dir": "$(nos_data_dir)",
+  "runtime": "$(lib_runtime)"
+}
+END
+}
+
+# Profile script to ensure symlinks for pip
+set_python_profile_script() {
+  mkdir -p "$(nos_etc_dir)/profile.d"
+  nos_template \
+    "profile.d/pip.sh" \
+    "$(nos_etc_dir)/profile.d/pip.sh" \
+    "$(python_profile_payload)"
 }
 
 # fetch the user-specified pip install command or use a default

templates/profile.d/pip.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+# Here we get to be a bit clever. pip will store it's cache (deps, etc)
+# in the site-packages dir. Changing this is not easily configurable, so we'll
+# essentially setup a cache dir in ~/.nanobox/pip_cache, and symlink
+# the site-packages to the cached location.
+
+site_packages="{{data_dir}}/lib/{{runtime}}/site-packages"
+cache_dir="{{code_dir}}/.nanobox/pip_cache/site-packages"
+
+# remove site-packages if it's a directory
+if [[ -d ${site_packages} ]]; then
+  rm -rf ${site_packages}
+fi
+
+# if site-packages isn't a symlink, create it
+if [[ ! -s ${site_packages} ]]; then
+  mkdir -p ${cache_dir}
+  ln -s ${cache_dir} ${site_packages}
+fi