From ea04a60b6e2e6cfed6b68338b5b020bdbc68ccce Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 08:43:36 -0600 Subject: [PATCH 01/11] Remove src from cache_dirs --- bin/boxfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bin/boxfile b/bin/boxfile index 2d5427c..ecb93b4 100755 --- a/bin/boxfile +++ b/bin/boxfile @@ -2,13 +2,9 @@ # -*- mode: bash; tab-width: 2; -*- # vim: ts=2 sw=2 ft=bash noet -cat <<-END -run.config: - cache_dirs: - - src -END if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then cat <<-END + run.config: build_triggers: - requirements.txt END From 15f39f57eef29b73e01790422e452ca5c5888fa5 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 08:47:36 -0600 Subject: [PATCH 02/11] Pull indentation for boxfile to the left --- bin/boxfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/boxfile b/bin/boxfile index ecb93b4..0a54a15 100755 --- a/bin/boxfile +++ b/bin/boxfile @@ -4,10 +4,10 @@ if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then cat <<-END - run.config: - build_triggers: - - requirements.txt - END +run.config: + build_triggers: + - requirements.txt +END fi exit 0 From 54232144b8b9ff6bea3ccc8df9d0c1f99637d7f1 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 08:57:53 -0600 Subject: [PATCH 03/11] Adjust default pip command to be more robust --- lib/python.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/python.sh b/lib/python.sh index 77068ec..2eeebbb 100644 --- a/lib/python.sh +++ b/lib/python.sh @@ -96,7 +96,12 @@ pip_install_cmd() { # the default pip install cmd when a user-specified cmd is not present default_pip_install() { - echo "pip install -I -r requirements.txt" + echo "\ + pip install \ + -I \ + -r requirements.txt \ + --disable-pip-version-check \ + --no-cache-dir" } # Install dependencies via pip from requirements.txt From f755af92be05bd12ac9eff9fb1ddfbce26d50d6a Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 11:32:33 -0600 Subject: [PATCH 04/11] Properly set the pip env and install pip modules into cache_dir --- bin/boxfile | 12 ++++++++---- bin/build | 3 +++ files/profile.d/pip.sh | 8 ++++++++ lib/python.sh | 13 ++++++++++++- 4 files changed, 31 insertions(+), 5 deletions(-) create mode 100755 files/profile.d/pip.sh diff --git a/bin/boxfile b/bin/boxfile index 0a54a15..d9b123e 100755 --- a/bin/boxfile +++ b/bin/boxfile @@ -2,12 +2,16 @@ # -*- mode: bash; tab-width: 2; -*- # vim: ts=2 sw=2 ft=bash noet -if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then - cat <<-END +cat <<-END run.config: - build_triggers: - - requirements.txt + cache_dirs: + - $(nos_code_dir)/.nanobox/pip END +if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then + cat <<-END + build_triggers: + - requirements.txt + END fi exit 0 diff --git a/bin/build b/bin/build index 4e89362..406bb87 100755 --- a/bin/build +++ b/bin/build @@ -14,6 +14,9 @@ nos_init "$@" # install python interpreter and pip install_runtime_packages +# set pip env +set_pip_env + # run pip install pip_install diff --git a/files/profile.d/pip.sh b/files/profile.d/pip.sh new file mode 100755 index 0000000..2dadbb7 --- /dev/null +++ b/files/profile.d/pip.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Nanobox stashes the pip module src and bin dirs inside of a cache_dir +# at /app/.nanobox/pip. We need to set PATH to include the bin dir +# and PYTHONPATH to include the src dir + +PYTHONPATH=/app/.nanobox/pip/src:$PYTHONPATH +PATH=/app/.nanobox/pip/bin diff --git a/lib/python.sh b/lib/python.sh index 2eeebbb..d103d54 100644 --- a/lib/python.sh +++ b/lib/python.sh @@ -101,7 +101,10 @@ default_pip_install() { -I \ -r requirements.txt \ --disable-pip-version-check \ - --no-cache-dir" + --no-cache-dir \ + --target=$(nos_code_dir)/.nanobox/pip/src \ + --upgrade \ + --install-option='--install-scripts=$(nos_code_dir)/.nanobox/pip/bin'" } # Install dependencies via pip from requirements.txt @@ -113,3 +116,11 @@ pip_install() { cd - >/dev/null fi } + +# Since we install pip modules into a custom cache_dir, we need +# to setup the environment (with env vars) for the python app to work +set_pip_env() { + nos_template_file \ + 'profile.d/pip.sh' \ + $(nos_etc_dir)/profile.d/pip.sh +} From 301a93924f08309103bd2dec35f7c72d84771c60 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 11:35:05 -0600 Subject: [PATCH 05/11] Fix path to prepend, not replace --- files/profile.d/pip.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/files/profile.d/pip.sh b/files/profile.d/pip.sh index 2dadbb7..9588e80 100755 --- a/files/profile.d/pip.sh +++ b/files/profile.d/pip.sh @@ -5,4 +5,4 @@ # and PYTHONPATH to include the src dir PYTHONPATH=/app/.nanobox/pip/src:$PYTHONPATH -PATH=/app/.nanobox/pip/bin +PATH=/app/.nanobox/pip/bin:$PATH From ec0c58c4f6e89b58385211cb00063b767b19217c Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 11:37:41 -0600 Subject: [PATCH 06/11] Export newly created env vars --- files/profile.d/pip.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/files/profile.d/pip.sh b/files/profile.d/pip.sh index 9588e80..2fb38d0 100755 --- a/files/profile.d/pip.sh +++ b/files/profile.d/pip.sh @@ -4,5 +4,5 @@ # at /app/.nanobox/pip. We need to set PATH to include the bin dir # and PYTHONPATH to include the src dir -PYTHONPATH=/app/.nanobox/pip/src:$PYTHONPATH -PATH=/app/.nanobox/pip/bin:$PATH +export PATH=/app/.nanobox/pip/bin:$PATH +export PYTHONPATH=/app/.nanobox/pip/src:$PYTHONPATH From 31be5551f99c7fa0c39253b00c8ff47b9078ae34 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 13:04:24 -0600 Subject: [PATCH 07/11] Switch pip profile from a file to a template --- files/profile.d/pip.sh | 8 -------- lib/python.sh | 16 +++++++++++++--- templates/profile.d/pip.sh | 8 ++++++++ 3 files changed, 21 insertions(+), 11 deletions(-) delete mode 100755 files/profile.d/pip.sh create mode 100755 templates/profile.d/pip.sh diff --git a/files/profile.d/pip.sh b/files/profile.d/pip.sh deleted file mode 100755 index 2fb38d0..0000000 --- a/files/profile.d/pip.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Nanobox stashes the pip module src and bin dirs inside of a cache_dir -# at /app/.nanobox/pip. We need to set PATH to include the bin dir -# and PYTHONPATH to include the src dir - -export PATH=/app/.nanobox/pip/bin:$PATH -export PYTHONPATH=/app/.nanobox/pip/src:$PYTHONPATH diff --git a/lib/python.sh b/lib/python.sh index d103d54..12c2fea 100644 --- a/lib/python.sh +++ b/lib/python.sh @@ -117,10 +117,20 @@ pip_install() { fi } +# Generate the payload to render the pip env profile.d template +pip_env_payload() { + cat <<-END +{ + "app_dir": "$(nos_app_dir)" +} +END +} + # Since we install pip modules into a custom cache_dir, we need # to setup the environment (with env vars) for the python app to work set_pip_env() { - nos_template_file \ - 'profile.d/pip.sh' \ - $(nos_etc_dir)/profile.d/pip.sh + nos_template \ + "profile.d/pip.sh" \ + "$(nos_etc_dir)/profile.d/pip.sh" \ + "$(pip_env_payload)" } diff --git a/templates/profile.d/pip.sh b/templates/profile.d/pip.sh new file mode 100755 index 0000000..7c67c5f --- /dev/null +++ b/templates/profile.d/pip.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Nanobox stashes the pip module src and bin dirs inside of a cache_dir +# at {{app_dir}}/.nanobox/pip. We need to set PATH to include the bin dir +# and PYTHONPATH to include the src dir + +export PATH={{app_dir}}/.nanobox/pip/bin:$PATH +export PYTHONPATH={{app_dir}}/.nanobox/pip/src:$PYTHONPATH From 0de0a922f3d2880627713488ea21deff18065ca4 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 13:06:54 -0600 Subject: [PATCH 08/11] Use the correct code_dir for pip env template --- lib/python.sh | 2 +- templates/profile.d/pip.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/python.sh b/lib/python.sh index 12c2fea..5372319 100644 --- a/lib/python.sh +++ b/lib/python.sh @@ -121,7 +121,7 @@ pip_install() { pip_env_payload() { cat <<-END { - "app_dir": "$(nos_app_dir)" + "code_dir": "$(nos_code_dir)" } END } diff --git a/templates/profile.d/pip.sh b/templates/profile.d/pip.sh index 7c67c5f..38abde5 100755 --- a/templates/profile.d/pip.sh +++ b/templates/profile.d/pip.sh @@ -1,8 +1,8 @@ #!/bin/bash # Nanobox stashes the pip module src and bin dirs inside of a cache_dir -# at {{app_dir}}/.nanobox/pip. We need to set PATH to include the bin dir +# at {{code_dir}}/.nanobox/pip. We need to set PATH to include the bin dir # and PYTHONPATH to include the src dir -export PATH={{app_dir}}/.nanobox/pip/bin:$PATH -export PYTHONPATH={{app_dir}}/.nanobox/pip/src:$PYTHONPATH +export PATH={{code_dir}}/.nanobox/pip/bin:$PATH +export PYTHONPATH={{code_dir}}/.nanobox/pip/src:$PYTHONPATH From cc42c618933172e99a7acd6356ae18aee66fe566 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Fri, 14 Jul 2017 13:50:16 -0600 Subject: [PATCH 09/11] Update tests to match new logic --- test/apps/simple-python/requirements.txt | 4 ++-- test/tests/integration/simple-python.bats | 5 ++++- test/util/env.sh | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/test/apps/simple-python/requirements.txt b/test/apps/simple-python/requirements.txt index bd4036f..bb7162c 100644 --- a/test/apps/simple-python/requirements.txt +++ b/test/apps/simple-python/requirements.txt @@ -1,3 +1,3 @@ --e git://github.com/webpy/webpy.git#egg=web.py +web.py gunicorn -Pillow \ No newline at end of file +Pillow diff --git a/test/tests/integration/simple-python.bats b/test/tests/integration/simple-python.bats index f9b1f14..14383b4 100644 --- a/test/tests/integration/simple-python.bats +++ b/test/tests/integration/simple-python.bats @@ -82,8 +82,11 @@ setup() { # cd into the app code_dir cd /tmp/code + # source the profile.d/pip.sh env + source /data/etc/profile.d/pip.sh + # start the server in the background - /data/bin/gunicorn -b 0.0.0.0:8080 app:wsgiapp & + /tmp/code/.nanobox/pip/bin/gunicorn -b 0.0.0.0:8080 app:wsgiapp & # grab the pid pid=$! diff --git a/test/util/env.sh b/test/util/env.sh index a599d3f..621db01 100644 --- a/test/util/env.sh +++ b/test/util/env.sh @@ -6,6 +6,7 @@ directories=( "/data" "/data/etc" "/data/etc/env.d" + "/data/etc/profile.d" ) prepare_environment() { From 38033aed294595f6fff92ce350ee6c0ed263e975 Mon Sep 17 00:00:00 2001 From: Tyler Flint Date: Thu, 31 Aug 2017 10:19:28 -0600 Subject: [PATCH 10/11] Update test env --- test/util/env.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/util/env.sh b/test/util/env.sh index 621db01..3f0e847 100644 --- a/test/util/env.sh +++ b/test/util/env.sh @@ -1,6 +1,8 @@ directories=( "/tmp/code" + "/tmp/code/.nanobox" + "/tmp/code/.nanobox/pip" "/tmp/app" "/tmp/cache" "/data" From ef62e0ee267788ab76bb44d0fee9ea97d36124f1 Mon Sep 17 00:00:00 2001 From: Braxton Huggins Date: Fri, 15 Sep 2017 10:34:30 -0600 Subject: [PATCH 11/11] Cache dirs are assumed to be in the app directory already. Also, the nos framework isnt included so nos_code_dir wouldnt return anything anyway --- bin/boxfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/boxfile b/bin/boxfile index d9b123e..0f5c68b 100755 --- a/bin/boxfile +++ b/bin/boxfile @@ -5,7 +5,7 @@ cat <<-END run.config: cache_dirs: - - $(nos_code_dir)/.nanobox/pip + - .nanobox/pip END if [[ $(grep -c build_triggers /opt/nanobox/hooks/lib/boxfile.rb) -gt 1 ]]; then cat <<-END