Skip to content

Commit 1f82a4e

Browse files
stevebachmeierstevebachmeier
authored
Sbachmei/hotfix/quick fix environment script (#106)
* Delete extranious environment.sh outside of cookiecutter * Fix environment.sh
1 parent 6544939 commit 1f82a4e

File tree

2 files changed

+100
-193
lines changed

2 files changed

+100
-193
lines changed

‎environment.sh

Lines changed: 0 additions & 146 deletions
This file was deleted.

‎{{cookiecutter.package_name}}/environment.sh

Lines changed: 100 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,27 @@
11
#!/bin/bash
22

3+
set -e # exit on error
4+
5+
# Initialize conda if not already initialized
6+
for conda_path in "$HOME/miniconda3" "$HOME/anaconda3" "/opt/conda" "/usr/local/miniconda3" "/usr/local/anaconda3"; do
7+
if [ -f "$conda_path/etc/profile.d/conda.sh" ]; then
8+
echo
9+
echo "Initializing conda from $conda_path"
10+
source "$conda_path/etc/profile.d/conda.sh"
11+
break
12+
else
13+
echo
14+
echo "ERROR: Unable to find conda in expected locations"
15+
exit 1
16+
fi
17+
done
18+
319
# Reset OPTIND so help can be invoked multiple times per shell session.
420
OPTIND=1
521
Help()
622
{
723
# Display Help
24+
echo
825
echo "Script to automatically create and validate conda environments."
926
echo
1027
echo "Syntax: source environment.sh [-h|t|v]"
@@ -20,22 +37,25 @@ username=$(whoami)
2037
env_type="simulation"
2138
make_new="no"
2239
install_git_lfs="no"
40+
days_until_stale=7 # Number of days until environment is considered stale
41+
2342

2443
# Process input options
2544
while getopts ":hflt:" option; do
2645
case $option in
2746
h) # display help
2847
Help
29-
return;;
48+
exit 0;;
3049
t) # Type of conda environment to build
3150
env_type=$OPTARG;;
3251
f) # Force creation of a new environment
3352
make_new="yes";;
3453
l) # Install git lfs
3554
install_git_lfs="yes";;
3655
\?) # Invalid option
37-
echo "Error: Invalid option"
38-
return;;
56+
echo
57+
echo "ERROR: Invalid option"
58+
exit 1;;
3959
esac
4060
done
4161

@@ -49,80 +69,101 @@ if [ $env_type == 'simulation' ]; then
4969
elif [ $env_type == 'artifact' ]; then
5070
install_file="artifact_requirements.txt"
5171
else
72+
echo
5273
echo "Invalid environment type. Valid argument types are 'simulation' and 'artifact'."
53-
return
74+
exit 1
5475
fi
5576

5677
# Pull repo to get latest changes from remote if remote exists
78+
set +e # Do not exit on error for git ls-remote
5779
git ls-remote --exit-code --heads origin $branch_name >/dev/null 2>&1
5880
exit_code=$?
81+
set -e # Re-enable exit on error
5982
if [[ $exit_code == '0' ]]; then
6083
git fetch --all
61-
echo "Git branch '$branch_name' exists in the remote repository, pulling latest changes..."
84+
echo
85+
echo "Git branch '$branch_name' exists in the remote repository; pulling latest changes"
6286
git pull origin $branch_name
6387
fi
6488

6589
# Check if environment exists already
66-
create_env=$(conda info --envs | grep $env_name | head -n 1)
67-
if [[ $create_env == '' ]]; then
90+
env_info=$(conda info --envs | grep $env_name | head -n 1)
91+
if [[ $env_info == '' ]]; then
6892
# No environment exists with this name
69-
echo "Environment $env_name does not exist."
93+
echo
94+
echo "Environment $env_name does not exist"
7095
create_env="yes"
7196
env_exists="no"
7297
elif [[ $make_new == 'yes' ]]; then
7398
# User has requested to make a new environment
74-
echo "Making a new environment."
99+
echo
100+
echo "Making a new environment"
75101
create_env="yes"
76102
env_exists="yes"
77103
else
78104
env_exists="yes"
79105
conda activate $env_name
80106
# Check if existing environment needs to be recreated
81-
echo "Existing environment found for $env_name."
82-
one_week_ago=$(date -d "7 days ago" '+%Y-%m-%d %H:%M:%S')
107+
echo
108+
echo "Existing environment found for $env_name"
109+
expiration_time=$(date -d "$days_until_stale days ago" +%s)
83110
creation_time="$(head -n1 $CONDA_PREFIX/conda-meta/history)"
84111
creation_time=$(echo $creation_time | sed -e 's/^==>\ //g' -e 's/\ <==//g')
85-
requirements_modification_time="$(date -r $install_file '+%Y-%m-%d %H:%M:%S')"
112+
creation_time="$(date -d "$creation_time" +%s)"
113+
requirements_modification_time="$(date -r $install_file +%s)"
86114
# Check if existing environment is older than a week or if environment was built
87115
# before last modification to requirements file. If so, mark for recreation.
88-
if [[ $one_week_ago > $creation_time ]] | [[ $creation_time < $requirements_modification_time ]]; then
89-
echo "Environment is stale. Deleting and remaking environment..."
116+
if [[ $creation_time < $expiration_time ]] || [[ $creation_time < $requirements_modification_time ]]; then
117+
echo
118+
echo "Environment is stale; deleting and remaking"
90119
create_env="yes"
91120
else
92-
# Install json parser if it is not installed
93-
jq_exists=$(conda list | grep -w jq)
94-
if [[ $jq_exists == '' ]]; then
95-
# Empty string is no return on grep
96-
conda install jq -c anaconda -y
97-
fi
98-
echo "Checking framework packages are up to date..."
99-
# Check if there has been an update to vivarium packages since last modification to requirements file
100-
# or more reccent than environment creation
101-
# Note: The lines we will return via grep will look like 'vivarium>=#.#.#' or will be of the format
102-
# 'vivarium @ git+https://github.com/ihmeuw/vivarium@SOME_BRANCH'
103-
framework_packages=$(grep -E 'vivarium|gbd|risk_distribution|layered_config' $install_file)
104-
num_packages=$(grep -E 'vivarium|gbd|risk_distribution|layered_config' -c $install_file)
121+
echo
122+
echo "Environment is up to date; no action needed"
123+
124+
##############################################################################
125+
# FIXME [MIC-6259]
126+
# This if/else block has never worked correctly due to using a single |
127+
# instead of double || (or) in the above 'if' check.
128+
# As such, we are commenting out the following complicated logic to not drastically
129+
# change behavior, but this needs to all be fixed.
130+
##############################################################################
131+
132+
# else
133+
# # Install json parser if it is not installed
134+
# jq_exists=$(conda list | grep -w jq)
135+
# if [[ $jq_exists == '' ]]; then
136+
# # Empty string is no return on grep
137+
# conda install jq -c anaconda -y
138+
# fi
139+
# echo "Checking framework packages are up to date..."
140+
# # Check if there has been an update to vivarium packages since last modification to requirements file
141+
# # or more reccent than environment creation
142+
# # Note: The lines we will return via grep will look like 'vivarium>=#.#.#' or will be of the format
143+
# # 'vivarium @ git+https://github.com/ihmeuw/vivarium@SOME_BRANCH'
144+
# framework_packages=$(grep -E 'vivarium|gbd|risk_distribution|layered_config' $install_file)
145+
# num_packages=$(grep -E 'vivarium|gbd|risk_distribution|layered_config' -c $install_file)
105146

106-
# Iterate through each return of the grep output
107-
for ((i = 1; i <= $num_packages; i++)); do
108-
line=$(echo "$framework_packages" | sed -n "${i}p")
109-
# Check if the line contains '@'
110-
if [[ "$line" == *"@"* ]]; then
111-
repo_info=(${line//@/ })
112-
repo=${repo_info[0]}
113-
repo_branch=${repo_info[2]}
114-
last_update_time=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/ihmeuw/$repo/commits?sha=$repo_branch | jq .[0].commit.committer.date)
115-
else
116-
repo=$(echo "$line" | cut -d '>' -f1)
117-
last_update_time=$(curl -s https://pypi.org/pypi/$repo/json | jq -r '.releases | to_entries | max_by(.key) | .value | .[0].upload_time')
118-
fi
119-
last_update_time=$(date -d "$last_update_time" '+%Y-%m-%d %H:%M:%S')
120-
if [[ $creation_time < $last_update_time ]]; then
121-
create_env="yes"
122-
echo "Last update time for $repo: $last_update_time. Environment is stale. Remaking environment..."
123-
break
124-
fi
125-
done
147+
# # Iterate through each return of the grep output
148+
# for ((i = 1; i <= $num_packages; i++)); do
149+
# line=$(echo "$framework_packages" | sed -n "${i}p")
150+
# # Check if the line contains '@'
151+
# if [[ "$line" == *"@"* ]]; then
152+
# repo_info=(${line//@/ })
153+
# repo=${repo_info[0]}
154+
# repo_branch=${repo_info[2]}
155+
# last_update_time=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/ihmeuw/$repo/commits?sha=$repo_branch | jq .[0].commit.committer.date)
156+
# else
157+
# repo=$(echo "$line" | cut -d '>' -f1)
158+
# last_update_time=$(curl -s https://pypi.org/pypi/$repo/json | jq -r '.releases | to_entries | max_by(.key) | .value | .[0].upload_time')
159+
# fi
160+
# last_update_time=$(date -d "$last_update_time" '+%Y-%m-%d %H:%M:%S')
161+
# if [[ $creation_time < $last_update_time ]]; then
162+
# create_env="yes"
163+
# echo "Last update time for $repo: $last_update_time. Environment is stale. Remaking environment..."
164+
# break
165+
# fi
166+
# done
126167
fi
127168
fi
128169

@@ -131,12 +172,17 @@ if [[ $create_env == 'yes' ]]; then
131172
if [[ $env_name == $CONDA_DEFAULT_ENV ]]; then
132173
conda deactivate
133174
fi
175+
echo
176+
echo "Removing existing environment $env_name"
134177
conda remove -n $env_name --all -y
135178
fi
136179
# Create conda environment
180+
echo
181+
echo "Creating new conda environment $env_name"
137182
conda create -n $env_name python=3.11 -c anaconda -y
138183
conda activate $env_name
139184
# NOTE: update branch name if you update requirements.txt in a branch
185+
echo
140186
echo "Installing packages for $env_type environment"
141187
pip install uv
142188
artifactory_url="https://artifactory.ihme.washington.edu/artifactory/api/pypi/pypi-shared/simple"
@@ -152,5 +198,12 @@ if [[ $create_env == 'yes' ]]; then
152198
git lfs install
153199
fi
154200
else
201+
echo
155202
echo "Existing environment validated"
156203
fi
204+
205+
echo
206+
echo "*** FINISHED ***"
207+
echo
208+
echo "Don't forget to activate the environment:"
209+
echo "conda activate $env_name"

0 commit comments

Comments
 (0)