Merge branch 'master' into release

Author: wleepang

README.md

@@ -43,13 +43,15 @@ TEMPLATE_ROOT_URL=https://<dist-bucketname>.s3-<region>.amazonaws.com/test/templ
 aws cloudformation create-stack \
     --region <region> \
     --stack-name <stackname> \
-    --template-url $TEMPLATE_ROOT_URL/gwfcore-root.template.yaml \
+    --template-url $TEMPLATE_ROOT_URL/gwfcore/gwfcore-root.template.yaml \
     --capabilities CAPABILITY_IAM CAPABILITY_AUTO_EXPAND \
     --parameters \
         ParameterKey=VpcId,ParameterValue=<vpc-id> \
         ParameterKey=SubnetIds,ParameterValue=\"<subnet-id-1>,<subnet-id-2>,...\" \
         ParameterKey=ArtifactBucketName,ParameterValue=<dist-bucketname> \
-        ParameterKey=TemplateRootUrl,ParameterValue=$TEMPLATE_ROOT_URL
+        ParameterKey=TemplateRootUrl,ParameterValue=$TEMPLATE_ROOT_URL \
+        ParameterKey=S3BucketName,ParameterValue=<store-buketname> \
+        ParameterKey=ExistingBucket,ParameterValue=false
 
 ```
 

docs/orchestration/cromwell/cromwell-overview.md

@@ -210,7 +210,9 @@ This value can also be supplied as a Java command line variable.
 
 ### Accessing the Cromwell server
 
-The Cromwell EC2 instance may be accessed using the [AWS Session Manager](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/session-manager.html). Please note that by default this will log you in as user `ec2-user` in the directory `/usr/bin`. You may prefer to become the ec2-user with the command `sudo su - ec2-user` which will switch you to that user's home directory.
+The Cromwell EC2 instance may be accessed using the [AWS Session Manager](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/session-manager.html) via console or in the terminal with the command `aws ssm start-session --target <instance-id>`.
+
+Please note that by default this will log you in as user `ec2-user` in the directory `/usr/bin`. You may prefer to become the ec2-user with the command `sudo su - ec2-user` which will switch you to that user's home directory.
 
 ### Stop / Start / Restart the Cromwell service
 

src/containers/nextflow/nextflow.aws.sh

@@ -52,7 +52,7 @@ cat ./nextflow.config
 
 # stage in session cache
 # .nextflow directory holds all session information for the current and past runs.
-# it should be `sync`'d with an s3 uri, so that runs from previous sessions can be 
+# it should be `sync`'d with an s3 uri, so that runs from previous sessions can be
 # resumed
 echo "== Restoring Session Cache =="
 aws s3 sync --no-progress $NF_LOGSDIR/.nextflow .nextflow

src/ecs-additions/awscli-shim.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This shim is for using the AWS ClI v2 with containers that do not have full glibc
+# it makes the shared libraries the AWS CLI v2 findable via LD_LIBRARY_PATH
+#
+# expect to be installed as /opt/aws-cli/bin/aws
+# expect to actually call /opt/aws-cli/dist/aws
+# expect that /opt/aws-cli is mapped to containers
+
+BIN_DIR=`dirname $0`
+DIST_DIR=`dirname $BIN_DIR`/dist
+AWS=$DIST_DIR/aws
+
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$DIST_DIR
+
+$AWS $@
+

src/ecs-additions/ecs-additions-common.sh

@@ -3,3 +3,10 @@
 # add fetch and run batch helper script
 chmod a+x /opt/ecs-additions/fetch_and_run.sh
 cp /opt/ecs-additions/fetch_and_run.sh /usr/local/bin
+
+# add awscli-shim
+mv /opt/aws-cli/bin /opt/aws-cli/dist
+chmod a+x /opt/ecs-additions/awscli-shim.sh
+mkdir /opt/aws-cli/bin
+cp /opt/ecs-additions/awscli-shim.sh /opt/aws-cli/bin/aws
+

src/ecs-additions/fetch_and_run.sh

@@ -16,6 +16,7 @@
 # See below for usage instructions.
 
 #PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/aws-cli/v2/current/dist
 BASENAME="${0##*/}"
 AWS="/usr/local/aws-cli/v2/current/bin/aws"
 

src/templates/cromwell/cromwell-resources.template.yaml

@@ -382,6 +382,20 @@ Resources:
               owner: "ec2-user"
               group: "ec2-user"
 
+            "/home/ec2-user/get_cromwell_tools.sh":
+              content: |
+                #!/bin/bash
+                url=$(curl --silent --retry 5 --retry-connrefused https://api.github.com/repos/broadinstitute/cromwell/releases/latest | jq -r .assets[1].browser_download_url)
+                curl --retry 5 --retry-connrefused -LO $url
+
+                curl --retry 5 --retry-connrefused -LO https://raw.githubusercontent.com/broadinstitute/cromshell/master/cromshell
+                chmod +x cromshell
+                sudo mv cromshell /usr/local/bin/
+
+              mode: "000755"
+              owner: "ec2-user"
+              group: "ec2-user"
+
             "/home/ec2-user/cromwell.conf":
               content: !Sub
                 - |
@@ -605,6 +619,16 @@ Resources:
             02_start_cromwell:
               cwd: "/home/ec2-user"
               command: "sudo -u ec2-user /usr/local/bin/supervisord"
+
+        config4:
+          commands:
+            00_get_cromwell_tools:
+              cwd: "/home/ec2-user/"
+              command: "./get_cromwell_tools.sh"
+            
+            01_chown_womtool:
+              cwd: "/home/ec2-user/"
+              command: "chown ec2-user:ec2-user womtool*.jar"
 
     Properties:
       ImageId: !Ref LatestAmazonLinuxAMI
@@ -623,7 +647,7 @@ Resources:
           #!/bin/bash -x
           yum -y update
           yum install -y aws-cfn-bootstrap
-          yum install -y jq awslogs python3 java
+          yum install -y jq awslogs python3 java mailx
 
           amazon-linux-extras install -y nginx1
           pip3 install supervisor

src/templates/gwfcore/gwfcore-batch.template.yaml

@@ -194,6 +194,8 @@ Resources:
       Type: MANAGED
       State: ENABLED
       ComputeResources:
+        Ec2Configuration:
+          - ImageType: ECS_AL2
         AllocationStrategy: BEST_FIT
         LaunchTemplate:
           LaunchTemplateId: !Ref LaunchTemplateId