Merge pull request #17 from Rathan8/cqlsh-expansion-python3

Cqlsh expansion python3 with Desc functionality for cassandra 3.11

Author: michaelraney

.gitignore

@@ -3,4 +3,5 @@
 /cqlsh-expansion/cqlsh_expansion.egg-info
 /cqlsh-expansion/build
 /cqlsh-expansion/dist
+/cqlsh-expansion/pylib/cqlshlib/__pycache__
 .idea

Dockerfile

@@ -1,6 +1,6 @@
 #Amazon Keyspaces toolkit
 
-ARG CLI_VERSION=latest
+ARG CLI_VERSION=2.1.27
 FROM amazon/aws-cli:$CLI_VERSION
 
 ENV AWS_KEYSPACES_WORKING_DIR=/root
@@ -9,9 +9,26 @@ ENV CQLSHRC_HOME=$AWS_KEYSPACES_WORKING_DIR/.cassandra
 
 WORKDIR $AWS_KEYSPACES_WORKING_DIR
 
+#Install python3 and set as python3 simlink 
+
+RUN yum install gcc openssl-devel bzip2-devel libffi-devel gzip make tar -y
+RUN curl https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tgz --output Python-3.7.9.tgz
+RUN tar xzf Python-3.7.9.tgz
+WORKDIR $AWS_KEYSPACES_WORKING_DIR/Python-3.7.9
+RUN ./configure --enable-optimizations
+RUN make altinstall
+RUN export PATH=${PATH}:/usr/local/lib/
+RUN ln -s /usr/local/bin/python3.7 /usr/bin/python3
+WORKDIR $AWS_KEYSPACES_WORKING_DIR
+RUN rm -f Python-3.7.9.tgz
+
 #Install jq
 RUN yum install -y jq && yum clean all
 
+#set as python3 as default interpreter
+RUN unlink /usr/bin/python
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
 #setup directory structure
 RUN mkdir $CASSANDRA_HOME && \
     mkdir $CASSANDRA_HOME/bin  && \

bin/aws-sm-cqlsh-expo-backoff.sh

@@ -23,14 +23,14 @@ backoff=0
 
 trap "echo Exited!; exit;" SIGINT SIGTERM
 
-mysecret=$(aws secretsmanager get-secret-value --secret-id "$1" --query SecretString --output text)
+mysecret=$(aws secretsmanager get-secret-value --secret-id "$1" --region "$4" --query SecretString --output text)
 
 username=$(jq --raw-output '.username' <<< $mysecret)
 password=$(jq --raw-output '.password' <<< $mysecret)
 host=$(jq --raw-output '.host' <<< $mysecret)
 port=$(jq --raw-output '.port' <<< $mysecret)
 
-echo "cqlsh $host $port -u **** -p **** ${@:4}"
+echo "cqlsh-expansion $host $port -u **** -p **** ${@:5}"
 
 while [ $success -ne 0 -a $attempts -le $3 -a $SECONDS -le $2  ]
  do
@@ -43,7 +43,7 @@ while [ $success -ne 0 -a $attempts -le $3 -a $SECONDS -le $2  ]
   echo ""
 
   #take paramters starting at $4
-  cqlsh $host $port -u $username -p $password "${@:4}"
+  cqlsh-expansion $host $port -u $username -p $password "${@:5}"
 
   success=$?
  ((attempts++))

bin/aws-sm-cqlsh.sh

@@ -14,12 +14,12 @@
 #$1 secret-id
 #$2 CQL statement
 
-mysecret=$(aws secretsmanager get-secret-value --secret-id "$1" --query SecretString --output text)
+mysecret=$(aws secretsmanager get-secret-value --secret-id "$1" --region "$2" --query SecretString --output text)
 
 username=$(jq --raw-output '.username' <<< $mysecret)
 password=$(jq --raw-output '.password' <<< $mysecret)
 host=$(jq --raw-output '.host' <<< $mysecret)
 port=$(jq --raw-output '.port' <<< $mysecret)
 
-echo "executing.. cqlsh" $host $port "-u *** -p *** ${@:2}"
-cqlsh $host $port -u $username -p $password "${@:2}"
+echo "executing.. cqlsh" $host $port "-u *** -p *** ${@:3}"
+cqlsh-expansion $host $port -u $username -p $password "${@:3}"

cqlsh-expansion/README.md

@@ -9,21 +9,15 @@ The Amazon Keyspaces toolkit contains common Cassandra tooling and helpers preco
 To install the cqlsh-expansion python package you can run the following pip command. The command below executes a “pip install” that will install the cqlsh-expansion scripts. It will also install a requirements file containing a list of dependencies. The --`user` flag tells pip to use the Python *user install directory* for your platform. Typically ~/.local/ on unix based systems. 
 
 ```
-
 pip install --user cqlsh-expansion 
-
 ```
 
 Alternatively, if you are using python3 as default you may have to use the following command to install the cqlsh-expansion package. 
 
 ```
-
-python2 -m pip install --user cqlsh-expansion
-
+python3 -m pip install --user cqlsh-expansion
 ```
 
-
-
 ## Setup cqlsh-expansion to connect to Amazon Keyspaces
 
 To use the cqlsh-expansion with Amazon Keyspaces you can use the following post install script or by following the instructions found in the official [Amazon Keyspaces documentation.](https://docs.aws.amazon.com/keyspaces/latest/devguide/programmatic.cqlsh.html) 
@@ -45,54 +39,64 @@ Now that you have you cqlsh-expansion installed and have setup up the configurat
 To connect to Amazon Keyspaces you will need to choose one of the [service endpoints](https://docs.aws.amazon.com/keyspaces/latest/devguide/programmatic.endpoints.html). You can also connect to Amazon Keyspaces using [Interface VPC endpoints](https://docs.aws.amazon.com/keyspaces/latest/devguide/vpc-endpoints.html) to enable private communication between your virtual private cloud (VPC) running in Amazon VPC and Amazon Keyspaces. For example, to connect to the Keyspaces service in US East (N. Virginia) (us-east-1) you will want to use the [cassandra.us-east-1.amazonaws.com](http://cassandra.us-east-1.amazonaws.com/) service endpoint.  All communication with Amazon Keyspaces will be over port 9142. 
 
 ### Choose authentication method and connect
-
- To provide users and applications with credentials for programmatic access to Amazon Keyspaces resources, you can do either of the following:
+To provide users and applications with credentials for programmatic access to Amazon Keyspaces resources, you can do either of the following:
 
 #### Connect with IAM access keys (users,roles, and federated identities)
 
-For enhanced security, we recommend to create IAM access keys for IAM users and roles that are used across all AWS services. To use IAM access keys to connect to Amazon Keyspaces, customers can use the Signature Version 4 Process (SigV4) authentication plugin for Cassandra client drivers. To learn more about how the Amazon Keyspaces SigV4 plugin enables [IAM users, roles, and federated identities](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) to authenticate in Amazon Keyspaces API requests, see [AWS Signature Version 4 process (SigV4)](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). You can use the Sigv4 plugin with the cqlsh-expansion script by providing the following flag. . `--auth-provider "SigV4AuthProvider"` . The Sigv4 plugin depends on the AWS SDK for Python (Boto3) which is included in the requirements file.  You will also need to set the the proper credentials to make service calls. You can use the following tutorial to [setup credentials using the AWS CLI.](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html) 
+For enhanced security, we recommend to create IAM access keys for IAM users and roles that are used across all AWS services. To use IAM access keys to connect to Amazon Keyspaces, customers can use the Signature Version 4 Process (SigV4) authentication plugin for Cassandra client drivers. To learn more about how the Amazon Keyspaces SigV4 plugin enables [IAM users, roles, and federated identities](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) to authenticate in Amazon Keyspaces API requests, see [AWS Signature Version 4 process (SigV4)](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html). 
 
 After you have the credentials setup with [privileges](https://docs.aws.amazon.com/keyspaces/latest/devguide/security_iam_service-with-iam.html) to access Amazon Keyspaces system tables, you can execute the following command to connect to Amazon Keyspaces with CQLSH using the Sigv4 process.  
 
-```
+Validate the module name and classname, region_name based on keyspaces endpoint in cqlshrc file. 
 
-cqlsh-expansion cassandra.us-east-1.amazonaws.com 9142 --ssl --auth-provider "SigV4AuthProvider"
+```
+[auth_provider]
+;; you can specify any auth provider found in your python environment
+;; module and class will be used to dynamically load the class
+;; all other properties found here and in the credentials file under the class name
+;; will be passed to the constructor
+module = cassandra_sigv4.auth
+classname = SigV4AuthProvider
+region_name = us-east-1
+```
+you can also set region as Environment variable
 
+```
+ export AWS_DEFAULT_REGION = us-east-1
 ```
 
+To connect to Amazon Keyspaces with cqlsh-expansion using Sigv4 authenticator.  
+```
+cqlsh-expansion cassandra.us-east-1.amazonaws.com 
+```
 
 #### Connect with service-specific credentials
 
-You can create service-specific credentials that are similar to the traditional username and password that Cassandra uses for authentication and access management. AWS service-specific credentials are associated with a specific AWS Identity and Access Management (IAM) user and can only be used for the service they were created for. For more information, see [Using IAM with Amazon Keyspaces (for Apache Cassandra)](http://using%20iam%20with%20amazon%20keyspaces%20%28for%20apache%20cassandra%29/) in the IAM User Guide. To connect to Amazon Keyspaces using the cqlsh-expansion and IAM service-specific credentials you can use the command below. In this command we are connecting to us-east-1 region with service specific user *‘mike-user-99’ *and service specific user password* ‘user-pass-01’. *You will need to replace these credentials with your own user name and password that were given to you when creating the service specific credentials. 
-
-```
+You can create service-specific credentials that are similar to the traditional username and password that Cassandra uses for authentication and access management. AWS service-specific credentials are associated with a specific AWS Identity and Access Management (IAM) user and can only be used for the service they were created for. For more information, see [Using IAM with Amazon Keyspaces (for Apache Cassandra)](http://using%20iam%20with%20amazon%20keyspaces%20%28for%20apache%20cassandra%29/) in the IAM User Guide. To connect to Amazon Keyspaces using the cqlsh-expansion and IAM service-specific credentials you can use the command below. In this command we are connecting to us-east-1 region with service specific user *‘Sri-user-99’ *and service specific user password* ‘user-pass-01’. *You will need to replace these credentials with your own user name and password that were given to you when creating the service specific credentials. 
 
-cqlsh-expansion cassandra.us-east-1.amazonaws.com 9142 --ssl -u mike-user-99 -p user-pass-01
 
 ```
-
-Alternatively, if you want to use the cqlsh without the additional functionality included in the cqlsh-expansion package you can execute the following. 
-
+[auth_provider]
+;; you can specify any auth provider found in your python environment
+;; module and class will be used to dynamically load the class
+;; all other properties found here and in the credentials file under the class name
+;; will be passed to the constructor
+module = cassandra.auth
+classname = PlainTextAuthProvider
 ```
 
-cqlsh cassandra.us-east-1.amazonaws.com 9142 --ssl -u mike-user-99 -p user-pass-01
-
 ```
+cqlsh-expansion cassandra.us-east-1.amazonaws.com -u Sri-user-99 -p user-pass-01
+```
+
 
 ## Cleanup
 To remove the cqlsh-expansion package you can use the pip uninstall api. Additionally, if you executed the post install script ```cqlsh-expansion.init```, you may want to delete the .cassandra directory which contains the cqlshrc file and the ssl certificate. Using pip uninstall will not remove changes made by the post install script. 
 
 ```
 pip uninstall cqlsh-expansion
-
 ```
 
-## Functional differences from CQLSH
-
-### Sigv4 authentication
-
-Instead of using the service specific credentials for an IAM user, you can use the `--auth-provider "SigV4AuthProvider"` parameter to leverage the Sigv4 authentication plugin for temporary credentials. This plugin enables IAM users, roles, and federated identities to add authentication information to Amazon Keyspaces (for Apache Cassandra) API requests using the AWS Signature Version 4 Process (SigV4). The plugin depends on the AWS SDK for Python (Boto3) and the [Amazon Keyspaces Sigv4 plugin for the DataStax python driver](https://github.com/aws/aws-sigv4-auth-cassandra-python-driver-plugin).
-
 ### New output for TTY
 
 When creating a new cqlsh session with the cqlsh-expansion utility, it will show the default consistency level after establishing a new connection. We find customers using cqlsh may not be aware of the default consistency level of `ONE`, and additional transparency will lead to better operational excellence.

cqlsh-expansion/bin/cqlsh

@@ -15,12 +15,85 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# bash code here; finds a suitable python interpreter and execs this file.
-# prefer unqualified "python" if suitable:
-python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \
-    && exec python "`python -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@"
-for pyver in 2.7; do
-    which python$pyver > /dev/null 2>&1 && exec python$pyver "`python$pyver -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh.py" "$@"
+# shell script to find a suitable Python interpreter and run cqlsh.py
+
+# Use the Python that is specified in the env
+if [ -n "$CQLSH_PYTHON" ]; then
+    USER_SPECIFIED_PYTHON="$CQLSH_PYTHON"
+fi
+
+
+# filter "--python" option and its value, and keep remaining arguments as it is
+USER_SPECIFIED_PYTHON_OPTION=false
+for arg do
+  shift
+  case "$arg" in
+    --python)
+        USER_SPECIFIED_PYTHON_OPTION=true
+        ;;
+    --)
+        break
+        ;;
+    *)
+        if [ "$USER_SPECIFIED_PYTHON_OPTION" = true ] ; then
+            USER_SPECIFIED_PYTHON_OPTION=false
+            USER_SPECIFIED_PYTHON="$arg"
+        else
+            set -- "$@" "$arg"
+        fi
+        ;;
+  esac
 done
-echo "No appropriate python interpreter found." >&2
+
+if [ "$USER_SPECIFIED_PYTHON_OPTION" = true ] ; then
+    echo "You must specify a python interpreter path with the --python option"
+    exit 1
+fi
+
+# get a version string for a Python interpreter
+get_python_version() {
+    interpreter=$1
+    version=$($interpreter -c "import os; print('{}.{}'.format(os.sys.version_info.major, os.sys.version_info.minor))" 2> /dev/null)
+    echo "$version"
+}
+
+# test whether a version string matches one of the supported versions for cqlsh
+is_supported_version() {
+    version=$1
+    major_version="${version%.*}"
+    minor_version="${version#*.}"
+    # python3.6+ is supported
+    if [ "$major_version" = 3 ] && [ "$minor_version" -ge 6 ]; then
+        echo "supported"
+    else
+        echo "unsupported"
+    fi
+}
+
+run_if_supported_version() {
+    # get the interpreter and remove it from argument
+    interpreter="$1" shift
+
+    version=$(get_python_version "$interpreter")
+    if [ -n "$version" ]; then
+        if [ "$(is_supported_version "$version")" = "supported" ]; then
+            exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" "$@"
+            exit
+        else
+            echo "Warning: unsupported version of Python:" $version >&2
+        fi
+    fi
+}
+
+
+if [ "$USER_SPECIFIED_PYTHON" != "" ]; then
+    # run a user specified Python interpreter
+    run_if_supported_version "$USER_SPECIFIED_PYTHON" "$@"
+else
+    for interpreter in python3 python; do
+        run_if_supported_version "$interpreter" "$@"
+    done
+fi
+
+echo "No appropriate Python interpreter found." >&2
 exit 1

cqlsh-expansion/bin/cqlsh-expansion

@@ -15,12 +15,85 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# bash code here; finds a suitable python interpreter and execs this file.
-# prefer unqualified "python" if suitable:
-python -c 'import sys; sys.exit(not (0x020700b0 < sys.hexversion < 0x03000000))' 2>/dev/null \
-    && exec python "`python -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh-expansion.py" "$@"
-for pyver in 2.7; do
-    which python$pyver > /dev/null 2>&1 && exec python$pyver "`python$pyver -c "import os;print(os.path.dirname(os.path.realpath('$0')))"`/cqlsh-expansion.py" "$@"
+# shell script to find a suitable Python interpreter and run cqlsh.py
+
+# Use the Python that is specified in the env
+if [ -n "$CQLSH_PYTHON" ]; then
+    USER_SPECIFIED_PYTHON="$CQLSH_PYTHON"
+fi
+
+
+# filter "--python" option and its value, and keep remaining arguments as it is
+USER_SPECIFIED_PYTHON_OPTION=false
+for arg do
+  shift
+  case "$arg" in
+    --python)
+        USER_SPECIFIED_PYTHON_OPTION=true
+        ;;
+    --)
+        break
+        ;;
+    *)
+        if [ "$USER_SPECIFIED_PYTHON_OPTION" = true ] ; then
+            USER_SPECIFIED_PYTHON_OPTION=false
+            USER_SPECIFIED_PYTHON="$arg"
+        else
+            set -- "$@" "$arg"
+        fi
+        ;;
+  esac
 done
-echo "No appropriate python interpreter found." >&2
+
+if [ "$USER_SPECIFIED_PYTHON_OPTION" = true ] ; then
+    echo "You must specify a python interpreter path with the --python option"
+    exit 1
+fi
+
+# get a version string for a Python interpreter
+get_python_version() {
+    interpreter=$1
+    version=$($interpreter -c "import os; print('{}.{}'.format(os.sys.version_info.major, os.sys.version_info.minor))" 2> /dev/null)
+    echo "$version"
+}
+
+# test whether a version string matches one of the supported versions for cqlsh
+is_supported_version() {
+    version=$1
+    major_version="${version%.*}"
+    minor_version="${version#*.}"
+    # python3.6+ is supported
+    if [ "$major_version" = 3 ] && [ "$minor_version" -ge 6 ]; then
+        echo "supported"
+    else
+        echo "unsupported"
+    fi
+}
+
+run_if_supported_version() {
+    # get the interpreter and remove it from argument
+    interpreter="$1" shift
+
+    version=$(get_python_version "$interpreter")
+    if [ -n "$version" ]; then
+        if [ "$(is_supported_version "$version")" = "supported" ]; then
+            exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh-expansion.py" "$@"
+            exit
+        else
+            echo "Warning: unsupported version of Python:" $version >&2
+        fi
+    fi
+}
+
+
+if [ "$USER_SPECIFIED_PYTHON" != "" ]; then
+    # run a user specified Python interpreter
+    run_if_supported_version "$USER_SPECIFIED_PYTHON" "$@"
+else
+    for interpreter in python3 python; do
+        run_if_supported_version "$interpreter" "$@"
+    done
+fi
+
+echo "No appropriate Python interpreter found." >&2
 exit 1