(fix): create all snowflake resources using scripts and related README updates

Author: kameshsampath

.envrc.example

@@ -6,6 +6,7 @@ export KEYS_DIR="$DEMO_HOME/keys"
 export SNOWSQL_ACCOUNT=<snowflake account>
 export TODOAPP_USER=<snowflake todoapp user>
 export TODOAPP_USER_PWD=<snowflake todoapp user password>
+export TODOAPP_USER_ROLE=todoapp_user
 export TODOAPP_DATABASE=<snowflake todoapp database>
 # update to different warehouse, this is default for trial accounts
 export SNOWSQL_WAREHOUSE=COMPUTE_WH

README.md

@@ -16,6 +16,22 @@ To run the demo you need the following tools on your local machine,
 - [direnv](https://direnv.net)
 - [miniconda](https://docs.anaconda.com/free/miniconda/)
 
+## Environment Setup
+
+The demo uses few environment variables, create and load it,
+
+```shell
+copy $DEMO_HOME/.envrc.example  $DEMO_HOME/.envrc
+```
+
+> **NOTE**: `$DEMO_HOME` is the directory where you have cloned this repo
+
+Adjust the values in the `$DEMO_HOME/.envrc` as per your requirements and load them,
+
+```shell
+direnv allow .
+```
+
 ## Data Streaming Platform Setup
 
 The entire demo is containerized and all the applications could be started using the Docker compose,
@@ -119,12 +135,31 @@ Once the task is added the terminal running the client should show an output sim
 ```
 
 > **TIP**: If you have downloaded the `add` binary then you can use it send Todo Tasks. Check `add -h` for possible options.
+> Add Todos from the file `$DEMO_HOME/samples/todos.jsonl`
+>
+> ```json
+> {"title": "Learn Redpanda CLI", "description": "Learn how to use rpk cli.","category": "learning","status": false}
+> {"title": "Learn Snowflake and Streamlit", "description": "Learn how to integrate snowflake  with streamlit.",> "category": "learning","status": false}
+> {"title": "Create GitHub Repo", "description": "Create GitHub Repository for todo-demo.","category": "code","status": false}
+> {"title": "Todo ID Gen", "description": "Use SQLite for Todo ID generation via sequence.","category": "code","status": false}
+> {"title": "Car Service", "description": "Get the car to service this weekend.","category": "personal","status": false}
+> {"title": "Buy Groceries", "description": "Buy groceries for home before 8pm this evening.","category": "personal","status": false}
+> {"title": "Book Tickets for Movie", "description": "Get the movie tickets for Dune2.","category": "family","status": false}
+> ```
+>
+> ```shell
+> $DEMO_HOME/bin/add -d $DEMO_HOME/samples/todos.jsonl
+> ```
+
+The added `todos` can ve viewed via the Redpanda [Web Console](http://localhost:8080).
 
 ## Snowflake Integration
 
 The following section details on steps required to integrate the application with Snowflake data cloud. As result of the integration the data from Redpanda topic `todo-list` gets synchronized on to Snowflake table `TODO_LIST`.
 
-### Create RSA Key Pair
+### RSA Key Pair
+
+The `$TODOAPP_USER` will be using the Private Key authentication. In the following section we will create a RSA Private Key and encrypt the same.
 
 Create directory to hold the keys,
 
@@ -166,31 +201,54 @@ end
 Export the Private Key and Public Key into variables,
 
 ```shell
-set -lx TODOAPP_USER_RSA_PRIVATE_KEY (awk 'NR > 2 {print last} {last=$0}' ORS='' "$KEYS_DIR/rsa_key.p8")
-set -lx TODOAPP_USER_RSA_PUBLIC_KEY (awk 'NR > 2 {print last} {last=$0}' ORS='' "$KEYS_DIR/rsa_key.pub")
+export TODOAPP_USER_RSA_PRIVATE_KEY=$(awk 'NR > 2 {print last} {last=$0}' ORS='' "$KEYS_DIR/rsa_key.p8")
+export TODOAPP_USER_RSA_PUBLIC_KEY=$(awk 'NR > 2 {print last} {last=$0}' ORS='' "$KEYS_DIR/rsa_key.pub")
+```
+
+> **NOTE**:
+> The variables strip the first and last line of the key as required by Snowflake Kafka Connector
+
+## Database Setup
+
+> **IMPORTANT**: The commands in this section will be executed as Snowflake Account Admin user
+
+Create Database `$TODOAPP_DATABASE`,
+
+```shell
+snowsql -c admin \
+--query "CREATE DATABASE IF NOT EXISTS $TODOAPP_DATABASE;"
 ```
 
-> **NOTE**: The variables strip the first and last line of the key as required by Snowflake Kafka Connector
+Create the `$TODOAPP_USER` user and provide the requried **GRANTs**,
 
-### Update the SnowSQL User
+```shell
+snowsql -c admin \
+--warehouse "$SNOWSQL_WAREHOUSE" \
+--variable todo_warehouse=$SNOWSQL_WAREHOUSE \
+--variable db_name=$TODOAPP_DATABASE \
+--variable todo_user_role=$TODOAPP_USER_ROLE \
+--variable todoapp_user=$TODOAPP_USER \
+--variable todo_pub_key=$TODOAPP_USER_RSA_PUBLIC_KEY \
+--filename "$DEMO_HOME/etc/snowflake/acl.sql"
+```
 
-Update the Snowflake `$TODOAPP_USER` to use the generated keys for authentication,
+Describe the user,
 
 ```shell
-snowsql -c admin -q "ALTER USER $TODOAPP_USER SET RSA_PUBLIC_KEY=\"$TODOAPP_USER_RSA_PUBLIC_KEY\";"
+snowsql -c admin \
+--warehouse "$SNOWSQL_WAREHOUSE" \
+--query "DESCRIBE USER $TODOAPP_USER;"
 ```
 
 Verify if `$TODOAPP_USER` is able to connect with key pair,
 
 ```shell
-snowsql -u $TODOAPP_USER --private-key-path="$DEMO_HOME/keys/rsa_key.p8" -q "SELECT CURRENT_DATE;"
+snowsql -u $TODOAPP_USER \
+--private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--query "SELECT CURRENT_DATE;"
 ```
 
-> **NOTE**: Can describe the user `$TODOAPP_USER` and check if `RSA_PUBLIC_KEY` column is same as `$TODOAPP_USER_RSA_PUBLIC_KEY`
->
-> ```shell
-> snowsql -c admin -q "DESCRIBE USER $TODOAPP_USER;"
-> ```
+> **NOTE**: First time login will ask for a password change.
 
 ## Kafka Connect API
 
@@ -289,9 +347,12 @@ Type SQL statements or !help
 1 Row(s) produced. Time Elapsed: 0.219s
 ```
 
+Select all rows from the `todo_list` table,
+
 ```shell
 snowsql -u $TODOAPP_USER \
 --private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--warehouse $SNOWSQL_WAREHOUSE \
 --dbname $TODOAPP_DATABASE \
 --variable redpanda_topic=$TOPICS \
 --filename "$DEMO_HOME/etc/snowflake/todo_list_query.sql"
@@ -300,25 +361,27 @@ snowsql -u $TODOAPP_USER \
 ```text
 * SnowSQL * v1.2.32
 Type SQL statements or !help
-+----+-------------------------------+---------------------------------------------------+----------+--------+---------------+
-| ID | TITLE                         | DESCRIPTION                                       | CATEGORY | STATUS |   CREATE_TIME |
-|----+-------------------------------+---------------------------------------------------+----------+--------+---------------|
-|  1 | Learn Redpanda CLI            | Learn how to use rpk cli.                         | learning | False  | 1712635699878 |
-|  2 | Learn Snowflake and Streamlit | Learn how to integrate snowflake  with streamlit. | learning | False  | 1712635700215 |
-|  3 | Create GitHub Repo            | Create GitHub Repository for todo-demo.           | code     | False  | 1712635700218 |
-|  4 | Todo ID Gen                   | Use SQLite for Todo ID generation via sequence.   | code     | False  | 1712635700222 |
-|  5 | Car Service                   | Get the car to service this weekend.              | personal | False  | 1712635700225 |
-|  6 | Buy Groceries                 | Buy groceries for home before 8pm this evening.   | personal | False  | 1712635700227 |
-|  7 | Book Tickets for Movie        | Get the movie tickets for Dune2.                  | family   | False  | 1712635700229 |
-+----+-------------------------------+---------------------------------------------------+----------+--------+---------------+
-7 Row(s) produced. Time Elapsed: 0.345s
++-----+---------------------------------+-----------------------------------------------------+------------+--------+---------------+
+| KEY | TITLE                           | DESCRIPTION                                         | CATEGORY   | STATUS | TZ            |
+|-----+---------------------------------+-----------------------------------------------------+------------+--------+---------------|
+| "1" | "Learn Redpanda CLI"            | "Learn how to use rpk cli."                         | "learning" | false  | 1712731713770 |
+| "2" | "Learn Snowflake and Streamlit" | "Learn how to integrate snowflake  with streamlit." | "learning" | false  | 1712731714112 |
+| "3" | "Create GitHub Repo"            | "Create GitHub Repository for todo-demo."           | "code"     | false  | 1712731714120 |
+| "4" | "Todo ID Gen"                   | "Use SQLite for Todo ID generation via sequence."   | "code"     | false  | 1712731714127 |
+| "5" | "Car Service"                   | "Get the car to service this weekend."              | "personal" | false  | 1712731714132 |
+| "6" | "Buy Groceries"                 | "Buy groceries for home before 8pm this evening."   | "personal" | false  | 1712731714138 |
+| "7" | "Book Tickets for Movie"        | "Get the movie tickets for Dune2."                  | "family"   | false  | 1712731714143 |
++-----+---------------------------------+-----------------------------------------------------+------------+--------+---------------+
+7 Row(s) produced. Time Elapsed: 1.548s
+Goodbye!
 ```
 
 **Optionally** you can run the following command to find the number of todos that has been inserted, it should return `7`.
 
 ```shell
 snowsql -u $TODOAPP_USER \
 --private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--warehouse $SNOWSQL_WAREHOUSE \
 --dbname $TODOAPP_DATABASE \
 --query "SELECT count(*) from TODO_LIST;"
 ```
@@ -327,6 +390,43 @@ snowsql -u $TODOAPP_USER \
 
 You can now use the synchronized data to build a dashboard using [Streamlit](https://streamlit.io). The demo builds a simple dashboard that shows task by category along with a tabular representation of the tasks data.
 
+#### Create the Stored Procedure
+
+Stored Procedure to extract the data the table synchronized with Redpanda topic to any custom table.
+
+```shell
+snowsql -u $TODOAPP_USER \
+--private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--warehouse $SNOWSQL_WAREHOUSE \
+--dbname $TODOAPP_DATABASE \
+--filename "$DEMO_HOME/etc/snowflake/todos_sp.sql"
+```
+
+#### Prepare Data for Dashboard
+
+Extract the data from `TODO_LIST` table and load the same on to `todos` which will be used by Streamlit dashboard,
+
+```shell
+snowsql -u $TODOAPP_USER \
+--private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--warehouse $SNOWSQL_WAREHOUSE \
+--dbname "$TODOAPP_DATABASE" \
+--query "CALL todos('todo_list','todos')"
+```
+
+Query the `todos` table,
+
+```shell
+snowsql -u $TODOAPP_USER \
+--private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--warehouse $SNOWSQL_WAREHOUSE \
+--dbname $TODOAPP_DATABASE \
+--query "SELECT * FROM TODOS;"
+```
+
+> **NOTE**: You need to refresh the data on the `TODOS` table on every new task added to th Redpanda topic.
+> **WIP**: Leveraging streams and dynamic tables
+
 #### Use predefined conda environment
 
 Create a Python virtual environment for SnowPark using the conda environment configuration file `$DEMO_HOME/etc/python/environment.yml`.
@@ -354,6 +454,8 @@ streamlit run etc/snowflake/todo_dashboard.py
 
 ## Clean up
 
+### Drop Table, User and Stored Procedures
+
 Destroy Redpanda containers and cleanup associated volumes,
 
 ```shell
@@ -363,16 +465,15 @@ docker compose down --volumes
 ```shell
 snowsql -u $TODOAPP_USER \
 --private-key-path="$DEMO_HOME/keys/rsa_key.p8" \
+--warehouse $SNOWSQL_WAREHOUSE \
 --dbname $TODOAPP_DATABASE \
 --filename "$DEMO_HOME/etc/snowflake/cleanup.sql"
 ```
 
-### Optional
-
-Drop the database
+### Drop Database
 
 ```shell
-snowsql -c admin --dbname $TODOAPP_DATABASE \
+snowsql -c admin --warehouse $SNOWSQL_WAREHOUSE \
 --query "DROP DATABASE IF EXISTS $SNOWSQL_DATABASE;"
 ```
 

etc/snowflake/acl.sql

@@ -1,37 +1,46 @@
--- Grant user the '&todo_user_role' role 
-GRANT ROLE '&todo_user_role' TO USER '&todoapp_user';
+-- The Role for Todo App
+CREATE ROLE IF NOT EXISTS &todo_user_role COMMENT="The Todo App User";
 
-GRANT USAGE ON WAREHOUSE COMPUTE_WH TO ROLE '&todo_user_role';
+-- Create Todo App User with default role of '&todo_user_role'
+CREATE USER IF NOT EXISTS &todoapp_user
+  PASSWORD = 'superS3cret!' 
+  MUST_CHANGE_PASSWORD = TRUE
+  RSA_PUBLIC_KEY = "&todo_pub_key"
+  DEFAULT_ROLE = &todo_user_role
+  DEFAULT_NAMESPACE = &db_name.PUBLIC
+  DEFAULT_WAREHOUSE = &todo_warehouse
+  COMMENT = "The Todo application user";
 
--- Set default warehouse
-ALTER USER '&todoapp_user' SET DEFAULT_WAREHOUSE = "&warehouse"
 
--- If you want to create a new database for Todo App, run
-CREATE DATABASE IF NOT EXISTS DEMODB;
+-- Grant user the "&todo_user_role" role 
+GRANT ROLE &todo_user_role TO USER &todoapp_user;
 
--- Allow the '&todo_user_role' role to use database and public schema
-GRANT USAGE ON DATABASE DEMODB TO ROLE '&todo_user_role';
-GRANT ALL ON SCHEMA DEMODB.PUBLIC TO ROLE '&todo_user_role';
+-- Grant usage on Warehouse to user role
+GRANT USAGE ON WAREHOUSE &todo_warehouse TO ROLE &todo_user_role;
 
--- Grant '&todo_user_role' to create, modify and delete table on demo public schema
-GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DEMODB.PUBLIC TO ROLE todo_app_user;
+-- Allow the "&todo_user_role" role to use database and public schema
+GRANT USAGE ON DATABASE &db_name TO ROLE &todo_user_role;
 
--- Grant CRUD on all tables on public schema of demodb to '&todo_user_role' role
-GRANT INSERT,DELETE,SELECT,UPDATE,TRUNCATE ON ALL TABLES IN SCHEMA DEMODB.PUBLIC
-TO ROLE '&todo_user_role';
+-- Allow all access on public schema of the demo db to the todo app user
+GRANT ALL ON SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
+
+-- Grant "&todo_user_role" to create, modify and delete table on demo public schema
+GRANT OWNERSHIP ON ALL TABLES IN SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
+
+-- Grant CRUD on all tables on public schema of demodb to "&todo_user_role" role
+GRANT INSERT,DELETE,SELECT,UPDATE,TRUNCATE ON ALL TABLES IN SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
 
 -- Grant table operations on all future tables
-GRANT INSERT,DELETE,SELECT,UPDATE,TRUNCATE ON FUTURE TABLES IN SCHEMA DEMODB.PUBLIC
-TO ROLE '&todo_user_role';
+GRANT INSERT,DELETE,SELECT,UPDATE,TRUNCATE ON FUTURE TABLES IN SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
 
 -- Ability to create stage(s)
-GRANT CREATE STAGE ON SCHEMA DEMODB.PUBLIC TO ROLE '&todo_user_role';
+GRANT CREATE STAGE ON SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
 
-GRANT OWNERSHIP ON PROCEDURE TODOS(string,string) TO ROLE '&todo_user_role';
+-- GRANT OWNERSHIP ON PROCEDURE TODOS(string,string) TO ROLE &todo_user_role;
 
 -- Create Streamlit apps in the PUBLIC schema
-GRANT CREATE STREAMLIT ON SCHEMA DEMODB.PUBLIC TO ROLE '&todo_user_role';
-GRANT CREATE STAGE ON SCHEMA DEMODB.PUBLIC TO ROLE '&todo_user_role';
+GRANT CREATE STREAMLIT ON SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
+GRANT CREATE STAGE ON SCHEMA &db_name.PUBLIC TO ROLE &todo_user_role;
 
 -- List all grants
-show grants to role '&todo_user_role';
+show grants to role &todo_user_role;

etc/snowflake/cleanup.sql

@@ -1,6 +1,3 @@
-USE WAREHOUSE COMPUTE_WH;
-USE SCHEMA public;
-
 -- Drop table 
 DROP TABLE IF EXISTS todo_list;
 DROP TABLE IF EXISTS todos;

etc/snowflake/core.sql

@@ -1,4 +1 @@
-use WAREHOUSE COMPUTE_WH;
-use SCHEMA public;
-
 CALL todos('todo_list','&to_table')

etc/snowflake/dashboard_data.sql

@@ -1,6 +1,3 @@
-use WAREHOUSE COMPUTE_WH;
-use SCHEMA public;
-
 select 
     record_metadata:key as key,
     record_content:title as title,

etc/snowflake/todo_list_query.sql

@@ -1,5 +1,3 @@
-use WAREHOUSE COMPUTE_WH;
-use SCHEMA public;
 CREATE OR REPLACE PROCEDURE todos(from_table string, to_table string)
   RETURNS string
   LANGUAGE python