From 8729788a9ce8416982ece2890cac17c32123af61 Mon Sep 17 00:00:00 2001 From: Damon LaCaille Date: Fri, 7 Feb 2025 18:46:17 -0600 Subject: [PATCH] Large changes --- .DS_Store | Bin 8196 -> 0 bytes .gitignore | 4 + lambda-feature-store/.DS_Store | Bin 6148 -> 0 bytes riot-example/LICENSE | 16 +++ riot-example/README.md | 141 +++++++++++++++---- riot-example/admin | 244 +++++++++++++++++++++++++++++++++ riot-example/env.sh | 193 +++++++++++++++++++++----- riot-example/generate | 32 +++++ riot-example/generate.sh | 32 ----- riot-example/replicate | 37 +++++ riot-example/replicate.sh | 47 ------- riot-example/userdata.sh | 106 ++++++++++++++ 12 files changed, 711 insertions(+), 141 deletions(-) delete mode 100644 .DS_Store delete mode 100644 lambda-feature-store/.DS_Store create mode 100644 riot-example/LICENSE create mode 100755 riot-example/admin create mode 100755 riot-example/generate delete mode 100755 riot-example/generate.sh create mode 100755 riot-example/replicate delete mode 100755 riot-example/replicate.sh create mode 100644 riot-example/userdata.sh diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 4ac369b0e5c4ec2c167ae8c202784d29e6b658f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMUu+ab82`R&DZ5Ld+fty196c#D1zNcN!Ght>-Sw1Ii&Um(g z;3ME8;3ME8;3M!~M1bCGmV`3zKG*uCkARQB{}KUqKZGeW8TI50m+sX;g&P8ptfX!? zQJvxd*C*`Bs3&K*bX{po9zCGvie89;+@17cW==Bd$r&!WJ0N!l^kzmcLP2+S@(a7> zfKe{}(nr8Y;93M&>|PHB#p@YJT@}AOkcS*a@@-&)0~V{Nigq?-SSj+kn93Tu?L;D9 zVxXdON!3z8kR)ljbTm0>6_bwQE?pPNTrl$Hg1@0R+8^|WOs^;6oITw@dHz0~Z_X$1m!C~X<}jA@U_ zwQ0pj=MBZtVurzt?O{hd;jr3a+sZjTuHl$0=WXqvMs|8ZOOQ3W*0GTeWF6fy?L%75 zCfO8?@a2fJ)vH1qZrc*se#hR#rR6JXWT{T7A1dg!o-(w-qOLlVBZ{5YOjS204%0T8 zj(%L%a$+s&;()1Sv`eekG|0iV!FA=l1i48V&ZqPT^SX1A&(RpXRg~Wqj;Qp7`z@7? zsJfb9W3X8eMp=JPPNR}{OD)08qO1r5Hm#Qr(I#yZWm_D0fVjq8JESf_R>eU>NofY} z>Xvp2@;%~EfiIhP?vnNhwb&{S)2?P{JvQ%-1$zYfL*W>kyHBy516j?a>D|>e2KNcV z2-Cz=Ra1qgXbZ>BML?9<6BvXI&;`4oA4cdf7U2v$1JA-MFbnU%`)~n1fluKx_#Eco zTlfKff?wcQ_zix?Dy%^n>u@E8aT9LFE!d4wjA0KZuon+v5(n`Rj-rYhPT(Y(Si}=} z3a9W9JcEzpQ}{H#fUn|f_&UCU@8LQ80MFw`_%VKgU*k9YI|K9QXa5y6rYc^hVli~~ zE|I{o>(M!M*FGu!uhRLeYR2K(x|Q{-);G1Zb#(8FEo^hi&R$6}msgxPE}`ZL<8JTO zEG0e!*WMz928nmtiP94?e+J~Gj9~&}Pp~l@3dxLGyt%QNngL=zZ*JNglH>~Fw&)sf z-6nUcAag@Rbh&$8@%6W)gNaFKX9 z2bbY{_z`|aAVyZ>YFtlzY{HG$j9W2+9oUIoxEuEpGvma~KD-zE@jl|`C?3Tz97hH3 zM}ydDVHQskJE!q9K8%mzW5m%X@JW20So$KqglCDTvn8zTEn($bi(zHPN+i6eO*kip zzSCD+h%L@v{agOo$-n+U6p}g9u*?bi-uV-HH56r96}-QJ#s7Df|NXy=x8Lm};3IG| zAb^#LWFk&0DR+CYSUXI4oH9$yZiY+Og$g$uC+UXcBp3c+NcAw8GM}g?XSk#uO8@u| S0e}AItpEP|@BdBz{{I2`I-WBC diff --git a/.gitignore b/.gitignore index 52e5f7e..b0bd105 100644 --- a/.gitignore +++ b/.gitignore @@ -326,3 +326,7 @@ Homestead.yaml Homestead.json /.vagrant .phpunit.result.cache + +# MacOS Directories + +.DS_Store diff --git a/lambda-feature-store/.DS_Store b/lambda-feature-store/.DS_Store deleted file mode 100644 index cbfda4e593af726d4d9bf8bfcf3fbfca0cd24a58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~Jx;?w5QX1BBqGtIL89CXP;i4)6i&ba_=#O0#W=!1?T%~n<_8hu&{CioY4+{T z&Uodo*&P6~erv9Q6@VpO6(2gxP50Gjb`lX)v2V1v!wXt$(a*Av2ekGA9rNGu#LDmb zj?os+Xz+kn?)97x&9L1)_Wh2>;S2+}I$UwwV}H1H{H%;7fh3Rwl0Xtj0(NIBhUIy} zNCHV93H%_SkL*HMt)Zi> ~/.bashrc -source ~/.bashrc +**`generate`** - calls the RIOT tool with the 'generate' action, passing all +parameters from the `env.sh` file. Execute via `./generate` + +**`replicate`** - calls the RIOT tool with the 'replicate' action, passing all +parameters from the `env.sh` file. Execute via `./replicate` + +**`admin`** - a shell script that allows you to perform some helpful tasks +against primary, such as summarizing key count, memory usage, etc. Run +`./admin --help` for more information. + +**`userdata.sh`** - This file can be pasted into an Amazon EC2 `userdata` +section to be used upon initial boot of an EC2 node. Useful to provide in +a Launch Template to automatically setup a Redis OSS environment across +multiple nodes. Currently deploys Redis OSS 5.0.10 in cluster mode. This +section will need comprehensive documentation in the future. + +## Requirements + +- Ensure Java is installed. +- Have at least one Valkey or Redis OSS cluster as a SOURCE clusterthat +you can reach from the node you install this on. (see `env.sh` for more). + +## Install + +1. Clone this repo + +2. Download the RIOT binaries into a local folder: + + ```sh + wget https://github.com/redis/riot/releases/download/early-access/riot-early-access.zip + ``` + +3. Unzip the zip file. + +4. Change directories into the newly created directory, and copy to the repo +directory from step 1. Example: + +```sh +cd riot-4.2.3-SNAPSHOT +cp -r * myhomedir/amazon-elasticache-samples/riot-example ``` -## Step 2 - Download riot docker image +## Running + +1. Modify the `env.sh` file appropriately -This docker image will perform the actual migration work: +2. Run a test via `./admin source primaries` to see if it can connect to show +the source cluster primary nodes. +3. Run a test via `./admin target primaries` to see if it can connect to show +the source cluster replica nodes. + +## admin usage + +As mentioned earlier, the `admin` script is provided as a helpful tool during a +migration to view source and target cluster information. Run `./admin --help` +for more information. Here are some examples of its usage: + +--- + +`./admin source dups` + +```text +Fetching keys from 172.31.20.133... +Fetching keys from 172.31.24.184... +Fetching keys from 172.31.30.162... +No duplicates ``` -docker pull riotx/riot + +`./admin source memory` + +```text +172.31.20.133 : 864 MB +172.31.24.184 : 864 MB +172.31.30.162 : 864 MB +Total : 2,592 MB ``` -## Step 3 - Update env variables +`./admin source keys` + +```text +172.31.20.133 : 184,182 +172.31.24.184 : 184,161 +172.31.30.162 : 184,257 +Total: 552,600 +``` -Update the file `env.sh`. It has most of the values that the tool uses, plus credentials for clusters. Update it with source and target cluster information, such as: +`./admin source bgsave` +```text +Executing BGSAVE on 172.31.20.133...Background saving started +Executing BGSAVE on 172.31.24.184...Background saving started +Executing BGSAVE on 172.31.30.162...Background saving started ``` -export SOURCE_CLUSTER=dns_name_of_cluster -export SOURCE_USER=default -export SOURCE_PASS=your_password -export SOURCE_PORT=6379 - -export ELASTICACHE_SLS=elasticache_endpoint.use1.cache.amazonaws.com -export ELASTICACHE_USER=default -export ELASTICACHE_PASS=elasticache_user_password -export ELASTICACHE_PORT=6379 + +`./admin source flush` + +```text +Flushing 172.31.20.133...OK +Flushing 172.31.24.184...OK +Flushing 172.31.30.162...OK ``` -## Step 4 - Generate data +## RIOT usage + +This repo is designed to call the RIOT application via scripts that include +parameters that are passed into RIOT. -Use the [generate.sh](generate.sh) script to create data in the source cluster. +Modify the `env.sh` file, ensuring the `GLOBAL_DRY_RUN` variable is set to +`--dry-run`. Then run the `./replicate` script to see a dry run of the +migration effort. -## Step 5 - Migrate data +## Populating data for testing -Use the [replicate.sh](replicate.sh) script to replicate data from source cluster to target cluster. +You may also use the `./generate` script after modifying the `env.sh` file to +generate mock data. This is helpful to see if the keys from the source cluster +(generated via `./generate`) exist after migration (via `./replicate`). diff --git a/riot-example/admin b/riot-example/admin new file mode 100755 index 0000000..072c478 --- /dev/null +++ b/riot-example/admin @@ -0,0 +1,244 @@ +#!/bin/bash + +# This script may be helpful to compare two different +# clusters. + +# WARNING: If you run the 'flush' command with this tool, +# it will flush all keys from all primary nodes in the +# specified cluster (source or target). + +if [[ ! -f './env.sh' ]]; then + echo "File 'env.sh' not found, exiting" + exit 1 +fi + +source env.sh + +run_command() { + # Normalize the ADMIN_DEBUG value + local debug_mode=${ADMIN_DEBUG,,} # Convert to lowercase for consistency + + # Construct the command as a single string + local cmd="$*" + + # Print the command to stderr if debugging is enabled + if [[ $debug_mode == "yes" || $debug_mode == "true" ]]; then + echo "Running: $cmd" >&2 + fi + + # Execute the command and return its output + eval "$cmd" +} + +set_cluster() { + if [[ ${CLUSTER} == "source" ]]; then + export ACTIVE_CLUSTER=${RIOT_SOURCE} + export ACTIVE_PORT=${SOURCE_PORT} + else + export ACTIVE_CLUSTER=${RIOT_TARGET} + export ACTIVE_PORT=${TARGET_PORT} + fi +} + +set_creds() { + if [[ ${CLUSTER} == "source" ]]; then + export AUTH_STRING="${SOURCE_AUTH_STRING}" + export TLS_FLAG="${SOURCE_TLS}" + fi + if [[ ${CLUSTER} == "target" ]]; then + export AUTH_STRING="${TARGET_AUTH_STRING}" + export TLS_FLAG="${TARGET_TLS}" + fi +} + +commify() { + number=$1 + run_command echo $number | sed ':a;s/\B[0-9]\{3\}\>/,&/;ta' +} + +bgsave () { + while IFS= read -r -t 5 node_name + do + echo -n "Executing BGSAVE on ${node_name}..." + set -o pipefail + run_command ${CLI} --raw --no-auth-warning -h $node_name -p ${ACTIVE_PORT} ${AUTH_STRING} ${TLS_FLAG} BGSAVE + done < <(get_primaries) +} + +get_dups() { + files="" + while IFS= read -r -t 5 node_name + do + echo "Fetching keys from $node_name..." + set -o pipefail + run_command ${CLI} --raw --no-auth-warning -h $node_name -p ${ACTIVE_PORT} ${AUTH_STRING} ${TLS_FLAG} KEYS \* > /tmp/$node_name.keys + files+=" /tmp/$node_name.keys " + done < <(get_primaries) + dup_keys=$(cat $files | sort | uniq -d) + if [[ -z "$dup_keys" ]]; then + echo No duplicates + else + echo "Duplicates:" + echo "$dup_keys" + fi + rm $files +} + +flush () { + while IFS= read -r -t 5 node_name + do + echo -n "Flushing ${node_name}..." + set -o pipefail + run_command ${CLI} --raw --no-auth-warning -h $node_name -p ${ACTIVE_PORT} ${AUTH_STRING} ${TLS_FLAG} flushall + done < <(get_primaries) +} + +parse_mb () { + line=$1 + value=$(echo "$line" | /usr/bin/grep -oE '^[0-9.]+') + unit=$(echo "$line" | /usr/bin/grep -oE '[MG]') + + if [[ -z "$value" || -z "$unit" ]]; then + echo "Unknown format in line: $line" >&2 + fi + + # Convert to MB + if [[ "$unit" == "G" ]]; then + # Convert GB to MB + value_in_mb=$(echo "$value * 1024" | bc -l) + elif [[ "$unit" == "M" ]]; then + value_in_mb="$value" + else + echo "Unknown unit in line: $line" >&2 + fi + + echo $value_in_mb +} + +memory () { + total_mb=0 + while IFS= read -r node_name + do + set -o pipefail + node_mb=$(run_command ${CLI} \ + --raw \ + --no-auth-warning \ + -h $node_name \ + -p ${ACTIVE_PORT} \ + ${AUTH_STRING} \ + ${TLS_FLAG} INFO MEMORY \ + | grep -i used_memory_human \ + | cut -d':' -f2) + numeric_mb=$(parse_mb $node_mb) + integer_part=${numeric_mb%.*} + numeric_mb=${integer_part} + with_comma=$(LC_NUMERIC="en_US.UTF-8" printf "%'d\n" "$numeric_mb") + echo "$node_name : $with_comma MB" + if [[ -z $total_mb ]]; then + total_mb=0 + fi + total_mb=$(echo "$total_mb + $numeric_mb" | bc -l) + done < <(get_primaries) + + print_mb=$(commify $total_mb) + echo "Total : $print_mb MB" +} + +keys() { + total_keys=0 + while IFS= read -r node_name + do + num_keys=$(run_command ${CLI} --no-auth-warning --raw -h $node_name -p ${ACTIVE_PORT} ${AUTH_STRING} ${TLS_FLAG} dbsize) + export dbsize=$(run_command commify $num_keys) + echo "$node_name : $dbsize" + if [[ -z $num_keys ]]; then + total_keys=0 + fi + total_keys=$(run_command echo "$total_keys + $num_keys" | bc -l) + done < <(get_primaries) + comma_keys=$(run_command commify $total_keys) + echo "Total: $comma_keys" +} + +usage() { + echo + echo "Usage: $0 [source|target] [bgsave|dups|flush|keys|memory|primaries|replicas]" + echo + echo " [source|target] which cluster to target for this operation" + echo + echo " bgsave - Start a background save on all self-managed primary nodes" + echo " keys - list key count for each primary node" + echo " memory - list memory used for each primary node" + echo " primaries - list all primary nodes" + echo " replicas - list all replica nodes" + echo " dups - list any duplicate keys in cluster" + echo " flush - delete all keys from each primary node" + echo +} + +get_primaries() { + set -o pipefail + run_command ${CLI} --no-auth-warning -u ${ACTIVE_CLUSTER} CLUSTER NODES \ + | grep master \ + | cut -d' ' -f2 \ + | cut -d':' -f1 +} + +get_replicas() { + set -o pipefail + run_command ${CLI} --no-auth-warning -u ${ACTIVE_CLUSTER} CLUSTER NODES \ + | grep slave \ + | cut -d' ' -f2 \ + | cut -d':' -f1 +} + +### BEGIN ACTUAL SHELL SCRIPT LOGIC + +CLUSTER=$1 +FUNCTION=$2 + +if [[ -z "$1" || -z "$2" ]]; then + usage + exit 1 +fi + +CLUSTER=${CLUSTER,,} + +if [ ${CLUSTER} != "source" ] && [ ${CLUSTER} != "target" ]; then + usage + exit 1 +fi + +set_cluster +set_creds + +case "$FUNCTION" in + bgsave) + bgsave + ;; + primaries) + get_primaries + ;; + replicas) + get_replicas + ;; + dups) + get_dups + ;; + flush) + flush + ;; + keys) + keys + ;; + memory) + memory + ;; + *) + echo "Error: Invalid function '$FUNCTION'." + usage + exit 1 + ;; +esac + +exit 0 diff --git a/riot-example/env.sh b/riot-example/env.sh index 89ca299..c77c987 100755 --- a/riot-example/env.sh +++ b/riot-example/env.sh @@ -1,37 +1,166 @@ -export SOURCE_CLUSTER=dns_name_of_cluster -export SOURCE_USER=default -export SOURCE_PASS=your_password -export SOURCE_PORT=6379 +# The `generate`, `replicate` and `admin` scripts source this file +# prior to execution. + +# Variables with prefix of SOURCE or TARGET will only +# be executed when source or target clusters are involved +# in the command being executed. + +# Variables with REPLICATE are only used during replication. +# Variables with GENERATE are only used when generating load. +# Variables with GLOBAL apply to both. + +#ADMIN_DEBUG="true" # set to "yes" or "true", either one + +#export CLI="redis-cli" +export CLI="valkey-cli" -export ELASTICACHE_CLUSTER=elasticache_endpoint.use1.cache.amazonaws.com -export ELASTICACHE_USER=default -export ELASTICACHE_PASS=elasticache_user_password -export ELASTICACHE_PORT=6379 +# SOURCE SETTINGS +export SOURCE_ENGINE_VERSION=5 # Connection options differ with engine versions +export SOURCE_ENDPOINT="your_oss_redis_ec2.ip.address.or.hostname" +export SOURCE_USER="default" +export SOURCE_PASS="your_user_password" +export SOURCE_PORT=6379 +# comment the next line to disable encryption on source +#export SOURCE_TLS="--tls" -export RIOT_SOURCE="rediss://${SOURCE_USER}:${SOURCE_PASS}@${SOURCE_CLUSTER}:${SOURCE_PORT}" -export RIOT_TARGET="rediss://${ELASTICACHE_USER}:${ELASTICACHE_PASS}@${ELASTICACHE_CLUSTER}:${ELASTICACHE_PORT}" +# TARGET SETTINGS +export TARGET_ENGINE_VERSION=8 +export TARGET_ENDPOINT="your_cluster....cache.amazonaws.com" # ElastiCache endpoint +export TARGET_USER="default" +export TARGET_PASS="your_user_password" +export TARGET_PORT=6379 +# comment the next line to disable encryption on target +export TARGET_TLS="--tls" +# OTHER SETTINGS export JAVA_OPTS="-Xmx16g" -export GENERATE_SIZE_KEY=1024 # in bytes -export KEY_COUNT=10000 -export KEYSPACE=100000 -export TTL=60480000 # 60480000 is 7 days -export BATCH_SIZE=1000 -export CMD_TIMEOUT=60 -export SLEEP_MS=1 -export NUM_THREADS=16 -export KEY_PREFIX="mykey" -export RESP_VERSION="2" -export RESP_PROTOCOL="RESP${RESP_VERSION}" -export EVENT_QUEUE=10000 -export FLUSH_INTERVAL=50 -export IDLE_TIMEOUT=1000 -export MEM_LIMIT=-1 -export READ_BATCH=10000 -export READ_POOL=16 -export READ_QUEUE=10000 -export READ_RETRY=2 -export READ_THREADS=1 -export SCAN_COUNT=1000 -export WRITE_POOL=32 +# DYNAMIC SETTINGS +epoch=$(date +%s) + +# Any values commented out will simply be replaced +# with empty "" values when executing command line. +# For example if you comment out the GLOBAL_DRY_RUN +# line below, the "--dry-run" option will not be +# added when executing RIOT. + +################################## +##### RIOT 'GLOBAL' SETTINGS ##### +################################## + +export GLOBAL_LETTUCE_LOG_LEVEL="--log=io.lettuce=INFO" +export GLOBAL_DRY_RUN="--dry-run" +export GLOBAL_BATCH_COUNT="--batch=50" +export GLOBAL_THREADS="--threads=8" +export GLOBAL_SLEEP_MS="--sleep=100" +export GLOBAL_SLEEP_MS="" +#export GLOBAL_DEBUG="--debug" +export GLOBAL_PROGRESS="--progress=LOG" + +#################################### +##### RIOT 'GENERATE' SETTINGS ##### +#################################### + +export GENERATE_DATA_TYPE="--type=STRING" +export GENERATE_KEY_PREFIX="--keyspace=myloadtest" +export GENERATE_CLUSTER="--cluster" +export GENERATE_TTL_EXPIRATION="--expiration=60480000" # in seconds - 60480000 is 7 days +export GENERATE_INSECURE_TLS="--insecure" +export GENERATE_SIZE_KEY="--string-value=1024-8192" +export GENERATE_KEY_COUNT="--count=2000" +export GENERATE_CLIENT_NAME="--client=riot_generate_${epoch}" +export GENERATE_RESP_VERSION="--resp=RESP2" +#export GENERATE_RESP_VERSION="--resp=RESP3" +export GENERATE_TIMEOUT="--timeout=30" +export GENERATE_LOG_FILE="--log-file=logs/riot_generate_${epoch}.log" +export GENERATE_POOL="--pool=8" + +##################################### +##### RIOT 'REPLICATE' SETTINGS ##### +##################################### + +export REPLICATE_URI="--uri=${RIOT_TARGET}" +export REPLICATE_MODE="--mode=live" +export REPLICATE_LOG_FILE="--log-file=logs/riot_replicate_${epoch}.log" +export REPLICATE_LOG_KEYS="--log-keys" +export REPLICATE_MEM_LIMIT="--mem-limit=0" +export REPLICATE_READ_QUEUE="--read-queue=10000" +export REPLICATE_FLUSH_INTERVAL="--flush-interval=50" +export REPLICATE_SCAN_COUNT="--scan-count=1000" +export REPLICATE_READ_FROM="--read-from=UPSTREAM" +export REPLICATE_PIPELINE_READ="--read-batch=50" +export REPLICATE_SOURCE_CLUSTER="--source-cluster" +export REPLICATE_TARGET_CLUSTER="--target-cluster" +export REPLICATE_READ_THREADS="--read-threads=1" +export REPLICATE_SOURCE_CLIENT_NAME="--source-client=riot_replicate_source_${epoch}" +export REPLICATE_TARGET_CLIENT_NAME="--target-client=riot_replicate_target_${epoch}" +export REPLICATE_READ_RETRY="--read-retry=1" +export REPLICATE_SOURCE_POOL="--source-pool=8" +export REPLICATE_TARGET_POOL="--target-pool=8" + +# Create logs dir if doesn't exist +if [[ ! -d ./logs ]]; then + mkdir ./logs + if [[ $? -ne 0 ]]; then + echo "Could not create logs directory" + else + echo "Created logs directory" + fi +fi + +## SOURCE CONNECTION SETTINGS +if [ ! -z "$SOURCE_TLS" ] && [ "$SOURCE_TLS" == "--tls" ]]; then + export SOURCE_URI_PREFIX="rediss" +else + export SOURCE_URI_PREFIX="redis" +fi + +if [[ $SOURCE_ENGINE_VERSION -gt 5 ]]; then + #Using 6.0 or later + if [[ ! -z "$SOURCE_PASS" ]]; then + export SOURCE_AUTH_STRING="--user ${SOURCE_USER} --pass ${SOURCE_PASS} " + export RIOT_SOURCE="${SOURCE_URI_PREFIX}://${SOURCE_USER}:${SOURCE_PASS}@${SOURCE_ENDPOINT}:${SOURCE_PORT}" + else + export SOURCE_AUTH_STRING="--user ${SOURCE_USER} " + export RIOT_SOURCE="${SOURCE_URI_PREFIX}://${SOURCE_USER}@${SOURCE_ENDPOINT}:${SOURCE_PORT}" + fi +else + #Using 5.0 or earlier + if [[ ! -z "$SOURCE_PASS" ]]; then + export SOURCE_AUTH_STRING="--pass ${SOURCE_PASS} " + export RIOT_SOURCE="${SOURCE_URI_PREFIX}://${SOURCE_PASS}@${SOURCE_ENDPOINT}:${SOURCE_PORT}" + else + export SOURCE_AUTH_STRING="" + export RIOT_SOURCE="${SOURCE_URI_PREFIX}://${SOURCE_ENDPOINT}:${SOURCE_PORT}" + fi +fi + +## TARGET CONNECTION SETTINGS +if [ ! -z "$TARGET_TLS" ] && [ "$TARGET_TLS" == "--tls" ]; then + export TARGET_URI_PREFIX="rediss" +else + export TARGET_URI_PREFIX="redis" +fi + +if [[ $TARGET_ENGINE_VERSION -gt 5 ]]; then + #Using 6.0 or later + if [[ ! -z "$TARGET_PASS" ]]; then + export TARGET_AUTH_STRING="--user ${TARGET_USER} --pass ${TARGET_PASS} " + export RIOT_TARGET="${TARGET_URI_PREFIX}://${TARGET_USER}:${TARGET_PASS}@${TARGET_ENDPOINT}:${TARGET_PORT}" + else + export TARGET_AUTH_STRING="--user ${TARGET_USER} " + export RIOT_TARGET="${TARGET_URI_PREFIX}://${TARGET_USER}@${TARGET_ENDPOINT}:${TARGET_PORT}" + fi +else + #Using 5.0 or earlier + if [[ ! -z "$TARGET_PASS" ]]; then + export TARGET_AUTH_STRING="--pass ${TARGET_PASS} " + export RIOT_TARGET="${TARGET_URI_PREFIX}://${TARGET_PASS}@${TARGET_ENDPOINT}:${TARGET_PORT}" + else + export TARGET_AUTH_STRING="" + export RIOT_TARGET="${TARGET_URI_PREFIX}://${TARGET_ENDPOINT}:${TARGET_PORT}" + fi +fi + +# How to connect +export GENERATE_URI="--uri=${RIOT_SOURCE}" diff --git a/riot-example/generate b/riot-example/generate new file mode 100755 index 0000000..f977d28 --- /dev/null +++ b/riot-example/generate @@ -0,0 +1,32 @@ +#!/bin/bash + +source env.sh + +cmd_to_run="./bin/riot generate \ +${GLOBAL_DRY_RUN} \ +${GLOBAL_DEBUG} \ +${GLOBAL_PROGRESS} \ +${GLOBAL_LETTUCE_LOG_LEVEL} \ +${GENERATE_KEY_PREFIX} \ +${GENERATE_TTL_EXPIRATION} \ +${GENERATE_CLUSTER} \ +${GLOBAL_SLEEP_MS} \ +${GENERATE_INSECURE_TLS} \ +${GENERATE_RESP_VERSION} \ +${GENERATE_TIMEOUT} \ +${GENERATE_SIZE_KEY} \ +${GENERATE_THREADS} \ +${GENERATE_CLIENT_NAME} \ +${GENERATE_KEY_COUNT} \ +${GENERATE_LOG_FILE} \ +${GENERATE_POOL} \ +${GENERATE_BATCH_COUNT} \ +${GENERATE_URI} \ +${GENERATE_DATA_TYPE}" + +echo +echo "Running command:" +echo +echo "$cmd_to_run" +eval "$cmd_to_run" + diff --git a/riot-example/generate.sh b/riot-example/generate.sh deleted file mode 100755 index f91d8d2..0000000 --- a/riot-example/generate.sh +++ /dev/null @@ -1,32 +0,0 @@ -source ./env.sh - -# SOURCE PING -valkey-cli -u ${RIOT_SOURCE} -c --tls PING - -epoch=$(date +%s) - -echo "Generating data..." -cmd_to_run="docker run riotx/riot generate \ - --dry-run \ - --uri=${RIOT_SOURCE} \ - --cluster \ - --insecure \ - --progress=LOG \ - --count=${KEY_COUNT} \ - --log="io.lettuce=INFO" \ - --log-file="generate_$epoch.log" \ - --expiration=${TTL} \ - --keys=1:${KEYSPACE} \ - --keyspace=${KEY_PREFIX} \ - --resp=RESP2 \ - --uri=${RIOT_SOURCE} \ - --string-value=${GENERATE_SIZE_KEY}-${GENERATES_SIZE_KEY} \ - --timeout=${CMD_TIMEOUT} \ - --tls \ - --type=STRING " -echo "Running command:" -echo "$cmd_to_run" - -eval "$cmd_to_run" - - diff --git a/riot-example/replicate b/riot-example/replicate new file mode 100755 index 0000000..5805267 --- /dev/null +++ b/riot-example/replicate @@ -0,0 +1,37 @@ +#!/bin/bash + +source ./env.sh + +cmd_to_run="./bin/riot replicate \ +${GLOBAL_DRY_RUN} \ +${GLOBAL_DEBUG} \ +${GLOBAL_LETTUCE_LOG_LEVEL} \ +${GLOBAL_BATCH_COUNT} \ +${GLOBAL_PROGRESS} \ +${GLOBAL_THREADS} \ +${GLOBAL_SLEEP_MS} \ +${REPLICATE_LOG_KEYS} \ +${REPLICATE_LOG_FILE} \ +${REPLICATE_MEM_LIMIT} \ +${REPLICATE_READ_QUEUE} \ +${REPLICATE_FLUSH_INTERVAL} \ +${REPLICATE_SCAN_COUNT} \ +${REPLICATE_READ_FROM} \ +${REPLICATE_PIPELINE_READ} \ +${REPLICATE_SOURCE_CLUSTER} \ +${REPLICATE_TARGET_CLUSTER} \ +${REPLICATE_MODE} \ +${REPLICATE_READ_THREADS} \ +${REPLICATE_SOURCE_CLIENT_NAME} \ +${REPLICATE_TARGET_CLIENT_NAME} \ +${REPLICATE_READ_RETRY} \ +${REPLICATE_SOURCE_POOL} \ +${REPLICATE_TARGET_POOL} \ +${RIOT_SOURCE} \ +${RIOT_TARGET}" + +echo "Running command:" +echo +echo "$cmd_to_run" +echo +eval "$cmd_to_run" diff --git a/riot-example/replicate.sh b/riot-example/replicate.sh deleted file mode 100755 index ad1e61d..0000000 --- a/riot-example/replicate.sh +++ /dev/null @@ -1,47 +0,0 @@ -source ./env.sh - -# SOURCE PING -valkey-cli -u ${RIOT_SOURCE} -c --tls PING - -# TARGET PING -valkey-cli -u ${RIOT_TARGET} -c --tls PING - -EPOCH_VALUE=$(date +%s) - -cmd_to_run="docker run riotx/riot replicate \ - --dry-run \ - --info \ - --progress=LOG \ - --log=io.lettuce=INFO \ - --source-cluster \ - --target-cluster \ - --compare="NONE" \ - --event-queue=${EVENT_QUEUE} \ - --flush-interval=${FLUSH_INTERVAL} \ - --idle-timeout=${IDLE_TIMEOUT} \ - --key-pattern="${KEY_PREFIX}:*" \ - --key-type="STRING" \ - --log-keys \ - --log-file="replicate_$epoch.log" \ - --mem-limit="${MEM_LIMIT}" \ - --mode="LIVE" \ - --read-batch=${READ_BATCH} \ - --read-from="ANY" \ - --read-pool=${READ_POOL} \ - --read-queue=${READ_QUEUE} \ - --read-retry=${READ_RETRY} \ - --read-threads=${READ_THREADS} \ - --scan-count=${SCAN_COUNT} \ - --target-resp=${RESP_PROTOCOL} \ - --write-pool=${WRITE_POOL} \ - --log-file=${LOG_FILE} \ - --log-time \ - --retry="LIMIT" \ - --retry-limit=2 \ - --threads=1 \ - ${RIOT_SOURCE} ${RIOT_TARGET}" - -echo "Running command:" -echo "$cmd_to_run" - -eval "$cmd_to_run" diff --git a/riot-example/userdata.sh b/riot-example/userdata.sh new file mode 100644 index 0000000..2e0b87e --- /dev/null +++ b/riot-example/userdata.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +dnf remove update-motd -y +rm -rf /etc/motd +rm -rf /etc/motd.d +rm /usr/lib/motd +rm -rf /usr/lib/motd.d + +# SETUP MY TOOLS + +dnf -y install htop wget gcc make git telnet python3.11 java-22-amazon-corretto-devel + +## Setup BTOP - not available via dnf or amazon-linux-extras +mkdir /home/ec2-user/bin +wget https://github.com/aristocratos/btop/releases/download/v1.4.0/btop-x86_64-linux-musl.tbz +tar xjf btop-x86_64-linux-musl.tbz +mv btop/bin/btop /home/ec2-user/bin +rm -rf btop-x86_64-linux-musl.tbz btop +chown -R ec2-user /home/ec2-user/bin +chmod -R 755 /home/ec2-user/bin + +#DOWNLOAD REDIS + +wget --quiet https://download.redis.io/releases/redis-5.0.10.tar.gz +tar xzf redis-5.0.10.tar.gz +rm redis-5.0.10.tar.gz +mv redis-5.0.10 /home/ec2-user +ln -s /home/ec2-user/redis-5.0.10 /home/ec2-user/redis +echo "export PATH=\$PATH:/home/ec2-user/redis/src" >> /home/ec2-user/.bashrc +cd /home/ec2-user/redis/deps +make --quiet hiredis jemalloc linenoise lua geohash-int > /dev/null 2>&1 +cd /home/ec2-user/redis +make --quiet > /dev/null 2>&1 +chown -R ec2-user /home/ec2-user/redis* + +# Update Python, PIP. Install lolcat for banner + +sudo -u ec2-user -i bash -c '\ +python3 -m ensurepip --upgrade; \ +python3 -m pip install -q --upgrade pip; \ +pip3 install -q --upgrade pip; \ +pip3 install -q lolcat; \ +pip3 install -q pyfiglet;' + +# SETUP MY BASH CUSTOMIZATIONS AND PRETTIFY WITH LOLCAT + +cat << 'EOF' >> /home/ec2-user/.bashrc + +get_token() { + curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" + echo +} + +ec2name() { + curl -s -H "X-aws-ec2-metadata-token: `get_token`" http://169.254.169.254/latest/meta-data/tags/instance/Name + echo +} + +refresh_banner() { + echo > /home/ec2-user/.banner.txt + pyfiglet -f ansi_shadow $(ec2name) >> /home/ec2-user/.banner.txt +} + +alias python3='python3.11' +alias python='python3.11' +alias c='cd ..' +set -o vi + +refresh_banner + +export PATH=/home/ec2-user/bin:$PATH +export PATH=/home/ec2-user/.local/bin:$PATH + +## Always keep lolcat after this check and at the end or it messes up SCP +[[ $- == *i* ]] || return +lolcat /home/ec2-user/.banner.txt + +EOF + + +cat << 'EOF' >> /home/ec2-user/redis/primary.conf +port 6379 +cluster-enabled yes +cluster-config-file cluster.conf +cluster-node-timeout 5000 +appendonly yes +protected-mode no + +EOF + +chown ec2-user /home/ec2-user/redis/primary.conf + +cat << 'EOF' >> /home/ec2-user/redis/replica.conf +replica-read-only yes +port 6379 +cluster-enabled yes +cluster-config-file cluster.conf +cluster-node-timeout 5000 +appendonly yes +protected-mode no + +EOF + +chown ec2-user /home/ec2-user/redis/replica.conf + +chown ec2-user /home/ec2-user/.bashrc /home/ec2-user/.banner.txt