feat: [NET-1417] Add utility scripts (#31)

These Bash scripts extract information from the Crawler logs, e.g.
- success/failure rates and version stats: `stats.sh`
- version stats for a node: `node-version.sh`
- version stats for a stream: `node-versions-for-stream.sh`
- topology sizes: `topologies.sh`

Author: teogeb

tools/node-types.sh

@@ -0,0 +1,5 @@
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+echo 'Success:'
+grep 'Queried' $LOGFILE | grep -o 'type":[0-9]' | tr -d '"' | sort | uniq -c
+echo 'Failure:'
+grep 'Query failed' $LOGFILE | grep -o 'type":[0-9]' | tr -d '"' | sort | uniq -c

tools/node-version.sh

@@ -0,0 +1,3 @@
+NODE_ID=$1
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+grep -e 'Queried '$1 $LOGFILE | jq -sr 'map(.info.applicationVersion) | .[]' | sort | uniq

tools/node-versions-for-stream.sh

@@ -0,0 +1,4 @@
+STREAM_ID=$1
+RUN_ID=$2
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+grep $RUN_ID $LOGFILE | grep 'Queried' | grep $STREAM_ID | jq -s 'map(.info.applicationVersion) | group_by(.) | map({(.[0]): length}) | add'

tools/nodes.sh

@@ -0,0 +1,3 @@
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+grep 'Queried ' $LOGFILE | jq -sr 'map("success " + .runId + " " + .info.peerDescriptor.nodeId) | .[]'
+grep 'Query failed' $LOGFILE | jq -sr 'map("failure " + .runId + " " + .peerDescriptor.nodeId) | .[]'

tools/peer-descriptors.sh

@@ -0,0 +1,5 @@
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+(
+grep 'Queried '$1 $LOGFILE | jq -s 'map({peerDescriptor: .info.peerDescriptor, runId, time: (.time/1000|todate), success: true}) | .[]'
+grep 'Query failed '$1 $LOGFILE | jq -s 'map({peerDescriptor: .peerDescriptor, runId, time: (.time/1000|todate), success: false}) | .[]'
+) | jq -s

tools/stats.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+DEFAULT_LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+LOGFILE="${1:-$DEFAULT_LOGFILE}"
+
+printStats() {
+    local RUN_ID=$1
+    echo "Run $RUN_ID"
+
+    local ITEMS=$(awk -v run_id="$RUN_ID" '$0 ~ run_id {print}' $LOGFILE)
+    SUCCESS_ITEMS=$(jq -s 'map(select(.msg | contains("Queried ")))' <<< $ITEMS)
+    FAILURE_ITEMS=$(jq -s 'map(select(.msg | contains("Query failed ")))' <<< $ITEMS)
+    SUCCESS_DESCRIPTORS=$(jq 'map(.info.peerDescriptor)' <<< $SUCCESS_ITEMS)
+    FAILURE_DESCRIPTORS=$(jq 'map(.peerDescriptor)' <<< $FAILURE_ITEMS)
+
+    printCategoryStats 'All' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '.'
+    printCategoryStats 'NodeJS-all' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '.type == 0'
+    printCategoryStats 'NodeJS-WebSocket' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '(.type == 0) and (.websocket)'
+    printCategoryStats 'NodeJS-AutoCertified' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '(.type == 0) and ((.websocket.host // "") | contains("streamr-nodes.xyz"))'
+    printCategoryStats 'NodeJS-non-WebSocket' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '(.type == 0) and (.websocket | not)'
+    printCategoryStats 'Browser' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" ".type == 1"
+
+    echo "Errors:"
+    jq -r --monochrome-output 'map(.err.code) | group_by(.) | map({(.[0]): length}) | add' <<< $FAILURE_ITEMS
+
+    echo "Versions:"
+    jq -r --monochrome-output 'map(.info.applicationVersion) | group_by(.) | map({(.[0]): length}) | add' <<< $SUCCESS_ITEMS
+
+    echo -e "\n\n"
+}
+
+printCategoryStats() {
+    local CATEGORY=$1
+    local SUCCESS_DESCRIPTORS=$2
+    local FAILURE_DESCRIPTORS=$3
+    local FILTER=$4
+
+    local SUCCESS_COUNT=$(jq 'length' <<< $SUCCESS_DESCRIPTORS)
+    local FAILURE_COUNT=$(jq 'length' <<< $FAILURE_DESCRIPTORS)
+    local TOTAL_COUNT=$((SUCCESS_COUNT + FAILURE_COUNT))
+    local FAILURE_PERCENTAGE=$((FAILURE_COUNT * 100 / TOTAL_COUNT))
+
+    echo "$CATEGORY:"
+    local CATEGORY_SUCCESS_COUNT=$(jq "map(select($FILTER)) | length" <<< $SUCCESS_DESCRIPTORS)
+    local CATEGORY_FAILURE_COUNT=$(jq "map(select($FILTER)) | length" <<< $FAILURE_DESCRIPTORS)
+    local CATEGORY_TOTAL_COUNT=$((CATEGORY_SUCCESS_COUNT + CATEGORY_FAILURE_COUNT))
+    if [ "$CATEGORY_TOTAL_COUNT" -eq 0 ]; then
+        local CATEGORY_FAILURE_PERCENTAGE=0
+    else
+        local CATEGORY_FAILURE_PERCENTAGE=$((CATEGORY_FAILURE_COUNT * 100 / CATEGORY_TOTAL_COUNT))
+    fi
+    echo "- total=$CATEGORY_TOTAL_COUNT success=$CATEGORY_SUCCESS_COUNT failure=$CATEGORY_FAILURE_COUNT ($CATEGORY_FAILURE_PERCENTAGE%)"
+}
+
+echo -e "\n\n"
+
+grep 'Queried ' $LOGFILE | grep -o '"runId":"full-[0-9]*' | cut -c 10- | sort -u | while read -r RUN_ID; do
+    printStats "$RUN_ID"
+done

tools/topologies.sh

@@ -0,0 +1,2 @@
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+grep 'Topology:' $LOGFILE  | grep '"runId":"full' | jq -s '.[] | {msg, runId, time: (.time/1000) | todate}'

tools/versions.sh

@@ -0,0 +1,8 @@
+if [ $# -eq 0 ]; then
+  echo "Error: No argument (runId) passed. Please provide an argument."
+  exit 1
+fi
+
+LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log
+
+grep $1 $LOGFILE | grep 'Queried' | jq .info.applicationVersion | sort | uniq -c | awk '{print $2, $1}'