diff --git a/tools/node-types.sh b/tools/node-types.sh new file mode 100755 index 0000000..15ffad4 --- /dev/null +++ b/tools/node-types.sh @@ -0,0 +1,5 @@ +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +echo 'Success:' +grep 'Queried' $LOGFILE | grep -o 'type":[0-9]' | tr -d '"' | sort | uniq -c +echo 'Failure:' +grep 'Query failed' $LOGFILE | grep -o 'type":[0-9]' | tr -d '"' | sort | uniq -c diff --git a/tools/node-version.sh b/tools/node-version.sh new file mode 100755 index 0000000..504d14b --- /dev/null +++ b/tools/node-version.sh @@ -0,0 +1,3 @@ +NODE_ID=$1 +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +grep -e 'Queried '$1 $LOGFILE | jq -sr 'map(.info.applicationVersion) | .[]' | sort | uniq diff --git a/tools/node-versions-for-stream.sh b/tools/node-versions-for-stream.sh new file mode 100755 index 0000000..d0a104e --- /dev/null +++ b/tools/node-versions-for-stream.sh @@ -0,0 +1,4 @@ +STREAM_ID=$1 +RUN_ID=$2 +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +grep $RUN_ID $LOGFILE | grep 'Queried' | grep $STREAM_ID | jq -s 'map(.info.applicationVersion) | group_by(.) | map({(.[0]): length}) | add' diff --git a/tools/nodes.sh b/tools/nodes.sh new file mode 100755 index 0000000..e6cec59 --- /dev/null +++ b/tools/nodes.sh @@ -0,0 +1,3 @@ +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +grep 'Queried ' $LOGFILE | jq -sr 'map("success " + .runId + " " + .info.peerDescriptor.nodeId) | .[]' +grep 'Query failed' $LOGFILE | jq -sr 'map("failure " + .runId + " " + .peerDescriptor.nodeId) | .[]' diff --git a/tools/peer-descriptors.sh b/tools/peer-descriptors.sh new file mode 100755 index 0000000..2bddaa6 --- /dev/null +++ b/tools/peer-descriptors.sh @@ -0,0 +1,5 @@ +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +( +grep 'Queried '$1 $LOGFILE | jq -s 'map({peerDescriptor: .info.peerDescriptor, runId, time: (.time/1000|todate), success: true}) | .[]' +grep 'Query failed '$1 $LOGFILE | jq -s 'map({peerDescriptor: .peerDescriptor, runId, time: (.time/1000|todate), success: false}) | .[]' +) | jq -s diff --git a/tools/stats.sh b/tools/stats.sh new file mode 100755 index 0000000..c35e54b --- /dev/null +++ b/tools/stats.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +DEFAULT_LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +LOGFILE="${1:-$DEFAULT_LOGFILE}" + +printStats() { + local RUN_ID=$1 + echo "Run $RUN_ID" + + local ITEMS=$(awk -v run_id="$RUN_ID" '$0 ~ run_id {print}' $LOGFILE) + SUCCESS_ITEMS=$(jq -s 'map(select(.msg | contains("Queried ")))' <<< $ITEMS) + FAILURE_ITEMS=$(jq -s 'map(select(.msg | contains("Query failed ")))' <<< $ITEMS) + SUCCESS_DESCRIPTORS=$(jq 'map(.info.peerDescriptor)' <<< $SUCCESS_ITEMS) + FAILURE_DESCRIPTORS=$(jq 'map(.peerDescriptor)' <<< $FAILURE_ITEMS) + + printCategoryStats 'All' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '.' + printCategoryStats 'NodeJS-all' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '.type == 0' + printCategoryStats 'NodeJS-WebSocket' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '(.type == 0) and (.websocket)' + printCategoryStats 'NodeJS-AutoCertified' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '(.type == 0) and ((.websocket.host // "") | contains("streamr-nodes.xyz"))' + printCategoryStats 'NodeJS-non-WebSocket' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" '(.type == 0) and (.websocket | not)' + printCategoryStats 'Browser' "$SUCCESS_DESCRIPTORS" "$FAILURE_DESCRIPTORS" ".type == 1" + + echo "Errors:" + jq -r --monochrome-output 'map(.err.code) | group_by(.) | map({(.[0]): length}) | add' <<< $FAILURE_ITEMS + + echo "Versions:" + jq -r --monochrome-output 'map(.info.applicationVersion) | group_by(.) | map({(.[0]): length}) | add' <<< $SUCCESS_ITEMS + + echo -e "\n\n" +} + +printCategoryStats() { + local CATEGORY=$1 + local SUCCESS_DESCRIPTORS=$2 + local FAILURE_DESCRIPTORS=$3 + local FILTER=$4 + + local SUCCESS_COUNT=$(jq 'length' <<< $SUCCESS_DESCRIPTORS) + local FAILURE_COUNT=$(jq 'length' <<< $FAILURE_DESCRIPTORS) + local TOTAL_COUNT=$((SUCCESS_COUNT + FAILURE_COUNT)) + local FAILURE_PERCENTAGE=$((FAILURE_COUNT * 100 / TOTAL_COUNT)) + + echo "$CATEGORY:" + local CATEGORY_SUCCESS_COUNT=$(jq "map(select($FILTER)) | length" <<< $SUCCESS_DESCRIPTORS) + local CATEGORY_FAILURE_COUNT=$(jq "map(select($FILTER)) | length" <<< $FAILURE_DESCRIPTORS) + local CATEGORY_TOTAL_COUNT=$((CATEGORY_SUCCESS_COUNT + CATEGORY_FAILURE_COUNT)) + if [ "$CATEGORY_TOTAL_COUNT" -eq 0 ]; then + local CATEGORY_FAILURE_PERCENTAGE=0 + else + local CATEGORY_FAILURE_PERCENTAGE=$((CATEGORY_FAILURE_COUNT * 100 / CATEGORY_TOTAL_COUNT)) + fi + echo "- total=$CATEGORY_TOTAL_COUNT success=$CATEGORY_SUCCESS_COUNT failure=$CATEGORY_FAILURE_COUNT ($CATEGORY_FAILURE_PERCENTAGE%)" +} + +echo -e "\n\n" + +grep 'Queried ' $LOGFILE | grep -o '"runId":"full-[0-9]*' | cut -c 10- | sort -u | while read -r RUN_ID; do + printStats "$RUN_ID" +done diff --git a/tools/topologies.sh b/tools/topologies.sh new file mode 100755 index 0000000..549161f --- /dev/null +++ b/tools/topologies.sh @@ -0,0 +1,2 @@ +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log +grep 'Topology:' $LOGFILE | grep '"runId":"full' | jq -s '.[] | {msg, runId, time: (.time/1000) | todate}' diff --git a/tools/versions.sh b/tools/versions.sh new file mode 100755 index 0000000..bf8f7c9 --- /dev/null +++ b/tools/versions.sh @@ -0,0 +1,8 @@ +if [ $# -eq 0 ]; then + echo "Error: No argument (runId) passed. Please provide an argument." + exit 1 +fi + +LOGFILE=~/.pm2/logs/stream-metrics-index-crawler-out.log + +grep $1 $LOGFILE | grep 'Queried' | jq .info.applicationVersion | sort | uniq -c | awk '{print $2, $1}'