Speed and performance improvements

Add compression support, optimize SQL queries, add support for sample resolution to reduce fetched data.

Author: j-broo

mqttdblg/config.ini

@@ -14,9 +14,11 @@ ROOT_TOPIC = solpiplog/pip
 # Subtopic(s) to subscribe to. For all subtopics use "+" or "#". Example: "+".
 SUB_TOPIC = +
 # QOS to use for subscription. 0, 1 or 2.
-QOS = 1
+QOS = 2
 # Days to retain data in database. Use with caution, database may become very large.
 AGE_LIMIT = 30
+# Sample resolution for graph. Increase this to speed up rendering. Load and siplay only every "n"th sample, where n = value below.
+SAMPLE_RESOLUTION = 6
 # Database file name to use for logging data. Database will be created automatically.
 DATABASE_FILE = /etc/mqttdblg/database/mqttdblogger.db
 # Run a vacuum on the database on startup. Good for maintenance but will slow down startup on a large DB.
@@ -30,4 +32,4 @@ DEFAULT_GRAPH = acin
 # Topics to graph, used in the dropdown selection box. JSON formatted, topic first, pretty label after.
 GRAPHED_TOPICS = {"load":"Load (%)", "acoutw":"Load (W)", "heatsinktemp":"Temp (°C)", "acin":"AC In (V)", "acinhz":"AC In (Hz)", "acout":"AC Out (V)", "acouthz":"AC Out (Hz)", "battv":"Battery (V)", "battcappa":"Capacity (%)", "battdischrg":"Discharge (A)"}
 # Page title
-PAGE_TITLE = Historical Data
+PAGE_TITLE = Synapse 3.0V+ History

mqttdblg/mqttdbgrapher.py

@@ -1,5 +1,6 @@
 import configparser
 from flask import Flask, render_template, request
+from flask_compress import Compress
 import sqlite3
 import json
 
@@ -12,16 +13,19 @@
 default_range = int(config.get('config', 'DEFAULT_RANGE'))
 default_graph = config.get('config', 'DEFAULT_GRAPH')
 graphed_topics = json.loads(config.get('config', 'GRAPHED_TOPICS'))
+sample_resolution = int(config.get('config', 'SAMPLE_RESOLUTION'))
 
 app = Flask(__name__)
+compress = Compress(app)
 
 @app.route("/<datasource>.json")
 def load(datasource):
     connection = sqlite3.connect(database_file)
     cursor = connection.cursor()
-    cursor.execute(f"SELECT (strftime('%s', DATETIME(timestamp, 'unixepoch', 'localtime')))*1000, CAST(payload AS DECIMAL) FROM mqtt_log WHERE topic = '{root_topic}/{datasource}' ORDER BY timestamp ASC")
+    cursor.execute(f"SELECT (strftime('%s', DATETIME(timestamp, 'unixepoch', 'localtime')))*1000 as localtimemillis, CAST(payload AS DECIMAL) FROM mqtt_log WHERE topic = '{root_topic}/{datasource}' AND (rowid % {sample_resolution}) = 0 ORDER BY localtimemillis ASC")
     results = cursor.fetchall()
     return json.dumps(results)
+    cursor.execute("PRAGMA OPTIMIZE")
 
 @app.route("/graph")
 def graph():

mqttdblg/mqttdblogger.py

@@ -63,7 +63,7 @@ def main():
     topic TEXT NOT NULL,
     payload TEXT NOT NULL,
     PRIMARY KEY (timestamp, topic));
-    CREATE INDEX IF NOT EXISTS mqtt_log_idx ON mqtt_log(topic, timestamp, payload);
+    CREATE INDEX IF NOT EXISTS mqtt_log_idx ON mqtt_log(topic, timestamp ASC, payload);
     '''
 
     cursor = db_conn.cursor()