Use auto-arima to forecast live data

Backend/components/component_router.py

@@ -1,7 +1,8 @@
 from fastapi import APIRouter
-from . import graph_api, record_data
+from . import graph_api, record_data, forecast
 
 router = APIRouter()
 
 router.include_router(graph_api.router)
-router.include_router(record_data.router)
+router.include_router(record_data.router)
+router.include_router(forecast.router)

Backend/components/forecast.py

@@ -0,0 +1,52 @@
+from fastapi import APIRouter
+from core import db
+import config
+from statsmodels.tsa.arima.model import ARIMA
+from pmdarima.arima import auto_arima
+import pandas as pd
+import time
+import asyncio
+from concurrent.futures import ProcessPoolExecutor
+
+router = APIRouter()
+
+def train_arima(df, forecast_step):
+    # Train the model
+    model = auto_arima(df, seasonal=False, m=12)
+    # Forecast the future data
+    forecast = model.predict(n_periods=forecast_step)
+    return forecast
+
+@router.get("/forecast")
+async def get_forecast(data: str, start_time: int = 0, end_time: int = 0, forecast_step: int = 0):
+    '''Using ARIMA to predict the future data on selected dataset
+    :param data: str: dataset name
+    :param start_time: int: start time of the training data
+    :param end_time: int: end time of the training data
+    :param forecast_step: int: the time to forecast
+    '''
+    # If time is not specified, use current time as end time and 5 min ago as start time for forecast 1 min
+    if end_time == 0:
+        end_time = round(time.time() * 1000)
+    if start_time == 0:
+        start_time = end_time - 300000
+    if forecast_step == 0:
+        forecast_step = 100
+
+    # Query the data
+    df = await db.query([data], start_time, end_time, ['avg'], (end_time - start_time) // 60)
+
+    # relabel the column to 'x','y'
+    df.columns = ['y']
+    # Spawn a new async task to train ARIMA in a separate process
+    loop = asyncio.get_event_loop()
+    with ProcessPoolExecutor() as pool:
+        future = loop.run_in_executor(pool, train_arima, df, forecast_step)
+        forecast = await future
+
+    # relabel the index to start from 1 to length
+    forecast.reset_index(drop=True, inplace=True)
+    forecast.index = (forecast.index * (end_time - start_time) // 60) + end_time
+    # create json response
+    response = [{'x': int(forecast.index[i]), 'y': forecast[forecast.index[i]]} for i in range(len(forecast))]
+    return {'response': {data: response}}

Backend/core/comms.py

@@ -32,6 +32,9 @@ def unpack_data(data):
     unpacked_data = struct.unpack(format_string, data)
     for i in range(len(properties)):
         fields[properties[i]] = unpacked_data[i]
+        # if the data is -inf or inf, we set it to -10000
+        if fields[properties[i]] == float('inf') or fields[properties[i]] == float('-inf'):
+            fields[properties[i]] = -10000
     return fields
 
 

Frontend/src/Components/Graph/CustomGraph.js

@@ -21,6 +21,7 @@ import {
   Text,
   useColorMode,
   useDisclosure,
+  useInterval,
 } from "@chakra-ui/react";
 import {
   Chart as ChartJS,
@@ -71,7 +72,7 @@ for (const category of GraphData.Output) {
 }
 
 // the options fed into the graph object, save regardless of datasets
-function getOptions(now, secondsRetained, colorMode, optionInfo, extremes) {
+function getOptions(now, secondsRetained, colorMode, optionInfo, extremes, xaxisbounds) {
   const gridColor = getColor("grid", colorMode);
   const gridBorderColor = getColor("gridBorder", colorMode);
   const ticksColor = getColor("ticks", colorMode);
@@ -125,12 +126,22 @@ function getOptions(now, secondsRetained, colorMode, optionInfo, extremes) {
           label: (item) => {
             // custom dataset label
             // name: value unit
+            if (item.dataset.key.indexOf("_forcast") !== -1) {
+              return `${item.dataset.label}: ${item.formattedValue}`;
+            }
             return `${item.dataset.label}: ${item.formattedValue} ${optionInfo[item.dataset.key].unit}`;
           },
           labelColor: (item) => {
-            return {
-              borderColor: optionInfo[item.dataset.key].borderColor,
-              backgroundColor: optionInfo[item.dataset.key].backgroundColor
+            if (item.dataset.key.indexOf("_forcast") !== -1) {
+              return {
+                borderColor: "red",
+                backgroundColor: "rgba(255, 0, 0, 0.5)"
+              }
+            } else {
+              return {
+                borderColor: optionInfo[item.dataset.key].borderColor,
+                backgroundColor: optionInfo[item.dataset.key].backgroundColor
+              }
             }
           },
         },
@@ -157,9 +168,9 @@ function getOptions(now, secondsRetained, colorMode, optionInfo, extremes) {
           borderWidth: 2,
         },
 
-        // show the last secondsRetained seconds
-        max: DateTime.fromMillis(Math.floor(now/1000) * 1000).toISO(),
-        min: DateTime.fromMillis((Math.floor(now/1000) - secondsRetained) * 1000).toISO(),
+        // round to the nearest second
+        max: Math.round(xaxisbounds.max/1000)*1000,
+        min: Math.round(xaxisbounds.min/1000)*1000,
       },
       y: {
         suggestedMin: extremes[0],
@@ -200,9 +211,10 @@ function getOptions(now, secondsRetained, colorMode, optionInfo, extremes) {
  * @constructor
  */
 function Graph(props) {
-  const { querylist, histLen, colorMode, optionInfo, extremes } = props;
+  const { querylist, histLen, colorMode, optionInfo, extremes, forcastKey } = props;
   // response from the server
   const [data, setData] = useState([]);
+  const [forcastData, setForcastData] = useState([]);
   const [fetchDep, setFetchDep] = useState(true);
 
   const fetchData = useCallback(async () => {
@@ -239,7 +251,19 @@ function Graph(props) {
     }
   }, [querylist, histLen]);
   useEffect(fetchData, [fetchDep]);
-
+
+  // fetch forcast data every 10 seconds
+  useInterval(() => {
+    if (forcastKey) {
+      const now = Date.now();
+      fetch(ROUTES.GET_FORECAST_DATA + `?data=${forcastKey}&start_time=${now - (histLen + 1) * 1000}&end_time=${now}&forecast_step=0`)
+      .then((response) => response.json())
+      .then((data) => {
+        setForcastData(data.response);
+      });
+    }
+  }, 10000);
+  console.log(forcastData);
   // get the latest timestamp in the packet
   let tstamp = 0;
   if ("timestamps" in data) {
@@ -250,6 +274,8 @@ function Graph(props) {
 
   let formattedData = {};
   formattedData['datasets'] = [];
+  let maximum = 0;
+  let minimum = Infinity;
   for(const key in data) {
     if(key !== 'timestamps' && optionInfo[key])
       formattedData['datasets'].push({
@@ -259,8 +285,26 @@ function Graph(props) {
         borderColor: optionInfo[key].borderColor,
         backgroundColor: optionInfo[key].backgroundColor
       });
+      maximum = Math.max(maximum, ...data[key].map((x) => x.x));
+      minimum = Math.min(minimum, ...data[key].map((x) => x.x));
+  }
+  if ('timestamps' in data) {
+    maximum = Math.max(...data['timestamps']);
+    minimum = Math.min(...data['timestamps']);
   }
 
+  if(forcastKey && forcastKey in forcastData) {
+    formattedData['datasets'].push({
+      key: forcastKey + "_forcast",
+      label: optionInfo[forcastKey].label,
+      data: forcastData[forcastKey],
+      borderColor: "red",
+      backgroundColor: "rgba(255, 0, 0, 0.5)"
+    });
+    maximum = Math.max(maximum, ...forcastData[forcastKey].map((x) => x.x));
+    minimum = Math.min(minimum, ...forcastData[forcastKey].map((x) => x.x));
+  }
+  const xbound = {'min': minimum, 'max': maximum};
   return (
       <Line
         data={formattedData}
@@ -269,7 +313,8 @@ function Graph(props) {
           histLen,
           colorMode,
           optionInfo,
-          extremes
+          extremes,
+          xbound
         )}
         parsing="false"
       />
@@ -303,6 +348,7 @@ export default function CustomGraph(props) {
   const [noShowKeys, setNoShowKeys] = useState({});
   // Information about options for styling purposes,
   // reduced and combined so that minimal information is passed to other components
+  const [forcastKey, setForcastKey] = useState(null);
   const [optionInfo, yRange] = useMemo(() => {
     return datasetKeys
         .filter((key) => !noShowKeys[key])
@@ -423,17 +469,29 @@ export default function CustomGraph(props) {
                 return (
                     <Button
                       key={`${key}-show-btn`}
-                      borderWidth={2}
-                      borderColor={packedData[key].borderColor}
+                      borderWidth={3}
+                      borderColor={forcastKey == key ? "red" : packedData[key].borderColor}
                       backgroundColor={noShowKeys[key] ? "transparent" : packedData[key].backgroundColor}
                       textDecoration={noShowKeys[key] ? "line-through" : "none"}
                       flexShrink={0}
                       size="xs"
                       onClick={() => {
-                        setNoShowKeys((oldKeys) => ({
-                          ...oldKeys,
-                          [key]: !oldKeys[key]
-                        }))
+                        // one click forcast, two clicks hide, click again to show
+                        if (forcastKey === key) {
+                          setForcastKey(null);
+                          setNoShowKeys((oldKeys) => ({
+                            ...oldKeys,
+                            [key]: !oldKeys[key]
+                          }));
+                        } else if (noShowKeys[key]) {
+                          setNoShowKeys((oldKeys) => ({
+                            ...oldKeys,
+                            [key]: false
+                          }));
+                        }
+                        else {
+                          setForcastKey(key);
+                        }
                       }}
                     >
                       {key}
@@ -450,6 +508,7 @@ export default function CustomGraph(props) {
               colorMode={colorMode}
               optionInfo={optionInfo}
               extremes={yRange}
+              forcastKey={forcastKey}
             />
           </Center>
         </VStack>

Frontend/src/Components/Shared/misc-constants.js

@@ -8,4 +8,5 @@ export const ROUTES = {
     GET_SINGLE_VALUES: "/single-values",
     GET_GRAPH_DATA: "components/graph",
     GET_PROCESSED_DATA: "components/get-processed-data",
+    GET_FORECAST_DATA: "components/forecast",
 }