chore: Update python, major deps, fix plotting (#234)

Author: tobiasmllr

.github/workflows/analysis.yml

@@ -56,7 +56,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-           python-version: "3.12"
+           python-version: "3.13"
       - name: Install requirements
         run: |
             pip install .

frontend/Dockerfile.app

@@ -1,6 +1,6 @@
 ### BUILDER IMAGE ###
 # syntax=docker/dockerfile:1
-FROM python:3.12-slim AS BUILDER
+FROM python:3.13-slim AS BUILDER
 
 # set environment variables
 ENV LANG=C.UTF-8
@@ -23,7 +23,7 @@ COPY rctool/ ./rctool
 RUN pip install . --no-cache-dir
 
 ### APP IMAGE ###
-FROM python:3.12-slim AS APP
+FROM python:3.13-slim AS APP
 WORKDIR /app
 
 # set environment variables

frontend/Dockerfile.nginx

@@ -1,6 +1,6 @@
 ### BUILDER IMAGE ###
 # syntax=docker/dockerfile:1
-FROM python:3.12-slim AS BUILDER
+FROM python:3.13-slim AS BUILDER
 WORKDIR /app
 
 # set environment variables

frontend/poetry.lock

@@ -8,7 +8,7 @@ authors = [
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = ">=3.12,<3.13"
+python = ">=3.13,<3.14"
 django = "^5.0.3"
 scikit-learn = "^1.4.1"
 scipy = "^1.12.0"
@@ -19,7 +19,7 @@ matplotlib = "^3.8.3"
 gunicorn = "^23.0.0"
 pip = "^24.0"
 setuptools = "^75.0.0"
-numpy = ">1.26.0,<2.0.0"
+numpy = "^2.2.2"
 
 [build-system]
 requires = ["poetry-core"]

frontend/pyproject.toml

@@ -37,9 +37,9 @@ def fit_linear_model(df, offset, label, weighted=None, intersect_points=None, *a
             params.add(constraint_name, expr=constraint_expression)
 
     result = plm.fit(df_data["Q"], params, x=df_data["H0"])
-
-    unw_const = result.best_values["amplitude"]
-    unw_exp = result.best_values["exponent"]
+    
+    unw_const = float(result.best_values["amplitude"])
+    unw_exp = float(result.best_values["exponent"])
     unw_best = result.best_fit
 
     unw_residual = list(
@@ -57,8 +57,8 @@ def fit_linear_model(df, offset, label, weighted=None, intersect_points=None, *a
 
     # try weighting
     result = plm.fit(df_data["Q"], params, x=df_data["H0"], weights=df_data["W"])
-    wgt_const = result.best_values["amplitude"]
-    wgt_exp = result.best_values["exponent"]
+    wgt_const = float(result.best_values["amplitude"])
+    wgt_exp = float(result.best_values["exponent"])
     wgt_best = result.best_fit
     wgt_sigs = result.eval_uncertainty(sigma=2)
     wgt_residual = list(
@@ -111,7 +111,7 @@ def fit_linear_model(df, offset, label, weighted=None, intersect_points=None, *a
             "mape": wgt_mape,
         }
         wgt_data = [
-            [a, b, c] for a, b, c in zip(df_data["H"].tolist(), wgt_best, wgt_residual)
+            [float(a), float(b), float(c)] for a, b, c in zip(df_data["H"].tolist(), wgt_best, wgt_residual)
         ]
         mdl_data = [{"label": label, "data": wgt_data}]
     else:
@@ -126,7 +126,7 @@ def fit_linear_model(df, offset, label, weighted=None, intersect_points=None, *a
             "mape": unw_mape,
         }
         unw_data = [
-            [a, b, c] for a, b, c in zip(df_data["H"].tolist(), unw_best, unw_residual)
+            [float(a), float(b), float(c)] for a, b, c in zip(df_data["H"].tolist(), unw_best, unw_residual)
         ]
         mdl_data = [{"label": label, "data": unw_data}]
 

frontend/rctool/functions/fit_linear_model.py

@@ -195,14 +195,14 @@
             var fieldDataFiltered = rawFieldData.filter(item => ( item[0] >= segmentData[0][0] && item[0] <= segmentData[1][0]));
 
 
-            // 3.2 Calculate model data from new rc equation, collect and prepaire output
+            // 3.2 Calculate model data from new rc equation, collect and prepare output
             var stage = fieldDataFiltered.map( item => (item[0]));
             var q_field = fieldDataFiltered.map( item => (item[1]));
             var q_model = stage.map(s => C * (s - offset) ** slope);
             // Calculate statistical parameters; residuals
             var newResiduals = fieldDataFiltered.map(item => ( 100 * (C * (item[0] - offset) ** slope - item[1]) / item[1] ) );
 
-            // Collect and prepaire output
+            // Collect and prepare output
             var newSegData = [];
             for (let t = 0; t < q_model.length; t++) {
                 newSegData.push([stage[t], q_model[t], newResiduals[t]]);
@@ -214,7 +214,13 @@
             });
 
             // 3.3 Update residualData
-            residualData[idx+1].data = newSegData;
+            try {
+                residualData[idx+1].data = newSegData;
+            } catch {
+                // this is likely a manual segment, skip it
+                continue
+            }
+
             // add boundary points to segment data (for plotting on rcChart)
             newSegData.unshift([segmentData[0][0], segmentData[0][1],  0]);
             newSegData.push([segmentData[1][0], segmentData[1][1], 0]);
@@ -361,6 +367,13 @@
             offset_thisline = offsetData[rc_trueidx]
             param = rcDict.parameters[rc_trueidx]
 
+            // in case a manual segment is defined, skip it
+            if (param == undefined) {
+                // TODO: this breaks interpolation for the manual segment
+                // FIX: get parameters from the correct data, rcDict shouldn't be used here
+                continue
+            }
+
             // create an array that will be used to calculate the line point positions
             const arr = [];
             for (let i = 0; i < 1; i += 1/interpolation_points)
@@ -987,13 +1000,13 @@
         // Get filtered field data within segment bounds (used with to calculate RMSE and residuals)
         var rawFieldData = rcData[0]['data'];
         var fieldDataFiltered = rawFieldData.filter(item => ( item[0] >= compStartBounds && item[0] <= compEndBounds));
-        // Calculate model data from new rc equation, collect and prepaire output
+        // Calculate model data from new rc equation, collect and prepare output
         var stage = fieldDataFiltered.map( item => (item[0]));
         var q_field_compare = fieldDataFiltered.map( item => (item[1]));
         var q_model_compare = stage.map(s => compConst * (s - compOffset) ** compExp);
         // Calculate statistical parameters; residuals
-        var compResiduals = fieldDataFiltered.map(item => ( 100 * (compConst * (item[0] - compOffset) ** compExp - item[1]) / item[1] ) );
-        // Collect and prepaire output
+        var compResiduals = fieldDataFiltered.map(item => ( -100 * (compConst * (item[0] - compOffset) ** compExp - item[1]) / item[1] ) );
+        // Collect and prepare output
         var compData = [];
         for (let t = 0; t < q_model_compare.length; t++) {
             compData.push([stage[t], q_model_compare[t], compResiduals[t]]);
@@ -1007,12 +1020,22 @@
         compRcData = [...compData];
         compRcData.unshift([lowerH, lowerQ, 0])
         compRcData.push([upperH, upperQ, 0])
+
+        // 2.9 interpolate data for residual chart
+        var stageInterpolated = [];
+        for (let i = 0; i < interpolation_points; i++) {
+            stageInterpolated.push(lowerH + i * (upperH - lowerH) / interpolation_points);
+        }
+        var q_model_interpolated = stageInterpolated.map(s => compConst * (s - compOffset) ** compExp);
+
         // 3. Calculate statistical parameters; RMSE
         stats_compare = calculate_stats(q_field_compare, q_model_compare)
 
-        // 4. Prepaire output
-        var compChartData = compRcData.map(item => ({ x: item[1], y: item[0] }));
+        // 4. Prepare output
+        var compChartData = stageInterpolated.map((item, index) => ({ x: q_model_interpolated[index], y: item }));
         var compResChartData = compData.map(item => ({ x: item[2], y: item[0] }));
+        // 4.1 Sort by x value
+        compChartData.sort((a, b) => a.x - b.x);
 
         // 5. If data is calculated, plot on RC chart and residual chart
         if (compChartData.length > 1) {
@@ -1201,6 +1224,11 @@
                             // prevent dragging
                             return false
                         }
+                        
+                        // prevent dragging of manually added comparison curves
+                        if (dataset.backgroundColor === '#DDCC77') {
+                            return false
+                        }
 
                         // hide interpolation line for current RC curve:
                         var datasetLabel = dataset.label

frontend/rctool/templates/rctool/rctool/develop/rctool_develop_js.html

@@ -196,10 +196,4 @@ def test_export_rc_data_form(self):
         )
         # check if the output is a pdf:
         assert response_output.status_code == 200
-        assert response_output["Content-Type"] == "application/pdf"
-
-    def test_numpy_version(self):
-        # numpy version has to be 1 to avoid formatting problems with float numbers output by numpy. Would take a lot of refactoring otherwise...
-        np_version = np.__version__.split(".")[0]
-        print(f"numpy version: {np_version}")
-        assert np_version == "1"
+        assert response_output["Content-Type"] == "application/pdf"