Fix(MCOL-4611): mod loses precision on huge narrow decimal (#3473)

Author: AestheticAkhmad

mysql-test/columnstore/bugfixes/mcol_4611.result

@@ -0,0 +1,17 @@
+DROP DATABASE IF EXISTS mcol_4611;
+CREATE DATABASE mcol_4611;
+USE mcol_4611;
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a DECIMAL(18,0)) ENGINE=ColumnStore;
+INSERT INTO t1 VALUES (999999999999999999);
+SELECT a MOD 10000 FROM t1;
+a MOD 10000
+9999
+DROP TABLE IF EXISTS t1;
+CREATE TABLE t1 (a DECIMAL(18,0)) ENGINE=ColumnStore;
+INSERT INTO t1 VALUES (999999999999999999);
+SELECT RAND(9999), RAND(a MOD 10000) FROM t1;
+RAND(9999)	RAND(a MOD 10000)
+0.7361292641015065	0.7361292641015065
+DROP TABLE IF EXISTS t1;
+DROP DATABASE mcol_4611;

mysql-test/columnstore/bugfixes/mcol_4611.test

@@ -0,0 +1,28 @@
+-- source ../include/have_columnstore.inc
+--disable_warnings
+DROP DATABASE IF EXISTS mcol_4611;
+--enable_warnings
+CREATE DATABASE mcol_4611;
+USE mcol_4611;
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+CREATE TABLE t1 (a DECIMAL(18,0)) ENGINE=ColumnStore;
+INSERT INTO t1 VALUES (999999999999999999);
+SELECT a MOD 10000 FROM t1;
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+CREATE TABLE t1 (a DECIMAL(18,0)) ENGINE=ColumnStore;
+INSERT INTO t1 VALUES (999999999999999999);
+SELECT RAND(9999), RAND(a MOD 10000) FROM t1;
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+--disable_warnings
+DROP DATABASE mcol_4611;
+--enable_warnings

utils/funcexp/func_mod.cpp

@@ -106,7 +106,7 @@ IDB_Decimal Func_mod::getDecimalVal(Row& row, FunctionParm& parm, bool& isNull,
   }
 
   IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
-  int64_t value = d.value / pow(10.0, d.scale);
+  int64_t value = d.value / static_cast<int64_t>(pow(10.0, d.scale));
   int lefto = d.value % (int)pow(10.0, d.scale);
 
   int64_t mod = (value % div) * pow(10.0, d.scale) + lefto;

utils/funcexp/functor_real.h

@@ -381,7 +381,7 @@ class Func_mod : public Func_Real
         return static_cast<ModType>(integralRemainder.toTFloat128() + intAndFract.second);
       }
     }
-    int64_t value = d.value / pow(10.0, d.scale);
+    int64_t value = d.value / static_cast<int64_t>(pow(10.0, d.scale));
     return value % div;
   }
 };