From b82dbc6d49ca58b38744dd8d27a5bcbcd05972f9 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 19 Aug 2025 08:15:30 +0800 Subject: [PATCH 01/20] enh: dumpQueryPlan skip nodesNodeToString. --- source/libs/planner/src/planner.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index f2d8db44bb5c..6b6fb07de60c 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -31,11 +31,13 @@ static int32_t debugPrintNode(SNode* pNode) { static int32_t dumpQueryPlan(SQueryPlan* pPlan) { int32_t code = 0; - char* pStr = NULL; - code = nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); - if (TSDB_CODE_SUCCESS == code) { - planDebugL("QID:0x%" PRIx64 ", Query Plan, JsonPlan: %s", pPlan->queryId, pStr); - taosMemoryFree(pStr); + if (qDebugFlag & DEBUG_DEBUG) { + char* pStr = NULL; + code = nodesNodeToString((SNode*)pPlan, false, &pStr, NULL); + if (TSDB_CODE_SUCCESS == code) { + planDebugL("QID:0x%" PRIx64 ", Query Plan, JsonPlan: %s", pPlan->queryId, pStr); + taosMemoryFree(pStr); + } } return code; } From ed22e44170debedb8983f44a01b38fd4640c2a36 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 19 Aug 2025 08:17:15 +0800 Subject: [PATCH 02/20] enh: make tbname in use opt. --- source/libs/executor/src/executil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 9784015acd97..b7de23a24984 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1419,8 +1419,8 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S return -1; } - if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_COLUMN && - ((SColumnNode*)pNode->pLeft)->colType == COLUMN_TYPE_TBNAME) && + if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_FUNCTION && + ((SFunctionNode*)pNode->pLeft)->funcType == FUNCTION_TYPE_TBNAME) && (pNode->pRight != NULL && nodeType(pNode->pRight) == QUERY_NODE_NODE_LIST)) { SNodeListNode* pList = (SNodeListNode*)pNode->pRight; From 373459136cb182ae88c7c0feed792c40c7a5b095 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 19 Aug 2025 08:18:23 +0800 Subject: [PATCH 03/20] feat: add query tools. --- lastTest/README.md | 147 ++++++++++++++++ lastTest/taos_query_a_simple | Bin 0 -> 21568 bytes lastTest/taos_query_a_simple.c | 301 +++++++++++++++++++++++++++++++++ 3 files changed, 448 insertions(+) create mode 100644 lastTest/README.md create mode 100755 lastTest/taos_query_a_simple create mode 100644 lastTest/taos_query_a_simple.c diff --git a/lastTest/README.md b/lastTest/README.md new file mode 100644 index 000000000000..eacb670ce0e9 --- /dev/null +++ b/lastTest/README.md @@ -0,0 +1,147 @@ +# TDengine Query Test Tool + +A C program to test TDengine's query functionality with multi-threading and performance analysis. + +## Features + +- Multi-threaded query execution using `taos_query_a` +- Performance testing with QPS statistics +- Support for different query patterns +- Thread-safe result collection +- Configurable test parameters + +## Usage + +### Compilation + +```bash +make all +``` + +### Basic Usage + +```bash +./taos_query_a_simple +``` + +### Test Modes + +#### Mode 1: Table Name IN Query Test + +```bash +./taos_query_a_simple 1 +``` + +**SQL Query:** + +```sql +SELECT tbname, last(*) +FROM test.meters +WHERE tbname IN ('d1','d2','d3','d4','d5','d6','d7','d8','d9','d10', + 'd11','d12','d13','d14','d15','d16','d17','d18','d19','d20', + 'd21','d22','d23','d24','d25','d26','d27','d28','d29','d30', + 'd31','d32','d33','d34','d35','d36','d37','d38','d39','d40', + 'd41','d42','d43','d44','d45','d46','d47','d48','d49','d50', + 'd51','d52','d53','d54','d55','d56','d57','d58','d59','d60', + 'd61','d62','d63','d64','d65','d66','d67','d68','d69','d70', + 'd71','d72','d73','d74','d75','d76','d77','d78','d79','d80', + 'd81','d82','d83','d84','d85','d86','d87','d88','d89','d90', + 'd91','d92','d93','d94','d95','d96','d97','d98','d99','d100') +PARTITION BY tbname; +``` + +**Parameters:** + +- **Thread Count**: 16 threads +- **Queries per Thread**: 100 queries +- **Max Concurrent Queries**: 5 per thread +- **QPS Rate Multiplier**: 100x +- **Purpose**: Tests query performance with `tbname IN` condition filtering multiple tables + +#### Mode 2: Sub-table Query Test + +```bash +./taos_query_a_simple 2 +``` + +**SQL Query:** + +```sql +SELECT last(ts, r32) FROM test.d1; +``` + +**Parameters:** + +- **Thread Count**: 16 threads +- **Queries per Thread**: 10,000 queries +- **Max Concurrent Queries**: 5 per thread +- **QPS Rate Multiplier**: 1x +- **Purpose**: Tests query performance on a single sub-table + +## Connection Parameters + +Both modes use the same connection settings: + +- **Library Path**: `../../debug/build/lib/libtaos.so` +- **Host**: `127.0.0.1` +- **User**: `root` +- **Password**: `taosdata` +- **Database**: `test` + +## Performance Metrics + +The program measures and reports: + +- **Individual Thread QPS**: Performance of each thread +- **Total QPS**: Sum of all threads' QPS (multiplied by rate factor) +- **Query Execution Time**: Duration for completing all queries +- **Concurrent Query Management**: Controls max concurrent queries per thread + +## Examples + +```bash +# Run table name IN query test (Mode 1) +./taos_query_a_simple 1 + +# Run sub-table query test (Mode 2) +./taos_query_a_simple 2 +``` + +## Test Scenarios + +### Mode 1: Table Name IN Query Testing + +Tests query performance with `tbname IN` conditions: + +- Queries 100 specific tables using `tbname IN` clause +- Uses `PARTITION BY tbname` for grouping +- Tests filtering performance on multiple tables + +### Mode 2: Sub-table Query Testing + +Tests standard query performance on sub-tables: + +- Simple aggregation query on a single sub-table (`test.d1`) +- High query volume (10,000 queries per thread) +- Baseline performance measurement for sub-table queries + +## Output + +The program displays: + +- Individual thread performance metrics +- Overall system performance statistics +- Query execution status and timing +- Error messages if any issues occur + +## Requirements + +- GCC compiler +- TDengine client library +- Linux system with pthread support +- Running TDengine server +- Proper database connection configuration + +## Configuration + +Make sure your TDengine server is running and accessible before executing the test program. The program will connect to the default TDengine instance unless modified in the source code. diff --git a/lastTest/taos_query_a_simple b/lastTest/taos_query_a_simple new file mode 100755 index 0000000000000000000000000000000000000000..203ad0714f2f2181b5fe37c12fb8fda59cb93bd5 GIT binary patch literal 21568 zcmeHPdvILUdA}=JV2sU5V1qFU+yIVcu-ChK3+zOGthsLGAZ)9M6tZ6JUP)W_A@%`w ziXFj58MbS+CNnhdG|4zqGPpwO1H6eZTYg&g0&D_U`$DzjJ$)%cZGMt$j+P)X>EO>5z({P8osd z(AujBLN+5lp-e$?K3j zC1<1~lvMsCpU|gG=I4}-YHLp3faE#lHsnb`Nh*WF?l;omk5XQjwr}jr6uOZHR`~MZZ?hYshAdFx8z^Ln5v6Bi(YTC;MA&-E;Vn{S%#+Jorx=A3yMR<4fEA`C+o5 zdJ~6iC{TW~ikznSW%!T{`@?k~;Z=mK#OFox$FcU=yqY$oD+|PvQ@ae0SHO2vz~_ML z_&CG;0Org8dlm58E8vY4@FNxQV-@gR1^kaI;LG749Xe+q-_F;rR0TX#0gqI`pQwOe zQ=vYetYCk*0)ABm`~u)tmaTyXPm>jE@HAWT#wIPWW6w_1 zJ!$qwQfV`}XXn;vB5v*p_C_h+{#YU|n-UPF^BTgHRhZFLn7!3lt*hlf!U(BiEyYz# z;Ln4n*0bf2$TC{#e=f1istCHUT62<*f;;9{ zA-dp!1rIEEV8H|bw>Mkoy7v9;app~Wz?(EbpR+=ufvs0U`G+kh4r+ikLF08XtpRm$& zgEBi+XQk-^J3HmF(r!+_^MPtFT~N_}D@_+tv|pv+t*?EGytPeZeaC9<(hDf)f6yCa{igTf5tdsSoR(QK zN^P2Y4}Zq}1&YJ)dVj$nB` zVoe{+cc01o&$=eEW3HV4-Ryz0+QFuI@*3G#m+yWzM>)QJ1?MQT8xh$}H~enSKiWCB z2~HNpqUam_r`?C{hC}Abfm?8&;94zMCFn-pKlkiYHZSUE(_L6;q(i1wU=_;y3+~7K zXUUP>XHh`zXK>~HWv;viHt^3#&DC7axM5jxoK&Y6TgD+{Guuh&&)bi_Rm(4u`K&?%21a0%74jv zO!rWMAq80W9l(4^?>2eei3C50}c$9e8inUFV;f zb=Uf5`rL0n+vnbJ;_h0+#%A5u`Jw!re`b0SrTYx`LHZLPyQ8fCjog6(+FJNNU4E#$ zYN_$oobeTge#D|&!Uc*!wY5wWuY%IF{Em(+dzXh4F<3?6Yu*J_P#}7D%n#-7u zF)^}YtfYKZD3_F!-zq7e7Roo;tZcttQa&P-r%TE&my~}dluwnELnUQQD8E%w#!JdB zq5LaW*7TB~&O`e@C*xkk6@IM^Uy1lPo8JuO2St(j4%~os9}^S$6t7p~n~M|EOMcLRq6*_E zM`l4)qsEMtm$I_`r29|`>z6cZv+6kg8wut+>S>nNiwanL4cO5@M%{!^68X(}t(^BIJM0VpeP_qulH8M>)Y#k;9T4`ep7V`mSyz?5-pCJ= z+Q_4`XvKdJ8xq#|`Ws|DN-M|IqqyBvrrEK@Y(8GN1b)j6r-?sOb2l^n4(1@)(skv<;gGF2{d4wt~8X;oTh>k68ZTdq!cY}Rpn0i zi?3n5z-qVN$k!N+)Xyirdo}VG7l)CW906WU_`?Dpb>fEvjzOA6YvErb-#mv&o*$u( z$Uj7W5u6>l86LIwV>{FuwQv3sD^7!ve}cwJWzN^_obP}mmV=R+W$c>q!b`x$hX;@e zHE-%q(QTdiWgActBfk^#wEL)$&m4ho4{shN3ship@kKQB6mCdQX&G|!t3JX{eD_M| zm76o7;KCcoU(BwZxUtQ1Ngs9}ueyfoq|oQ8&Si)CoHBA-7Axm%J{LjiF+-QC`?NgO zI={$;d}^n!KUW4=7s%6Od@j>G&+^=smFF@vM*J8@TXbjT&cT~NuVCaoXXLt2E9yGi z{TeO^7?iv&yX$@rf74*gZRt8kk+Nf`9O=Kjf7EcH59q-lhc&^v%sl-LPd^XF#j_jG z;}3BKN0w!=Ic(%gw+r{eP7I7DO!_Rh!zHS5BTq71#&E+wJ{*-}bJ~;Vcv)jn3AjVE zO8*ja&9WKtBBsde-lx^LU+v{FTzF*$R~){_tn&|z4-cwBym^k76+#}xwG7w9+MPq{ zqBxXgLli??S{SPG9|1D$3vt#*ZLP!lMR(_---Z*zQ< z;xCK%7{yYlT+Rw&OiXGpwf*dcPSrlXwAB zPt!|}dKB+L>V1i%zWbI=eRWv9+g7sY7edKCX2uf9{SMod88t&`J>44*#>{omU@BdI z-CDgbnTY9Wydmm|nRv@GrQb7vS1zTM9*OI#Q~DrY7>%SOiMZaoUuy6bTdCY=A{a*P zdXqsEx;k|OK1;N8Fp&y`67e_+6^XktW^%tR8VHI6y>c2znyE}w8jANzIRm^{8czr_ zMDZf(hssSQVy51giHE3yvTQC%rx#A)$WSn4-k^sPVN>74ZtG43`+=Zg8|6#0p+qL0 z{*-j6$Fl+d!)9-$e?xC35)IRf>GYur;oViu*U;+m;-61TCK95`;b0n#ffvz@0Ft>rZ4a{v|ZX%dEzE6UZVPi;nA^hK7Us4vw}rW_%xz_ED|1_!+sfQl z=C(4omAS3VZDnp7bK98P#@sgMwlTMjxoymCV{RLB+nC$V+;--+Gq;_&?aXawZaZ_^ zncL3XcINuL-Zg70Om8jq`>hxAE)KRS#&a1HrApqu;5P)4aJ|A+}yEVwTq3X5lAv=D@s=&8G+Vl|?9`9IEYU0Hsm zcF0wZmTc9UCAQ^JyhShDP+GW`>`A19Q5}`-HIsUxPY;`SM?z*wuTN!Sl*V$d8om~- z@HF)4vauUdCd#y~0Gc-RMbBo}71cM=Z+Pfct5d%y7LP*C{Y$ZU3Ucww#o{Dn{h4C% z0m!;nip4p|TmQ9KT#3!mul`%H*ao?BvRLeaycKdE_Ax9y*UN08UKvGce zCc8erOLOh3bzQOa{Kbc%BOG5)YDZA!C&_Tz`SdNCb}c?l_~@vcm-4u%cKbyu|G<6W zJ&T95Pp`V+y2fki>m%mh4q1=7Gm$_9$~TRV4qTUYAe_oNgijsXww-YQMYUh5+Pdt# z>P;?GiS>U6xD{>om+3cJ`X51mJA833q{XjF5uN#G0gpnDH}XpQyIIZprO5Z^&>t_$ z{}C(yb-?@K(}>x>J!2MZ%=VhzstHprm?hhSS6+U zeHL9$C@B4}J}4$AeVpE@MxfU}D0ty|TYMr0kM%uatvQ4oP`f%A-;qmvT(X z2`ML~oRJbg%H&Whr7opc$_^==^*dKu{m$>Ety^!<>$`jL2%6D-K2MX!yWW@Kr0-xu zgU8#nR>HqKM;b1Rw0HRR61ye!zYCf}vV(M~q~8M0XHVm!N~_h5Un~u!&?P?U!0~yYKDasGnD4F~?J#Je2$I}-14;OCnc;5ycqLM*KqZb{i{^Nu&)JP z>%b3D{#A9Fx^5}^FIOn{n-y>`%AK!WV-@UQtAPKe0$z*!ss7G(eY`@s8!F&if$OS~ zHn|IUUDf59S|61vKsdF`D-e3CkwUye{`Uc|rQg%baW1Si?NYX{wa#1NXH+P^B<-F3 z@?~j%Ox`H)^B3&D!nn${B)?q&KUo3)X$Ab>fs-Gc^_c~(mx{AuOW~0D`u*Apcq{O$ zEp{ogi|nhGmKF@D>Zy>=oxtZkw_tZDnf7@S__2PlHxft(`!xWmbf&M*6VggQ><^@4 z0qpq3%@i!di9mle(Ho2g!s$dZ6$obbX`w`HFpACTu*W;U1)Yfy2?UeL;QoLaPh*q0 zFNrj3R8m+f1B2q{_EHnVM3yrXSP z^R{+*8hhIU=nr7mijFo&n=!N|g@YBu*8p%Kz6QV{4%lY4Fo!J{@4%!(kt5E1pd%VI ziJ0-QNQZ*aP$r5319uIkG}_KbS=hd}w$(Lu&{ph7_2M`OkqU@SIrw+`uFX6B0sm*V z1#lF@w%b0ldFPI;fN`Koz;8&cv2B+YxT*8zEt@+7H*ep*+rKBUXY-a$e?T2?(V-56 zz|jxX3M);=LVW1x3QK#ggCb1cn&%NB&#|zmB4fD4c0h%tY_bl)C_6;M(i2&ofKdk8 zkHIKYU~uDG0&!-AJWIpUQ=7$U8WtpKSANVzAe>4B27+;%2!b;&Zbn|=NIZ~9nPGVv zNI4t*3(xhYQnFA!^FtjJB6cM$0&2jgfY<>#5X4GTeZ;9AWl$i1gE>^+(}^AP9>bvu zrPE0)Iv2?^!w!|7a$;%N9W;66fhT1OaRNyhq5peTgGrohl1%TnLHy0n`bbzpo44#(Pb(Ok4+K*KnkT$JPN$}@#P?FHxf|c0;cKX} zL;$*^84Z%5j1A(OE)O@xgQ%w;-y(4Y+q#+uN46kIOB$>bJbXRofLz1|!pKdfL>^)_ z6ATqcQNdUQnF%_Efabw6O6y5!q5p_6dVY4{gWCfR&hy07^Rra5t%{Sp1|NEKRQyWk zW|1ugb^k6>n%07do$ow9>snPUW8eD(ZJ_u04wu=r{pLP@pn022N7AmdeaR+~|_n=RR`~l+kc$*wJqN4v3uk2g<+QT$ zQ~mfTxMZUE>bck}ANVJrvjUy z=(adT>ty*}2VUDDfa@K&%FZsWv1haGko;qE{Cym{Yw%I^SL^Quz*OB4wL)RLh#qtd XI90APr0F^zzwZx)#VQBEk<|Vh@A24S literal 0 HcmV?d00001 diff --git a/lastTest/taos_query_a_simple.c b/lastTest/taos_query_a_simple.c new file mode 100644 index 000000000000..5ccb415a070e --- /dev/null +++ b/lastTest/taos_query_a_simple.c @@ -0,0 +1,301 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef void* TAOS; +typedef void* TAOS_RES; +typedef TAOS* (*taos_connect_f)(const char*, const char*, const char*, const char*, int); +typedef int (*taos_query_a_f)(TAOS*, const char*, void (*)(void*, TAOS_RES*, int), void*); +typedef int (*taos_query_f)(TAOS*, const char*); +typedef void (*taos_free_result_f)(TAOS_RES*); +typedef void (*taos_close_f)(TAOS*); +typedef int (*taos_errno_f)(TAOS_RES*); +typedef const char* (*taos_errstr_f)(TAOS_RES*); +static taos_connect_f taos_connect_func = NULL; +static taos_query_a_f taos_query_a_func = NULL; +static taos_query_f taos_query_func = NULL; +static taos_free_result_f taos_free_result_func = NULL; +static taos_close_f taos_close_func = NULL; +static taos_errno_f taos_errno_func = NULL; +static taos_errstr_f taos_errstr_func = NULL; + +// Global array to store 16 SQL statements +static char* g_sql_statements[16] = {NULL}; + +// Function to generate table names for a given range +static char* generate_table_list(int start_idx, int count) { + int max_len = count * 10 + 1000; // Estimate buffer size + char* buffer = malloc(max_len); + if (!buffer) return NULL; + + int pos = 0; + pos += snprintf(buffer + pos, max_len - pos, "("); + + for (int i = 0; i < count; i++) { + int table_num = start_idx + i + 1; + if (i > 0) { + pos += snprintf(buffer + pos, max_len - pos, ",'d%d'", table_num); + } else { + pos += snprintf(buffer + pos, max_len - pos, "'d%d'", table_num); + } + } + + pos += snprintf(buffer + pos, max_len - pos, ")"); + return buffer; +} + +// Function to generate 16 SQL statements +static int generate_sql_statements() { + const int tables_per_sql = 625; // 10000 / 16 = 625 + + for (int i = 0; i < 16; i++) { + int start_table = i * tables_per_sql; + char* table_list = generate_table_list(start_table, tables_per_sql); + if (!table_list) { + printf("Failed to generate table list for SQL %d\n", i); + return -1; + } + + // Generate the complete SQL statement + int sql_len = strlen("select tbname,last(*) from test.meters where tbname in ") + strlen(table_list) + + strlen(" partition by tbname;") + 100; + + g_sql_statements[i] = malloc(sql_len); + if (!g_sql_statements[i]) { + printf("Failed to allocate memory for SQL %d\n", i); + free(table_list); + return -1; + } + + snprintf(g_sql_statements[i], sql_len, + "select tbname,last(*) from test.meters where tbname in %s partition by tbname;", table_list); + + free(table_list); + } + + return 0; +} + +// Function to free SQL statements +static void free_sql_statements() { + for (int i = 0; i < 16; i++) { + if (g_sql_statements[i]) { + free(g_sql_statements[i]); + g_sql_statements[i] = NULL; + } + } +} + +typedef struct { + int thread_id; + TAOS* taos; + const char* sql; + int queries_per_thread; + pthread_mutex_t* mutex; + struct timeval start_time; + struct timeval end_time; + double* thread_qps; + int max_query_nums; + atomic_int current_query_nums; + atomic_int completed_query_nums; +} ThreadParam; + +static struct timeval global_start_time; +static struct timeval global_end_time; + +static double calculate_qps(int count, double duration) { + return duration > 0 ? count / duration : 0.0; +} + +static int load_taos_functions(const char* lib_path) { + void* handle = dlopen(lib_path, RTLD_LAZY); + if (!handle) { + fprintf(stderr, "Failed to load library %s: %s\n", lib_path, dlerror()); + return -1; + } + + taos_connect_func = (taos_connect_f)dlsym(handle, "taos_connect"); + taos_query_func = (taos_query_f)dlsym(handle, "taos_query"); + taos_query_a_func = (taos_query_a_f)dlsym(handle, "taos_query_a"); + taos_free_result_func = (taos_free_result_f)dlsym(handle, "taos_free_result"); + taos_close_func = (taos_close_f)dlsym(handle, "taos_close"); + taos_errno_func = (taos_errno_f)dlsym(handle, "taos_errno"); + taos_errstr_func = (taos_errstr_f)dlsym(handle, "taos_errstr"); + + if (!taos_connect_func || !taos_query_a_func || !taos_free_result_func || + !taos_close_func || !taos_errno_func || !taos_errstr_func) { + fprintf(stderr, "Failed to load some functions: %s\n", dlerror()); + dlclose(handle); + return -1; + } + + return 0; +} + +static void queryCallback(void* param, TAOS_RES* res, int code) { + ThreadParam* thread_param = (ThreadParam*)param; + if (code != 0) { + printf("query failed case: code = %d\n", code); + } + atomic_fetch_sub(&thread_param->current_query_nums, 1); + atomic_fetch_add(&thread_param->completed_query_nums, 1); + + taos_free_result_func(res); +} + +static void* query_thread(void* arg) { + ThreadParam* param = (ThreadParam*)arg; + + // 初始化线程参数 + atomic_init(¶m->current_query_nums, 0); + atomic_init(¶m->completed_query_nums, 0); + + gettimeofday(¶m->start_time, NULL); + + while (atomic_load(¶m->completed_query_nums) < param->queries_per_thread) { + if (atomic_load(¶m->current_query_nums) < param->max_query_nums) { + atomic_fetch_add(¶m->current_query_nums, 1); + taos_query_a_func(param->taos, param->sql, queryCallback, param); + } else { + usleep(50); + } + } + + gettimeofday(¶m->end_time, NULL); + + double start_sec = param->start_time.tv_sec + param->start_time.tv_usec / 1000000.0; + double end_sec = param->end_time.tv_sec + param->end_time.tv_usec / 1000000.0; + double duration = end_sec - start_sec; + *param->thread_qps = calculate_qps(param->queries_per_thread, duration); + return NULL; +} + +int main(int argc, char* argv[]) { + if (argc != 2) { + printf("Usage: %s \n", argv[0]); + return 1; + } + + int query_mode = atoi(argv[1]); + const char* lib_path = "../../debug/build/lib/libtaos.so"; + const char* host = "127.0.0.1"; + const char* user = "root"; + const char* pass = "taosdata"; + const char* db = "test"; + int thread_count = 0; + int queries_per_thread = 0; + int max_query_nums = 0; + int qps_rate = 0; + const char* sql = ""; + + if (query_mode == 0) { + sql = "SELECT last(ts,r32) FROM test.d1;"; + thread_count = 16; + queries_per_thread = 10000; + max_query_nums = 5; + qps_rate = 1; + } else if (query_mode == 1) { + sql = + "select tbname,last(*) from test.meters where tbname in " + "('d1','d2','d3','d4','d5','d6','d7','d8','d9','d10','d11','d12','d13','d14','d15','d16','d17','d18','d19','" + "d20'," + "'d21','d22','d23','d24','d25','d26','d27','d28','d29','d30','d31','d32','d33','d34','d35','d36','d37','d38','" + "d39','d40','d41','d42','d43','d44','d45','d46','d47','d48','d49','d50','d51','d52','d53','d54','d55','d56','" + "d57','d58','d59','d60','d61','d62','d63','d64','d65','d66','d67','d68','d69','d70','d71','d72','d73','d74','" + "d75','d76','d77','d78','d79','d80','d81','d82','d83','d84','d85','d86','d87','d88','d89','d90','d91','d92','" + "d93','d94','d95','d96','d97','d98','d99','d100') " + "partition by tbname;"; + thread_count = 16; + queries_per_thread = 100; + max_query_nums = 5; + qps_rate = 100; + } else if (query_mode == 2) { + generate_sql_statements(); + thread_count = 16; + queries_per_thread = 100; + max_query_nums = 5; + qps_rate = 625; + } else if (query_mode == 3) { + sql = "select tbname,last(*) from test.meters partition by tbname;"; + thread_count = 4; + queries_per_thread = 1; + max_query_nums = 1; + qps_rate = 1000000; + } else { + printf("Usage: %s \n", argv[0]); + return 1; + } + + if (load_taos_functions(lib_path) != 0) { + fprintf(stderr, "Failed to load TDengine functions\n"); + return 1; + } + + pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + + ThreadParam* params = malloc(thread_count * sizeof(ThreadParam)); + pthread_t* threads = malloc(thread_count * sizeof(pthread_t)); + double* thread_qps = malloc(thread_count * sizeof(double)); + + if (query_mode == 2) { + printf("Starting %d threads, each executing %d queries, max_query_nums: %d\n sql: %s\n", thread_count, + queries_per_thread, max_query_nums, g_sql_statements[0]); + } else { + printf("Starting %d threads, each executing %d queries, max_query_nums: %d, sql: %s\n", thread_count, + queries_per_thread, max_query_nums, sql); + } + + gettimeofday(&global_start_time, NULL); + + for (int i = 0; i < thread_count; i++) { + TAOS* taos = taos_connect_func(host, user, pass, db, 0); + if (taos == NULL) { + fprintf(stderr, "Failed to connect to TDengine\n"); + return 1; + } + params[i].thread_id = i; + params[i].taos = taos; + if (query_mode == 2) { + params[i].sql = g_sql_statements[i]; + } else { + params[i].sql = (const char*)sql; + } + params[i].queries_per_thread = queries_per_thread; + params[i].mutex = &mutex; + params[i].thread_qps = &thread_qps[i]; + thread_qps[i] = 0.0; + params[i].max_query_nums = max_query_nums; + if (pthread_create(&threads[i], NULL, query_thread, ¶ms[i]) != 0) { + fprintf(stderr, "Failed to create thread %d\n", i); + return 1; + } + } + + for (int i = 0; i < thread_count; i++) { + pthread_join(threads[i], NULL); + } + + gettimeofday(&global_end_time, NULL); + + // 计算每个线程QPS的总和 + double total_thread_qps = 0.0; + for (int i = 0; i < thread_count; i++) { + total_thread_qps += thread_qps[i]; + } + total_thread_qps = total_thread_qps * qps_rate; + + printf("\nTotal number of devices (sum of all threads): %.2f queries/second\n", total_thread_qps); + + free(params); + free(threads); + free(thread_qps); + pthread_mutex_destroy(&mutex); + return 0; +} \ No newline at end of file From 0ff47c36f96013a901624e29ab628df4a853554d Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 20 Aug 2025 08:48:22 +0800 Subject: [PATCH 04/20] feat: add tag filter test. --- lastTest/taos_query_a_simple | Bin 21568 -> 21568 bytes lastTest/taos_query_a_simple.c | 6 ++++++ 2 files changed, 6 insertions(+) diff --git a/lastTest/taos_query_a_simple b/lastTest/taos_query_a_simple index 203ad0714f2f2181b5fe37c12fb8fda59cb93bd5..036b756756543d3947bb790bec2f6413da8219f4 100755 GIT binary patch delta 1096 zcmYjRTTBx{6x~^hKtoj!l>)5`D&UjyNOyfdu(Bp5!Uu{PgVnB5!$aC7ikfI*N+b)3 z=Ef){qK1g_p*F4`)gUaIh#I5DFW>n3C^4p@exQDgcUmD#GC61NJ#+4zn{AD&#JEa= ziNtJ8<)X`)3aMmql93oY8u|uOKvE3n|Kk`!D<4TqnN)q_*@Z3Ijb94oxdp!80sQfi#HWB^}wAR)bueSI!G~xa3YLkZ6XI@jculNCmPAEu8KFo96M$PiI5T@7{+Rhg;(ZC%$rpEf~h&hHYL6yCOl<7b08;H|2I_8X)$3~(L zT#OG46mXn&nz_sEmMhbl+tcy!NYE%$lC>}>2uUAN$lB?E@4}HSy=l=?bavsP2%E1f z$kJUg9%B4Vj2nz=jJLaU7S_7`YX1Cr!X95kolvK!iq9_`s8M{%1d-sa7xMFpizSCsl1d8m z<_Y`VKGmyw8|sB!%@c&nxJti!ud-B_>lcbeGxveJyD^}i)vvTf%pN-yH+ z&Ol2?!cM0y$OW?)aA_J>O<1B{@N6x9ZLl_s(ZER&BX~W&~2DXWI z{@GhNgz)PfoD=P&9BzxVNCUhQXYq6h{$NN3t2BF>348G*;5FCesV#h|Wm4RBeUsE| z0>=u8PtMWzt+*afdcj&+MhtM5_On>dx=uno`haW5Oxoa5d4V~Z{V;GP=fLao^{C9Q YC`cOr3xKT^qS+zFBo0wOSJBP?10blBeEJY3jDUdRX7~O&uWfLM-Nl7$9%``EZluXGV^VjJ;cW&8%!}Ff!eb4hg@3~z6F!B#0 zuM(+(8b(?{4NgNvNls<-=oD*_5ow&J^Ns8tSjyJ3nFHa`k=1lZ)ZfVD;xA zu}{{u4OQXfEE7^;YnC59!1C;Ic8S5CvJa~7gHHF$c0vf-QHD8P$>v|J(X*_>!uKaw z@hM$t<4#_19e^5MV0iZ%p!a1k5RLh$i_qdKAwC4r;))z-!PX0@(EJ)9E`R%OUP$0e zY4zyyJg6C=g`kigRS4N}x&CnG3y|R5Hm~4v(l#D`bWTP^hVU)DHlr_^^~a6!LV2Ww zVXoz5rCio-aI%C~^#yweH&W035HbQSX-~D}V=NQ|Zp_sTooHh50?ROVL2WF-M{;)Q zzwe2f&4Eov6rLQ{y&O_2c3NIj$|&iOuCF(ec0}A-7?k$P^&~K;<+cbAW|k3F!sY z%n%X*Z1_e<27I7_Swbv;8FPd<0UH3%0V?JR@c^~~`T*+|aDjOT@?eKq%Nizei@Bhy z5PGD7R&2poOgn4?@CLXOTRI5(U~9lb4drbqsXeS!ohYkB$qFSy^ABO0!JF;Bd__Dz z4*5p#+zGgTKTVy;o~8FCf#1OUFlWg{UvQNrmpcuEocUJ_wI`%n18{a|;>DLN`Px<( z&!7M*Zb_?vGOrsFRA7m9#trT_o{ diff --git a/lastTest/taos_query_a_simple.c b/lastTest/taos_query_a_simple.c index 5ccb415a070e..dd483f60107d 100644 --- a/lastTest/taos_query_a_simple.c +++ b/lastTest/taos_query_a_simple.c @@ -228,6 +228,12 @@ int main(int argc, char* argv[]) { queries_per_thread = 1; max_query_nums = 1; qps_rate = 1000000; + } else if (query_mode == 4) { + sql = "select channel_id,last(*) from meters where channel_id in ('338068841') partition by channel_id;"; + thread_count = 16; + queries_per_thread = 100; + max_query_nums = 5; + qps_rate = 1; } else { printf("Usage: %s \n", argv[0]); return 1; From 2c6850dad3c25772affc7c55aa88e4b180926422 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 20 Aug 2025 09:17:03 +0800 Subject: [PATCH 05/20] fix: ci problems. --- source/libs/executor/src/executil.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b7de23a24984..76273d86c5bf 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1419,8 +1419,9 @@ static int32_t optimizeTbnameInCondImpl(void* pVnode, SArray* pExistedUidList, S return -1; } - if ((pNode->pLeft != NULL && nodeType(pNode->pLeft) == QUERY_NODE_FUNCTION && - ((SFunctionNode*)pNode->pLeft)->funcType == FUNCTION_TYPE_TBNAME) && + if ((pNode->pLeft != NULL && ((nodeType(pNode->pLeft) == QUERY_NODE_FUNCTION && + ((SFunctionNode*)pNode->pLeft)->funcType == FUNCTION_TYPE_TBNAME)) || + (nodeType(pNode->pLeft) == QUERY_NODE_COLUMN && ((SColumnNode*)pNode->pLeft)->colType == COLUMN_TYPE_TBNAME)) && (pNode->pRight != NULL && nodeType(pNode->pRight) == QUERY_NODE_NODE_LIST)) { SNodeListNode* pList = (SNodeListNode*)pNode->pRight; From 9112b05b874ba4fba17c913a8d85517c1c7b4fbe Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 20 Aug 2025 18:40:56 +0800 Subject: [PATCH 06/20] enh: where tag in use tag index. --- lastTest/taos_query_a_simple | Bin 21568 -> 21568 bytes lastTest/taos_query_a_simple.c | 10 +++-- source/libs/index/src/indexFilter.c | 60 ++++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/lastTest/taos_query_a_simple b/lastTest/taos_query_a_simple index 036b756756543d3947bb790bec2f6413da8219f4..00a0d1f5a8c3df660cf1911210708737c9832734 100755 GIT binary patch delta 1044 zcmY+DO=w+36vywK*u;cBTO@{;mk(mmK=n^@=X-{{L1=ps}wWMirZO_ipk(Y7Ky zp|nI(B`^q~iy8%E7hMGJ!fb4_P-#KvM(npn-Bf&*hbwpEd9O9K1LywEJ^yK*x$v_*?p8d_&|P}k2aJ!1SAwR3@jWxcm~eF1-JxP z;2PY3TW|;NAu3B$bqq3Q|KFNDJv8JyZt8p#&6lR8ShK zY@jTZgYwWBG=~<@5}GE`&<5HBViN_4Qya6jDzv887zktuo70mYFOHj zg>|qVE`#H60#3pyI1OjuXuIxdI1it}b9ezS;T61wH}Dpoj-z%SS>C75rjyyV7+2_W zrmn|$iFlZ}NnE-S;{x&c%^3e7Uc41!F-?+3evZ)*-zQEGFA|RtXE$S9CSD`16W4zW zz4n<>LS=d>JsED=S;JFgifrL(IB0vzOVqs{uW7Ve$>zxxDBeR={#%{ooF+R?A|M>2`jaC}f`=-g>W*J*I1}y|$|_7bY)c-{(gM7qh1>mk&Lk=Q&HtO-nL! z`cBZ>GVAyKZppbNDgG#yk3X?yH`6hsna#9-G|kTWJG|#jAu}R`NWUG%Q@Yy&Gs2er zKK9ywn6x5GlL)&mM`)jyP52KmU3Pg14)^P5Je!H2CZ5Hpd6Zr9vQ_NUyi94T(4vP` zn>y&6hsn+N=`#<1+T1|t)P7zJI0&6_@mB}e8mx+PTv6DYU=QC2MlHMBJ3BQstZAWO zJHHa1AjzT6LKHA9|3!#*U;$hL^{+xa1WVt9*Z|kR3t=29^g{>@EQ1%pb#MfnToz&s zw66$J0jodRpmxNkqD>phrT)4`1FeVPUH&Z@Ne)p$ zy8%Ow?P(mPQK!w)OX}(Tfo{PK?j3lXDd^}O4e*s));5+~T diff --git a/lastTest/taos_query_a_simple.c b/lastTest/taos_query_a_simple.c index dd483f60107d..3f65265194cf 100644 --- a/lastTest/taos_query_a_simple.c +++ b/lastTest/taos_query_a_simple.c @@ -184,7 +184,7 @@ int main(int argc, char* argv[]) { } int query_mode = atoi(argv[1]); - const char* lib_path = "../../debug/build/lib/libtaos.so"; + const char* lib_path = "/root/workspace/TDinternal/debug/build/lib/libtaos.so"; const char* host = "127.0.0.1"; const char* user = "root"; const char* pass = "taosdata"; @@ -229,11 +229,15 @@ int main(int argc, char* argv[]) { max_query_nums = 1; qps_rate = 1000000; } else if (query_mode == 4) { - sql = "select channel_id,last(*) from meters where channel_id in ('338068841') partition by channel_id;"; + sql = + "select channel_id,last(*) from meters where channel_id in " + "(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39," + "40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75," + "76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100) partition by channel_id;"; thread_count = 16; queries_per_thread = 100; max_query_nums = 5; - qps_rate = 1; + qps_rate = 100; } else { printf("Usage: %s \n", argv[0]); return 1; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 937218b53160..b69a01eb2cae 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -103,6 +103,8 @@ static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst = QUERY_LESS_EQUAL; } else if (src == OP_TYPE_EQUAL) { *dst = QUERY_TERM; + } else if (src == OP_TYPE_IN) { + *dst = QUERY_TERM; } else if (src == OP_TYPE_LIKE || src == OP_TYPE_MATCH || src == OP_TYPE_NMATCH) { *dst = QUERY_REGEX; } else if (src == OP_TYPE_JSON_CONTAINS) { @@ -135,7 +137,7 @@ static FORCE_INLINE int32_t sifGetOperParamNum(EOperatorType ty) { return 2; } static FORCE_INLINE int32_t sifValidOp(EOperatorType ty) { - if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || + if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { return TSDB_CODE_INVALID_PARA; } @@ -332,12 +334,8 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type, 0, 0)); - if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { - taosHashCleanup(param->pFilter); - param->pFilter = NULL; - indexError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); - SIF_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } + // Don't store NODE_LIST to context to avoid double free + // The pFilter will be managed by the caller (sifExecOper) break; } case QUERY_NODE_FUNCTION: @@ -675,6 +673,52 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIndexMetaArg *arg = &output->arg; EIndexQueryType qtype = 0; SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); + // Handle IN operator with multiple values + if (operType == OP_TYPE_IN && right->pFilter != NULL) { + // For IN operator, we need to iterate through all values in the set + void *pIter = taosHashIterate(right->pFilter, NULL); + while (pIter != NULL) { + SDataTypeBuf typedata; + memset(&typedata, 0, sizeof(typedata)); + + bool reverse = false, equal = true; // IN operator is essentially multiple equality checks + FilterFunc filterFunc = sifEqual; + + SMetaFltParam param = {.suid = arg->suid, + .cid = left->colId, + .type = left->colValType, + .val = pIter, + .reverse = reverse, + .equal = equal, + .filterFunc = filterFunc}; + + SArray *tempResult = taosArrayInit(8, sizeof(uint64_t)); + if (tempResult == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + ret = left->api.metaFilterTableIds(arg->metaEx, ¶m, tempResult); + if (ret == 0) { + // Add results to output array + if (taosArrayAddAll(output->result, tempResult) == NULL) { + taosArrayDestroy(tempResult); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + taosArrayDestroy(tempResult); + pIter = taosHashIterate(right->pFilter, pIter); + } + + // Remove duplicates and sort + if (taosArrayGetSize(output->result) > 0) { + taosArraySort(output->result, uidCompare); + taosArrayRemoveDuplicate(output->result, uidCompare, NULL); + } + + return TSDB_CODE_SUCCESS; + } + if (left->colValType == TSDB_DATA_TYPE_JSON) { SIndexTerm *tm = indexTermCreate(arg->suid, DEFAULT, right->colValType, left->colName, strlen(left->colName), right->condValue, strlen(right->condValue)); @@ -821,7 +865,7 @@ static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxF *func = sifNotEqualFunc; return 0; case OP_TYPE_IN: - *status = SFLT_NOT_INDEX; + *status = SFLT_ACCURATE_INDEX; *func = sifInFunc; return 0; case OP_TYPE_NOT_IN: From fa49a2be7f50c71616b6dad134715c7ba358b2f5 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 21 Aug 2025 09:50:35 +0800 Subject: [PATCH 07/20] feat: add function to get value hash from nodelist. --- lastTest/taos_query_a_simple | Bin 21568 -> 25664 bytes lastTest/taos_query_a_simple.c | 27 +++++++++++- source/libs/index/src/indexFilter.c | 63 +++++++++++++++++++++++++--- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/lastTest/taos_query_a_simple b/lastTest/taos_query_a_simple index 00a0d1f5a8c3df660cf1911210708737c9832734..a2a90f9413bb38b0fc18d4a70553ba492cd7f17f 100755 GIT binary patch delta 5101 zcmZA5du$xl702!ut+qBU|cBENKsHFXw?!`v<0k65u#w;QPl!{@Mxq;@CU*z{Lw=HNTaIL@9ZO) z>`M3Ex##}o&g^`=iM_Lj!`Y+ZLub`ZcSo$5ZuXsr+EMHfdq#N!3gB`~qspV{W{5r0SUshs)svfTy!NK*$weBk8&mbhQ%cruP1P4R!pZgTPA#;ZTnK{uQuWWK>JO#r zi&FK2srnhI`lG4(;#B=`Qm@dGyRZL1R+pblop5HgQtOVS>Yb_jY^uIARX>`lcctoY zrs`q$%!;9(wmjUiXyDQyDD9h_%*1b&?s>WOrRoW>c2fQ%Ub*DbnR7eVhm}i$VDZ3c z5PZ70{{-c~@exVS-)frwp=tg~)BLwh^WQYhf7vvz{H&?)k=bBYi|NV^(iwC}6eT=2);V)4sO}AX048`MaXL>!m)x3ryF{MD(&murh3l%k1c zePJoOtJ*(O?T;+$+gVC>Y`Hoa_;EUqZrX}GbdKfjP^d1>3NoaPCh!fG_`Lt52rN6 zdwG?I@QTax$o!ZSlWndg#!j#!x;iZ-16$h-@J;Nkw>%!Z+8?rfL2XejFI z?-`(&Jcl2C8pp5g+sl5*BmMjLxWe|S_K!xQEBe=W@!vqQWq)eR6+_X1{>Q?Rnf$;+ zc*dq6m^zT0T$*m*I2iqW@ZNA!w0^}snbConr&m0&w6ZRj&Cwjq(}bopqvdG@T5(-2 zm!*-*k;{?Gk;{?Gk;{?Gk;{>DiG##J z;vjL5I7l2M&gCm~1Tk@#I82-;2#JHlLE<2BkT^&jBn}b>iG##J;vjL5I7l2M4iX26 zgTz7NAaRg5NE{>%5(kNc#A$^fCJqyaiNnM};vjL5I7l2M4iX26gTz7NAaRg5NE{>% z5(kNc#6jX9agaDj93&1B2Z@8kLE<2Bkhlsa4ikro!^C0YAaRg5NE{>{5)X-o#6#jC z@sM~(JR}|x4~d7wL*gOvka$QuBpwnEiHF2P;vw;nct|Q96OW0<#AD+5n0QD$BpwnE ziHF2P;vw;nct|`X9ug0Uhr~nTA@PuSNIWDS5)X-o#6#jC@sM~(JS1MFLPro2kBP^` z^D*&|ct|`X9ug0Uhr~nTA@PuSNIWDS5)X-o#6#jC@sM~(JR}|x4~d7wL*gOvka$RZ zp+XQ7kBP^`W8xw4ka$QuBpwnEiHF2P;vo@81QLNnAQ4Ce5`jb@5l93QfkYq?NCXmr zL?97J1QMxWBA5s!f{9=vkO(9Ki9jNd2qXfDKq8O`Bm#**B9I6q0*OE(kO(9Ki9jNd z2qXfDKq8O`BmzlAFcC}y6Tw9Im4F8iSaQpNDLB# z#2_(93=)IHATdY`5`)AbF-Qy&gTx>)NDLB##2_(93=)IHATdY`5|au+Obipl#4s^P z3=)IHATdY`5`)AbF-Qy&gTx>)NDLB##2_(93=)IHATdY`5`)AbF-Qy&gTyMB7$$~^ zVPcpVBnF8=Vvral28lspkQgKei9uqJ7$gRXL1K^?BnF8=Vvral28ltE&t})$IC0bD zb(7b9`-Z`*Z;P(Cl`{wJnsC>|uP5vGB(@zK?OmpW^s9oiJ*B7zz zhpaz&+65jzd4YA*Z{cxVmp(sJu5K_$Z9D6;tiN~i{OGx2cjk_FX5KFDnm3ajIy^th z4qq7Vj&2_wpg%NR3HQwOp1&+)@>HffDs6fpEJtmdKU`i;E^+GHg)6Ja-9ce@)Hbq_ z^cO}pR?0~N*xC@h!59X;L8)9TH0tly>pPn2|Ekw_Hq{q!_sKrxruwpaeQ&+KwsC>g z^@SUoPAJstcbB8i3-&CZ2&%8fPs1bi)sBhi{tJGP8Ql`~ZrPZte>GH#20=_H2$lqm z@1q*^PX1a6`A#OgC%S#hEsdwuSl!2JN3cBT;8~77&lCOK=pOpDW81=s=%%ra9RFnO zL3+3KalWfMwsmV;rgUaBblHZs#)V2F(XPv^?Z`-N&m$wzq090M>vxwOkB*(16~?39 z%TKOt9*=f0tA7oVy)-(+tX#jd9nrBgEn4WS)dX^|g3m9ma8GMmUYe&{qY5;DRM zBPGERpj0NuC1k{i7?lttKx48Hbp!b^NfiE&U*m^ELoBkyIs9Yp53lFFcb~hwo4otI z=X1_I=iZyw``SN1`;XDCQo8(fnVnXTWD~XNN-Ua9Q6}T#3u?A6qAP5ov!`+z<}@6E z;<~z$Xbj&3RmzkSjPwgbBQ+>Kzk! zp%7op7i2=_5V*|aGm$CBthcDZB|_plC(VLGNv5|}fh&vETf4xOB6`~>aQ&1N-&TS1 z*8wTk-zhlEdKT0@0)Ix}T>{S$_#uH$5%>{-=L!6n!j=6itLHDI>OLh*n5q)3>lL_F z;QazG5cq(=Z34e0aB5FYFZs-L(30Z~5h8c?cS!7-yz4^N_v!>T&x&}I&CCy_=2*&U zd?6uu-X(-Q$@7dOzK1^%(Y|i5e>B(^4feMN`)h;!mBEgGZU7$}>~4ddG}xU6`#pob z*lJz4_tj&RlfhS!$Vt;mMQX-6Y;wY655j=e zouxYFl>}kf>ZWy&o$o2F!&#f>G|9Vy=ar$n_hCnloCsb*Bc1&(W9jvbu}h}wb*X6n zICa)4AvscFX<9xcCkrkrHL=#TT3BaWK^MRoTRZmC>~8D>b}#nL_5-vA()NY5H5i&$ zIwmJ={Z1v#4SDb7^n-E6^Ig!ANt^MdAl7=X4yHx0lJwLh%(y@l@PfRfR)RuO!i z>TtY|};X_yq4Qyj^CRhY5#+k& zNk@xPOI%%&V?FNr@f7r_B)B$df!;4TkhYz1HcDb`1^WhZo=v}<9mM3x_S3#qv+*YNf^av?` za6q@`Q(Besd)w%gNutscP&c9up>_>2wg+|F9md8``|dL4z>Q40&sYF;Cu%clAL>@rcZL`{ zgnA3L7q$0SIJv0IMKH}QoKBWxsyEII96pcFo|`bbXjXL>Ru^v~V1+{lYDi{9Kg z*pJV)L3sHEkMzkMcnjsiUD$_Gd=HMJgzmu=l&kmQ4;05QU|w7z{XCSaSRBg$U*shk zf$b45_C1j}txvgD6jC%!OLma!&(SEDqrW@(w?(hqge57llRyN_4V4(ZprJC(WAv;3 zsv6@WvVlL~JZj*7ao%j;nOKw(7il%{LeAScFVhRp=8lbq2?5S`M!~wW#o2)Muvs)A zhq>3%0Q**cAPv`mtFf|*fBbn2VHiY6KGFYi=-i6$BgL<-uO7BHZqZLm_jaP!LY%~c zXSu2mPxQ*F7VOKKUZf4Msi_jjN1F~{U%mQE{5LYVdYM@o$%K-&3bS5l#1FBy`Q}#q zJu0m=06lH~Y`!{Q5C$i#KoDGO|M!K15JQW9XnY~)L5m*@3B%xoWeJ08U5JJuruIG9 SFN48#p{!T5nl5Nt\n", argv[0]); return 1; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index b69a01eb2cae..8d94517b611a 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -213,6 +213,61 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { return TSDB_CODE_SUCCESS; } +static int32_t sifCreateHashFromNodeList(SNode *node, SHashObj **pFilter) { + SNodeListNode *nl = (SNodeListNode *)node; + + if (LIST_LENGTH(nl->pNodeList) <= 0) { + indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + // Create hash table with appropriate hash function for the node list type + uint32_t type = nl->node.resType.type; + SHashObj *pObj = taosHashInit(256, taosGetDefaultHashFunction(type), true, false); + if (NULL == pObj) { + indexError("taosHashInit failed, size:%d", 256); + return terrno; + } + + taosHashSetEqualFp(pObj, taosGetDefaultEqualFunction(type)); + + // Iterate through each node in the list + SNode *nodeItem = NULL; + FOREACH(nodeItem, nl->pNodeList) { + if (nodeType(nodeItem) != QUERY_NODE_VALUE) { + indexError("Only value nodes are supported in node list"); + taosHashCleanup(pObj); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + SValueNode *valueNode = (SValueNode *)nodeItem; + char *pData = nodesGetValueFromNode(valueNode); + SDataType *pType = &valueNode->node.resType; + int32_t valLen = 0; + + // Calculate value length + if (IS_VAR_DATA_TYPE(pType->type)) { + if (IS_STR_DATA_BLOB(pType->type)) { + valLen = blobDataTLen(pData); + } else { + valLen = varDataTLen(pData); + } + } else { + valLen = pType->bytes; + } + + // Add to hash table (duplicates will be automatically handled) + if (taosHashPut(pObj, pData, (size_t)valLen, NULL, 0)) { + indexError("taosHashPut to set failed"); + taosHashCleanup(pObj); + return terrno; + } + } + + *pFilter = pObj; + return TSDB_CODE_SUCCESS; +} + static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SOperatorNode *nd = (SOperatorNode *)node; if (nodeType(node) != QUERY_NODE_OPERATOR) { @@ -328,12 +383,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { break; } case QUERY_NODE_NODE_LIST: { - SNodeListNode *nl = (SNodeListNode *)node; - if (LIST_LENGTH(nl->pNodeList) <= 0) { - indexError("invalid length for node:%p, length: %d", node, LIST_LENGTH(nl->pNodeList)); - SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - SIF_ERR_RET(scalarGenerateSetFromList((void **)¶m->pFilter, node, nl->node.resType.type, 0, 0)); + SIF_ERR_RET(sifCreateHashFromNodeList(node, ¶m->pFilter)); // Don't store NODE_LIST to context to avoid double free // The pFilter will be managed by the caller (sifExecOper) break; @@ -668,6 +718,7 @@ static int8_t sifShouldUseIndexBasedOnType(SIFParam *left, SIFParam *right) { } return 1; } + static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { int ret = 0; SIndexMetaArg *arg = &output->arg; From 7b5797d2234d66539f884689ca8e143fffb36b62 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 21 Aug 2025 14:35:13 +0800 Subject: [PATCH 08/20] fix: tag in add more check. --- lastTest/taos_query_a_simple | Bin 25664 -> 25664 bytes lastTest/taos_query_a_simple.c | 1 + source/libs/index/src/indexFilter.c | 89 +++++++++++++++++++++++++++- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/lastTest/taos_query_a_simple b/lastTest/taos_query_a_simple index a2a90f9413bb38b0fc18d4a70553ba492cd7f17f..39ff53d04a76b4c4a202b885118ecfe3947e5e08 100755 GIT binary patch delta 581 zcmXw%O=uHQ5XW~mUE_+Cq^)fdNxP{ZMS4h!K}!|GHVtl&(nBg3D0Y`5VnnUb3IPuu z1Ti!PVH6bv^;A8G!6z8-ka%g+OPka$3W7aGD7GGi9tz`ZQwQe%`_H_cop*TU3Y05Q z*bA!OBJG5ZZV0MoYojtn41(Y`_1*G6F(+@d-Q4`>e{^;IeDD0(rG)iteI+}0LgXEH z+98JfeJACX9vt?S~Nf8^^0gZr+$x%q}i+3q|`1^$#0F#!R%pjr#3|FWP>x;mfv9+I}|5;|ul2_&eZ% zOTyV*Ykt>v#@(LIrjy!q=6ZTiJ9;gpO{J5QH&RnLsQVJ-C88I+WOJFQVsEPwtueG3 z(S4@6LbSyk{Y>NmNqYE&Xn^VZN_3Gq%FHt98_^TymG4AFrh#wtUhf0`+cBP&hMg`O zM>s>A9{i*Ccs1U+)3%Q|+Bl~9Zx*&)d)-lYn7VtIeGA^=C;4ju!s74OA zrg&%G$%D}Yu6uH%d5`0+2uD^`>>C+?kjRX@RbU*w@pE9J6)$lsF!mfEFFsxPxewms kOEaj3x_O=>gsZ0gUBKc8p*MOaCahCxb+@XDr`DYO4+1H~_W%F@ delta 561 zcmXw$O=uHQ5XX0RlO|iXVl*^OYuaq2Hbpckhz2U?=7T;DIn{#VB@4k)EaIV{h|&sD zEQxrSgL=~7K@UQU&%@r5;=%QzONwOba)!^257bN=l`nMyQ!@J4s?P5NwTzI|^+ zS-5}Ff&p!OmKz4=*N;TcnZY$8m)Xbs!OS#?9x%;yqFv_n22md<%Ii&{3^Vze=n`|9 zd5anOLiB{Wz;u|UEuwEsFT(u74J!t(aL0;38ujc5WN<7SiKagh#XD2p?ViBP_8Hyj LI7dFUU#kBATbH*o diff --git a/lastTest/taos_query_a_simple.c b/lastTest/taos_query_a_simple.c index 653dc21b2a0e..25ff342572a5 100644 --- a/lastTest/taos_query_a_simple.c +++ b/lastTest/taos_query_a_simple.c @@ -325,6 +325,7 @@ int main(int argc, char* argv[]) { total_thread_qps = total_thread_qps * qps_rate; printf("\nTotal number of devices (sum of all threads): %.2f queries/second\n", total_thread_qps); + printf("exec time: %ld seconds\n", global_end_time.tv_sec - global_start_time.tv_sec); free(params); free(threads); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 8d94517b611a..485de2a814de 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -213,6 +213,13 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { return TSDB_CODE_SUCCESS; } +// Structure to store value and type information in hash table for IN operations +typedef struct SInFilterValue { + uint8_t valType; // Data type of the value + int32_t valLen; // Length of the value + char data[0]; // Variable length data +} SInFilterValue; + static int32_t sifCreateHashFromNodeList(SNode *node, SHashObj **pFilter) { SNodeListNode *nl = (SNodeListNode *)node; @@ -256,12 +263,28 @@ static int32_t sifCreateHashFromNodeList(SNode *node, SHashObj **pFilter) { valLen = pType->bytes; } + // Create extended value structure with type information + SInFilterValue *inValue = taosMemoryCalloc(1, sizeof(SInFilterValue) + valLen); + if (inValue == NULL) { + indexError("taosMemoryCalloc failed for SInFilterValue"); + taosHashCleanup(pObj); + return terrno; + } + + inValue->valType = pType->type; + inValue->valLen = valLen; + memcpy(inValue->data, pData, valLen); + // Add to hash table (duplicates will be automatically handled) - if (taosHashPut(pObj, pData, (size_t)valLen, NULL, 0)) { + // Use the actual data as key, store the extended value structure as value + if (taosHashPut(pObj, pData, (size_t)valLen, inValue, sizeof(SInFilterValue) + valLen)) { indexError("taosHashPut to set failed"); + taosMemoryFree(inValue); taosHashCleanup(pObj); return terrno; } + + taosMemoryFree(inValue); } *pFilter = pObj; @@ -719,6 +742,57 @@ static int8_t sifShouldUseIndexBasedOnType(SIFParam *left, SIFParam *right) { return 1; } +// Function to check if all values in IN operation are compatible with column type for index usage +static int8_t sifShouldUseIndexForInOperator(SIFParam *left, SIFParam *right) { + // Basic checks first + if (left->colValType == TSDB_DATA_TYPE_FLOAT) return 0; + + if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || left->colValType == TSDB_DATA_TYPE_JSON) { + return 0; + } + + if (right->pFilter == NULL) { + return 0; + } + + // Iterate through all values in the hash table and check type compatibility + void *pIter = taosHashIterate(right->pFilter, NULL); + while (pIter != NULL) { + // Get the stored value information + SInFilterValue *inValue = (SInFilterValue *)pIter; + uint8_t valueType = inValue->valType; + + // Check for unsupported types in values + if (valueType == TSDB_DATA_TYPE_GEOMETRY || valueType == TSDB_DATA_TYPE_JSON) { + return 0; + } + + // Type compatibility check - similar logic to sifShouldUseIndexBasedOnType + if (IS_VAR_DATA_TYPE(left->colValType)) { + if (!IS_VAR_DATA_TYPE(valueType)) { + return 0; // Incompatible: column is var type but value is not + } + } else if (IS_NUMERIC_TYPE(left->colValType)) { + // For IN operations, be more permissive with numeric type compatibility + // This addresses the issue where parser assigns different types to values in IN vs == operations + if (IS_NUMERIC_TYPE(valueType)) { + // Allow any numeric type conversion - the actual value range checking + // will be handled during execution if the value doesn't fit + return 1; + } else if (IS_VAR_DATA_TYPE(valueType)) { + // Allow string values for numeric columns (they can be converted during execution) + return 1; + } else { + return 0; // Incompatible: column is numeric but value is neither numeric nor string + } + } + + pIter = taosHashIterate(right->pFilter, pIter); + } + + return 1; // All values are compatible with column type +} + static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFParam *output) { int ret = 0; SIndexMetaArg *arg = &output->arg; @@ -726,19 +800,30 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP SIF_ERR_RET(sifGetFuncFromSql(operType, &qtype)); // Handle IN operator with multiple values if (operType == OP_TYPE_IN && right->pFilter != NULL) { + // First check if all values in the IN operation are compatible with column type for index usage + int8_t useIndex = sifShouldUseIndexForInOperator(left, right); + if (!useIndex) { + output->status = SFLT_NOT_INDEX; + return TSDB_CODE_INVALID_PARA; + } + // For IN operator, we need to iterate through all values in the set void *pIter = taosHashIterate(right->pFilter, NULL); while (pIter != NULL) { SDataTypeBuf typedata; memset(&typedata, 0, sizeof(typedata)); + // Get the stored value information + SInFilterValue *inValue = (SInFilterValue *)pIter; + void *actualValue = inValue->data; + bool reverse = false, equal = true; // IN operator is essentially multiple equality checks FilterFunc filterFunc = sifEqual; SMetaFltParam param = {.suid = arg->suid, .cid = left->colId, .type = left->colValType, - .val = pIter, + .val = actualValue, .reverse = reverse, .equal = equal, .filterFunc = filterFunc}; From 1b94fdf3f0e97e6c13a5ad27d76de8930245154a Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 26 Aug 2025 17:03:21 +0800 Subject: [PATCH 09/20] feat: tsdb cache support row format cache. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 687 +++++++++++++++++++++++- 1 file changed, 671 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 4c8ad379be34..57fde85028ed 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -22,6 +22,10 @@ #define ROCKS_BATCH_SIZE (4096) +// Row-based cache control macro +// Set to 1 for row-based cache, 0 for column-based cache +#define TSDB_CACHE_ROW_BASED 1 + void tsdbLRUCacheRelease(SLRUCache *cache, LRUHandle *handle, bool eraseIfLastRef) { if (!taosLRUCacheRelease(cache, handle, eraseIfLastRef)) { tsdbTrace(" release lru cache failed"); @@ -112,6 +116,13 @@ static void tsdbClosePgCache(STsdb *pTsdb) { #define ROCKS_KEY_LEN (sizeof(tb_uid_t) + sizeof(int16_t) + sizeof(int8_t)) +// Macro to get the appropriate key length based on cache type +#if TSDB_CACHE_ROW_BASED +#define TSDB_CACHE_KEY_LEN ROCKS_ROW_KEY_LEN +#else +#define TSDB_CACHE_KEY_LEN ROCKS_KEY_LEN +#endif + enum { LFLAG_LAST_ROW = 0, LFLAG_LAST = 1, @@ -126,6 +137,27 @@ typedef struct { #define IS_LAST_ROW_KEY(k) (((k).lflag & LFLAG_LAST) == LFLAG_LAST_ROW) #define IS_LAST_KEY(k) (((k).lflag & LFLAG_LAST) == LFLAG_LAST) +#if TSDB_CACHE_ROW_BASED +// Row-based cache key structure +#define ROCKS_ROW_KEY_LEN (sizeof(tb_uid_t) + sizeof(int8_t)) + +typedef struct { + tb_uid_t uid; + int8_t lflag; // LFLAG_LAST_ROW or LFLAG_LAST +} SLastRowKey; + +// Row-based cache value structure +typedef struct { + STsdbRowKey rowKey; // Row key with timestamp and primary keys + SRow *pRow; // Complete row data + int8_t dirty; // Dirty flag for write-back + uint8_t cacheStatus; // Cache status +} SLastRow; + +#define IS_ROW_LAST_ROW_KEY(k) (((k).lflag & LFLAG_LAST) == LFLAG_LAST_ROW) +#define IS_ROW_LAST_KEY(k) (((k).lflag & LFLAG_LAST) == LFLAG_LAST) +#endif + static void tsdbGetRocksPath(STsdb *pTsdb, char *path) { SVnode *pVnode = pTsdb->pVnode; vnodeGetPrimaryPath(pVnode, false, path, TSDB_FILENAME_LEN); @@ -145,6 +177,25 @@ static int myCmp(void *state, const char *a, size_t alen, const char *b, size_t (void)state; (void)alen; (void)blen; + +#if TSDB_CACHE_ROW_BASED + SLastRowKey *lhs = (SLastRowKey *)a; + SLastRowKey *rhs = (SLastRowKey *)b; + + if (lhs->uid < rhs->uid) { + return -1; + } else if (lhs->uid > rhs->uid) { + return 1; + } + + if ((lhs->lflag & LFLAG_LAST) < (rhs->lflag & LFLAG_LAST)) { + return -1; + } else if ((lhs->lflag & LFLAG_LAST) > (rhs->lflag & LFLAG_LAST)) { + return 1; + } + + return 0; +#else SLastKey *lhs = (SLastKey *)a; SLastKey *rhs = (SLastKey *)b; @@ -167,6 +218,7 @@ static int myCmp(void *state, const char *a, size_t alen, const char *b, size_t } return 0; +#endif } static int32_t tsdbOpenRocksCache(STsdb *pTsdb) { @@ -492,6 +544,114 @@ static int32_t tsdbCacheSerialize(SLastCol *pLastCol, char **value, size_t *size TAOS_RETURN(TSDB_CODE_SUCCESS); } +#if TSDB_CACHE_ROW_BASED +// Forward declarations for row-based cache functions +static int32_t tsdbRowCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, + SRow **aRow); +static int32_t tsdbRowCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData); +static int32_t tsdbRowCacheUpdateFromCtxArray(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray); +static void tsdbRowCacheDeleter(const void *key, size_t klen, void *value, void *ud); +static void tsdbRowCacheOverWriter(const void *key, size_t klen, void *value, void *ud); +int tsdbRowCacheFlushDirty(const void *key, size_t klen, void *value, void *ud); + +// Row-based cache serialization functions +static int32_t tsdbRowCacheSerialize(SLastRow *pLastRow, char **value, size_t *size) { + if (!pLastRow || !pLastRow->pRow) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + // Calculate total size needed + *size = sizeof(STsdbRowKey) + sizeof(int8_t) + sizeof(uint8_t); // rowKey + dirty + cacheStatus + *size += TD_ROW_LEN(pLastRow->pRow); // Row data size + + *value = taosMemoryMalloc(*size); + if (NULL == *value) { + TAOS_RETURN(terrno); + } + + int32_t offset = 0; + + // Serialize row key + memcpy(*value + offset, &pLastRow->rowKey, sizeof(STsdbRowKey)); + offset += sizeof(STsdbRowKey); + + // Serialize dirty flag + *((int8_t *)(*value + offset)) = pLastRow->dirty; + offset += sizeof(int8_t); + + // Serialize cache status + *((uint8_t *)(*value + offset)) = pLastRow->cacheStatus; + offset += sizeof(uint8_t); + + // Serialize row data + int32_t rowLen = TD_ROW_LEN(pLastRow->pRow); + memcpy(*value + offset, pLastRow->pRow, rowLen); + + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +static int32_t tsdbRowCacheDeserialize(char const *value, size_t size, SLastRow **ppLastRow) { + if (!value || size == 0) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + SLastRow *pLastRow = taosMemoryCalloc(1, sizeof(SLastRow)); + if (NULL == pLastRow) { + TAOS_RETURN(terrno); + } + + int32_t offset = 0; + + // Deserialize row key + if (offset + sizeof(STsdbRowKey) > size) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + } + memcpy(&pLastRow->rowKey, value + offset, sizeof(STsdbRowKey)); + offset += sizeof(STsdbRowKey); + + // Deserialize dirty flag + if (offset + sizeof(int8_t) > size) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + } + pLastRow->dirty = *((int8_t *)(value + offset)); + offset += sizeof(int8_t); + + // Deserialize cache status + if (offset + sizeof(uint8_t) > size) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + } + pLastRow->cacheStatus = *((uint8_t *)(value + offset)); + offset += sizeof(uint8_t); + + // Deserialize row data + int32_t remainingSize = size - offset; + if (remainingSize <= 0) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + } + + pLastRow->pRow = taosMemoryMalloc(remainingSize); + if (!pLastRow->pRow) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(terrno); + } + memcpy(pLastRow->pRow, value + offset, remainingSize); + + *ppLastRow = pLastRow; + TAOS_RETURN(TSDB_CODE_SUCCESS); +} + +static void tsdbRowCacheFreeItem(void *pItem) { + SLastRow *pRow = (SLastRow *)pItem; + if (pRow && pRow->pRow) { + taosMemoryFree(pRow->pRow); + } +} +#endif + static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol); int tsdbCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { @@ -795,7 +955,11 @@ int32_t tsdbCacheCommit(STsdb *pTsdb) { (void)taosThreadMutexLock(&pTsdb->lruMutex); +#if TSDB_CACHE_ROW_BASED + taosLRUCacheApply(pCache, tsdbRowCacheFlushDirty, pTsdb); +#else taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); +#endif #ifdef USE_ROCKSDB rocksMayWrite(pTsdb, true); @@ -926,11 +1090,29 @@ static void tsdbCacheOverWriter(const void *key, size_t klen, void *value, void static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol, int8_t dirty); +#if TSDB_CACHE_ROW_BASED +static int32_t tsdbRowCachePutToLRU(STsdb *pTsdb, SLastRowKey *pLastRowKey, SLastRow *pLastRow, int8_t dirty); +#endif + static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, int8_t col_type, int8_t lflag) { int32_t code = 0, lino = 0; +#if TSDB_CACHE_ROW_BASED + // Use row-based cache instead of column-based cache + SLRUCache *pCache = pTsdb->lruCache; + STsdbRowKey emptyRowKey = {.key = {.ts = TSKEY_MIN, .numOfPKs = 0}}; + SLastRow emptyRow = {.rowKey = emptyRowKey, .pRow = NULL, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + + // Convert lflag to row-based cache flag + int8_t rowLflag = (lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW; + SLastRowKey *pLastRowKey = &(SLastRowKey){.lflag = rowLflag, .uid = uid}; + code = tsdbRowCachePutToLRU(pTsdb, pLastRowKey, &emptyRow, 1); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + } +#else + // Original column-based cache logic SLRUCache *pCache = pTsdb->lruCache; - // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; SRowKey emptyRowKey = {.ts = TSKEY_MIN, .numOfPKs = 0}; SLastCol emptyCol = { .rowKey = emptyRowKey, .colVal = COL_VAL_NONE(cid, col_type), .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; @@ -940,6 +1122,7 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); } +#endif TAOS_RETURN(code); } @@ -951,7 +1134,11 @@ int32_t tsdbCacheCommitNoLock(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->lruCache; // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; +#if TSDB_CACHE_ROW_BASED + taosLRUCacheApply(pCache, tsdbRowCacheFlushDirty, pTsdb); +#else taosLRUCacheApply(pCache, tsdbCacheFlushDirty, pTsdb); +#endif #ifdef USE_ROCKSDB rocksMayWrite(pTsdb, true); rocksdb_flush(pTsdb->rCache.db, pTsdb->rCache.flushoptions, &err); @@ -1006,7 +1193,7 @@ static int32_t tsdbCacheDropTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, taosMemoryFree(keys_list); return terrno; } - const size_t klen = ROCKS_KEY_LEN; + const size_t klen = TSDB_CACHE_KEY_LEN; char *keys = taosMemoryCalloc(2, sizeof(SLastKey)); if (!keys) { @@ -1397,7 +1584,7 @@ static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; (void)taosThreadMutexLock(&pTsdb->rCache.writeBatchMutex); - rocksdb_writebatch_put(wb, (char *)pLastKey, ROCKS_KEY_LEN, rocks_value, vlen); + rocksdb_writebatch_put(wb, (char *)pLastKey, TSDB_CACHE_KEY_LEN, rocks_value, vlen); (void)taosThreadMutexUnlock(&pTsdb->rCache.writeBatchMutex); taosMemoryFree(rocks_value); @@ -1435,11 +1622,128 @@ static int32_t tsdbCachePutToLRU(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLa TAOS_RETURN(code); } +#if TSDB_CACHE_ROW_BASED +// Row-based cache RocksDB operations +static int32_t tsdbRowCachePutToRocksdb(STsdb *pTsdb, SLastRowKey *pLastRowKey, SLastRow *pLastRow) { + int32_t code = 0; +#ifdef USE_ROCKSDB + char *rocks_value = NULL; + size_t vlen = 0; + + code = tsdbRowCacheSerialize(pLastRow, &rocks_value, &vlen); + if (code) { + tsdbError("tsdb/rowcache/putrocks: vgId:%d, serialize failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + TAOS_RETURN(code); + } + + rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; + (void)taosThreadMutexLock(&pTsdb->rCache.writeBatchMutex); + rocksdb_writebatch_put(wb, (char *)pLastRowKey, ROCKS_ROW_KEY_LEN, rocks_value, vlen); + (void)taosThreadMutexUnlock(&pTsdb->rCache.writeBatchMutex); + + taosMemoryFree(rocks_value); +#endif + TAOS_RETURN(code); +} + +static int32_t tsdbRowCachePutToLRU(STsdb *pTsdb, SLastRowKey *pLastRowKey, SLastRow *pLastRow, int8_t dirty) { + int32_t code = 0, lino = 0; + + SLastRow *pLRULastRow = taosMemoryCalloc(1, sizeof(SLastRow)); + if (!pLRULastRow) { + return terrno; + } + + *pLRULastRow = *pLastRow; + pLRULastRow->dirty = dirty; + + // Allocate memory for row data + if (pLastRow->pRow) { + int32_t rowLen = TD_ROW_LEN(pLastRow->pRow); + pLRULastRow->pRow = taosMemoryMalloc(rowLen); + if (!pLRULastRow->pRow) { + taosMemoryFree(pLRULastRow); + TAOS_RETURN(terrno); + } + memcpy(pLRULastRow->pRow, pLastRow->pRow, rowLen); + } + + size_t charge = sizeof(SLastRow) + (pLRULastRow->pRow ? TD_ROW_LEN(pLRULastRow->pRow) : 0); + + LRUStatus status = + taosLRUCacheInsert(pTsdb->lruCache, pLastRowKey, ROCKS_ROW_KEY_LEN, pLRULastRow, charge, tsdbRowCacheDeleter, + tsdbRowCacheOverWriter, NULL, TAOS_LRU_PRIORITY_LOW, pTsdb); + if (TAOS_LRU_STATUS_OK != status && TAOS_LRU_STATUS_OK_OVERWRITTEN != status) { + tsdbError("vgId:%d, %s failed at line %d status %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__, status); + code = TSDB_CODE_FAILED; + pLRULastRow = NULL; + } + +_exit: + if (TSDB_CODE_SUCCESS != code) { + if (pLRULastRow) { + tsdbRowCacheFreeItem(pLRULastRow); + taosMemoryFree(pLRULastRow); + } + tsdbError("tsdb/rowcache/putlru: vgId:%d, failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, + tstrerror(code)); + } + + TAOS_RETURN(code); +} + +// Row-based cache deleter and overwriter for LRU +static void tsdbRowCacheDeleter(const void *key, size_t klen, void *value, void *ud) { + SLastRow *pLastRow = (SLastRow *)value; + + if (pLastRow->dirty) { + STsdb *pTsdb = (STsdb *)ud; + int32_t code = tsdbRowCachePutToRocksdb(pTsdb, (SLastRowKey *)key, pLastRow); + if (code) { + tsdbTrace("tsdb/rowcache: vgId:%d, flush cache %s failed at line %d.", TD_VID(pTsdb->pVnode), __func__, __LINE__); + } + } + + tsdbRowCacheFreeItem(pLastRow); + taosMemoryFree(value); +} + +static void tsdbRowCacheOverWriter(const void *key, size_t klen, void *value, void *ud) { + SLastRow *pLastRow = (SLastRow *)value; + pLastRow->dirty = 0; +} + +// Row-based cache flush dirty function +int tsdbRowCacheFlushDirty(const void *key, size_t klen, void *value, void *ud) { + SLastRow *pLastRow = (SLastRow *)value; + + if (pLastRow->dirty) { + STsdb *pTsdb = (STsdb *)ud; + + int32_t code = tsdbRowCachePutToRocksdb(pTsdb, (SLastRowKey *)key, pLastRow); + if (code) { + tsdbError("tsdb/rowcache: vgId:%d, flush dirty lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + return code; + } + + pLastRow->dirty = 0; + rocksMayWrite(pTsdb, false); + } + + return 0; +} +#endif + static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray) { if (!updCtxArray || TARRAY_SIZE(updCtxArray) == 0) { TAOS_RETURN(TSDB_CODE_SUCCESS); } +#if TSDB_CACHE_ROW_BASED + // For row-based cache, we need to handle this differently + // Group the updates by lflag and process as complete rows + return tsdbRowCacheUpdateFromCtxArray(pTsdb, suid, uid, updCtxArray); +#else int32_t code = 0, lino = 0; int num_keys = TARRAY_SIZE(updCtxArray); @@ -1458,7 +1762,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray } SLastKey *key = &(SLastKey){.lflag = lflag, .uid = uid, .cid = pColVal->cid}; - LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN); + LRUHandle *h = taosLRUCacheLookup(pCache, key, TSDB_CACHE_KEY_LEN); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h); if (pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { @@ -1509,7 +1813,7 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; keys_list[i] = (char *)&idxKey->key; - keys_list_sizes[i] = ROCKS_KEY_LEN; + keys_list_sizes[i] = TSDB_CACHE_KEY_LEN; } rocksMayWrite(pTsdb, true); // flush writebatch cache @@ -1613,10 +1917,14 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray } TAOS_RETURN(code); +#endif } int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, SRow **aRow) { +#if TSDB_CACHE_ROW_BASED + return tsdbRowCacheRowFormatUpdate(pTsdb, suid, uid, version, nRow, aRow); +#else int32_t code = 0, lino = 0; // 1. prepare last @@ -1709,9 +2017,13 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6 taosArrayClear(ctxArray); TAOS_RETURN(code); +#endif } int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) { +#if TSDB_CACHE_ROW_BASED + return tsdbRowCacheColFormatUpdate(pTsdb, suid, uid, pBlockData); +#else int32_t code = 0, lino = 0; STSDBRowIter iter = {0}; STSchema *pTSchema = NULL; @@ -1783,8 +2095,309 @@ int32_t tsdbCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlo taosMemoryFreeClear(pTSchema); taosArrayDestroy(ctxArray); + TAOS_RETURN(code); +#endif +} + +#if TSDB_CACHE_ROW_BASED +// Row-based cache update functions +static int32_t tsdbRowCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, STsdbRowKey *pRowKey, SRow *pRow, + int8_t lflag) { + int32_t code = 0, lino = 0; + SLRUCache *pCache = pTsdb->lruCache; + + (void)taosThreadMutexLock(&pTsdb->lruMutex); + + SLastRowKey key = {.uid = uid, .lflag = lflag}; + LRUHandle *h = taosLRUCacheLookup(pCache, &key, ROCKS_ROW_KEY_LEN); + + if (h) { + SLastRow *pLastRow = (SLastRow *)taosLRUCacheValue(pCache, h); + if (pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { + int32_t cmp_res = tRowKeyCompare(&pLastRow->rowKey.key, &pRowKey->key); + if (cmp_res < 0 || (cmp_res == 0 && pRow != NULL)) { + SLastRow newLastRow = {.rowKey = *pRowKey, .pRow = pRow, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + code = tsdbRowCachePutToLRU(pTsdb, &key, &newLastRow, 1); + } + } + tsdbLRUCacheRelease(pCache, h, false); + } else { + // Load from RocksDB and update + char *keys_list[1]; + size_t keys_list_sizes[1]; + char **values_list = NULL; + size_t *values_list_sizes = NULL; + + keys_list[0] = (char *)&key; + keys_list_sizes[0] = ROCKS_ROW_KEY_LEN; + + rocksMayWrite(pTsdb, true); + code = tsdbCacheGetValuesFromRocks(pTsdb, 1, (const char *const *)keys_list, keys_list_sizes, &values_list, + &values_list_sizes); + + if (code == TSDB_CODE_SUCCESS && values_list && values_list[0]) { + SLastRow *pLastRow = NULL; + code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pLastRow); + if (code == TSDB_CODE_SUCCESS && pLastRow) { + if (pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { + int32_t cmp_res = tRowKeyCompare(&pLastRow->rowKey.key, &pRowKey->key); + if (cmp_res < 0 || (cmp_res == 0 && pRow != NULL)) { + SLastRow lastRowTmp = {.rowKey = *pRowKey, .pRow = pRow, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID}; + code = tsdbRowCachePutToRocksdb(pTsdb, &key, &lastRowTmp); + if (code == TSDB_CODE_SUCCESS) { + code = tsdbRowCachePutToLRU(pTsdb, &key, &lastRowTmp, 0); + } + } + } + if (pLastRow->pRow) { + taosMemoryFree(pLastRow->pRow); + } + taosMemoryFree(pLastRow); + } + } + + if (values_list) { +#ifdef USE_ROCKSDB + if (values_list[0]) { + rocksdb_free(values_list[0]); + } +#endif + taosMemoryFree(values_list); + } + if (values_list_sizes) { + taosMemoryFree(values_list_sizes); + } + + rocksMayWrite(pTsdb, false); + } + + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + + if (code) { + tsdbError("tsdb/rowcache: vgId:%d, update failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, + tstrerror(code)); + } + + TAOS_RETURN(code); +} + +int32_t tsdbRowCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int64_t version, int32_t nRow, + SRow **aRow) { + int32_t code = 0, lino = 0; + + if (!aRow || nRow <= 0) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + // Update last_row cache with the last row + TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(&lRow, &tsdbRowKey); + + TAOS_CHECK_GOTO(tsdbRowCacheUpdate(pTsdb, suid, uid, &tsdbRowKey, aRow[nRow - 1], LFLAG_LAST_ROW), &lino, _exit); + + // Find the most recent row with non-null values for last cache + for (int32_t iRow = nRow - 1; iRow >= 0; --iRow) { + TSDBROW tRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[iRow], .version = version}; + STsdbRowKey tRowKey = {0}; + tsdbRowGetKey(&tRow, &tRowKey); + + // Check if this row has any non-null values + STSchema *pTSchema = NULL; + int32_t sver = TSDBROW_SVERSION(&tRow); + TAOS_CHECK_GOTO(tsdbUpdateSkm(pTsdb, suid, uid, sver), &lino, _exit); + pTSchema = pTsdb->rCache.pTSchema; + + STSDBRowIter iter = {0}; + TAOS_CHECK_GOTO(tsdbRowIterOpen(&iter, &tRow, pTSchema), &lino, _exit); + + bool hasValue = false; + for (SColVal *pColVal = tsdbRowIterNext(&iter); pColVal; pColVal = tsdbRowIterNext(&iter)) { + if (COL_VAL_IS_VALUE(pColVal)) { + hasValue = true; + break; + } + } + tsdbRowClose(&iter); + + if (hasValue) { + TAOS_CHECK_GOTO(tsdbRowCacheUpdate(pTsdb, suid, uid, &tRowKey, aRow[iRow], LFLAG_LAST), &lino, _exit); + break; // Found the most recent row with values, stop here + } + } + +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + } + + TAOS_RETURN(code); +} + +int32_t tsdbRowCacheColFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SBlockData *pBlockData) { + int32_t code = 0, lino = 0; + + if (!pBlockData || pBlockData->nRow <= 0) { + TAOS_RETURN(TSDB_CODE_INVALID_PARA); + } + + // Update last_row cache with the last row + TSDBROW lRow = tsdbRowFromBlockData(pBlockData, pBlockData->nRow - 1); + STsdbRowKey tsdbRowKey = {0}; + tsdbRowGetKey(&lRow, &tsdbRowKey); + + // For block data, we need to convert to row format + // Since we're working with block data, we can't directly get SRow + // For now, we'll skip row cache update for block data format + // TODO: Implement proper block data to row conversion if needed + TAOS_CHECK_GOTO(TSDB_CODE_SUCCESS, &lino, _exit); // Placeholder - skip for now + + // TODO: Implement proper block data processing for row-based cache + // For now, skip the last cache update for block data + +_exit: + + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + } + + TAOS_RETURN(code); +} + +// Row cache update from context array function +static int32_t tsdbRowCacheUpdateFromCtxArray(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray *updCtxArray) { + int32_t code = 0, lino = 0; + + if (!updCtxArray || TARRAY_SIZE(updCtxArray) == 0) { + TAOS_RETURN(TSDB_CODE_SUCCESS); + } + + // Group updates by lflag and timestamp to find the most recent entry for each type + STsdbRowKey lastRowKey = {.key.ts = TSKEY_MIN}; + STsdbRowKey lastKey = {.key.ts = TSKEY_MIN}; + bool hasLastRow = false; + bool hasLast = false; + + // Arrays to collect column values for row building + SArray *lastRowColVals = NULL; + SArray *lastColVals = NULL; + STSchema *pTSchema = NULL; + + // Find the most recent timestamp for each cache type and collect column values + int num_keys = TARRAY_SIZE(updCtxArray); + for (int i = 0; i < num_keys; ++i) { + SLastUpdateCtx *updCtx = &((SLastUpdateCtx *)TARRAY_DATA(updCtxArray))[i]; + + if (updCtx->lflag == LFLAG_LAST_ROW) { + if (tRowKeyCompare(&lastRowKey.key, &updCtx->tsdbRowKey.key) < 0) { + lastRowKey = updCtx->tsdbRowKey; + hasLastRow = true; + } + } else if (updCtx->lflag == LFLAG_LAST && COL_VAL_IS_VALUE(&updCtx->colVal)) { + if (tRowKeyCompare(&lastKey.key, &updCtx->tsdbRowKey.key) < 0) { + lastKey = updCtx->tsdbRowKey; + hasLast = true; + } + } + } + + // Get table schema needed for row building + if (hasLastRow || hasLast) { + // Use the version from the most recent update + STsdbRowKey *pMostRecentKey = (hasLastRow && hasLast) + ? (tRowKeyCompare(&lastRowKey.key, &lastKey.key) >= 0 ? &lastRowKey : &lastKey) + : (hasLastRow ? &lastRowKey : &lastKey); + + TAOS_CHECK_GOTO(metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema), &lino, _exit); + } + + // Process LAST_ROW updates + if (hasLastRow && pTSchema) { + lastRowColVals = taosArrayInit(16, sizeof(SColVal)); + if (!lastRowColVals) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + // Collect all column values with the latest timestamp for LAST_ROW + for (int i = 0; i < num_keys; ++i) { + SLastUpdateCtx *updCtx = &((SLastUpdateCtx *)TARRAY_DATA(updCtxArray))[i]; + if (updCtx->lflag == LFLAG_LAST_ROW && tRowKeyCompare(&updCtx->tsdbRowKey.key, &lastRowKey.key) == 0) { + if (!taosArrayPush(lastRowColVals, &updCtx->colVal)) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + } + } + + // Build SRow and update cache + if (TARRAY_SIZE(lastRowColVals) > 0) { + SRow *pRow = NULL; + SRowBuildScanInfo scanInfo = {0}; + + TAOS_CHECK_GOTO(tRowBuild(lastRowColVals, pTSchema, &pRow, &scanInfo), &lino, _exit); + if (pRow) { + code = tsdbRowCacheUpdate(pTsdb, suid, uid, &lastRowKey, pRow, LFLAG_LAST_ROW); + // Free pRow since tsdbRowCacheUpdate makes a copy + taosMemoryFree(pRow); + pRow = NULL; + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } + + // Process LAST updates + if (hasLast && pTSchema) { + lastColVals = taosArrayInit(16, sizeof(SColVal)); + if (!lastColVals) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + // Collect all column values with the latest timestamp for LAST + for (int i = 0; i < num_keys; ++i) { + SLastUpdateCtx *updCtx = &((SLastUpdateCtx *)TARRAY_DATA(updCtxArray))[i]; + if (updCtx->lflag == LFLAG_LAST && COL_VAL_IS_VALUE(&updCtx->colVal) && + tRowKeyCompare(&updCtx->tsdbRowKey.key, &lastKey.key) == 0) { + if (!taosArrayPush(lastColVals, &updCtx->colVal)) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + } + } + + // Build SRow and update cache + if (TARRAY_SIZE(lastColVals) > 0) { + SRow *pRow = NULL; + SRowBuildScanInfo scanInfo = {0}; + + TAOS_CHECK_GOTO(tRowBuild(lastColVals, pTSchema, &pRow, &scanInfo), &lino, _exit); + if (pRow) { + code = tsdbRowCacheUpdate(pTsdb, suid, uid, &lastKey, pRow, LFLAG_LAST); + // Free pRow since tsdbRowCacheUpdate makes a copy + taosMemoryFree(pRow); + pRow = NULL; + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } + +_exit: + // Cleanup + if (lastRowColVals) { + taosArrayDestroy(lastRowColVals); + } + if (lastColVals) { + taosArrayDestroy(lastColVals); + } + if (pTSchema) { + taosMemoryFree(pTSchema); + } + + if (code) { + tsdbError("tsdb/rowcache: vgId:%d, update from ctx array failed at line %d since %s.", TD_VID(pTsdb->pVnode), lino, + tstrerror(code)); + } + TAOS_RETURN(code); } +#endif static int32_t mergeLastCid(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCacheRowsReader *pr, int16_t *aCols, int nCols, int16_t *slotIds); @@ -1937,7 +2550,15 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr TAOS_CHECK_EXIT(code); } +#if TSDB_CACHE_ROW_BASED + // Convert column to row-based cache + SLastRowKey rowKey = {.uid = uid, .lflag = (idxKey->key.lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; + STsdbRowKey tsdbRowKey = {.key = pLastCol->rowKey}; + SLastRow tmpRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .cacheStatus = pLastCol->cacheStatus}; + code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &tmpRow, 0); +#else code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); +#endif if (code) { tsdbError("vgId:%d, %s failed at line %d since %s.", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); TAOS_CHECK_EXIT(code); @@ -1974,7 +2595,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA int num_keys = TARRAY_SIZE(remainCols); char **keys_list = taosMemoryMalloc(num_keys * sizeof(char *)); size_t *keys_list_sizes = taosMemoryMalloc(num_keys * sizeof(size_t)); - char *key_list = taosMemoryMalloc(num_keys * ROCKS_KEY_LEN); + char *key_list = taosMemoryMalloc(num_keys * TSDB_CACHE_KEY_LEN); if (!keys_list || !keys_list_sizes || !key_list) { taosMemoryFree(keys_list); taosMemoryFree(keys_list_sizes); @@ -1983,9 +2604,9 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA char **values_list = NULL; size_t *values_list_sizes = NULL; for (int i = 0; i < num_keys; ++i) { - memcpy(key_list + i * ROCKS_KEY_LEN, &((SIdxKey *)taosArrayGet(remainCols, i))->key, ROCKS_KEY_LEN); - keys_list[i] = key_list + i * ROCKS_KEY_LEN; - keys_list_sizes[i] = ROCKS_KEY_LEN; + memcpy(key_list + i * TSDB_CACHE_KEY_LEN, &((SIdxKey *)taosArrayGet(remainCols, i))->key, TSDB_CACHE_KEY_LEN); + keys_list[i] = key_list + i * TSDB_CACHE_KEY_LEN; + keys_list_sizes[i] = TSDB_CACHE_KEY_LEN; } rocksMayWrite(pTsdb, true); // flush writebatch cache @@ -2019,7 +2640,15 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA SLastCol *pToFree = pLastCol; SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[j]; if (pLastCol && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { +#if TSDB_CACHE_ROW_BASED + // Convert column to row-based cache + SLastRowKey rowKey = {.uid = uid, .lflag = (idxKey->key.lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; + STsdbRowKey tsdbRowKey = {.key = pLastCol->rowKey}; + SLastRow tmpRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .cacheStatus = pLastCol->cacheStatus}; + code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &tmpRow, 0); +#else code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); +#endif if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); taosMemoryFreeClear(pToFree); @@ -2092,13 +2721,13 @@ static int32_t tsdbCacheGetBatchFromLru(STsdb *pTsdb, tb_uid_t uid, SArray *pLas TAOS_CHECK_EXIT(terrno); } - LRUHandle *h = taosLRUCacheLookup(pCache, &key, ROCKS_KEY_LEN); + LRUHandle *h = taosLRUCacheLookup(pCache, &key, TSDB_CACHE_KEY_LEN); SLastCol *pLastCol = h ? (SLastCol *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { SLastCol lastCol = *pLastCol; if (TSDB_CODE_SUCCESS != (code = tsdbCacheReallocSLastCol(&lastCol, NULL))) { tsdbLRUCacheRelease(pCache, h, false); - TAOS_CHECK_GOTO(code, NULL, _exit); + TAOS_CHECK_GOTO(code, &lino, _exit); } if (taosArrayPush(pLastArray, &lastCol) == NULL) { @@ -2154,7 +2783,7 @@ static int32_t tsdbCacheGetBatchFromLru(STsdb *pTsdb, tb_uid_t uid, SArray *pLas for (int i = 0; i < TARRAY_SIZE(remainCols);) { SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; - LRUHandle *h = taosLRUCacheLookup(pCache, &idxKey->key, ROCKS_KEY_LEN); + LRUHandle *h = taosLRUCacheLookup(pCache, &idxKey->key, TSDB_CACHE_KEY_LEN); SLastCol *pLastCol = h ? (SLastCol *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastCol->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { SLastCol lastCol = *pLastCol; @@ -2697,15 +3326,23 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE int16_t cid = pTSchema->columns[i].colId; for (int8_t lflag = LFLAG_LAST_ROW; lflag <= LFLAG_LAST; ++lflag) { SLastKey lastKey = {.lflag = lflag, .uid = uid, .cid = cid}; - LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, &lastKey, ROCKS_KEY_LEN); + LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, &lastKey, TSDB_CACHE_KEY_LEN); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); if (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey) { +#if TSDB_CACHE_ROW_BASED + // For row-based cache, invalidate the entire row instead of individual columns + SLastRowKey rowKey = {.uid = uid, .lflag = (lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; + STsdbRowKey tsdbRowKey = {.key = {.ts = TSKEY_MIN, .numOfPKs = 0}}; + SLastRow noneRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; + code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &noneRow, 1); +#else SLastCol noneCol = {.rowKey.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(cid, pTSchema->columns[i].type), .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; code = tsdbCachePutToLRU(pTsdb, &lastKey, &noneCol, 1); +#endif } tsdbLRUCacheRelease(pTsdb->lruCache, h, false); TAOS_CHECK_EXIT(code); @@ -2733,7 +3370,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE code = terrno; goto _exit; } - const size_t klen = ROCKS_KEY_LEN; + const size_t klen = TSDB_CACHE_KEY_LEN; for (int i = 0; i < numKeys; ++i) { char *key = taosMemoryCalloc(1, sizeof(SLastKey)); @@ -2753,7 +3390,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE TAOS_CHECK_GOTO(tsdbCacheGetValuesFromRocks(pTsdb, numKeys, (const char *const *)keys_list, keys_list_sizes, &values_list, &values_list_sizes), - NULL, _exit); + &lino, _exit); // rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; for (int i = 0; i < numKeys; ++i) { @@ -2769,6 +3406,23 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE SIdxKey *idxKey = taosArrayGet(remainCols, i); SLastKey *pLastKey = &idxKey->key; if (NULL != pLastCol && (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey)) { +#if TSDB_CACHE_ROW_BASED + // For row-based cache, invalidate the entire row + SLastRowKey rowKey = {.uid = uid, .lflag = (pLastKey->lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; + STsdbRowKey tsdbRowKey = {.key = {.ts = TSKEY_MIN, .numOfPKs = 0}}; + SLastRow noCacheRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; + + if ((code = tsdbRowCachePutToRocksdb(pTsdb, &rowKey, &noCacheRow)) != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pLastCol); + tsdbError("tsdb/rowcache/del: vgId:%d, put to rocks failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + goto _exit; + } + if ((code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &noCacheRow, 0)) != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pLastCol); + tsdbError("tsdb/rowcache/del: vgId:%d, put to lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); + goto _exit; + } +#else SLastCol noCacheCol = {.rowKey.ts = TSKEY_MIN, .colVal = COL_VAL_NONE(pLastKey->cid, pTSchema->columns[idxKey->idx].type), .dirty = 0, @@ -2784,6 +3438,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE tsdbError("tsdb/cache/del: vgId:%d, put to lru failed since %s.", TD_VID(pTsdb->pVnode), tstrerror(code)); goto _exit; } +#endif } if (pLastCol == NULL) { @@ -4180,7 +4835,7 @@ static int32_t tsdbCacheLoadBlockSs(STsdbFD *pFD, uint8_t **ppBlock) { taosMemoryFree(buf); goto _exit; } - *ppBlock = buf; + *ppBlock = (uint8_t *)buf; _exit: return code; From 65e1da6f11928d57a111cc57d48c31c28c1ddf6c Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 27 Aug 2025 15:13:51 +0800 Subject: [PATCH 10/20] fix: put new table column failed. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 50 ++++++++++++++++++------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 57fde85028ed..06437bccb6f9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -556,13 +556,18 @@ int tsdbRowCacheFlushDirty(const void *key, size_t klen, void *value, // Row-based cache serialization functions static int32_t tsdbRowCacheSerialize(SLastRow *pLastRow, char **value, size_t *size) { - if (!pLastRow || !pLastRow->pRow) { + if (!pLastRow) { TAOS_RETURN(TSDB_CODE_INVALID_PARA); } // Calculate total size needed - *size = sizeof(STsdbRowKey) + sizeof(int8_t) + sizeof(uint8_t); // rowKey + dirty + cacheStatus - *size += TD_ROW_LEN(pLastRow->pRow); // Row data size + *size = sizeof(STsdbRowKey) + sizeof(int8_t) + sizeof(uint8_t) + + sizeof(int32_t); // rowKey + dirty + cacheStatus + rowLen + int32_t rowLen = 0; + if (pLastRow->pRow) { + rowLen = TD_ROW_LEN(pLastRow->pRow); + *size += rowLen; // Row data size + } *value = taosMemoryMalloc(*size); if (NULL == *value) { @@ -583,9 +588,14 @@ static int32_t tsdbRowCacheSerialize(SLastRow *pLastRow, char **value, size_t *s *((uint8_t *)(*value + offset)) = pLastRow->cacheStatus; offset += sizeof(uint8_t); - // Serialize row data - int32_t rowLen = TD_ROW_LEN(pLastRow->pRow); - memcpy(*value + offset, pLastRow->pRow, rowLen); + // Serialize row length + *((int32_t *)(*value + offset)) = rowLen; + offset += sizeof(int32_t); + + // Serialize row data (only if pRow is not NULL) + if (pLastRow->pRow && rowLen > 0) { + memcpy(*value + offset, pLastRow->pRow, rowLen); + } TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -626,19 +636,31 @@ static int32_t tsdbRowCacheDeserialize(char const *value, size_t size, SLastRow pLastRow->cacheStatus = *((uint8_t *)(value + offset)); offset += sizeof(uint8_t); - // Deserialize row data - int32_t remainingSize = size - offset; - if (remainingSize <= 0) { + // Deserialize row length + if (offset + sizeof(int32_t) > size) { taosMemoryFreeClear(pLastRow); TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } + int32_t rowLen = *((int32_t *)(value + offset)); + offset += sizeof(int32_t); - pLastRow->pRow = taosMemoryMalloc(remainingSize); - if (!pLastRow->pRow) { - taosMemoryFreeClear(pLastRow); - TAOS_RETURN(terrno); + // Deserialize row data (only if rowLen > 0) + if (rowLen > 0) { + if (offset + rowLen > size) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + } + + pLastRow->pRow = taosMemoryMalloc(rowLen); + if (!pLastRow->pRow) { + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(terrno); + } + memcpy(pLastRow->pRow, value + offset, rowLen); + } else { + // pRow is NULL when rowLen is 0 + pLastRow->pRow = NULL; } - memcpy(pLastRow->pRow, value + offset, remainingSize); *ppLastRow = pLastRow; TAOS_RETURN(TSDB_CODE_SUCCESS); From ba60286d1cba66acf7ac893af44d019f53c72f4b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Aug 2025 14:19:35 +0800 Subject: [PATCH 11/20] fix: write last & last_row to rocksdb. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 281 +++++++++++++++++++++++- 1 file changed, 276 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 06437bccb6f9..902d903bccd2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1127,11 +1127,17 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i // Convert lflag to row-based cache flag int8_t rowLflag = (lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW; - SLastRowKey *pLastRowKey = &(SLastRowKey){.lflag = rowLflag, .uid = uid}; + SLastRowKey *pLastRowKey = &(SLastRowKey){.lflag = LFLAG_LAST_ROW, .uid = uid}; code = tsdbRowCachePutToLRU(pTsdb, pLastRowKey, &emptyRow, 1); if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); } + + SLastRowKey *pLastKey = &(SLastRowKey){.lflag = LFLAG_LAST, .uid = uid}; + code = tsdbRowCachePutToLRU(pTsdb, pLastKey, &emptyRow, 1); + if (code) { + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + } #else // Original column-based cache logic SLRUCache *pCache = pTsdb->lruCache; @@ -3304,10 +3310,274 @@ static int32_t tsdbCacheGetBatchFromMem(STsdb *pTsdb, tb_uid_t uid, SArray *pLas TAOS_RETURN(code); } +#if TSDB_CACHE_ROW_BASED +static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, + int8_t ltype) { + int32_t code = 0, lino = 0; + SLRUCache *pCache = pTsdb->lruCache; + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + bool needLoadFromRocks = false; + + // Step 1: Try to get from row-based LRU cache + SLastRowKey rowKey = {.lflag = ltype, .uid = uid}; + LRUHandle *h = taosLRUCacheLookup(pCache, &rowKey, ROCKS_ROW_KEY_LEN); + SLastRow *pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; + + if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { + // Found valid row in LRU cache, extract column values + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + if (code != 0) { + tsdbLRUCacheRelease(pCache, h, false); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + + // Find column in schema + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pLastRow->rowKey.key; + lastCol.cacheStatus = pLastRow->cacheStatus; + + if (pCol) { + // Extract column value from row + SColVal colVal; + code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + + // Reallocate if needed for variable length data + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } else { + // Column not found in row, set as null + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + // Column not found in schema, set as null + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + if (taosArrayPush(pLastArray, &lastCol) == NULL) { + code = terrno; + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + goto _exit; + } + } + + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + } else { + // Row not found in LRU cache or invalid, need to load from RocksDB + needLoadFromRocks = true; + + if (h) { + tsdbLRUCacheRelease(pCache, h, false); + } + + // Fill with NONE values first + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + SLastCol noneCol = {.rowKey.ts = TSKEY_MIN, + .colVal = COL_VAL_NONE(cid, pr->pSchema->columns[pr->pSlotIds[i]].type), + .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; + + if (taosArrayPush(pLastArray, &noneCol) == NULL) { + code = terrno; + goto _exit; + } + } + } + + if (needLoadFromRocks) { + // Step 2: Try to load from RocksDB using row-based key + (void)taosThreadMutexLock(&pTsdb->lruMutex); + + // Double check LRU cache after acquiring mutex + h = taosLRUCacheLookup(pCache, &rowKey, ROCKS_ROW_KEY_LEN); + pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; + + if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { + // Found in LRU after double check, extract columns and update array + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + if (code == 0) { + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pLastRow->rowKey.key; + lastCol.cacheStatus = pLastRow->cacheStatus; + + if (pCol) { + SColVal colVal; + code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + taosArraySet(pLastArray, i, &lastCol); + } + taosMemoryFree(pTSchema); + } + tsdbLRUCacheRelease(pCache, h, false); + } else { + // Not in LRU, try RocksDB + if (h) { + tsdbLRUCacheRelease(pCache, h, false); + } + + rocksMayWrite(pTsdb, true); // flush writebatch cache + + // Load from RocksDB + const char *keys_list[1] = {(const char *)&rowKey}; + size_t keys_list_sizes[1] = {ROCKS_ROW_KEY_LEN}; + char **values_list = NULL; + size_t *values_list_sizes = NULL; + + code = tsdbCacheGetValuesFromRocks(pTsdb, 1, keys_list, keys_list_sizes, &values_list, &values_list_sizes); + if (code == 0 && values_list && values_list[0]) { + // Found in RocksDB, deserialize and extract columns + SLastRow *pRocksRow = NULL; + code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pRocksRow); + if (code == 0 && pRocksRow) { + // Put back to LRU + code = tsdbRowCachePutToLRU(pTsdb, &rowKey, pRocksRow, 0); + + if (pRocksRow->pRow) { + // Extract columns from row and update array + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + if (code == 0) { + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pRocksRow->rowKey.key; + lastCol.cacheStatus = pRocksRow->cacheStatus; + + if (pCol) { + SColVal colVal; + code = tRowGet(pRocksRow->pRow, pTSchema, cid, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + taosArraySet(pLastArray, i, &lastCol); + } + taosMemoryFree(pTSchema); + } + } + + // Free the deserialized row + if (pRocksRow->pRow) { + taosMemoryFree(pRocksRow->pRow); + } + taosMemoryFree(pRocksRow); + } + } else { + // Not in RocksDB either, need to load from raw TSDB data + // For row-based cache, we would need to implement row-based raw data loading + // For now, keep the NONE values that were already set + } + + // Clean up RocksDB values + if (values_list) { +#ifdef USE_ROCKSDB + for (int i = 0; i < 1; ++i) { + if (values_list[i]) { + rocksdb_free(values_list[i]); + } + } +#endif + taosMemoryFree(values_list); + } + if (values_list_sizes) { + taosMemoryFree(values_list_sizes); + } + } + + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); + } + + TAOS_RETURN(code); +} +#endif + int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { int32_t code = 0; int32_t lino = 0; +#if TSDB_CACHE_ROW_BASED + // Use row-based cache implementation + TAOS_CHECK_EXIT(tsdbCacheGetBatchFromRowLru(pTsdb, uid, pLastArray, pr, ltype)); +#else + // Use original column-based cache implementation SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); if (!keyArray) { TAOS_CHECK_EXIT(terrno); @@ -3319,15 +3589,16 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache TAOS_CHECK_EXIT(tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray)); } + if (keyArray) { + taosArrayDestroy(keyArray); + } +#endif + _exit: if (code) { tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } - if (keyArray) { - taosArrayDestroy(keyArray); - } - TAOS_RETURN(code); } From d0746c489f8b3a2b8fc7fd45ff8e77e62ad46d19 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Thu, 28 Aug 2025 18:49:03 +0800 Subject: [PATCH 12/20] fix: select row cache err. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 902d903bccd2..59034019a45e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3336,11 +3336,13 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p for (int i = 0; i < numKeys; ++i) { int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; STColumn *pCol = NULL; + int colIndex = -1; // Find column in schema for (int j = 0; j < pTSchema->numOfCols; ++j) { if (pTSchema->columns[j].colId == cid) { pCol = &pTSchema->columns[j]; + colIndex = j; break; } } @@ -3352,7 +3354,7 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p if (pCol) { // Extract column value from row SColVal colVal; - code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); + code = tRowGet(pLastRow->pRow, pTSchema, colIndex, &colVal); if (code == 0) { lastCol.colVal = colVal; From 710a766b790747bc6880bed313bded87b326c2bd Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Fri, 29 Aug 2025 13:46:17 +0800 Subject: [PATCH 13/20] fix: select last res err. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 59034019a45e..7de417348a9f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3562,6 +3562,46 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p (void)taosThreadMutexUnlock(&pTsdb->lruMutex); } + // Add memory query logic if tsUpdateCacheBatch is enabled + if (tsUpdateCacheBatch) { + // Create and populate keyArray for compatibility with tsdbCacheGetBatchFromMem + SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); + if (!keyArray) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + // Fill keyArray with keys for all columns being queried + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + // Handle function type for last_row vs last + int32_t funcType = FUNCTION_TYPE_CACHE_LAST; + if (pr->pFuncTypeList != NULL && taosArrayGetSize(pr->pFuncTypeList) > i) { + funcType = ((int32_t *)TARRAY_DATA(pr->pFuncTypeList))[i]; + } + if (((pr->type & CACHESCAN_RETRIEVE_LAST) == CACHESCAN_RETRIEVE_LAST) && + FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { + int8_t tempType = CACHESCAN_RETRIEVE_LAST_ROW | (pr->type ^ CACHESCAN_RETRIEVE_LAST); + key.lflag = (tempType & CACHESCAN_RETRIEVE_LAST) >> 3; + } + + if (!taosArrayPush(keyArray, &key)) { + taosArrayDestroy(keyArray); + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + } + + // Use the existing tsdbCacheGetBatchFromMem function which works for both row and col cache + code = tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray); + taosArrayDestroy(keyArray); + if (code) { + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + _exit: if (code) { tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); From 04e3c63e063a088d98b0a3e20ea6cf592935d186 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 2 Sep 2025 10:31:59 +0800 Subject: [PATCH 14/20] feat: get last from raw data file. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 7de417348a9f..53ff4ec4d476 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3541,6 +3541,18 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p // Not in RocksDB either, need to load from raw TSDB data // For row-based cache, we would need to implement row-based raw data loading // For now, keep the NONE values that were already set + SArray *remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + SIdxKey idxKey = {.idx = i, .key = key}; + taosArrayPush(remainCols, &idxKey); + } + code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); + if (code) { + TAOS_CHECK_GOTO(code, &lino, _exit); + } + taosArrayDestroy(remainCols); } // Clean up RocksDB values From c134740db3d7bbca5c0e8b81bd938633ddc31f7c Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 2 Sep 2025 10:50:16 +0800 Subject: [PATCH 15/20] feat: refactor some last code. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 426 +++++++++++------------- 1 file changed, 187 insertions(+), 239 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 53ff4ec4d476..c83319539cf6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3311,6 +3311,176 @@ static int32_t tsdbCacheGetBatchFromMem(STsdb *pTsdb, tb_uid_t uid, SArray *pLas } #if TSDB_CACHE_ROW_BASED +// Helper function to extract column values from row data +static int32_t tsdbExtractColsFromRow(SLastRow *pLastRow, STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, + SCacheRowsReader *pr) { + int32_t code = 0; + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + + if (!pLastRow->pRow) { + return 0; + } + + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + if (code != 0) { + return code; + } + + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pLastRow->rowKey.key; + lastCol.cacheStatus = pLastRow->cacheStatus; + + if (pCol) { + SColVal colVal; + code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + return code; + } + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + taosArraySet(pLastArray, i, &lastCol); + } + + taosMemoryFree(pTSchema); + return 0; +} + +// Load row cache data from RocksDB +static int32_t tsdbCacheLoadFromRocksDB(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, + int8_t ltype, SLastRowKey *rowKey) { + int32_t code = 0, lino = 0; + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + + rocksMayWrite(pTsdb, true); // flush writebatch cache + + // Load from RocksDB + const char *keys_list[1] = {(const char *)rowKey}; + size_t keys_list_sizes[1] = {ROCKS_ROW_KEY_LEN}; + char **values_list = NULL; + size_t *values_list_sizes = NULL; + + code = tsdbCacheGetValuesFromRocks(pTsdb, 1, keys_list, keys_list_sizes, &values_list, &values_list_sizes); + if (code == 0 && values_list && values_list[0]) { + // Found in RocksDB, deserialize and extract columns + SLastRow *pRocksRow = NULL; + code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pRocksRow); + if (code == 0 && pRocksRow) { + // Put back to LRU + code = tsdbRowCachePutToLRU(pTsdb, rowKey, pRocksRow, 0); + + // Extract columns from row using helper function + if (code == 0) { + code = tsdbExtractColsFromRow(pRocksRow, pTsdb, uid, pLastArray, pr); + } + + // Free the deserialized row + if (pRocksRow->pRow) { + taosMemoryFree(pRocksRow->pRow); + } + taosMemoryFree(pRocksRow); + } + } else { + // Not in RocksDB either, need to load from raw TSDB data + SArray *remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + SIdxKey idxKey = {.idx = i, .key = key}; + taosArrayPush(remainCols, &idxKey); + } + code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); + if (code) { + taosArrayDestroy(remainCols); + goto _exit; + } + taosArrayDestroy(remainCols); + } + +_exit: + // Clean up RocksDB values + if (values_list) { +#ifdef USE_ROCKSDB + for (int i = 0; i < 1; ++i) { + if (values_list[i]) { + rocksdb_free(values_list[i]); + } + } +#endif + taosMemoryFree(values_list); + } + if (values_list_sizes) { + taosMemoryFree(values_list_sizes); + } + + return code; +} + +// Load cache data from memory table +static int32_t tsdbCacheLoadFromMemTable(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, + int8_t ltype) { + int32_t code = 0, lino = 0; + + // Create and populate keyArray for compatibility with tsdbCacheGetBatchFromMem + SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); + if (!keyArray) { + TAOS_RETURN(terrno); + } + + // Fill keyArray with keys for all columns being queried + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + // Handle function type for last_row vs last + int32_t funcType = FUNCTION_TYPE_CACHE_LAST; + if (pr->pFuncTypeList != NULL && taosArrayGetSize(pr->pFuncTypeList) > i) { + funcType = ((int32_t *)TARRAY_DATA(pr->pFuncTypeList))[i]; + } + if (((pr->type & CACHESCAN_RETRIEVE_LAST) == CACHESCAN_RETRIEVE_LAST) && FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { + int8_t tempType = CACHESCAN_RETRIEVE_LAST_ROW | (pr->type ^ CACHESCAN_RETRIEVE_LAST); + key.lflag = (tempType & CACHESCAN_RETRIEVE_LAST) >> 3; + } + + if (!taosArrayPush(keyArray, &key)) { + taosArrayDestroy(keyArray); + TAOS_RETURN(terrno); + } + } + + // Use the existing tsdbCacheGetBatchFromMem function which works for both row and col cache + code = tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray); + taosArrayDestroy(keyArray); + + TAOS_RETURN(code); +} + static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { int32_t code = 0, lino = 0; @@ -3325,67 +3495,12 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p SLastRow *pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { - // Found valid row in LRU cache, extract column values - STSchema *pTSchema = NULL; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + // Found valid row in LRU cache, extract column values using helper function + code = tsdbExtractColsFromRow(pLastRow, pTsdb, uid, pLastArray, pr); + tsdbLRUCacheRelease(pCache, h, false); if (code != 0) { - tsdbLRUCacheRelease(pCache, h, false); TAOS_CHECK_GOTO(code, &lino, _exit); } - - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - STColumn *pCol = NULL; - int colIndex = -1; - - // Find column in schema - for (int j = 0; j < pTSchema->numOfCols; ++j) { - if (pTSchema->columns[j].colId == cid) { - pCol = &pTSchema->columns[j]; - colIndex = j; - break; - } - } - - SLastCol lastCol; - lastCol.rowKey = pLastRow->rowKey.key; - lastCol.cacheStatus = pLastRow->cacheStatus; - - if (pCol) { - // Extract column value from row - SColVal colVal; - code = tRowGet(pLastRow->pRow, pTSchema, colIndex, &colVal); - if (code == 0) { - lastCol.colVal = colVal; - - // Reallocate if needed for variable length data - if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { - code = tsdbCacheReallocSLastCol(&lastCol, NULL); - if (code != 0) { - taosMemoryFree(pTSchema); - tsdbLRUCacheRelease(pCache, h, false); - TAOS_CHECK_GOTO(code, &lino, _exit); - } - } - } else { - // Column not found in row, set as null - lastCol.colVal = COL_VAL_NONE(cid, pCol->type); - } - } else { - // Column not found in schema, set as null - lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); - } - - if (taosArrayPush(pLastArray, &lastCol) == NULL) { - code = terrno; - taosMemoryFree(pTSchema); - tsdbLRUCacheRelease(pCache, h, false); - goto _exit; - } - } - - taosMemoryFree(pTSchema); - tsdbLRUCacheRelease(pCache, h, false); } else { // Row not found in LRU cache or invalid, need to load from RocksDB needLoadFromRocks = true; @@ -3407,9 +3522,8 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p } } } - + // Step 2: Try to load from RocksDB using row-based key if (needLoadFromRocks) { - // Step 2: Try to load from RocksDB using row-based key (void)taosThreadMutexLock(&pTsdb->lruMutex); // Double check LRU cache after acquiring mutex @@ -3417,198 +3531,32 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { - // Found in LRU after double check, extract columns and update array - STSchema *pTSchema = NULL; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); - if (code == 0) { - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - STColumn *pCol = NULL; - - for (int j = 0; j < pTSchema->numOfCols; ++j) { - if (pTSchema->columns[j].colId == cid) { - pCol = &pTSchema->columns[j]; - break; - } - } - - SLastCol lastCol; - lastCol.rowKey = pLastRow->rowKey.key; - lastCol.cacheStatus = pLastRow->cacheStatus; - - if (pCol) { - SColVal colVal; - code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); - if (code == 0) { - lastCol.colVal = colVal; - if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { - code = tsdbCacheReallocSLastCol(&lastCol, NULL); - if (code != 0) { - taosMemoryFree(pTSchema); - tsdbLRUCacheRelease(pCache, h, false); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_CHECK_GOTO(code, &lino, _exit); - } - } - } else { - lastCol.colVal = COL_VAL_NONE(cid, pCol->type); - } - } else { - lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); - } - - taosArraySet(pLastArray, i, &lastCol); - } - taosMemoryFree(pTSchema); - } + // Found in LRU after double check, extract columns using helper function + code = tsdbExtractColsFromRow(pLastRow, pTsdb, uid, pLastArray, pr); tsdbLRUCacheRelease(pCache, h, false); + if (code != 0) { + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } } else { - // Not in LRU, try RocksDB + // Not in LRU, try RocksDB using encapsulated function if (h) { tsdbLRUCacheRelease(pCache, h, false); } - rocksMayWrite(pTsdb, true); // flush writebatch cache - - // Load from RocksDB - const char *keys_list[1] = {(const char *)&rowKey}; - size_t keys_list_sizes[1] = {ROCKS_ROW_KEY_LEN}; - char **values_list = NULL; - size_t *values_list_sizes = NULL; - - code = tsdbCacheGetValuesFromRocks(pTsdb, 1, keys_list, keys_list_sizes, &values_list, &values_list_sizes); - if (code == 0 && values_list && values_list[0]) { - // Found in RocksDB, deserialize and extract columns - SLastRow *pRocksRow = NULL; - code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pRocksRow); - if (code == 0 && pRocksRow) { - // Put back to LRU - code = tsdbRowCachePutToLRU(pTsdb, &rowKey, pRocksRow, 0); - - if (pRocksRow->pRow) { - // Extract columns from row and update array - STSchema *pTSchema = NULL; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); - if (code == 0) { - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - STColumn *pCol = NULL; - - for (int j = 0; j < pTSchema->numOfCols; ++j) { - if (pTSchema->columns[j].colId == cid) { - pCol = &pTSchema->columns[j]; - break; - } - } - - SLastCol lastCol; - lastCol.rowKey = pRocksRow->rowKey.key; - lastCol.cacheStatus = pRocksRow->cacheStatus; - - if (pCol) { - SColVal colVal; - code = tRowGet(pRocksRow->pRow, pTSchema, cid, &colVal); - if (code == 0) { - lastCol.colVal = colVal; - if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { - code = tsdbCacheReallocSLastCol(&lastCol, NULL); - if (code != 0) { - taosMemoryFree(pTSchema); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_CHECK_GOTO(code, &lino, _exit); - } - } - } else { - lastCol.colVal = COL_VAL_NONE(cid, pCol->type); - } - } else { - lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); - } - - taosArraySet(pLastArray, i, &lastCol); - } - taosMemoryFree(pTSchema); - } - } - - // Free the deserialized row - if (pRocksRow->pRow) { - taosMemoryFree(pRocksRow->pRow); - } - taosMemoryFree(pRocksRow); - } - } else { - // Not in RocksDB either, need to load from raw TSDB data - // For row-based cache, we would need to implement row-based raw data loading - // For now, keep the NONE values that were already set - SArray *remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; - SIdxKey idxKey = {.idx = i, .key = key}; - taosArrayPush(remainCols, &idxKey); - } - code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); - if (code) { - TAOS_CHECK_GOTO(code, &lino, _exit); - } - taosArrayDestroy(remainCols); - } - - // Clean up RocksDB values - if (values_list) { -#ifdef USE_ROCKSDB - for (int i = 0; i < 1; ++i) { - if (values_list[i]) { - rocksdb_free(values_list[i]); - } - } -#endif - taosMemoryFree(values_list); - } - if (values_list_sizes) { - taosMemoryFree(values_list_sizes); + code = tsdbCacheLoadFromRocksDB(pTsdb, uid, pLastArray, pr, ltype, &rowKey); + if (code != 0) { + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); } } (void)taosThreadMutexUnlock(&pTsdb->lruMutex); } - // Add memory query logic if tsUpdateCacheBatch is enabled + // Step 3: Add memory query logic if tsUpdateCacheBatch is enabled if (tsUpdateCacheBatch) { - // Create and populate keyArray for compatibility with tsdbCacheGetBatchFromMem - SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); - if (!keyArray) { - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } - - // Fill keyArray with keys for all columns being queried - SArray *pCidList = pr->pCidList; - int numKeys = TARRAY_SIZE(pCidList); - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - - SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; - // Handle function type for last_row vs last - int32_t funcType = FUNCTION_TYPE_CACHE_LAST; - if (pr->pFuncTypeList != NULL && taosArrayGetSize(pr->pFuncTypeList) > i) { - funcType = ((int32_t *)TARRAY_DATA(pr->pFuncTypeList))[i]; - } - if (((pr->type & CACHESCAN_RETRIEVE_LAST) == CACHESCAN_RETRIEVE_LAST) && - FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { - int8_t tempType = CACHESCAN_RETRIEVE_LAST_ROW | (pr->type ^ CACHESCAN_RETRIEVE_LAST); - key.lflag = (tempType & CACHESCAN_RETRIEVE_LAST) >> 3; - } - - if (!taosArrayPush(keyArray, &key)) { - taosArrayDestroy(keyArray); - TAOS_CHECK_GOTO(terrno, &lino, _exit); - } - } - - // Use the existing tsdbCacheGetBatchFromMem function which works for both row and col cache - code = tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray); - taosArrayDestroy(keyArray); + code = tsdbCacheLoadFromMemTable(pTsdb, uid, pLastArray, pr, ltype); if (code) { TAOS_CHECK_GOTO(code, &lino, _exit); } From 90fd1308fd583178640d164cd001ab912e4bea70 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 2 Sep 2025 11:15:11 +0800 Subject: [PATCH 16/20] fix: ci problems. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index c83319539cf6..ecd04584b542 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3411,7 +3411,11 @@ static int32_t tsdbCacheLoadFromRocksDB(STsdb *pTsdb, tb_uid_t uid, SArray *pLas int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; SIdxKey idxKey = {.idx = i, .key = key}; - taosArrayPush(remainCols, &idxKey); + if (!taosArrayPush(remainCols, &idxKey)) { + taosArrayDestroy(remainCols); + code = terrno; + goto _exit; + } } code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); if (code) { From 61f45611ae10133c805a0a9614f04ef7a6521804 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 2 Sep 2025 14:13:09 +0800 Subject: [PATCH 17/20] Revert "feat: refactor some last code." This reverts commit c134740db3d7bbca5c0e8b81bd938633ddc31f7c. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 430 +++++++++++++----------- 1 file changed, 239 insertions(+), 191 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ecd04584b542..53ff4ec4d476 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3311,180 +3311,6 @@ static int32_t tsdbCacheGetBatchFromMem(STsdb *pTsdb, tb_uid_t uid, SArray *pLas } #if TSDB_CACHE_ROW_BASED -// Helper function to extract column values from row data -static int32_t tsdbExtractColsFromRow(SLastRow *pLastRow, STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, - SCacheRowsReader *pr) { - int32_t code = 0; - SArray *pCidList = pr->pCidList; - int numKeys = TARRAY_SIZE(pCidList); - - if (!pLastRow->pRow) { - return 0; - } - - STSchema *pTSchema = NULL; - code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); - if (code != 0) { - return code; - } - - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - STColumn *pCol = NULL; - - for (int j = 0; j < pTSchema->numOfCols; ++j) { - if (pTSchema->columns[j].colId == cid) { - pCol = &pTSchema->columns[j]; - break; - } - } - - SLastCol lastCol; - lastCol.rowKey = pLastRow->rowKey.key; - lastCol.cacheStatus = pLastRow->cacheStatus; - - if (pCol) { - SColVal colVal; - code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); - if (code == 0) { - lastCol.colVal = colVal; - if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { - code = tsdbCacheReallocSLastCol(&lastCol, NULL); - if (code != 0) { - taosMemoryFree(pTSchema); - return code; - } - } - } else { - lastCol.colVal = COL_VAL_NONE(cid, pCol->type); - } - } else { - lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); - } - - taosArraySet(pLastArray, i, &lastCol); - } - - taosMemoryFree(pTSchema); - return 0; -} - -// Load row cache data from RocksDB -static int32_t tsdbCacheLoadFromRocksDB(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, - int8_t ltype, SLastRowKey *rowKey) { - int32_t code = 0, lino = 0; - SArray *pCidList = pr->pCidList; - int numKeys = TARRAY_SIZE(pCidList); - - rocksMayWrite(pTsdb, true); // flush writebatch cache - - // Load from RocksDB - const char *keys_list[1] = {(const char *)rowKey}; - size_t keys_list_sizes[1] = {ROCKS_ROW_KEY_LEN}; - char **values_list = NULL; - size_t *values_list_sizes = NULL; - - code = tsdbCacheGetValuesFromRocks(pTsdb, 1, keys_list, keys_list_sizes, &values_list, &values_list_sizes); - if (code == 0 && values_list && values_list[0]) { - // Found in RocksDB, deserialize and extract columns - SLastRow *pRocksRow = NULL; - code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pRocksRow); - if (code == 0 && pRocksRow) { - // Put back to LRU - code = tsdbRowCachePutToLRU(pTsdb, rowKey, pRocksRow, 0); - - // Extract columns from row using helper function - if (code == 0) { - code = tsdbExtractColsFromRow(pRocksRow, pTsdb, uid, pLastArray, pr); - } - - // Free the deserialized row - if (pRocksRow->pRow) { - taosMemoryFree(pRocksRow->pRow); - } - taosMemoryFree(pRocksRow); - } - } else { - // Not in RocksDB either, need to load from raw TSDB data - SArray *remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; - SIdxKey idxKey = {.idx = i, .key = key}; - if (!taosArrayPush(remainCols, &idxKey)) { - taosArrayDestroy(remainCols); - code = terrno; - goto _exit; - } - } - code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); - if (code) { - taosArrayDestroy(remainCols); - goto _exit; - } - taosArrayDestroy(remainCols); - } - -_exit: - // Clean up RocksDB values - if (values_list) { -#ifdef USE_ROCKSDB - for (int i = 0; i < 1; ++i) { - if (values_list[i]) { - rocksdb_free(values_list[i]); - } - } -#endif - taosMemoryFree(values_list); - } - if (values_list_sizes) { - taosMemoryFree(values_list_sizes); - } - - return code; -} - -// Load cache data from memory table -static int32_t tsdbCacheLoadFromMemTable(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, - int8_t ltype) { - int32_t code = 0, lino = 0; - - // Create and populate keyArray for compatibility with tsdbCacheGetBatchFromMem - SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); - if (!keyArray) { - TAOS_RETURN(terrno); - } - - // Fill keyArray with keys for all columns being queried - SArray *pCidList = pr->pCidList; - int numKeys = TARRAY_SIZE(pCidList); - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - - SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; - // Handle function type for last_row vs last - int32_t funcType = FUNCTION_TYPE_CACHE_LAST; - if (pr->pFuncTypeList != NULL && taosArrayGetSize(pr->pFuncTypeList) > i) { - funcType = ((int32_t *)TARRAY_DATA(pr->pFuncTypeList))[i]; - } - if (((pr->type & CACHESCAN_RETRIEVE_LAST) == CACHESCAN_RETRIEVE_LAST) && FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { - int8_t tempType = CACHESCAN_RETRIEVE_LAST_ROW | (pr->type ^ CACHESCAN_RETRIEVE_LAST); - key.lflag = (tempType & CACHESCAN_RETRIEVE_LAST) >> 3; - } - - if (!taosArrayPush(keyArray, &key)) { - taosArrayDestroy(keyArray); - TAOS_RETURN(terrno); - } - } - - // Use the existing tsdbCacheGetBatchFromMem function which works for both row and col cache - code = tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray); - taosArrayDestroy(keyArray); - - TAOS_RETURN(code); -} - static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) { int32_t code = 0, lino = 0; @@ -3499,12 +3325,67 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p SLastRow *pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { - // Found valid row in LRU cache, extract column values using helper function - code = tsdbExtractColsFromRow(pLastRow, pTsdb, uid, pLastArray, pr); - tsdbLRUCacheRelease(pCache, h, false); + // Found valid row in LRU cache, extract column values + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); if (code != 0) { + tsdbLRUCacheRelease(pCache, h, false); TAOS_CHECK_GOTO(code, &lino, _exit); } + + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + int colIndex = -1; + + // Find column in schema + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + colIndex = j; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pLastRow->rowKey.key; + lastCol.cacheStatus = pLastRow->cacheStatus; + + if (pCol) { + // Extract column value from row + SColVal colVal; + code = tRowGet(pLastRow->pRow, pTSchema, colIndex, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + + // Reallocate if needed for variable length data + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } else { + // Column not found in row, set as null + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + // Column not found in schema, set as null + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + if (taosArrayPush(pLastArray, &lastCol) == NULL) { + code = terrno; + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + goto _exit; + } + } + + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); } else { // Row not found in LRU cache or invalid, need to load from RocksDB needLoadFromRocks = true; @@ -3526,8 +3407,9 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p } } } - // Step 2: Try to load from RocksDB using row-based key + if (needLoadFromRocks) { + // Step 2: Try to load from RocksDB using row-based key (void)taosThreadMutexLock(&pTsdb->lruMutex); // Double check LRU cache after acquiring mutex @@ -3535,32 +3417,198 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { - // Found in LRU after double check, extract columns using helper function - code = tsdbExtractColsFromRow(pLastRow, pTsdb, uid, pLastArray, pr); - tsdbLRUCacheRelease(pCache, h, false); - if (code != 0) { - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_CHECK_GOTO(code, &lino, _exit); + // Found in LRU after double check, extract columns and update array + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + if (code == 0) { + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pLastRow->rowKey.key; + lastCol.cacheStatus = pLastRow->cacheStatus; + + if (pCol) { + SColVal colVal; + code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + taosArraySet(pLastArray, i, &lastCol); + } + taosMemoryFree(pTSchema); } + tsdbLRUCacheRelease(pCache, h, false); } else { - // Not in LRU, try RocksDB using encapsulated function + // Not in LRU, try RocksDB if (h) { tsdbLRUCacheRelease(pCache, h, false); } - code = tsdbCacheLoadFromRocksDB(pTsdb, uid, pLastArray, pr, ltype, &rowKey); - if (code != 0) { - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_CHECK_GOTO(code, &lino, _exit); + rocksMayWrite(pTsdb, true); // flush writebatch cache + + // Load from RocksDB + const char *keys_list[1] = {(const char *)&rowKey}; + size_t keys_list_sizes[1] = {ROCKS_ROW_KEY_LEN}; + char **values_list = NULL; + size_t *values_list_sizes = NULL; + + code = tsdbCacheGetValuesFromRocks(pTsdb, 1, keys_list, keys_list_sizes, &values_list, &values_list_sizes); + if (code == 0 && values_list && values_list[0]) { + // Found in RocksDB, deserialize and extract columns + SLastRow *pRocksRow = NULL; + code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pRocksRow); + if (code == 0 && pRocksRow) { + // Put back to LRU + code = tsdbRowCachePutToLRU(pTsdb, &rowKey, pRocksRow, 0); + + if (pRocksRow->pRow) { + // Extract columns from row and update array + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); + if (code == 0) { + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + STColumn *pCol = NULL; + + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + break; + } + } + + SLastCol lastCol; + lastCol.rowKey = pRocksRow->rowKey.key; + lastCol.cacheStatus = pRocksRow->cacheStatus; + + if (pCol) { + SColVal colVal; + code = tRowGet(pRocksRow->pRow, pTSchema, cid, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + } + } else { + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + taosArraySet(pLastArray, i, &lastCol); + } + taosMemoryFree(pTSchema); + } + } + + // Free the deserialized row + if (pRocksRow->pRow) { + taosMemoryFree(pRocksRow->pRow); + } + taosMemoryFree(pRocksRow); + } + } else { + // Not in RocksDB either, need to load from raw TSDB data + // For row-based cache, we would need to implement row-based raw data loading + // For now, keep the NONE values that were already set + SArray *remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + SIdxKey idxKey = {.idx = i, .key = key}; + taosArrayPush(remainCols, &idxKey); + } + code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); + if (code) { + TAOS_CHECK_GOTO(code, &lino, _exit); + } + taosArrayDestroy(remainCols); + } + + // Clean up RocksDB values + if (values_list) { +#ifdef USE_ROCKSDB + for (int i = 0; i < 1; ++i) { + if (values_list[i]) { + rocksdb_free(values_list[i]); + } + } +#endif + taosMemoryFree(values_list); + } + if (values_list_sizes) { + taosMemoryFree(values_list_sizes); } } (void)taosThreadMutexUnlock(&pTsdb->lruMutex); } - // Step 3: Add memory query logic if tsUpdateCacheBatch is enabled + // Add memory query logic if tsUpdateCacheBatch is enabled if (tsUpdateCacheBatch) { - code = tsdbCacheLoadFromMemTable(pTsdb, uid, pLastArray, pr, ltype); + // Create and populate keyArray for compatibility with tsdbCacheGetBatchFromMem + SArray *keyArray = taosArrayInit(16, sizeof(SLastKey)); + if (!keyArray) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + // Fill keyArray with keys for all columns being queried + SArray *pCidList = pr->pCidList; + int numKeys = TARRAY_SIZE(pCidList); + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + // Handle function type for last_row vs last + int32_t funcType = FUNCTION_TYPE_CACHE_LAST; + if (pr->pFuncTypeList != NULL && taosArrayGetSize(pr->pFuncTypeList) > i) { + funcType = ((int32_t *)TARRAY_DATA(pr->pFuncTypeList))[i]; + } + if (((pr->type & CACHESCAN_RETRIEVE_LAST) == CACHESCAN_RETRIEVE_LAST) && + FUNCTION_TYPE_CACHE_LAST_ROW == funcType) { + int8_t tempType = CACHESCAN_RETRIEVE_LAST_ROW | (pr->type ^ CACHESCAN_RETRIEVE_LAST); + key.lflag = (tempType & CACHESCAN_RETRIEVE_LAST) >> 3; + } + + if (!taosArrayPush(keyArray, &key)) { + taosArrayDestroy(keyArray); + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + } + + // Use the existing tsdbCacheGetBatchFromMem function which works for both row and col cache + code = tsdbCacheGetBatchFromMem(pTsdb, uid, pLastArray, pr, keyArray); + taosArrayDestroy(keyArray); if (code) { TAOS_CHECK_GOTO(code, &lino, _exit); } From e45c136aa9373e37c41951632c168afb5b1c0a5b Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Tue, 2 Sep 2025 15:37:43 +0800 Subject: [PATCH 18/20] fix: ci problems. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 5 ++++- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 53ff4ec4d476..679039055f1e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3477,7 +3477,10 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p size_t *values_list_sizes = NULL; code = tsdbCacheGetValuesFromRocks(pTsdb, 1, keys_list, keys_list_sizes, &values_list, &values_list_sizes); - if (code == 0 && values_list && values_list[0]) { + if (code) { + TAOS_RETURN(code); + } + if (values_list && values_list[0]) { // Found in RocksDB, deserialize and extract columns SLastRow *pRocksRow = NULL; code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pRocksRow); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f93485d5c63e..4e4aeb1a46f2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -581,7 +581,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } TSDB_CHECK_CODE(code, lino, _end); - if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal) || + COL_VAL_IS_NULL(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); continue; } @@ -693,7 +694,8 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 TSDB_CHECK_CODE(code, lino, _end); } - if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal) || + COL_VAL_IS_NULL(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); continue; } From e3ed89cd551798c0cb7eceaa35441c34ed2872e4 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 3 Sep 2025 08:40:30 +0800 Subject: [PATCH 19/20] feat: add cache status in tsdb cache row. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 557 +++++++++++++----- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 6 +- .../2-query/test_last_cache_scan.py | 1 + 3 files changed, 420 insertions(+), 144 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 679039055f1e..c9f468fd3314 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -146,12 +146,19 @@ typedef struct { int8_t lflag; // LFLAG_LAST_ROW or LFLAG_LAST } SLastRowKey; -// Row-based cache value structure +// Column cache status structure - tracks cache status for each column typedef struct { - STsdbRowKey rowKey; // Row key with timestamp and primary keys - SRow *pRow; // Complete row data - int8_t dirty; // Dirty flag for write-back - uint8_t cacheStatus; // Cache status + int16_t cid; // Column ID + ELastCacheStatus status; // Cache status for this column +} SColCacheStatus; + +// Row-based cache value structure with per-column cache status +typedef struct { + STsdbRowKey rowKey; // Row key with timestamp and primary keys + SRow *pRow; // Complete row data + int8_t dirty; // Dirty flag for write-back + int32_t numCols; // Number of columns with cache status + SColCacheStatus *colStatus; // Array of column cache statuses } SLastRow; #define IS_ROW_LAST_ROW_KEY(k) (((k).lflag & LFLAG_LAST) == LFLAG_LAST_ROW) @@ -561,13 +568,16 @@ static int32_t tsdbRowCacheSerialize(SLastRow *pLastRow, char **value, size_t *s } // Calculate total size needed - *size = sizeof(STsdbRowKey) + sizeof(int8_t) + sizeof(uint8_t) + - sizeof(int32_t); // rowKey + dirty + cacheStatus + rowLen + *size = + sizeof(STsdbRowKey) + sizeof(int8_t) + sizeof(int32_t) + sizeof(int32_t); // rowKey + dirty + numCols + rowLen int32_t rowLen = 0; if (pLastRow->pRow) { rowLen = TD_ROW_LEN(pLastRow->pRow); *size += rowLen; // Row data size } + if (pLastRow->numCols > 0) { + *size += pLastRow->numCols * sizeof(SColCacheStatus); // Column status array size + } *value = taosMemoryMalloc(*size); if (NULL == *value) { @@ -584,9 +594,9 @@ static int32_t tsdbRowCacheSerialize(SLastRow *pLastRow, char **value, size_t *s *((int8_t *)(*value + offset)) = pLastRow->dirty; offset += sizeof(int8_t); - // Serialize cache status - *((uint8_t *)(*value + offset)) = pLastRow->cacheStatus; - offset += sizeof(uint8_t); + // Serialize number of columns + *((int32_t *)(*value + offset)) = pLastRow->numCols; + offset += sizeof(int32_t); // Serialize row length *((int32_t *)(*value + offset)) = rowLen; @@ -595,6 +605,12 @@ static int32_t tsdbRowCacheSerialize(SLastRow *pLastRow, char **value, size_t *s // Serialize row data (only if pRow is not NULL) if (pLastRow->pRow && rowLen > 0) { memcpy(*value + offset, pLastRow->pRow, rowLen); + offset += rowLen; + } + + // Serialize column cache statuses + if (pLastRow->numCols > 0 && pLastRow->colStatus) { + memcpy(*value + offset, pLastRow->colStatus, pLastRow->numCols * sizeof(SColCacheStatus)); } TAOS_RETURN(TSDB_CODE_SUCCESS); @@ -628,13 +644,13 @@ static int32_t tsdbRowCacheDeserialize(char const *value, size_t size, SLastRow pLastRow->dirty = *((int8_t *)(value + offset)); offset += sizeof(int8_t); - // Deserialize cache status - if (offset + sizeof(uint8_t) > size) { + // Deserialize number of columns + if (offset + sizeof(int32_t) > size) { taosMemoryFreeClear(pLastRow); TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); } - pLastRow->cacheStatus = *((uint8_t *)(value + offset)); - offset += sizeof(uint8_t); + pLastRow->numCols = *((int32_t *)(value + offset)); + offset += sizeof(int32_t); // Deserialize row length if (offset + sizeof(int32_t) > size) { @@ -662,16 +678,98 @@ static int32_t tsdbRowCacheDeserialize(char const *value, size_t size, SLastRow pLastRow->pRow = NULL; } + // Deserialize column cache statuses + if (pLastRow->numCols > 0) { + size_t colStatusSize = pLastRow->numCols * sizeof(SColCacheStatus); + if (offset + colStatusSize > size) { + if (pLastRow->pRow) { + taosMemoryFree(pLastRow->pRow); + } + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(TSDB_CODE_INVALID_DATA_FMT); + } + + pLastRow->colStatus = taosMemoryMalloc(colStatusSize); + if (!pLastRow->colStatus) { + if (pLastRow->pRow) { + taosMemoryFree(pLastRow->pRow); + } + taosMemoryFreeClear(pLastRow); + TAOS_RETURN(terrno); + } + memcpy(pLastRow->colStatus, value + offset, colStatusSize); + } else { + pLastRow->colStatus = NULL; + } + *ppLastRow = pLastRow; TAOS_RETURN(TSDB_CODE_SUCCESS); } static void tsdbRowCacheFreeItem(void *pItem) { SLastRow *pRow = (SLastRow *)pItem; - if (pRow && pRow->pRow) { - taosMemoryFree(pRow->pRow); + if (pRow) { + if (pRow->pRow) { + taosMemoryFree(pRow->pRow); + } + if (pRow->colStatus) { + taosMemoryFree(pRow->colStatus); + } } } + +// Helper functions for column cache status management +static int32_t tsdbLastRowInitColStatus(SLastRow *pLastRow, STSchema *pTSchema) { + if (!pLastRow || !pTSchema) { + return TSDB_CODE_INVALID_PARA; + } + + pLastRow->numCols = pTSchema->numOfCols; + pLastRow->colStatus = taosMemoryCalloc(pLastRow->numCols, sizeof(SColCacheStatus)); + if (!pLastRow->colStatus) { + return terrno; + } + + for (int32_t i = 0; i < pLastRow->numCols; i++) { + pLastRow->colStatus[i].cid = pTSchema->columns[i].colId; + pLastRow->colStatus[i].status = TSDB_LAST_CACHE_VALID; + } + + return TSDB_CODE_SUCCESS; +} + +static ELastCacheStatus tsdbLastRowGetColStatus(SLastRow *pLastRow, int16_t cid) { + if (!pLastRow || !pLastRow->colStatus) { + return TSDB_LAST_CACHE_NO_CACHE; + } + + for (int32_t i = 0; i < pLastRow->numCols; i++) { + if (pLastRow->colStatus[i].cid == cid) { + return pLastRow->colStatus[i].status; + } + } + + return TSDB_LAST_CACHE_NO_CACHE; +} + +static int32_t tsdbLastRowSetColStatus(SLastRow *pLastRow, int16_t cid, ELastCacheStatus status) { + if (!pLastRow || !pLastRow->colStatus) { + return TSDB_CODE_INVALID_PARA; + } + + for (int32_t i = 0; i < pLastRow->numCols; i++) { + if (pLastRow->colStatus[i].cid == cid) { + pLastRow->colStatus[i].status = status; + return TSDB_CODE_SUCCESS; + } + } + + return TSDB_CODE_NOT_FOUND; +} + +static int32_t tsdbLastRowInvalidateCol(SLastRow *pLastRow, int16_t cid) { + return tsdbLastRowSetColStatus(pLastRow, cid, TSDB_LAST_CACHE_NO_CACHE); +} #endif static int32_t tsdbCachePutToRocksdb(STsdb *pTsdb, SLastKey *pLastKey, SLastCol *pLastCol); @@ -1120,24 +1218,72 @@ static int32_t tsdbCacheNewTableColumn(STsdb *pTsdb, int64_t uid, int16_t cid, i int32_t code = 0, lino = 0; #if TSDB_CACHE_ROW_BASED - // Use row-based cache instead of column-based cache - SLRUCache *pCache = pTsdb->lruCache; - STsdbRowKey emptyRowKey = {.key = {.ts = TSKEY_MIN, .numOfPKs = 0}}; - SLastRow emptyRow = {.rowKey = emptyRowKey, .pRow = NULL, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + // Create empty column similar to line 1242-1244, then encode to row and write to LRU + STSchema *pTSchema = NULL; + code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, 0, uid, -1, &pTSchema); + if (code != 0) { + TAOS_RETURN(code); + } + + // Create empty column for the new column + SRowKey emptyRowKey = {.ts = TSKEY_MIN, .numOfPKs = 0}; + SLastCol emptyCol = { + .rowKey = emptyRowKey, .colVal = COL_VAL_NONE(cid, col_type), .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; + + // Create array with just this one column to build row + SArray *colVals = taosArrayInit(1, sizeof(SColVal)); + if (!colVals) { + taosMemoryFree(pTSchema); + TAOS_RETURN(terrno); + } + + taosArrayPush(colVals, &emptyCol.colVal); + + // Build SRow from the column values + SRow *pRow = NULL; + SRowBuildScanInfo scanInfo = {0}; + code = tRowBuild(colVals, pTSchema, &pRow, &scanInfo); - // Convert lflag to row-based cache flag - int8_t rowLflag = (lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW; - SLastRowKey *pLastRowKey = &(SLastRowKey){.lflag = LFLAG_LAST_ROW, .uid = uid}; - code = tsdbRowCachePutToLRU(pTsdb, pLastRowKey, &emptyRow, 1); + taosArrayDestroy(colVals); + + if (code != 0) { + taosMemoryFree(pTSchema); + TAOS_RETURN(code); + } + + // Create row cache entry with the built row + STsdbRowKey rowKey = {.key = emptyRowKey}; + SLastRow newRow = {.rowKey = rowKey, .pRow = pRow, .dirty = 1, .numCols = 0, .colStatus = NULL}; + + // Initialize column status for the new row + code = tsdbLastRowInitColStatus(&newRow, pTSchema); + if (code != 0) { + taosMemoryFree(pTSchema); + if (pRow) taosMemoryFree(pRow); + TAOS_RETURN(code); + } + + // Set this specific column as valid + tsdbLastRowSetColStatus(&newRow, cid, TSDB_LAST_CACHE_VALID); + + // Write to LRU for both LAST_ROW and LAST flags + SLastRowKey lastRowKey = {.lflag = LFLAG_LAST_ROW, .uid = uid}; + code = tsdbRowCachePutToLRU(pTsdb, &lastRowKey, &newRow, 1); if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); + taosMemoryFree(pTSchema); + if (pRow) taosMemoryFree(pRow); + if (newRow.colStatus) taosMemoryFree(newRow.colStatus); + TAOS_RETURN(code); } - SLastRowKey *pLastKey = &(SLastRowKey){.lflag = LFLAG_LAST, .uid = uid}; - code = tsdbRowCachePutToLRU(pTsdb, pLastKey, &emptyRow, 1); + SLastRowKey lastKey = {.lflag = LFLAG_LAST, .uid = uid}; + code = tsdbRowCachePutToLRU(pTsdb, &lastKey, &newRow, 1); if (code) { tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, __LINE__, tstrerror(code)); } + + taosMemoryFree(pTSchema); #else // Original column-based cache logic SLRUCache *pCache = pTsdb->lruCache; @@ -2141,12 +2287,19 @@ static int32_t tsdbRowCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, STs if (h) { SLastRow *pLastRow = (SLastRow *)taosLRUCacheValue(pCache, h); - if (pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { - int32_t cmp_res = tRowKeyCompare(&pLastRow->rowKey.key, &pRowKey->key); - if (cmp_res < 0 || (cmp_res == 0 && pRow != NULL)) { - SLastRow newLastRow = {.rowKey = *pRowKey, .pRow = pRow, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_VALID}; - code = tsdbRowCachePutToLRU(pTsdb, &key, &newLastRow, 1); + // Always check for row cache updates - individual column status will be handled separately + int32_t cmp_res = tRowKeyCompare(&pLastRow->rowKey.key, &pRowKey->key); + if (cmp_res < 0 || (cmp_res == 0 && pRow != NULL)) { + SLastRow newLastRow = {.rowKey = *pRowKey, .pRow = pRow, .dirty = 1, .numCols = 0, .colStatus = NULL}; + + // Initialize column status array for the new row + STSchema *pTSchema = NULL; + if (metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema) == 0 && pTSchema) { + tsdbLastRowInitColStatus(&newLastRow, pTSchema); + taosMemoryFree(pTSchema); } + + code = tsdbRowCachePutToLRU(pTsdb, &key, &newLastRow, 1); } tsdbLRUCacheRelease(pCache, h, false); } else { @@ -2167,19 +2320,29 @@ static int32_t tsdbRowCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, STs SLastRow *pLastRow = NULL; code = tsdbRowCacheDeserialize(values_list[0], values_list_sizes[0], &pLastRow); if (code == TSDB_CODE_SUCCESS && pLastRow) { - if (pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE) { - int32_t cmp_res = tRowKeyCompare(&pLastRow->rowKey.key, &pRowKey->key); - if (cmp_res < 0 || (cmp_res == 0 && pRow != NULL)) { - SLastRow lastRowTmp = {.rowKey = *pRowKey, .pRow = pRow, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_VALID}; - code = tsdbRowCachePutToRocksdb(pTsdb, &key, &lastRowTmp); - if (code == TSDB_CODE_SUCCESS) { - code = tsdbRowCachePutToLRU(pTsdb, &key, &lastRowTmp, 0); - } + // Always check for row cache updates - individual column status will be handled separately + int32_t cmp_res = tRowKeyCompare(&pLastRow->rowKey.key, &pRowKey->key); + if (cmp_res < 0 || (cmp_res == 0 && pRow != NULL)) { + SLastRow lastRowTmp = {.rowKey = *pRowKey, .pRow = pRow, .dirty = 0, .numCols = 0, .colStatus = NULL}; + + // Initialize column status array for the new row + STSchema *pTSchema = NULL; + if (metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema) == 0 && pTSchema) { + tsdbLastRowInitColStatus(&lastRowTmp, pTSchema); + taosMemoryFree(pTSchema); + } + + code = tsdbRowCachePutToRocksdb(pTsdb, &key, &lastRowTmp); + if (code == TSDB_CODE_SUCCESS) { + code = tsdbRowCachePutToLRU(pTsdb, &key, &lastRowTmp, 0); } } if (pLastRow->pRow) { taosMemoryFree(pLastRow->pRow); } + if (pLastRow->colStatus) { + taosMemoryFree(pLastRow->colStatus); + } taosMemoryFree(pLastRow); } } @@ -2582,7 +2745,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr // Convert column to row-based cache SLastRowKey rowKey = {.uid = uid, .lflag = (idxKey->key.lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; STsdbRowKey tsdbRowKey = {.key = pLastCol->rowKey}; - SLastRow tmpRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .cacheStatus = pLastCol->cacheStatus}; + SLastRow tmpRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .numCols = 0, .colStatus = NULL}; code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &tmpRow, 0); #else code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); @@ -2672,7 +2835,7 @@ static int32_t tsdbCacheLoadFromRocks(STsdb *pTsdb, tb_uid_t uid, SArray *pLastA // Convert column to row-based cache SLastRowKey rowKey = {.uid = uid, .lflag = (idxKey->key.lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; STsdbRowKey tsdbRowKey = {.key = pLastCol->rowKey}; - SLastRow tmpRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .cacheStatus = pLastCol->cacheStatus}; + SLastRow tmpRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .numCols = 0, .colStatus = NULL}; code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &tmpRow, 0); #else code = tsdbCachePutToLRU(pTsdb, &idxKey->key, pLastCol, 0); @@ -3317,15 +3480,28 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p SLRUCache *pCache = pTsdb->lruCache; SArray *pCidList = pr->pCidList; int numKeys = TARRAY_SIZE(pCidList); - bool needLoadFromRocks = false; + SArray *remainCols = NULL; // Track columns that need to be loaded from subsequent layers // Step 1: Try to get from row-based LRU cache SLastRowKey rowKey = {.lflag = ltype, .uid = uid}; LRUHandle *h = taosLRUCacheLookup(pCache, &rowKey, ROCKS_ROW_KEY_LEN); SLastRow *pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; - if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { - // Found valid row in LRU cache, extract column values + // Initialize remainCols with all requested columns + remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); + if (!remainCols) { + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + for (int i = 0; i < numKeys; ++i) { + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; + SIdxKey idxKey = {.idx = i, .key = key}; + taosArrayPush(remainCols, &idxKey); + } + + if (h && pLastRow) { + // Found valid row in LRU cache, check each column and remove valid ones from remainCols STSchema *pTSchema = NULL; code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); if (code != 0) { @@ -3333,8 +3509,18 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p TAOS_CHECK_GOTO(code, &lino, _exit); } - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + // Create a new array to store columns that are still invalid after LRU check + SArray *newRemainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); + if (!newRemainCols) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } + + // Check each column in remainCols + for (int i = 0; i < TARRAY_SIZE(remainCols); ++i) { + SIdxKey *idxKey = (SIdxKey *)taosArrayGet(remainCols, i); + int16_t cid = idxKey->key.cid; STColumn *pCol = NULL; int colIndex = -1; @@ -3347,11 +3533,31 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p } } + // Check column-specific cache status + ELastCacheStatus colStatus = tsdbLastRowGetColStatus(pLastRow, cid); + + if (colStatus == TSDB_LAST_CACHE_NO_CACHE) { + // Column is invalid in LRU cache, keep it in remainCols for RocksDB/TSDB loading + int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; + SLastCol noneCol = {.rowKey.ts = TSKEY_MIN, + .colVal = COL_VAL_NONE(cid, pr->pSchema->columns[pr->pSlotIds[i]].type), + .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; + + if (taosArrayPush(pLastArray, &noneCol) == NULL) { + code = terrno; + taosArrayDestroy(newRemainCols); + TAOS_CHECK_GOTO(code, &lino, _exit); + } + taosArrayPush(newRemainCols, idxKey); + continue; + } + + // Column is valid in LRU cache, extract it and add to result SLastCol lastCol; lastCol.rowKey = pLastRow->rowKey.key; - lastCol.cacheStatus = pLastRow->cacheStatus; + lastCol.cacheStatus = colStatus; - if (pCol) { + if (pCol && colIndex >= 0) { // Extract column value from row SColVal colVal; code = tRowGet(pLastRow->pRow, pTSchema, colIndex, &colVal); @@ -3364,6 +3570,7 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p if (code != 0) { taosMemoryFree(pTSchema); tsdbLRUCacheRelease(pCache, h, false); + taosArrayDestroy(newRemainCols); TAOS_CHECK_GOTO(code, &lino, _exit); } } @@ -3376,20 +3583,23 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); } + // Add valid column to result array if (taosArrayPush(pLastArray, &lastCol) == NULL) { - code = terrno; taosMemoryFree(pTSchema); tsdbLRUCacheRelease(pCache, h, false); - goto _exit; + taosArrayDestroy(newRemainCols); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } } taosMemoryFree(pTSchema); tsdbLRUCacheRelease(pCache, h, false); - } else { - // Row not found in LRU cache or invalid, need to load from RocksDB - needLoadFromRocks = true; + // Replace remainCols with newRemainCols (contains only invalid columns) + taosArrayDestroy(remainCols); + remainCols = newRemainCols; + } else { + // Row not found in LRU cache, all columns remain in remainCols for RocksDB/TSDB loading if (h) { tsdbLRUCacheRelease(pCache, h, false); } @@ -3408,56 +3618,85 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p } } - if (needLoadFromRocks) { - // Step 2: Try to load from RocksDB using row-based key + // Step 2: If remainCols > 0, try to load from RocksDB + if (remainCols && TARRAY_SIZE(remainCols) > 0) { (void)taosThreadMutexLock(&pTsdb->lruMutex); // Double check LRU cache after acquiring mutex h = taosLRUCacheLookup(pCache, &rowKey, ROCKS_ROW_KEY_LEN); pLastRow = h ? (SLastRow *)taosLRUCacheValue(pCache, h) : NULL; - if (h && pLastRow && pLastRow->cacheStatus != TSDB_LAST_CACHE_NO_CACHE && pLastRow->pRow) { - // Found in LRU after double check, extract columns and update array + if (h && pLastRow && pLastRow->pRow) { + // Found in LRU after double check, re-check remainCols and update them STSchema *pTSchema = NULL; code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); if (code == 0) { - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - STColumn *pCol = NULL; - - for (int j = 0; j < pTSchema->numOfCols; ++j) { - if (pTSchema->columns[j].colId == cid) { - pCol = &pTSchema->columns[j]; - break; + SArray *newRemainCols = taosArrayInit(TARRAY_SIZE(remainCols), sizeof(SIdxKey)); + if (newRemainCols) { + // Re-check each column in remainCols + for (int i = 0; i < TARRAY_SIZE(remainCols); ++i) { + SIdxKey *idxKey = (SIdxKey *)taosArrayGet(remainCols, i); + int16_t cid = idxKey->key.cid; + STColumn *pCol = NULL; + int colIndex = -1; + + // Find column in schema + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + colIndex = j; + break; + } } - } - SLastCol lastCol; - lastCol.rowKey = pLastRow->rowKey.key; - lastCol.cacheStatus = pLastRow->cacheStatus; + // Check if this column is still invalid in the cached row + ELastCacheStatus colStatus = tsdbLastRowGetColStatus(pLastRow, cid); + if (colStatus == TSDB_LAST_CACHE_NO_CACHE) { + // Column is still invalid, keep it in remainCols + taosArrayPush(newRemainCols, idxKey); + continue; + } - if (pCol) { - SColVal colVal; - code = tRowGet(pLastRow->pRow, pTSchema, cid, &colVal); - if (code == 0) { - lastCol.colVal = colVal; - if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { - code = tsdbCacheReallocSLastCol(&lastCol, NULL); - if (code != 0) { - taosMemoryFree(pTSchema); - tsdbLRUCacheRelease(pCache, h, false); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_CHECK_GOTO(code, &lino, _exit); + // Column is now valid, extract from cached row + SLastCol lastCol; + lastCol.rowKey = pLastRow->rowKey.key; + lastCol.cacheStatus = colStatus; + + if (pCol && colIndex >= 0) { + SColVal colVal; + code = tRowGet(pLastRow->pRow, pTSchema, colIndex, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + taosArrayDestroy(newRemainCols); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); } } else { - lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + } + + // Add to result array + if (taosArrayPush(pLastArray, &lastCol) == NULL) { + taosMemoryFree(pTSchema); + tsdbLRUCacheRelease(pCache, h, false); + taosArrayDestroy(newRemainCols); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(terrno, &lino, _exit); } - } else { - lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); } - taosArraySet(pLastArray, i, &lastCol); + // Update remainCols with only invalid columns + taosArrayDestroy(remainCols); + remainCols = newRemainCols; } taosMemoryFree(pTSchema); } @@ -3489,46 +3728,74 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p code = tsdbRowCachePutToLRU(pTsdb, &rowKey, pRocksRow, 0); if (pRocksRow->pRow) { - // Extract columns from row and update array + // Extract only the invalid columns from RocksDB row STSchema *pTSchema = NULL; code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, pr->info.suid, uid, -1, &pTSchema); if (code == 0) { - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - STColumn *pCol = NULL; - - for (int j = 0; j < pTSchema->numOfCols; ++j) { - if (pTSchema->columns[j].colId == cid) { - pCol = &pTSchema->columns[j]; - break; + SArray *newRemainCols = taosArrayInit(TARRAY_SIZE(remainCols), sizeof(SIdxKey)); + if (newRemainCols) { + // Check each column in remainCols + for (int i = 0; i < TARRAY_SIZE(remainCols); ++i) { + SIdxKey *idxKey = (SIdxKey *)taosArrayGet(remainCols, i); + int16_t cid = idxKey->key.cid; + STColumn *pCol = NULL; + int colIndex = -1; + + // Find column in schema + for (int j = 0; j < pTSchema->numOfCols; ++j) { + if (pTSchema->columns[j].colId == cid) { + pCol = &pTSchema->columns[j]; + colIndex = j; + break; + } } - } - SLastCol lastCol; - lastCol.rowKey = pRocksRow->rowKey.key; - lastCol.cacheStatus = pRocksRow->cacheStatus; - - if (pCol) { - SColVal colVal; - code = tRowGet(pRocksRow->pRow, pTSchema, cid, &colVal); - if (code == 0) { - lastCol.colVal = colVal; - if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { - code = tsdbCacheReallocSLastCol(&lastCol, NULL); - if (code != 0) { - taosMemoryFree(pTSchema); - (void)taosThreadMutexUnlock(&pTsdb->lruMutex); - TAOS_CHECK_GOTO(code, &lino, _exit); + // Check column status in RocksDB row + ELastCacheStatus colStatus = tsdbLastRowGetColStatus(pRocksRow, cid); + if (colStatus == TSDB_LAST_CACHE_NO_CACHE) { + // Column is still invalid in RocksDB, keep it for TSDB loading + taosArrayPush(newRemainCols, idxKey); + continue; + } + + // Column is valid in RocksDB, extract it + SLastCol lastCol; + lastCol.rowKey = pRocksRow->rowKey.key; + lastCol.cacheStatus = colStatus; + + if (pCol && colIndex >= 0) { + SColVal colVal; + code = tRowGet(pRocksRow->pRow, pTSchema, colIndex, &colVal); + if (code == 0) { + lastCol.colVal = colVal; + if (IS_VAR_DATA_TYPE(colVal.value.type) && COL_VAL_IS_VALUE(&colVal)) { + code = tsdbCacheReallocSLastCol(&lastCol, NULL); + if (code != 0) { + taosMemoryFree(pTSchema); + taosArrayDestroy(newRemainCols); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(code, &lino, _exit); + } } + } else { + lastCol.colVal = COL_VAL_NONE(cid, pCol->type); } } else { - lastCol.colVal = COL_VAL_NONE(cid, pCol->type); + lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); } - } else { - lastCol.colVal = COL_VAL_NONE(cid, TSDB_DATA_TYPE_NULL); + + // Add to result array + if (taosArrayPush(pLastArray, &lastCol) == NULL) { + taosMemoryFree(pTSchema); + taosArrayDestroy(newRemainCols); + (void)taosThreadMutexUnlock(&pTsdb->lruMutex); + TAOS_CHECK_GOTO(terrno, &lino, _exit); + } } - taosArraySet(pLastArray, i, &lastCol); + // Update remainCols with only invalid columns + taosArrayDestroy(remainCols); + remainCols = newRemainCols; } taosMemoryFree(pTSchema); } @@ -3541,21 +3808,8 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p taosMemoryFree(pRocksRow); } } else { - // Not in RocksDB either, need to load from raw TSDB data - // For row-based cache, we would need to implement row-based raw data loading - // For now, keep the NONE values that were already set - SArray *remainCols = taosArrayInit(numKeys, sizeof(SIdxKey)); - for (int i = 0; i < numKeys; ++i) { - int16_t cid = ((int16_t *)TARRAY_DATA(pCidList))[i]; - SLastKey key = {.lflag = ltype, .uid = uid, .cid = cid}; - SIdxKey idxKey = {.idx = i, .key = key}; - taosArrayPush(remainCols, &idxKey); - } - code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); - if (code) { - TAOS_CHECK_GOTO(code, &lino, _exit); - } - taosArrayDestroy(remainCols); + // Not in RocksDB either, remainCols still contains columns that need TSDB loading + // These will be processed in Step 3 below } // Clean up RocksDB values @@ -3577,6 +3831,14 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p (void)taosThreadMutexUnlock(&pTsdb->lruMutex); } + // Step 3: If remainCols still > 0, load from TSDB + if (remainCols && TARRAY_SIZE(remainCols) > 0) { + code = tsdbCacheLoadFromRaw(pTsdb, uid, pLastArray, remainCols, pr, ltype); + if (code) { + TAOS_CHECK_GOTO(code, &lino, _exit); + } + } + // Add memory query logic if tsUpdateCacheBatch is enabled if (tsUpdateCacheBatch) { // Create and populate keyArray for compatibility with tsdbCacheGetBatchFromMem @@ -3618,6 +3880,11 @@ static int32_t tsdbCacheGetBatchFromRowLru(STsdb *pTsdb, tb_uid_t uid, SArray *p } _exit: + // Clean up remainCols if it exists + if (remainCols) { + taosArrayDestroy(remainCols); + } + if (code) { tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code)); } @@ -3681,17 +3948,27 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); if (pLastCol->rowKey.ts <= eKey && pLastCol->rowKey.ts >= sKey) { #if TSDB_CACHE_ROW_BASED - // For row-based cache, invalidate the entire row instead of individual columns - SLastRowKey rowKey = {.uid = uid, .lflag = (lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; - STsdbRowKey tsdbRowKey = {.key = {.ts = TSKEY_MIN, .numOfPKs = 0}}; - SLastRow noneRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 1, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; - code = tsdbRowCachePutToLRU(pTsdb, &rowKey, &noneRow, 1); + // For row-based cache, get the entire row and invalidate specific column + SLastRowKey rowKey = {.uid = uid, .lflag = lflag}; + LRUHandle *rowHandle = taosLRUCacheLookup(pTsdb->lruCache, &rowKey, ROCKS_ROW_KEY_LEN); + if (rowHandle) { + SLastRow *pLastRow = (SLastRow *)taosLRUCacheValue(pTsdb->lruCache, rowHandle); + if (pLastRow && pLastRow->rowKey.key.ts <= eKey && pLastRow->rowKey.key.ts >= sKey) { + // Invalidate this specific column in the row + tsdbLastRowInvalidateCol(pLastRow, cid); + pLastRow->dirty = 1; // Mark row as dirty for flush + + tsdbDebug("vgId:%d, invalidated column %d in row cache for uid:%" PRIu64 ", ts:%" PRId64, + TD_VID(pTsdb->pVnode), cid, uid, pLastRow->rowKey.key.ts); + } + tsdbLRUCacheRelease(pTsdb->lruCache, rowHandle, false); + } #else - SLastCol noneCol = {.rowKey.ts = TSKEY_MIN, - .colVal = COL_VAL_NONE(cid, pTSchema->columns[i].type), - .dirty = 1, - .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; - code = tsdbCachePutToLRU(pTsdb, &lastKey, &noneCol, 1); + // Column-level invalidation: mark this specific column as invalid + pLastCol->cacheStatus = TSDB_LAST_CACHE_NO_CACHE; + pLastCol->dirty = 1; // Mark as dirty for flush + tsdbDebug("vgId:%d, invalidated column cache for uid:%" PRIu64 ", cid:%d, lflag:%d, ts:%" PRId64, + TD_VID(pTsdb->pVnode), uid, cid, lflag, pLastCol->rowKey.ts); #endif } tsdbLRUCacheRelease(pTsdb->lruCache, h, false); @@ -3760,7 +4037,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE // For row-based cache, invalidate the entire row SLastRowKey rowKey = {.uid = uid, .lflag = (pLastKey->lflag == LFLAG_LAST) ? LFLAG_LAST_ROW : LFLAG_LAST_ROW}; STsdbRowKey tsdbRowKey = {.key = {.ts = TSKEY_MIN, .numOfPKs = 0}}; - SLastRow noCacheRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .cacheStatus = TSDB_LAST_CACHE_NO_CACHE}; + SLastRow noCacheRow = {.rowKey = tsdbRowKey, .pRow = NULL, .dirty = 0, .numCols = 0, .colStatus = NULL}; if ((code = tsdbRowCachePutToRocksdb(pTsdb, &rowKey, &noCacheRow)) != TSDB_CODE_SUCCESS) { taosMemoryFreeClear(pLastCol); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 4e4aeb1a46f2..f93485d5c63e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -581,8 +581,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } TSDB_CHECK_CODE(code, lino, _end); - if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal) || - COL_VAL_IS_NULL(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); continue; } @@ -694,8 +693,7 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 TSDB_CHECK_CODE(code, lino, _end); } - if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal) || - COL_VAL_IS_NULL(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { + if (TARRAY_SIZE(pRow) <= 0 || COL_VAL_IS_NONE(&((SLastCol*)TARRAY_DATA(pRow))[0].colVal)) { taosArrayClearEx(pRow, tsdbCacheFreeSLastColItem); continue; } diff --git a/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py b/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py index 06ef75f16058..dd2d2dec65ee 100644 --- a/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py +++ b/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py @@ -350,6 +350,7 @@ def check_cache_scan_with_drop_and_add_column2(self): tdSql.checkData(0, 0, '999') p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) p.check_returncode() + tdLog.exit("error") tdSql.query_success_failed("select ts, last(c1), c1, ts, c1 from meters", queryTimes=10, expectErrInfo="Invalid column name: c1") tdSql.query('select last(c12), c12, ts from meters', queryTimes=1) tdSql.checkRows(0) From 188fabe1576837967797ef492f63d53bf23a7170 Mon Sep 17 00:00:00 2001 From: xiao-77 Date: Wed, 3 Sep 2025 08:40:52 +0800 Subject: [PATCH 20/20] fix: ci problems. --- test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py b/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py index dd2d2dec65ee..06ef75f16058 100644 --- a/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py +++ b/test/cases/uncatalog/system-test/2-query/test_last_cache_scan.py @@ -350,7 +350,6 @@ def check_cache_scan_with_drop_and_add_column2(self): tdSql.checkData(0, 0, '999') p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) p.check_returncode() - tdLog.exit("error") tdSql.query_success_failed("select ts, last(c1), c1, ts, c1 from meters", queryTimes=10, expectErrInfo="Invalid column name: c1") tdSql.query('select last(c12), c12, ts from meters', queryTimes=1) tdSql.checkRows(0)