logic test

Author: dantengsky

src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs

@@ -122,31 +122,34 @@ pub async fn do_vacuum2(
                 start.elapsed(),
                 lvt
             ));
-            let snapshots_before_lvt = if is_vacuum_all {
-                list_until_prefix(
-                    fuse_table,
-                    fuse_table
-                        .meta_location_generator()
-                        .snapshot_location_prefix(),
-                    fuse_table.snapshot_loc().unwrap().as_str(),
-                    true,
-                    None,
-                )
-                .await?
-            } else {
-                list_until_timestamp(
-                    fuse_table,
-                    fuse_table
-                        .meta_location_generator()
-                        .snapshot_location_prefix(),
-                    lvt,
-                    true,
-                    None,
-                )
-                .await?
-            };
 
             let elapsed = start.elapsed();
+            let snapshots_before_lvt =
+                collect_gc_candidate_by_retention_period(fuse_table, lvt, is_vacuum_all).await?;
+            // let snapshots_before_lvt = if is_vacuum_all {
+            //    list_until_prefix(
+            //        fuse_table,
+            //        fuse_table
+            //            .meta_location_generator()
+            //            .snapshot_location_prefix(),
+            //        fuse_table.snapshot_loc().unwrap().as_str(),
+            //        true,
+            //        None,
+            //    )
+            //    .await?
+            //} else {
+            //    list_until_timestamp(
+            //        fuse_table,
+            //        fuse_table
+            //            .meta_location_generator()
+            //            .snapshot_location_prefix(),
+            //        lvt,
+            //        true,
+            //        None,
+            //    )
+            //    .await?
+            //};
+
             ctx.set_status_info(&format!(
                 "list snapshots before lvt for table {} takes {:?}, snapshots_dir: {:?}, lvt: {:?}, snapshots: {:?}",
                 fuse_table.get_table_info().desc,
@@ -389,6 +392,38 @@ pub async fn do_vacuum2(
     Ok(files_to_gc)
 }
 
+async fn collect_gc_candidate_by_retention_period(
+    fuse_table: &FuseTable,
+    lvt: DateTime<Utc>,
+    is_vacuum_all: bool,
+) -> Result<Vec<Entry>> {
+    let snapshots_before_lvt = if is_vacuum_all {
+        list_until_prefix(
+            fuse_table,
+            fuse_table
+                .meta_location_generator()
+                .snapshot_location_prefix(),
+            fuse_table.snapshot_loc().unwrap().as_str(),
+            true,
+            None,
+        )
+        .await?
+    } else {
+        list_until_timestamp(
+            fuse_table,
+            fuse_table
+                .meta_location_generator()
+                .snapshot_location_prefix(),
+            lvt,
+            true,
+            None,
+        )
+        .await?
+    };
+
+    Ok(snapshots_before_lvt)
+}
+
 /// Try set lvt as min(latest_snapshot.timestamp, now - retention_time).
 ///
 /// Return `None` means we stop vacuumming, but don't want to report error to user.

tests/sqllogictests/suites/ee/03_ee_vacuum/03_0004_auto_vacuum.test

@@ -145,12 +145,75 @@ select count() from list_stage(location=> '@stage_av') where name like '%_ss%';
 ----
 1
 
-# ---------------------------------------------
 
-# We do not check default retention period works as expected, i.e. by default table historical data
-# will be cleaned according to the retention period settings. As tests of streams will check it implicitly
+#################################################
+# Test autovacuum policy `ByNumSnapshotsToKeep` #
+#################################################
+
+# CASE 1: Create table with data_retention_num_snapshots_to_keep table option
+statement ok
+create or replace table t (c int) 'fs:///tmp/auto_vacuum_case3/' data_retention_num_snapshots_to_keep = 3;
+
+statement ok
+create or replace stage stage_av url = 'fs:///tmp/auto_vacuum_case3/';
+
+statement ok
+set enable_auto_vacuum = 1;
+
+statement ok
+insert into t values(1);
+
+statement ok
+insert into t values(2);
+
+statement ok
+insert into t values(3);
+
+statement ok
+insert into t values(4);
+
+
+# Insert 4 time, but only 3 snapshots will be kept
+onlyif mysql
+query I
+select count() from list_stage(location=> '@stage_av') where name like '%_ss%';
+----
+3
+
+# CASE 2: Alter table option
+statement ok
+alter table t set options(data_retention_num_snapshots_to_keep = 1);
+
+statement ok
+insert into t values(1);
+
+statement ok
+optimize table t compact;
+
+onlyif mysql
+query I
+select count() from list_stage(location=> '@stage_av') where name like '%_sg%';
+----
+1
+
+onlyif mysql
+query I
+select count() from list_stage(location=> '@stage_av') where name like '%\/_b\/%';
+----
+1
+
+onlyif mysql
+query I
+select count() from list_stage(location=> '@stage_av') where name like '%_ss%';
+----
+1
+
 
+# CASE 3: Create table with invalid data_retention_num_snapshots_to_keep table option
 
+# data_retention_num_snapshots_to_keep must be greater than 0
+statement error
+create or replace table t (c int) 'fs:///tmp/auto_vacuum_case3/' data_retention_num_snapshots_to_keep = 0;
 
 statement ok
 remove @stage_av;