refactor: reorganize default catalog module structure

Cargo.toml

@@ -208,6 +208,7 @@ databend-functions-scalar-math = { path = "src/query/functions/src/scalars/mathe
 databend-functions-scalar-numeric-basic-arithmetic = { path = "src/query/functions/src/scalars/numeric_basic_arithmetic" }
 databend-meta = { path = "src/meta/service" }
 databend-query = { path = "src/query/service" }
+databend-query-default-catalog = { path = "src/query/default_catalog" }
 databend-sqllogictests = { path = "tests/sqllogictests" }
 databend-storages-common-blocks = { path = "src/query/storages/common/blocks" }
 databend-storages-common-cache = { path = "src/query/storages/common/cache" }

src/query/default_catalog/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "databend-query-default-catalog"
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+publish = { workspace = true }
+edition = { workspace = true }
+
+[dependencies]
+
+async-backtrace = { workspace = true }
+async-trait = { workspace = true }
+dashmap = { workspace = true }
+fastrace = { workspace = true }
+log = { workspace = true }
+
+databend-common-catalog = { workspace = true }
+databend-common-config = { workspace = true }
+databend-common-exception = { workspace = true }
+databend-common-meta-api = { workspace = true }
+databend-common-meta-app = { workspace = true }
+databend-common-meta-store = { workspace = true }
+databend-common-meta-types = { workspace = true }
+databend-common-storages-factory = { workspace = true }
+databend-common-storages-information-schema = { workspace = true }
+databend-common-storages-system = { workspace = true }
+databend-storages-common-session = { workspace = true }
+databend-storages-common-table-meta = { workspace = true }
+
+[features]
+jemalloc = ["databend-common-storages-system/jemalloc"]

src/query/service/src/databases/database_context.rs → src/query/default_catalog/src/databases/database_context.rs

@@ -16,8 +16,7 @@ use std::sync::Arc;
 
 use databend_common_meta_app::tenant::Tenant;
 use databend_common_meta_store::MetaStore;
-
-use crate::storages::StorageFactory;
+use databend_common_storages_factory::StorageFactory;
 
 /// Database Context.
 #[derive(Clone)]

src/query/service/src/databases/information_schema/information_schema_database.rs → src/query/default_catalog/src/databases/information_schema/information_schema_database.rs

@@ -14,6 +14,7 @@
 
 use std::sync::Arc;
 
+use databend_common_catalog::table::Table;
 use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent;
 use databend_common_meta_app::schema::DatabaseId;
 use databend_common_meta_app::schema::DatabaseInfo;
@@ -28,9 +29,8 @@ use databend_common_storages_information_schema::StatisticsTable;
 use databend_common_storages_information_schema::TablesTable;
 use databend_common_storages_information_schema::ViewsTable;
 
-use crate::catalogs::InMemoryMetas;
 use crate::databases::Database;
-use crate::storages::Table;
+use crate::default::InMemoryMetas;
 
 #[derive(Clone)]
 pub struct InformationSchemaDatabase {

src/query/default_catalog/src/databases/mod.rs

@@ -0,0 +1,26 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+mod database;
+mod database_context;
+mod database_factory;
+mod default;
+mod information_schema;
+mod system;
+
+pub use database::Database;
+pub use database_context::DatabaseContext;
+pub use database_factory::DatabaseFactory;
+pub use information_schema::InformationSchemaDatabase;
+pub use system::SystemDatabase;

src/query/service/src/databases/system/system_database.rs → src/query/default_catalog/src/databases/system/system_database.rs

@@ -15,6 +15,7 @@
 use std::collections::HashMap;
 use std::sync::Arc;
 
+use databend_common_catalog::table::Table;
 use databend_common_config::InnerConfig;
 use databend_common_meta_app::schema::database_name_ident::DatabaseNameIdent;
 use databend_common_meta_app::schema::DatabaseId;
@@ -71,9 +72,8 @@ use databend_common_storages_system::ViewsTableWithHistory;
 use databend_common_storages_system::ViewsTableWithoutHistory;
 use databend_common_storages_system::VirtualColumnsTable;
 
-use crate::catalogs::InMemoryMetas;
 use crate::databases::Database;
-use crate::storages::Table;
+use crate::default::InMemoryMetas;
 
 #[derive(Clone)]
 pub struct SystemDatabase {

src/query/service/src/catalogs/default/catalog_context.rs → src/query/default_catalog/src/default/catalog_context.rs

@@ -15,9 +15,9 @@
 use std::sync::Arc;
 
 use databend_common_meta_store::MetaStore;
+use databend_common_storages_factory::StorageFactory;
 
 use crate::databases::DatabaseFactory;
-use crate::storages::StorageFactory;
 
 #[derive(Clone)]
 pub struct CatalogContext {

src/query/service/src/catalogs/default/database_catalog.rs → src/query/default_catalog/src/default/database_catalog.rs

@@ -20,6 +20,7 @@ use std::sync::Arc;
 use databend_common_catalog::catalog::Catalog;
 use databend_common_catalog::catalog::StorageDescription;
 use databend_common_catalog::database::Database;
+use databend_common_catalog::table::Table;
 use databend_common_catalog::table_args::TableArgs;
 use databend_common_catalog::table_function::TableFunction;
 use databend_common_config::InnerConfig;
@@ -113,11 +114,11 @@ use databend_common_meta_types::SeqV;
 use databend_storages_common_session::SessionState;
 use log::info;
 
-use crate::catalogs::default::ImmutableCatalog;
-use crate::catalogs::default::MutableCatalog;
-use crate::catalogs::default::SessionCatalog;
-use crate::storages::Table;
-use crate::table_functions::TableFunctionFactory;
+use crate::default::table_function_factory::TableFunctionFactory;
+use crate::default::ImmutableCatalog;
+use crate::default::MutableCatalog;
+use crate::default::SessionCatalog;
+
 /// Combine two catalogs together
 /// - read/search like operations are always performed at
 ///   upper layer first, and bottom layer later(if necessary)
@@ -129,7 +130,7 @@ pub struct DatabaseCatalog {
     /// bottom layer, writing goes here
     mutable_catalog: Arc<SessionCatalog>,
     /// table function engine factories
-    table_function_factory: Arc<TableFunctionFactory>,
+    table_function_factory: Arc<dyn TableFunctionFactory + Send + Sync>,
 }
 
 impl Debug for DatabaseCatalog {
@@ -140,15 +141,17 @@ impl Debug for DatabaseCatalog {
 
 impl DatabaseCatalog {
     #[async_backtrace::framed]
-    pub async fn try_create_with_config(conf: InnerConfig) -> Result<DatabaseCatalog> {
+    pub async fn try_create_with_config(
+        conf: InnerConfig,
+        table_function_factory: Arc<dyn TableFunctionFactory + Send + Sync>,
+    ) -> Result<DatabaseCatalog> {
         let immutable_catalog = ImmutableCatalog::try_create_with_config(&conf).await?;
         let mutable_catalog = MutableCatalog::try_create_with_config(conf).await?;
         let session_catalog = SessionCatalog::create(mutable_catalog, SessionState::default());
-        let table_function_factory = TableFunctionFactory::create();
         let res = DatabaseCatalog {
             immutable_catalog: Arc::new(immutable_catalog),
             mutable_catalog: Arc::new(session_catalog),
-            table_function_factory: Arc::new(table_function_factory),
+            table_function_factory,
         };
         Ok(res)
     }

src/query/service/src/catalogs/default/immutable_catalog.rs → src/query/default_catalog/src/default/immutable_catalog.rs

@@ -18,6 +18,7 @@ use std::fmt::Formatter;
 use std::sync::Arc;
 
 use databend_common_catalog::catalog::Catalog;
+use databend_common_catalog::table::Table;
 use databend_common_config::InnerConfig;
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
@@ -99,11 +100,10 @@ use databend_common_meta_types::SeqV;
 use databend_storages_common_table_meta::table_id_ranges::SYS_DB_ID_BEGIN;
 use databend_storages_common_table_meta::table_id_ranges::SYS_TBL_ID_BEGIN;
 
-use crate::catalogs::InMemoryMetas;
 use crate::databases::Database;
 use crate::databases::InformationSchemaDatabase;
 use crate::databases::SystemDatabase;
-use crate::storages::Table;
+use crate::default::table_memory_meta::InMemoryMetas;
 
 /// System Catalog contains ... all the system databases (no surprise :)
 #[derive(Clone)]

src/query/service/src/catalogs/default/mod.rs → src/query/default_catalog/src/default/mod.rs

@@ -17,10 +17,13 @@ mod database_catalog;
 mod immutable_catalog;
 mod mutable_catalog;
 mod session_catalog;
-pub mod table_memory_meta;
+mod table_function_factory;
+mod table_memory_meta;
 
 pub use catalog_context::CatalogContext;
 pub use database_catalog::DatabaseCatalog;
 pub use immutable_catalog::ImmutableCatalog;
 pub use mutable_catalog::MutableCatalog;
 pub use session_catalog::SessionCatalog;
+pub use table_function_factory::TableFunctionFactory;
+pub use table_memory_meta::InMemoryMetas;

src/query/service/src/catalogs/default/mutable_catalog.rs → src/query/default_catalog/src/default/mutable_catalog.rs

@@ -19,6 +19,8 @@ use std::sync::Arc;
 use std::time::Instant;
 
 use databend_common_catalog::catalog::Catalog;
+use databend_common_catalog::catalog::StorageDescription;
+use databend_common_catalog::table::Table;
 use databend_common_config::InnerConfig;
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
@@ -123,17 +125,15 @@ use databend_common_meta_app::KeyWithTenant;
 use databend_common_meta_store::MetaStoreProvider;
 use databend_common_meta_types::seq_value::SeqV;
 use databend_common_meta_types::MetaId;
+use databend_common_storages_factory::StorageFactory;
 use fastrace::func_name;
 use log::info;
 use log::warn;
 
-use crate::catalogs::default::catalog_context::CatalogContext;
 use crate::databases::Database;
 use crate::databases::DatabaseContext;
 use crate::databases::DatabaseFactory;
-use crate::storages::StorageDescription;
-use crate::storages::StorageFactory;
-use crate::storages::Table;
+use crate::default::CatalogContext;
 
 /// Catalog based on MetaStore
 /// - System Database NOT included

src/query/service/src/catalogs/default/session_catalog.rs → src/query/default_catalog/src/default/session_catalog.rs

@@ -16,6 +16,7 @@ use std::any::Any;
 use std::fmt::Debug;
 use std::sync::Arc;
 
+use databend_common_catalog::catalog::Catalog;
 use databend_common_catalog::catalog::StorageDescription;
 use databend_common_catalog::database::Database;
 use databend_common_catalog::table::Table;
@@ -114,8 +115,8 @@ use databend_storages_common_session::TxnState;
 use databend_storages_common_table_meta::table::OPT_KEY_TEMP_PREFIX;
 use databend_storages_common_table_meta::table_id_ranges::is_temp_table_id;
 
-use crate::catalogs::default::MutableCatalog;
-use crate::catalogs::Catalog;
+use crate::default::MutableCatalog;
+
 #[derive(Clone, Debug)]
 pub struct SessionCatalog {
     inner: MutableCatalog,

src/query/default_catalog/src/default/table_function_factory.rs

@@ -0,0 +1,24 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use databend_common_catalog::table_args::TableArgs;
+use databend_common_catalog::table_function::TableFunction;
+use databend_common_exception::Result;
+pub trait TableFunctionFactory {
+    fn get(&self, func_name: &str, tbl_args: TableArgs) -> Result<Arc<dyn TableFunction>>;
+    fn exists(&self, func_name: &str) -> bool;
+    fn list(&self) -> Vec<String>;
+}

src/query/service/src/catalogs/default/table_memory_meta.rs → src/query/default_catalog/src/default/table_memory_meta.rs

@@ -17,12 +17,11 @@ use std::sync::atomic::Ordering;
 use std::sync::Arc;
 
 use dashmap::DashMap;
+use databend_common_catalog::table::Table;
 use databend_common_exception::ErrorCode;
 use databend_common_exception::Result;
 use databend_common_meta_types::MetaId;
 
-use crate::storages::Table;
-
 pub struct DbTables {
     name_to_table: DashMap<String, Arc<dyn Table>>,
     id_to_table: DashMap<MetaId, Arc<dyn Table>>,

src/query/default_catalog/src/lib.rs

@@ -0,0 +1,19 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(clippy::result_large_err)]
+pub mod databases;
+mod default;
+
+pub use default::*;

src/query/service/Cargo.toml

@@ -12,7 +12,7 @@ default = ["simd"]
 simd = ["databend-common-column/simd"]
 python-udf = ["arrow-udf-runtime/python"]
 disable_initial_exec_tls = ["databend-common-base/disable_initial_exec_tls"]
-jemalloc = ["databend-common-storages-system/jemalloc"]
+jemalloc = ["databend-common-storages-system/jemalloc", "databend-query-default-catalog/jemalloc"]
 memory-profiling = ["databend-common-base/memory-profiling", "databend-common-http/memory-profiling"]
 storage-hdfs = ["opendal/services-hdfs", "databend-common-storage/storage-hdfs"]
 io-uring = [
@@ -85,7 +85,6 @@ databend-common-storages-factory = { workspace = true }
 databend-common-storages-fuse = { workspace = true }
 databend-common-storages-hive = { workspace = true }
 databend-common-storages-iceberg = { workspace = true }
-databend-common-storages-information-schema = { workspace = true }
 databend-common-storages-memory = { workspace = true }
 databend-common-storages-null = { workspace = true }
 databend-common-storages-orc = { workspace = true }
@@ -106,6 +105,7 @@ databend-enterprise-resources-management = { workspace = true }
 databend-enterprise-stream-handler = { workspace = true }
 databend-enterprise-vacuum-handler = { workspace = true }
 databend-enterprise-virtual-column = { workspace = true }
+databend-query-default-catalog = { workspace = true }
 databend-storages-common-blocks = { workspace = true }
 databend-storages-common-cache = { workspace = true }
 databend-storages-common-index = { workspace = true }