From a3b5b38ad04990f6c06c172a71f5dfb083999845 Mon Sep 17 00:00:00 2001 From: coldWater Date: Fri, 18 Jul 2025 16:42:35 +0800 Subject: [PATCH] config --- Cargo.lock | 1 - src/binaries/query/cmd.rs | 75 ++++++++++++++++++++++++++ src/binaries/query/ee_main.rs | 12 +++-- src/binaries/query/entry.rs | 18 ++++--- src/binaries/query/oss_main.rs | 12 +++-- src/query/config/Cargo.toml | 1 - src/query/config/src/config.rs | 96 +++++++++++----------------------- src/query/config/src/inner.rs | 16 ++---- src/query/config/src/mask.rs | 3 -- 9 files changed, 138 insertions(+), 96 deletions(-) create mode 100644 src/binaries/query/cmd.rs diff --git a/Cargo.lock b/Cargo.lock index d95a92b13bef2..f0379b97d9890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3265,7 +3265,6 @@ dependencies = [ "databend-common-meta-app", "databend-common-storage", "databend-common-tracing", - "databend-common-version", "log", "pretty_assertions", "serde", diff --git a/src/binaries/query/cmd.rs b/src/binaries/query/cmd.rs new file mode 100644 index 0000000000000..7008327b861c4 --- /dev/null +++ b/src/binaries/query/cmd.rs @@ -0,0 +1,75 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::Parser; +use clap::Subcommand; +use databend_common_config::Config; +use databend_common_config::InnerConfig; +use databend_common_exception::Result; +use databend_common_version::DATABEND_COMMIT_VERSION; + +/// Databend +#[derive(Parser, Clone)] +#[command(name = "databend-query")] +#[command(about = "Databend: The Next-Gen Cloud [Data+AI] Analytics.")] +#[command(version = &**DATABEND_COMMIT_VERSION)] +pub struct Cmd { + /// Run a command and quit + #[command(subcommand)] + pub subcommand: Option, + + /// To be compatible with the old version, we keep the `cmd` arg + /// We should always use `databend-query ver` instead `databend-query --cmd ver` in latest version + #[clap(long)] + pub cmd: Option, + + #[clap(long, short = 'c', value_name = "PATH", default_value_t)] + pub config_file: String, + + #[clap(flatten)] + pub config: Config, +} + +impl Cmd { + pub fn normalize(&mut self) { + if self.cmd == Some("ver".to_string()) { + self.subcommand = Some(Commands::Ver); + } + } + + pub async fn init_inner_config(self, check_meta: bool) -> Result { + let Cmd { + config, + config_file, + .. + } = self; + + let config = config.merge(&config_file).unwrap(); + InnerConfig::init(config, check_meta).await + } +} + +#[derive(Subcommand, Clone)] +pub enum Commands { + #[command(about = "Print version and quit")] + Ver, + Local { + #[clap(long, short = 'q', default_value_t)] + query: String, + #[clap(long, default_value_t)] + output_format: String, + #[clap(long, short = 'c')] + config: String, + }, +} diff --git a/src/binaries/query/ee_main.rs b/src/binaries/query/ee_main.rs index bf69ecce396f5..ab23c273b7d94 100644 --- a/src/binaries/query/ee_main.rs +++ b/src/binaries/query/ee_main.rs @@ -15,12 +15,13 @@ #![allow(clippy::uninlined_format_args)] #![feature(try_blocks)] +mod cmd; mod entry; +use clap::Parser; use databend_common_base::mem_allocator::TrackingGlobalAllocator; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::ThreadTracker; -use databend_common_config::InnerConfig; use databend_common_exception::Result; use databend_common_exception::ResultExt; use databend_common_tracing::pipe_file; @@ -30,6 +31,7 @@ use databend_common_version::DATABEND_COMMIT_VERSION; use databend_enterprise_query::enterprise_services::EnterpriseServices; use entry::MainError; +use self::cmd::Cmd; use crate::entry::init_services; use crate::entry::run_cmd; use crate::entry::start_services; @@ -63,11 +65,15 @@ fn main() { pub async fn main_entrypoint() -> Result<(), MainError> { let make_error = || "an fatal error occurred in query"; - let conf: InnerConfig = InnerConfig::load().await.with_context(make_error)?; - if run_cmd(&conf).await.with_context(make_error)? { + // if the usage is print, std::process::exit() will be called. + let mut cmd = Cmd::parse(); + cmd.normalize(); + + if run_cmd(&cmd).await.with_context(make_error)? { return Ok(()); } + let conf = cmd.init_inner_config(true).await.with_context(make_error)?; init_services(&conf, true).await.with_context(make_error)?; EnterpriseServices::init(conf.clone()) .await diff --git a/src/binaries/query/entry.rs b/src/binaries/query/entry.rs index 29cae01359ad0..878aef9668c50 100644 --- a/src/binaries/query/entry.rs +++ b/src/binaries/query/entry.rs @@ -18,7 +18,6 @@ use std::time::Duration; use databend_common_base::mem_allocator::TrackingGlobalAllocator; use databend_common_base::runtime::set_alloc_error_hook; use databend_common_base::runtime::GLOBAL_MEM_STAT; -use databend_common_config::Commands; use databend_common_config::InnerConfig; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -47,12 +46,15 @@ use databend_query::servers::ShutdownHandle; use databend_query::GlobalServices; use log::info; +use super::cmd::Cmd; +use super::cmd::Commands; + pub struct MainError; -pub async fn run_cmd(conf: &InnerConfig) -> Result { +pub async fn run_cmd(cmd: &Cmd) -> Result { let make_error = || "failed to run cmd"; - match &conf.subcommand { + match &cmd.subcommand { None => return Ok(false), Some(Commands::Ver) => { println!("version: {}", *DATABEND_SEMVER); @@ -63,12 +65,14 @@ pub async fn run_cmd(conf: &InnerConfig) -> Result { output_format, config, }) => { - let mut conf = conf.clone(); + let mut cmd = cmd.clone(); if !config.is_empty() { - let c = - databend_common_config::Config::load_with_config_file(config.as_str()).unwrap(); - conf = c.try_into().unwrap(); + cmd.config_file = config.to_string(); } + let conf = cmd + .init_inner_config(false) + .await + .with_context(make_error)?; local::query_local(conf, query, output_format) .await .with_context(make_error)? diff --git a/src/binaries/query/oss_main.rs b/src/binaries/query/oss_main.rs index c3ede11d2ae8b..bf5be80dd8996 100644 --- a/src/binaries/query/oss_main.rs +++ b/src/binaries/query/oss_main.rs @@ -15,12 +15,13 @@ #![allow(clippy::uninlined_format_args)] #![feature(try_blocks)] +mod cmd; mod entry; +use clap::Parser; use databend_common_base::mem_allocator::TrackingGlobalAllocator; use databend_common_base::runtime::Runtime; use databend_common_base::runtime::ThreadTracker; -use databend_common_config::InnerConfig; use databend_common_exception::Result; use databend_common_exception::ResultExt; use databend_common_license::license_manager::LicenseManager; @@ -31,6 +32,7 @@ use databend_common_tracing::SignalListener; use databend_common_version::DATABEND_COMMIT_VERSION; use entry::MainError; +use self::cmd::Cmd; use crate::entry::init_services; use crate::entry::run_cmd; use crate::entry::start_services; @@ -64,11 +66,15 @@ fn main() { async fn main_entrypoint() -> Result<(), MainError> { let make_error = || "an fatal error occurred in query"; - let conf: InnerConfig = InnerConfig::load().await.with_context(make_error)?; - if run_cmd(&conf).await.with_context(make_error)? { + // if the usage is print, std::process::exit() will be called. + let mut cmd = Cmd::parse(); + cmd.normalize(); + + if run_cmd(&cmd).await.with_context(make_error)? { return Ok(()); } + let conf = cmd.init_inner_config(true).await.with_context(make_error)?; init_services(&conf, false).await?; // init oss license manager OssLicenseManager::init(conf.query.tenant_id.tenant_name().to_string()) diff --git a/src/query/config/Cargo.toml b/src/query/config/Cargo.toml index bb763182ad8a9..5a635acae3015 100644 --- a/src/query/config/Cargo.toml +++ b/src/query/config/Cargo.toml @@ -24,7 +24,6 @@ databend-common-grpc = { workspace = true } databend-common-meta-app = { workspace = true } databend-common-storage = { workspace = true } databend-common-tracing = { workspace = true } -databend-common-version = { workspace = true } log = { workspace = true } serde = { workspace = true } serde_ignored = { workspace = true } diff --git a/src/query/config/src/config.rs b/src/query/config/src/config.rs index 7a4329633b86d..f29c967a370b3 100644 --- a/src/query/config/src/config.rs +++ b/src/query/config/src/config.rs @@ -21,7 +21,6 @@ use std::fmt::Formatter; use clap::ArgAction; use clap::Args; -use clap::Parser; use clap::Subcommand; use clap::ValueEnum; use databend_common_base::base::mask_string; @@ -57,7 +56,6 @@ use databend_common_tracing::StderrConfig as InnerStderrLogConfig; use databend_common_tracing::StructLogConfig as InnerStructLogConfig; use databend_common_tracing::TracingConfig as InnerTracingConfig; use databend_common_tracing::CONFIG_DEFAULT_LOG_LEVEL; -use databend_common_version::DATABEND_COMMIT_VERSION; use serde::Deserialize; use serde::Serialize; use serde_with::with_prefix; @@ -90,23 +88,9 @@ const CATALOG_HIVE: &str = "hive"; /// It's forbidden to do any breaking changes on this struct. /// Only adding new fields is allowed. /// This same rules should be applied to all fields of this struct. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Parser)] -#[clap(name = "databend-query", about, version = & * * DATABEND_COMMIT_VERSION, author)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Args)] #[serde(default)] pub struct Config { - /// Run a command and quit - #[command(subcommand)] - #[serde(skip)] - pub subcommand: Option, - - // To be compatible with the old version, we keep the `cmd` arg - // We should always use `databend-query ver` instead `databend-query --cmd ver` in latest version - #[clap(long)] - pub cmd: Option, - - #[clap(long, short = 'c', value_name = "VALUE", default_value_t)] - pub config_file: String, - // Query engine config. #[clap(flatten)] pub query: QueryConfig, @@ -184,32 +168,13 @@ impl Config { /// with_args is to control whether we need to load from args or not. /// We should set this to false during tests because we don't want /// our test binary to parse cargo's args. - #[no_sanitize(address)] - pub fn load(with_args: bool) -> Result { - let mut arg_conf = Self::default(); - - if with_args { - arg_conf = Self::parse(); - } - - if arg_conf.cmd == Some("ver".to_string()) { - arg_conf.subcommand = Some(Commands::Ver); - } - - if arg_conf.subcommand.is_some() { - return Ok(arg_conf); - } - + pub fn merge(self, config_file: &str) -> Result { let mut builder: serfig::Builder = serfig::Builder::default(); // Load from config file first. { - let final_config_file = if !arg_conf.config_file.is_empty() { - // TODO: remove this `allow(clippy::redundant_clone)` - // as soon as this issue is fixed: - // https://github.com/rust-lang/rust-clippy/issues/10940 - #[allow(clippy::redundant_clone)] - arg_conf.config_file.clone() + let final_config_file = if !config_file.is_empty() { + config_file.to_string() } else if let Ok(path) = env::var("CONFIG_FILE") { path } else { @@ -228,9 +193,7 @@ impl Config { builder = builder.collect(from_env()); // Finally, load from args. - if with_args { - builder = builder.collect(from_self(arg_conf)); - } + builder = builder.collect(from_self(self)); // Check obsoleted. let conf = builder.build()?; @@ -667,7 +630,9 @@ pub struct HiveCatalogConfig { default_value_t )] pub address: String, + /// Deprecated fields, used for catching error, will be removed later. + #[clap(skip)] pub meta_store_address: Option, #[clap(long = "hive-thrift-protocol", value_name = "VALUE", default_value_t)] @@ -1811,39 +1776,39 @@ pub struct QueryConfig { // ----- the following options/args are all deprecated ---- // ----- and turned into Option, to help user migrate the configs ---- /// OBSOLETED: Table disk cache size (mb). - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_disk_cache_mb_size: Option, /// OBSOLETED: Table Meta Cached enabled - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_meta_cache_enabled: Option, /// OBSOLETED: Max number of cached table block meta - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_cache_block_meta_count: Option, /// OBSOLETED: Table memory cache size (mb) - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_memory_cache_mb_size: Option, /// OBSOLETED: Table disk cache folder root - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_disk_cache_root: Option, /// OBSOLETED: Max number of cached table snapshot - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_cache_snapshot_count: Option, /// OBSOLETED: Max number of cached table snapshot statistics - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_cache_statistic_count: Option, /// OBSOLETED: Max number of cached table segment - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_cache_segment_count: Option, /// OBSOLETED: Max number of cached bloom index meta objects - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_cache_bloom_index_meta_count: Option, /// OBSOLETED: @@ -1852,12 +1817,12 @@ pub struct QueryConfig { /// /// For example, a table of 1024 columns, with 800 data blocks, a query that triggers a full /// table filter on 2 columns, might populate 2 * 800 bloom index filter cache items (at most) - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub table_cache_bloom_index_filter_count: Option, /// OBSOLETED: (cache of raw bloom filter data is no longer supported) /// Max bytes of cached bloom filter bytes. - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub(crate) table_cache_bloom_index_data_bytes: Option, /// OBSOLETED: use settings['parquet_fast_read_bytes'] instead @@ -1866,11 +1831,11 @@ pub struct QueryConfig { /// parquet_fast_read_bytes = 52428800 /// will let databend read whole file for parquet file less than 50MB and read column by column /// if file size is greater than 50MB - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub parquet_fast_read_bytes: Option, /// OBSOLETED: use settings['max_storage_io_requests'] instead - #[clap(long, value_name = "VALUE")] + #[clap(long, value_name = "VALUE", hide = true)] pub max_storage_io_requests: Option, /// Disable some system load(For example system.configs) for cloud security. @@ -3550,9 +3515,6 @@ mod cache_config_converters { impl From for Config { fn from(inner: InnerConfig) -> Self { Self { - subcommand: inner.subcommand, - cmd: None, - config_file: inner.config_file, query: inner.query.into(), log: inner.log.into(), meta: inner.meta.into(), @@ -3575,8 +3537,6 @@ mod cache_config_converters { fn try_into(self) -> Result { let Config { - subcommand, - config_file, query, log, meta, @@ -3605,8 +3565,6 @@ mod cache_config_converters { let spill = convert_local_spill_config(spill, &cache.disk_cache_config)?; Ok(InnerConfig { - subcommand, - config_file, query: query.try_into()?, log: log.try_into()?, meta: meta.try_into()?, @@ -3811,16 +3769,22 @@ mod test { use std::ffi::OsString; use clap::Parser; - use pretty_assertions::assert_eq; - use crate::Config; - use crate::InnerConfig; + use super::*; + + #[derive(Parser)] + #[clap(name = "databend-query", about, author)] + pub struct Cmd { + #[clap(flatten)] + pub config: Config, + } /// It's required to make sure setting's default value is the same with clap. #[test] fn test_config_default() { let setting_default = InnerConfig::default(); - let config_default: InnerConfig = Config::parse_from(Vec::::new()) + let config_default: InnerConfig = Cmd::parse_from(Vec::::new()) + .config .try_into() .expect("parse from args must succeed"); diff --git a/src/query/config/src/inner.rs b/src/query/config/src/inner.rs index 2807e1aaee817..4afdae40d74da 100644 --- a/src/query/config/src/inner.rs +++ b/src/query/config/src/inner.rs @@ -35,7 +35,6 @@ use databend_common_meta_app::tenant::TenantQuota; use databend_common_storage::StorageConfig; use databend_common_tracing::Config as LogConfig; -use super::config::Commands; use super::config::Config; use super::config::ResourcesManagementConfig; use crate::BuiltInConfig; @@ -45,9 +44,6 @@ use crate::BuiltInConfig; /// All function should implement based on this Config. #[derive(Clone, Default, PartialEq, Eq)] pub struct InnerConfig { - pub subcommand: Option, - pub config_file: String, - // Query engine config. pub query: QueryConfig, @@ -75,8 +71,8 @@ impl InnerConfig { /// As requires by [RFC: Config Backward Compatibility](https://github.com/datafuselabs/databend/pull/5324), we will load user's config via wrapper [`ConfigV0`] and then convert from [`ConfigV0`] to [`InnerConfig`]. /// /// In the future, we could have `ConfigV1` and `ConfigV2`. - pub async fn load() -> Result { - let mut cfg: Self = Config::load(true)?.try_into()?; + pub async fn init(cfg: Config, check_meta: bool) -> Result { + let mut cfg: Self = cfg.try_into()?; // Handle the node_id and node_secret for query node. cfg.query.node_id = GlobalUniqName::unique(); @@ -85,8 +81,7 @@ impl InnerConfig { // Handle auto detect for storage params. cfg.storage.params = cfg.storage.params.auto_detect().await?; - // Only check meta config when cmd is empty. - if cfg.subcommand.is_none() { + if check_meta { cfg.meta.check_valid()?; } Ok(cfg) @@ -96,8 +91,7 @@ impl InnerConfig { /// /// This function is served for tests only. pub fn load_for_test() -> Result { - let cfg: Self = Config::load(false)?.try_into()?; - Ok(cfg) + Config::load_with_config_file("")?.try_into() } pub fn tls_query_cli_enabled(&self) -> bool { @@ -136,8 +130,6 @@ impl InnerConfig { impl Debug for InnerConfig { fn fmt(&self, f: &mut Formatter) -> fmt::Result { f.debug_struct("InnerConfig") - .field("subcommand", &self.subcommand) - .field("config_file", &self.config_file) .field("query", &self.query.sanitize()) .field("log", &self.log) .field("meta", &self.meta) diff --git a/src/query/config/src/mask.rs b/src/query/config/src/mask.rs index 85fcb1dcec3b0..1f3cf8d50d436 100644 --- a/src/query/config/src/mask.rs +++ b/src/query/config/src/mask.rs @@ -43,9 +43,6 @@ fn mask_sensitive_field(field: &str) -> String { impl Config { pub fn with_mask(self) -> Self { Config { - subcommand: self.subcommand, - cmd: self.cmd, - config_file: self.config_file, query: self.query.mask_display(), log: self.log, meta: self.meta.mask_display(),