From 3caffd12810050df7888d8d4dd54f67b15a0948c Mon Sep 17 00:00:00 2001 From: amochin Date: Tue, 24 Sep 2024 19:38:11 +0200 Subject: [PATCH 01/30] Keyword "Connect to database" - custom params and config file refactoring --- src/DatabaseLibrary/__init__.py | 45 +++ src/DatabaseLibrary/connection_manager.py | 347 ++++++++++-------- test/resources/common.resource | 19 +- .../config_files/connect_config_file.resource | 11 + .../oracledb/custom_param_password.cfg | 6 + .../oracledb/invalid_custom_params.cfg | 8 + .../oracledb/simple_default_alias.cfg | 8 + .../oracledb/some_basic_params_missing.cfg | 3 + .../config_files/oracledb/thick_mode.cfg | 8 + .../oracledb/valid_custom_params.cfg | 7 + .../config_files/oracledb/wrong_password.cfg | 7 + .../psycopg2/custom_param_password.cfg | 6 + .../psycopg2/invalid_custom_params.cfg | 8 + .../psycopg2/simple_default_alias.cfg | 7 + .../psycopg2/some_basic_params_missing.cfg | 3 + .../psycopg2/valid_custom_params.cfg | 7 + .../config_files/psycopg2/wrong_password.cfg | 7 + .../config_files/pymssql/charset_invalid.cfg | 8 + .../pymssql/custom_param_password.cfg | 6 + .../pymssql/invalid_custom_params.cfg | 8 + .../pymssql/simple_default_alias.cfg | 7 + .../pymssql/some_basic_params_missing.cfg | 3 + .../pymssql/valid_custom_params.cfg | 7 + .../config_files/pymssql/wrong_password.cfg | 7 + .../config_files/pymysql/charset_invalid.cfg | 8 + .../pymysql/custom_param_password.cfg | 6 + .../pymysql/invalid_custom_params.cfg | 8 + .../pymysql/simple_default_alias.cfg | 7 + .../pymysql/some_basic_params_missing.cfg | 3 + .../pymysql/valid_custom_params.cfg | 7 + .../config_files/pymysql/wrong_password.cfg | 7 + .../config_files/pyodbc/charset_invalid.cfg | 8 + .../pyodbc/custom_param_password.cfg | 6 + .../pyodbc/invalid_custom_params.cfg | 8 + .../pyodbc/simple_default_alias.cfg | 7 + .../pyodbc/some_basic_params_missing.cfg | 3 + .../pyodbc/valid_custom_params.cfg | 7 + .../config_files/pyodbc/wrong_password.cfg | 7 + .../sqlite3/simple_default_alias.cfg | 4 + .../common_tests/connection_params.robot | 153 ++++++++ 40 files changed, 655 insertions(+), 147 deletions(-) create mode 100644 test/resources/config_files/connect_config_file.resource create mode 100644 test/resources/config_files/oracledb/custom_param_password.cfg create mode 100644 test/resources/config_files/oracledb/invalid_custom_params.cfg create mode 100644 test/resources/config_files/oracledb/simple_default_alias.cfg create mode 100644 test/resources/config_files/oracledb/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/oracledb/thick_mode.cfg create mode 100644 test/resources/config_files/oracledb/valid_custom_params.cfg create mode 100644 test/resources/config_files/oracledb/wrong_password.cfg create mode 100644 test/resources/config_files/psycopg2/custom_param_password.cfg create mode 100644 test/resources/config_files/psycopg2/invalid_custom_params.cfg create mode 100644 test/resources/config_files/psycopg2/simple_default_alias.cfg create mode 100644 test/resources/config_files/psycopg2/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/psycopg2/valid_custom_params.cfg create mode 100644 test/resources/config_files/psycopg2/wrong_password.cfg create mode 100644 test/resources/config_files/pymssql/charset_invalid.cfg create mode 100644 test/resources/config_files/pymssql/custom_param_password.cfg create mode 100644 test/resources/config_files/pymssql/invalid_custom_params.cfg create mode 100644 test/resources/config_files/pymssql/simple_default_alias.cfg create mode 100644 test/resources/config_files/pymssql/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/pymssql/valid_custom_params.cfg create mode 100644 test/resources/config_files/pymssql/wrong_password.cfg create mode 100644 test/resources/config_files/pymysql/charset_invalid.cfg create mode 100644 test/resources/config_files/pymysql/custom_param_password.cfg create mode 100644 test/resources/config_files/pymysql/invalid_custom_params.cfg create mode 100644 test/resources/config_files/pymysql/simple_default_alias.cfg create mode 100644 test/resources/config_files/pymysql/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/pymysql/valid_custom_params.cfg create mode 100644 test/resources/config_files/pymysql/wrong_password.cfg create mode 100644 test/resources/config_files/pyodbc/charset_invalid.cfg create mode 100644 test/resources/config_files/pyodbc/custom_param_password.cfg create mode 100644 test/resources/config_files/pyodbc/invalid_custom_params.cfg create mode 100644 test/resources/config_files/pyodbc/simple_default_alias.cfg create mode 100644 test/resources/config_files/pyodbc/some_basic_params_missing.cfg create mode 100644 test/resources/config_files/pyodbc/valid_custom_params.cfg create mode 100644 test/resources/config_files/pyodbc/wrong_password.cfg create mode 100644 test/resources/config_files/sqlite3/simple_default_alias.cfg create mode 100644 test/tests/common_tests/connection_params.robot diff --git a/src/DatabaseLibrary/__init__.py b/src/DatabaseLibrary/__init__.py index 6f6c986..e8eb25c 100644 --- a/src/DatabaseLibrary/__init__.py +++ b/src/DatabaseLibrary/__init__.py @@ -93,6 +93,51 @@ class DatabaseLibrary(ConnectionManager, Query, Assertion): | Execute Sql String drop table XYZ | + = Using configuration file = + The `Connect To Database` keyword allows providing the connection parameters in two ways: + - As keyword arguments + - In a configuration file - a simple list of _key=value_ pairs, set inside an _alias_ section. + + You can use only one way or you can combine them: + - The keyword arguments are taken by default + - If no keyword argument is provided, a parameter value is searched in the config file + + Along with commonly used connection parameters, named exactly as keyword arguments, a config file + can contain any other DB module specific parameters as key/value pairs. + If same custom parameter is provided both as a keyword argument *and* in config file, + the *keyword argument value takes precedence*. + + The path to the config file is set by default to `./resources/db.cfg`. + You can change it using an according parameter in the `Connect To Database` keyword. + + == Config file examples == + === Config file with default alias (equal to using no aliases at all) === + | [default] + | dbapiModuleName=psycopg2 + | dbName=yourdbname + | dbUsername=yourusername + | dbPassword=yourpassword + | dbHost=yourhost + | dbPort=yourport + + === Config file with a specific alias === + | [myoracle] + | dbapiModuleName=oracledb + | dbName=yourdbname + | dbUsername=yourusername + | dbPassword=yourpassword + | dbHost=yourhost + | dbPort=yourport + + === Config file with some params only === + | [default] + | dbPassword=mysecret + + === Config file with some custom DB module specific params === + | [default] + | my_custom_param=value + + = Inline assertions = Keywords, that accept arguments ``assertion_operator`` <`AssertionOperator`> and ``expected_value``, perform a check according to the specified condition - using the [https://github.com/MarketSquare/AssertionEngine|Assertion Engine]. diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 6c5cbba..5463305 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -88,23 +88,48 @@ def __init__(self, config_file: Optional[str], alias: str): @staticmethod def _load_config(config_file: str) -> Optional[ConfigParser]: config_path = Path(config_file) + logger.info(f"Looking for configuration file: '{config_path}'") if not config_path.exists(): + logger.info("Configuration file doesn't exist") return None config = ConfigParser() config.read([config_path]) + logger.info("Successfully loaded configuration file") return config - def get(self, param: str) -> str: + def pop(self, param: str) -> Optional[str]: + """ + Returns the `param` value read from the config file and deletes it from the list of all params read + """ if self.config is None: - raise ValueError(f"Required '{param}' parameter was not provided in keyword arguments.") from None + logger.debug("Configuration file not loaded") + return None try: - return self.config.get(self.alias, param) + logger.debug(f"Looking for parameter '{param}' in configuration file") + param_value = self.config.get(self.alias, param) + logger.info(f"Found parameter '{param}' in configuration file") + self.config.remove_option(self.alias, param) + return param_value except NoSectionError: - raise ValueError(f"Configuration file does not have [{self.alias}] section.") from None + logger.debug(f"Configuration file does not have [{self.alias}] section.") except NoOptionError: - raise ValueError( - f"Required '{param}' parameter missing in both keyword arguments and configuration file." - ) from None + logger.debug(f"Parameter '{param}' missing in configuration file.") + return None + + def get_all_available_params(self) -> Dict: + """ + Returns a dictionary of all params read from the config file, which are currently available + (some of them might have been removed using the `pop` function) + """ + if self.config is None: + logger.debug("Configuration file not loaded") + return {} + try: + all_options = dict(self.config.items(self.alias)) + return all_options + except NoSectionError: + logger.debug(f"Configuration file does not have [{self.alias}] section.") + return {} class ConnectionManager: @@ -116,6 +141,21 @@ def __init__(self): self.omit_trailing_semicolon: bool = False self.connection_store: ConnectionStore = ConnectionStore() + @staticmethod + def _hide_password_values(string_with_pass): + string_with_hidden_pass = string_with_pass + for pass_param_name in ["pass", "passwd", "password", "pwd", "PWD"]: + pass_param_name += "=" + splitted = string_with_hidden_pass.split(pass_param_name) + if len(splitted) < 2: + continue + splitted = splitted[1].split(",") + value_to_hide = splitted[0] + string_with_hidden_pass = string_with_hidden_pass.replace( + f"{pass_param_name}{value_to_hide}", f"{pass_param_name}***" + ) + return string_with_hidden_pass + def connect_to_database( self, dbapiModuleName: Optional[str] = None, @@ -129,13 +169,30 @@ def connect_to_database( dbConfigFile: Optional[str] = None, driverMode: Optional[str] = None, alias: str = "default", + **custom_connection_params, ): """ - Loads the DB API 2.0 module given ``dbapiModuleName`` then uses it to - connect to the database using provided parameters such as ``dbName``, ``dbUsername``, and ``dbPassword``. + Creates a database connection using the DB API 2.0 module ``dbapiModuleName`` and the parameters provided. + Along with listed commonly used arguments (`dbName`, `dbHost` etc.) + you can set any other DB module specific parameters as key/value pairs. + + Use ``dbConfigFile`` to provide a path to configuration file with connection parameters + to be used along with / instead of keyword arguments. + If no specified, it defaults to `./resources/db.cfg`. + See `Using configuration file` for more details. + + All params are optional, although ``dbapiModuleName`` must be set - either as keyword argument or in config file. + If some of the listed keyword arguments (`dbName`, `dbHost` etc.) are not provided (i.e. left on dafault value `None`), + they are normally not passed to the Python DB module at all, except: + - _dbPort_ - commonly used port number for known databases is set as fallback + - _dbCharset_ - _UTF8_ is used as fallback for _pymysql_, _pymssql_ and _pyodbc_ + - _dbDriver_ - _{SQL Server}_ is used as fallback for _pyodbc_ + - _driverMode_ - _thin_ is used as fallback for _oracledb_ + + Other params are passed to the Python DB module module as provided. Optional ``alias`` parameter can be used for creating multiple open connections, even for different databases. - If the same alias is given twice then previous connection will be overriden. + If the same alias is given twice then previous connection will be overridden. The ``driverMode`` is used to select the *oracledb* client mode. Allowed values are: @@ -143,50 +200,80 @@ def connect_to_database( - _thick_ - _thick,lib_dir=_ - Optionally, you can specify a ``dbConfigFile`` wherein it will load the - alias (or alias will be "default") property values for ``dbapiModuleName``, ``dbName`` ``dbUsername`` - and ``dbPassword`` (note: specifying ``dbapiModuleName``, ``dbName`` - `dbUsername` or `dbPassword` directly will override the properties of - the same key in `dbConfigFile`). If no `dbConfigFile` is specified, it - defaults to `./resources/db.cfg`. - - The `dbConfigFile` is useful if you don't want to check into your SCM - your database credentials. - - Example db.cfg file - | [alias] - | dbapiModuleName=pymysqlforexample - | dbName=yourdbname - | dbUsername=yourusername - | dbPassword=yourpassword - | dbHost=yourhost - | dbPort=yourport - - Example usage: - | # explicitly specifies all db property values | + Examples | Connect To Database | psycopg2 | my_db | postgres | s3cr3t | tiger.foobar.com | 5432 | + | Connect To Database | psycopg2 | my_db | postgres | s3cr3t | tiger.foobar.com | 5432 | my_custom_param=value | | Connect To Database | psycopg2 | my_db | postgres | s3cr3t | tiger.foobar.com | 5432 | alias=my_alias | - - | # loads all property values from default.cfg | - | Connect To Database | dbConfigFile=default.cfg | - - | # loads all property values from ./resources/db.cfg | - | Connect To Database | - - | # uses explicit `dbapiModuleName` and `dbName` but uses the `dbUsername` and `dbPassword` in 'default.cfg' | - | Connect To Database | psycopg2 | my_db_test | dbConfigFile=default.cfg | - - | # uses explicit `dbapiModuleName` and `dbName` but uses the `dbUsername` and `dbPassword` in './resources/db.cfg' | - | Connect To Database | psycopg2 | my_db_test | + | Connect To Database | dbConfigFile=my_db_params.cfg | """ config = ConfigReader(dbConfigFile, alias) - dbapiModuleName = dbapiModuleName or config.get("dbapiModuleName") - dbName = dbName or config.get("dbName") - dbUsername = dbUsername or config.get("dbUsername") - dbPassword = dbPassword if dbPassword is not None else config.get("dbPassword") - dbHost = dbHost or config.get("dbHost") or "localhost" - dbPort = int(dbPort if dbPort is not None else config.get("dbPort")) + def _build_connection_params(custom_params=True, **basic_params): + con_params = basic_params.copy() + for param_name, param_val in basic_params.items(): + if param_val is None: + con_params.pop(param_name, None) + if custom_params: + con_params.update(custom_connection_params) + con_params.update(other_config_file_params) + + return con_params + + def _log_all_connection_params(*, connection_object=None, connection_string=None, **connection_params): + connection_object = connection_object or dbapiModuleName + msg = f"Connect to DB using : {connection_object}.connect(" + if connection_string: + msg += f'"{connection_string}"' + for param_name, param_value in connection_params.items(): + msg += f", {param_name}=" + if isinstance(param_value, str): + msg += f"'{param_value}'" + else: + msg += f"{param_value}" + msg += ")" + if dbPassword: + msg = msg.replace(f"'{dbPassword}'", "***") + msg = self._hide_password_values(msg) + msg = msg.replace("connect(, ", "connect(") + logger.info(msg) + + def _arg_or_config(arg_value, param_name, mandatory=False): + val_from_config = config.pop(param_name) + if arg_value: + final_value = arg_value + if val_from_config: + logger.info( + f"Parameter '{param_name}' set both as keyword argument and in config file, " + "but keyword arguments take precedence" + ) + else: + final_value = val_from_config + if final_value is None and mandatory: + raise ValueError( + f"Required parameter '{param_name}' was not provided - " + "neither in keyword arguments nor in config file" + ) + return final_value + + # mandatory parameter + dbapiModuleName = _arg_or_config(dbapiModuleName, "dbapiModuleName", mandatory=True) + # optional named params - named because of custom module specific handling + dbName = _arg_or_config(dbName, "dbName") + dbUsername = _arg_or_config(dbUsername, "dbUsername") + dbPassword = _arg_or_config(dbPassword, "dbPassword") + dbHost = _arg_or_config(dbHost, "dbHost") + dbPort = _arg_or_config(dbPort, "dbPort") + if dbPort: + dbPort = int(dbPort) + dbCharset = _arg_or_config(dbCharset, "dbCharset") + dbDriver = _arg_or_config(dbDriver, "dbDriver") + driverMode = _arg_or_config(driverMode, "driverMode") + + for param_name, param_value in custom_connection_params.items(): + _arg_or_config(param_value, param_name) + other_config_file_params = config.get_all_available_params() + if other_config_file_params: + logger.info(f"Other params from configuration file: {list(other_config_file_params.keys())}") if dbapiModuleName == "excel" or dbapiModuleName == "excelrw": db_api_module_name = "pyodbc" @@ -197,31 +284,30 @@ def connect_to_database( if dbapiModuleName in ["MySQLdb", "pymysql"]: dbPort = dbPort or 3306 - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"db={dbName}, user={dbUsername}, passwd=***, host={dbHost}, port={dbPort}, charset={dbCharset})" + dbCharset = dbCharset or "utf8mb4" + con_params = _build_connection_params( + db=dbName, user=dbUsername, passwd=dbPassword, host=dbHost, port=dbPort, charset=dbCharset ) - db_connection = db_api_2.connect( - db=dbName, - user=dbUsername, - passwd=dbPassword, - host=dbHost, - port=dbPort, - charset="utf8mb4" or dbCharset, + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) + + elif dbapiModuleName in ["pymssql"]: + dbPort = dbPort or 1433 + dbCharset = dbCharset or "UTF-8" + con_params = _build_connection_params( + database=dbName, user=dbUsername, password=dbPassword, host=dbHost, port=dbPort, charset=dbCharset ) + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) + elif dbapiModuleName in ["psycopg2"]: dbPort = dbPort or 5432 - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"database={dbName}, user={dbUsername}, password=***, host={dbHost}, port={dbPort})" - ) - db_connection = db_api_2.connect( - database=dbName, - user=dbUsername, - password=dbPassword, - host=dbHost, - port=dbPort, + con_params = _build_connection_params( + database=dbName, user=dbUsername, password=dbPassword, host=dbHost, port=dbPort ) + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) + elif dbapiModuleName in ["pyodbc", "pypyodbc"]: dbPort = dbPort or 1433 dbCharset = dbCharset or "utf8mb4" @@ -231,47 +317,41 @@ def connect_to_database( con_str += f"SERVER={dbHost}:{dbPort}" else: con_str += f"SERVER={dbHost},{dbPort}" - logger.info(f'Connecting using : {dbapiModuleName}.connect({con_str.replace(dbPassword, "***")})') + for param_name, param_value in custom_connection_params.items(): + con_str += f";{param_name}={param_value}" + for param_name, param_value in other_config_file_params.items(): + con_str += f";{param_name}={param_value}" + _log_all_connection_params(connection_string=con_str) db_connection = db_api_2.connect(con_str) - elif dbapiModuleName in ["excel"]: - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"DRIVER={{Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)}};DBQ={dbName};" - f'ReadOnly=1;Extended Properties="Excel 8.0;HDR=YES";)' - ) - db_connection = db_api_2.connect( - f"DRIVER={{Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)}};DBQ={dbName};" - f'ReadOnly=1;Extended Properties="Excel 8.0;HDR=YES";)', - autocommit=True, - ) - elif dbapiModuleName in ["excelrw"]: - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"DRIVER={{Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)}};DBQ={dbName};" - f'ReadOnly=0;Extended Properties="Excel 8.0;HDR=YES";)', - ) - db_connection = db_api_2.connect( - f"DRIVER={{Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)}};DBQ={dbName};" - f'ReadOnly=0;Extended Properties="Excel 8.0;HDR=YES";)', - autocommit=True, - ) + + elif dbapiModuleName in ["excel", "excelrw"]: + con_str = f"DRIVER={{Microsoft Excel Driver (*.xls, *.xlsx, *.xlsm, *.xlsb)}};DBQ={dbName};" + con_str += "ReadOnly=" + if dbapiModuleName == "excel": + con_str += "1;" + elif dbapiModuleName == "excelrw": + con_str += "0;" + con_str += 'Extended Properties="Excel 8.0;HDR=YES";)' + logger.info(f"Connecting using : {dbapiModuleName}.connect({con_str}, autocommit=True)") + db_connection = db_api_2.connect(con_str, autocommit=True) + elif dbapiModuleName in ["ibm_db", "ibm_db_dbi"]: dbPort = dbPort or 50000 - conn_str = f"DATABASE={dbName};HOSTNAME={dbHost};PORT={dbPort};PROTOCOL=TCPIP;UID={dbUsername};" - logger.info(f"Connecting using : {dbapiModuleName}.connect(" f"{conn_str};PWD=***;)") - db_connection = db_api_2.connect( - f"{conn_str};PWD={dbPassword};", - "", - "", + con_str = ( + f"DATABASE={dbName};HOSTNAME={dbHost};PORT={dbPort};PROTOCOL=TCPIP;UID={dbUsername};PWD={dbPassword};" ) + con_params = _build_connection_params(userID="", userPassword="") + _log_all_connection_params(connection_string=con_str, **con_params) + db_connection = db_api_2.connect(con_str, **con_params) + elif dbapiModuleName in ["cx_Oracle"]: dbPort = dbPort or 1521 oracle_dsn = db_api_2.makedsn(host=dbHost, port=dbPort, service_name=dbName) - logger.info( - f"Connecting using: {dbapiModuleName}.connect(user={dbUsername}, password=***, dsn={oracle_dsn})" - ) - db_connection = db_api_2.connect(user=dbUsername, password=dbPassword, dsn=oracle_dsn) + con_params = _build_connection_params(user=dbUsername, password=dbPassword, dsn=oracle_dsn) + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) self.omit_trailing_semicolon = True + elif dbapiModuleName in ["oracledb"]: dbPort = dbPort or 1521 driverMode = driverMode or "thin" @@ -292,24 +372,19 @@ def connect_to_database( logger.info("Using thin Oracle client mode") else: raise ValueError(f"Invalid Oracle client mode provided: {driverMode}") - logger.info( - f"Connecting using: {dbapiModuleName}.connect(" - f"user={dbUsername}, password=***, params={oracle_connection_params})" - ) - db_connection = db_api_2.connect(user=dbUsername, password=dbPassword, params=oracle_connection_params) + con_params = _build_connection_params(user=dbUsername, password=dbPassword, params=oracle_connection_params) + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) assert db_connection.thin == oracle_thin_mode, ( "Expected oracledb to run in thin mode: {oracle_thin_mode}, " f"but the connection has thin mode: {db_connection.thin}" ) self.omit_trailing_semicolon = True + elif dbapiModuleName in ["teradata"]: dbPort = dbPort or 1025 teradata_udaExec = db_api_2.UdaExec(appName="RobotFramework", version="1.0", logConsole=False) - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"database={dbName}, user={dbUsername}, password=***, host={dbHost}, port={dbPort})" - ) - db_connection = teradata_udaExec.connect( + con_params = _build_connection_params( method="odbc", system=dbHost, database=dbName, @@ -318,37 +393,33 @@ def connect_to_database( host=dbHost, port=dbPort, ) + _log_all_connection_params(connection_object=f"{dbapiModuleName}.UdaExec", **con_params) + db_connection = teradata_udaExec.connect(**con_params) + elif dbapiModuleName in ["ksycopg2"]: dbPort = dbPort or 54321 - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"database={dbName}, user={dbUsername}, password=***, host={dbHost}, port={dbPort})" - ) - db_connection = db_api_2.connect( - database=dbName, - user=dbUsername, - password=dbPassword, - host=dbHost, - port=dbPort, + con_params = _build_connection_params( + database=dbName, user=dbUsername, password=dbPassword, host=dbHost, port=dbPort ) + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) + else: - logger.info( - f"Connecting using : {dbapiModuleName}.connect(" - f"database={dbName}, user={dbUsername}, password=***, host={dbHost}, port={dbPort}) " - ) - db_connection = db_api_2.connect( - database=dbName, - user=dbUsername, - password=dbPassword, - host=dbHost, - port=dbPort, + con_params = _build_connection_params( + database=dbName, user=dbUsername, password=dbPassword, host=dbHost, port=dbPort ) + _log_all_connection_params(**con_params) + db_connection = db_api_2.connect(**con_params) + self.connection_store.register_connection(db_connection, db_api_module_name, alias) def connect_to_database_using_custom_params( self, dbapiModuleName: Optional[str] = None, db_connect_string: str = "", alias: str = "default" ): """ + *DEPRECATED* Use new `Connect To Database` keyword with custom parameters instead. + The deprecated keyword will be removed in future versions. + Loads the DB API 2.0 module given `dbapiModuleName` then uses it to connect to the database using the map string `db_connect_string` (parsed as a list of named arguments). @@ -364,20 +435,10 @@ def connect_to_database_using_custom_params( """ db_api_2 = importlib.import_module(dbapiModuleName) db_api_module_name = dbapiModuleName - db_connect_string = f"db_api_2.connect({db_connect_string})" - - connection_string_with_hidden_pass = db_connect_string - for pass_param_name in ["pass", "passwd", "password", "pwd", "PWD"]: - splitted = connection_string_with_hidden_pass.split(pass_param_name + "=") - if len(splitted) < 2: - continue - splitted = splitted[1].split(",") - value_to_hide = splitted[0] - connection_string_with_hidden_pass = connection_string_with_hidden_pass.replace(value_to_hide, "***") logger.info( f"Executing : Connect To Database Using Custom Params : {dbapiModuleName}.connect(" - f"{connection_string_with_hidden_pass})" + f"{self._hide_password_values(db_connect_string)})" ) db_connection = eval(db_connect_string) diff --git a/test/resources/common.resource b/test/resources/common.resource index 0ba69a8..d98ca37 100644 --- a/test/resources/common.resource +++ b/test/resources/common.resource @@ -6,6 +6,7 @@ Library Collections Library OperatingSystem Library DatabaseLibrary Library DateTime +Resource config_files/connect_config_file.resource *** Variables *** @@ -26,8 +27,10 @@ Connect To DB [Documentation] Connects to the database based on the current DB module under test ... and connection params set in global variables with alias [Arguments] ${alias}=${None} - ${DB_KWARGS} Create Dictionary - IF $alias is not None Set To Dictionary ${DB_KWARGS} alias=${alias} + ${DB_KWARGS}= Create Dictionary + IF $alias is not None + Set To Dictionary ${DB_KWARGS} alias=${alias} + END IF "${DB_MODULE_MODE}" == "custom" IF "${DB_MODULE}" == "sqlite3" Remove File ${DBName}.db @@ -38,8 +41,16 @@ Connect To DB Connect To Database Using Custom Connection String ${DB_MODULE} ${Connection String} &{DB_KWARGS} END ELSE IF "${DB_MODULE_MODE}" == "standard" - ${DB_ARGS} Create List ${DB_MODULE} ${DB_NAME} ${DB_USER} ${DB_PASS} ${DB_HOST} ${DB_PORT} - IF "${DB_MODULE}" == "pyodbc" Set To Dictionary ${DB_KWARGS} dbDriver=${DB_DRIVER} + ${DB_ARGS}= Create List + ... ${DB_MODULE} + ... ${DB_NAME} + ... ${DB_USER} + ... ${DB_PASS} + ... ${DB_HOST} + ... ${DB_PORT} + IF "${DB_MODULE}" == "pyodbc" + Set To Dictionary ${DB_KWARGS} dbDriver=${DB_DRIVER} + END Connect To Database @{DB_ARGS} &{DB_KWARGS} ELSE Fail Unexpected mode - ${DB_MODULE_MODE} diff --git a/test/resources/config_files/connect_config_file.resource b/test/resources/config_files/connect_config_file.resource new file mode 100644 index 0000000..569c1f3 --- /dev/null +++ b/test/resources/config_files/connect_config_file.resource @@ -0,0 +1,11 @@ +*** Settings *** +Resource ../common.resource + + +*** Keywords *** +Connect Using Config File + [Documentation] `File name` is only name without extension, + ... the path is build relative to the resource directory + [Arguments] ${File name}=${None} &{Params} + ${Path}= Set Variable ${CURDIR}/${File name}.cfg + Connect To Database dbConfigFile=${Path} &{Params} diff --git a/test/resources/config_files/oracledb/custom_param_password.cfg b/test/resources/config_files/oracledb/custom_param_password.cfg new file mode 100644 index 0000000..4835811 --- /dev/null +++ b/test/resources/config_files/oracledb/custom_param_password.cfg @@ -0,0 +1,6 @@ +[default] +dbapiModuleName=oracledb +dbName=db +password=pass +dbHost=127.0.0.1 +dbPort=1521 \ No newline at end of file diff --git a/test/resources/config_files/oracledb/invalid_custom_params.cfg b/test/resources/config_files/oracledb/invalid_custom_params.cfg new file mode 100644 index 0000000..3f2a48b --- /dev/null +++ b/test/resources/config_files/oracledb/invalid_custom_params.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=oracledb +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=1521 +blah=blah \ No newline at end of file diff --git a/test/resources/config_files/oracledb/simple_default_alias.cfg b/test/resources/config_files/oracledb/simple_default_alias.cfg new file mode 100644 index 0000000..2d81cd4 --- /dev/null +++ b/test/resources/config_files/oracledb/simple_default_alias.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=oracledb +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=1521 +driverMode=thin \ No newline at end of file diff --git a/test/resources/config_files/oracledb/some_basic_params_missing.cfg b/test/resources/config_files/oracledb/some_basic_params_missing.cfg new file mode 100644 index 0000000..83454d0 --- /dev/null +++ b/test/resources/config_files/oracledb/some_basic_params_missing.cfg @@ -0,0 +1,3 @@ +[default] +dbapiModuleName=oracledb +dbName=db \ No newline at end of file diff --git a/test/resources/config_files/oracledb/thick_mode.cfg b/test/resources/config_files/oracledb/thick_mode.cfg new file mode 100644 index 0000000..7878033 --- /dev/null +++ b/test/resources/config_files/oracledb/thick_mode.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=oracledb +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=1521 +driverMode=thick \ No newline at end of file diff --git a/test/resources/config_files/oracledb/valid_custom_params.cfg b/test/resources/config_files/oracledb/valid_custom_params.cfg new file mode 100644 index 0000000..6538367 --- /dev/null +++ b/test/resources/config_files/oracledb/valid_custom_params.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=oracledb +dbName=db +user=db_user +password=pass +dbHost=127.0.0.1 +dbPort=1521 \ No newline at end of file diff --git a/test/resources/config_files/oracledb/wrong_password.cfg b/test/resources/config_files/oracledb/wrong_password.cfg new file mode 100644 index 0000000..2b6b05d --- /dev/null +++ b/test/resources/config_files/oracledb/wrong_password.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=oracledb +dbName=db +dbUsername=db_user +dbPassword=wrong +dbHost=127.0.0.1 +dbPort=1521 \ No newline at end of file diff --git a/test/resources/config_files/psycopg2/custom_param_password.cfg b/test/resources/config_files/psycopg2/custom_param_password.cfg new file mode 100644 index 0000000..0b0fa77 --- /dev/null +++ b/test/resources/config_files/psycopg2/custom_param_password.cfg @@ -0,0 +1,6 @@ +[default] +dbapiModuleName=psycopg2 +dbName=db +password=pass +dbHost=127.0.0.1 +dbPort=5432 \ No newline at end of file diff --git a/test/resources/config_files/psycopg2/invalid_custom_params.cfg b/test/resources/config_files/psycopg2/invalid_custom_params.cfg new file mode 100644 index 0000000..343eeb4 --- /dev/null +++ b/test/resources/config_files/psycopg2/invalid_custom_params.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=psycopg2 +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=5432 +blah=blah \ No newline at end of file diff --git a/test/resources/config_files/psycopg2/simple_default_alias.cfg b/test/resources/config_files/psycopg2/simple_default_alias.cfg new file mode 100644 index 0000000..d9faef7 --- /dev/null +++ b/test/resources/config_files/psycopg2/simple_default_alias.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=psycopg2 +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=5432 \ No newline at end of file diff --git a/test/resources/config_files/psycopg2/some_basic_params_missing.cfg b/test/resources/config_files/psycopg2/some_basic_params_missing.cfg new file mode 100644 index 0000000..7d82bff --- /dev/null +++ b/test/resources/config_files/psycopg2/some_basic_params_missing.cfg @@ -0,0 +1,3 @@ +[default] +dbapiModuleName=psycopg2 +dbName=db \ No newline at end of file diff --git a/test/resources/config_files/psycopg2/valid_custom_params.cfg b/test/resources/config_files/psycopg2/valid_custom_params.cfg new file mode 100644 index 0000000..c143601 --- /dev/null +++ b/test/resources/config_files/psycopg2/valid_custom_params.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=psycopg2 +dbName=db +user=db_user +password=pass +dbHost=127.0.0.1 +dbPort=5432 \ No newline at end of file diff --git a/test/resources/config_files/psycopg2/wrong_password.cfg b/test/resources/config_files/psycopg2/wrong_password.cfg new file mode 100644 index 0000000..074080f --- /dev/null +++ b/test/resources/config_files/psycopg2/wrong_password.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=psycopg2 +dbName=db +dbUsername=db_user +dbPassword=wrong +dbHost=127.0.0.1 +dbPort=5432 \ No newline at end of file diff --git a/test/resources/config_files/pymssql/charset_invalid.cfg b/test/resources/config_files/pymssql/charset_invalid.cfg new file mode 100644 index 0000000..c8ed37c --- /dev/null +++ b/test/resources/config_files/pymssql/charset_invalid.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=pymssql +dbName=db +user=SA +password=MyPass1234! +dbHost=127.0.0.1 +dbPort=1433 +dbCharset=wrong \ No newline at end of file diff --git a/test/resources/config_files/pymssql/custom_param_password.cfg b/test/resources/config_files/pymssql/custom_param_password.cfg new file mode 100644 index 0000000..e11f91b --- /dev/null +++ b/test/resources/config_files/pymssql/custom_param_password.cfg @@ -0,0 +1,6 @@ +[default] +dbapiModuleName=pymssql +dbName=db +password=MyPass1234! +dbHost=127.0.0.1 +dbPort=1433 \ No newline at end of file diff --git a/test/resources/config_files/pymssql/invalid_custom_params.cfg b/test/resources/config_files/pymssql/invalid_custom_params.cfg new file mode 100644 index 0000000..7935942 --- /dev/null +++ b/test/resources/config_files/pymssql/invalid_custom_params.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=pymssql +dbName=db +dbUsername=SA +dbPassword=MyPass1234! +dbHost=127.0.0.1 +dbPort=1433 +blah=blah \ No newline at end of file diff --git a/test/resources/config_files/pymssql/simple_default_alias.cfg b/test/resources/config_files/pymssql/simple_default_alias.cfg new file mode 100644 index 0000000..00a68ad --- /dev/null +++ b/test/resources/config_files/pymssql/simple_default_alias.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pymssql +dbName=db +dbUsername=SA +dbPassword=MyPass1234! +dbHost=127.0.0.1 +dbPort=1433 \ No newline at end of file diff --git a/test/resources/config_files/pymssql/some_basic_params_missing.cfg b/test/resources/config_files/pymssql/some_basic_params_missing.cfg new file mode 100644 index 0000000..af0f2d7 --- /dev/null +++ b/test/resources/config_files/pymssql/some_basic_params_missing.cfg @@ -0,0 +1,3 @@ +[default] +dbapiModuleName=pymssql +dbName=db \ No newline at end of file diff --git a/test/resources/config_files/pymssql/valid_custom_params.cfg b/test/resources/config_files/pymssql/valid_custom_params.cfg new file mode 100644 index 0000000..4ba6dd2 --- /dev/null +++ b/test/resources/config_files/pymssql/valid_custom_params.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pymssql +dbName=db +user=SA +password=MyPass1234! +dbHost=127.0.0.1 +dbPort=1433 \ No newline at end of file diff --git a/test/resources/config_files/pymssql/wrong_password.cfg b/test/resources/config_files/pymssql/wrong_password.cfg new file mode 100644 index 0000000..df8b822 --- /dev/null +++ b/test/resources/config_files/pymssql/wrong_password.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pymssql +dbName=db +dbUsername=SA +dbPassword=wrong +dbHost=127.0.0.1 +dbPort=1433 \ No newline at end of file diff --git a/test/resources/config_files/pymysql/charset_invalid.cfg b/test/resources/config_files/pymysql/charset_invalid.cfg new file mode 100644 index 0000000..e64b4d8 --- /dev/null +++ b/test/resources/config_files/pymysql/charset_invalid.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=pymysql +dbName=db +user=db_user +password=pass +dbHost=127.0.0.1 +dbPort=3306 +dbCharset=wrong \ No newline at end of file diff --git a/test/resources/config_files/pymysql/custom_param_password.cfg b/test/resources/config_files/pymysql/custom_param_password.cfg new file mode 100644 index 0000000..dd211e0 --- /dev/null +++ b/test/resources/config_files/pymysql/custom_param_password.cfg @@ -0,0 +1,6 @@ +[default] +dbapiModuleName=pymysql +dbName=db +password=pass +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pymysql/invalid_custom_params.cfg b/test/resources/config_files/pymysql/invalid_custom_params.cfg new file mode 100644 index 0000000..31e3a20 --- /dev/null +++ b/test/resources/config_files/pymysql/invalid_custom_params.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=pymysql +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=3306 +blah=blah \ No newline at end of file diff --git a/test/resources/config_files/pymysql/simple_default_alias.cfg b/test/resources/config_files/pymysql/simple_default_alias.cfg new file mode 100644 index 0000000..0d73312 --- /dev/null +++ b/test/resources/config_files/pymysql/simple_default_alias.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pymysql +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pymysql/some_basic_params_missing.cfg b/test/resources/config_files/pymysql/some_basic_params_missing.cfg new file mode 100644 index 0000000..e341dc9 --- /dev/null +++ b/test/resources/config_files/pymysql/some_basic_params_missing.cfg @@ -0,0 +1,3 @@ +[default] +dbapiModuleName=pymysql +dbName=db \ No newline at end of file diff --git a/test/resources/config_files/pymysql/valid_custom_params.cfg b/test/resources/config_files/pymysql/valid_custom_params.cfg new file mode 100644 index 0000000..4df80c1 --- /dev/null +++ b/test/resources/config_files/pymysql/valid_custom_params.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pymysql +dbName=db +user=db_user +password=pass +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pymysql/wrong_password.cfg b/test/resources/config_files/pymysql/wrong_password.cfg new file mode 100644 index 0000000..9ce8ca7 --- /dev/null +++ b/test/resources/config_files/pymysql/wrong_password.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pymysql +dbName=db +dbUsername=db_user +dbPassword=wrong +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/charset_invalid.cfg b/test/resources/config_files/pyodbc/charset_invalid.cfg new file mode 100644 index 0000000..3e7b0d7 --- /dev/null +++ b/test/resources/config_files/pyodbc/charset_invalid.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=pyodbc +dbName=db +user=db_user +password=pass +dbHost=127.0.0.1 +dbPort=3306 +dbCharset=wrong \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/custom_param_password.cfg b/test/resources/config_files/pyodbc/custom_param_password.cfg new file mode 100644 index 0000000..6989a2d --- /dev/null +++ b/test/resources/config_files/pyodbc/custom_param_password.cfg @@ -0,0 +1,6 @@ +[default] +dbapiModuleName=pyodbc +dbName=db +password=pass +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/invalid_custom_params.cfg b/test/resources/config_files/pyodbc/invalid_custom_params.cfg new file mode 100644 index 0000000..e7001c5 --- /dev/null +++ b/test/resources/config_files/pyodbc/invalid_custom_params.cfg @@ -0,0 +1,8 @@ +[default] +dbapiModuleName=pyodbc +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=3306 +blah=blah \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/simple_default_alias.cfg b/test/resources/config_files/pyodbc/simple_default_alias.cfg new file mode 100644 index 0000000..91b1569 --- /dev/null +++ b/test/resources/config_files/pyodbc/simple_default_alias.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pyodbc +dbName=db +dbUsername=db_user +dbPassword=pass +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/some_basic_params_missing.cfg b/test/resources/config_files/pyodbc/some_basic_params_missing.cfg new file mode 100644 index 0000000..436ebc9 --- /dev/null +++ b/test/resources/config_files/pyodbc/some_basic_params_missing.cfg @@ -0,0 +1,3 @@ +[default] +dbapiModuleName=pyodbc +dbName=db \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/valid_custom_params.cfg b/test/resources/config_files/pyodbc/valid_custom_params.cfg new file mode 100644 index 0000000..eb8d71f --- /dev/null +++ b/test/resources/config_files/pyodbc/valid_custom_params.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pyodbc +dbName=db +user=db_user +password=pass +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/wrong_password.cfg b/test/resources/config_files/pyodbc/wrong_password.cfg new file mode 100644 index 0000000..622e1d8 --- /dev/null +++ b/test/resources/config_files/pyodbc/wrong_password.cfg @@ -0,0 +1,7 @@ +[default] +dbapiModuleName=pyodbc +dbName=db +dbUsername=db_user +dbPassword=wrong +dbHost=127.0.0.1 +dbPort=3306 \ No newline at end of file diff --git a/test/resources/config_files/sqlite3/simple_default_alias.cfg b/test/resources/config_files/sqlite3/simple_default_alias.cfg new file mode 100644 index 0000000..4a8047b --- /dev/null +++ b/test/resources/config_files/sqlite3/simple_default_alias.cfg @@ -0,0 +1,4 @@ +[default] +dbapiModuleName=sqlite3 +database=./${DBName}.db +isolation_level= \ No newline at end of file diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot new file mode 100644 index 0000000..c0514f5 --- /dev/null +++ b/test/tests/common_tests/connection_params.robot @@ -0,0 +1,153 @@ +*** Settings *** +Documentation Tests for the basic _Connect To Database_ keyword - with and without config files. +... The parameter handling is partly DB module specific. + +Resource ../../resources/common.resource + +Test Setup Skip If $DB_MODULE == "sqlite3" +Test Teardown Disconnect From Database + +*** Variables *** +&{Errors psycopg2} +... missing basic params=OperationalError: connection to server on socket * +... invalid custom param=ProgrammingError: invalid dsn: invalid connection option "blah"* +&{Errors oracledb} +... missing basic params=DatabaseError: DPY-4001: no credentials specified +... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' +&{Errors pymssql} +... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* +... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' +&{Errors pymysql} +... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* +... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' +&{Errors pyodbc} +... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* +... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' + +&{Errors} +... psycopg2=${Errors psycopg2} +... oracledb=${Errors oracledb} +... pymssql=${Errors pymssql} +... pymysql=${Errors pymysql} +... pyodbc=${Errors pyodbc} + + +*** Test Cases *** +Mandatory params can't be missing + Run Keyword And Expect Error + ... ValueError: Required parameter 'dbapiModuleName' was not provided* + ... Connect To Database dbName=${DB_NAME} + +All basic params, no config file + Connect To Database + ... dbapiModuleName=${DB_MODULE} + ... dbName=${DB_NAME} + ... dbUsername=${DB_USER} + ... dbPassword=${DB_PASS} + ... dbHost=${DB_HOST} + ... dbPort=${DB_PORT} + +Missing basic params are accepted, error from Python DB module + Run Keyword And Expect Error + ... ${Errors}[${DB_MODULE}][missing basic params] + ... Connect To Database + ... dbapiModuleName=${DB_MODULE} + +Custom params as keyword args - valid + Connect To Database + ... dbapiModuleName=${DB_MODULE} + ... dbName=${DB_NAME} + ... dbHost=${DB_HOST} + ... dbPort=${DB_PORT} + ... user=${DB_USER} + ... password=${DB_PASS} + +Custom params as keyword args - invalid, error from Python DB module + Run Keyword And Expect Error + ... ${Errors}[${DB_MODULE}][invalid custom param] + ... Connect To Database + ... dbapiModuleName=${DB_MODULE} + ... dbName=${DB_NAME} + ... dbHost=${DB_HOST} + ... dbPort=${DB_PORT} + ... dbUsername=${DB_USER} + ... dbPassword=${DB_PASS} + ... blah=blah + +All basic params in config file + Connect Using Config File ${DB_MODULE}/simple_default_alias + +Missing basic params in config file are accepted, error from Python DB module + Run Keyword And Expect Error + ... ${Errors}[${DB_MODULE}][missing basic params] + ... Connect Using Config File + ... ${DB_MODULE}/some_basic_params_missing + +Custom params from config file - valid + Connect Using Config File ${DB_MODULE}/valid_custom_params + +Custom params from config file - invalid, error from Python DB module + Run Keyword And Expect Error + ... ${Errors}[${DB_MODULE}][invalid custom param] + ... Connect Using Config File ${DB_MODULE}/invalid_custom_params + +Custom params as keyword args combined with custom params from config file + Connect Using Config File ${DB_MODULE}/custom_param_password + ... user=${DB_USER} + + +Keyword args override config file values - basic params + Connect Using Config File ${DB_MODULE}/wrong_password + ... dbPassword=${DB_PASS} + +Keyword args override config file values - custom params + Connect Using Config File ${DB_MODULE}/valid_custom_params + ... user=${DB_USER} + +Oracle specific - basic params, no config file, driverMode + Skip If $DB_MODULE != "oracledb" + Connect To Database + ... dbapiModuleName=${DB_MODULE} + ... dbName=${DB_NAME} + ... dbUsername=${DB_USER} + ... dbPassword=${DB_PASS} + ... dbHost=${DB_HOST} + ... dbPort=${DB_PORT} + ... driverMode=thin + +Oracle specific - thick mode in config file - invalid + [Documentation] Invalid as mode switch during test execution is not supported + ... This test must run the last one in the suite, after others used thin mode already. + Skip If $DB_MODULE != "oracledb" + Run Keyword And Expect Error ProgrammingError: DPY-2019: python-oracledb thick mode cannot be used because a thin mode connection has already been created + ... Connect Using Config File ${DB_MODULE}/thick_mode + + +MSSQL / MySQL / PyODBC specific - charset as keyword argument + Skip If $DB_MODULE not in ["pymssql", "pymysql", "pyodbc"] + Connect To Database + ... dbapiModuleName=${DB_MODULE} + ... dbName=${DB_NAME} + ... dbUsername=${DB_USER} + ... dbPassword=${DB_PASS} + ... dbHost=${DB_HOST} + ... dbPort=${DB_PORT} + ... dbCharset=LATIN1 + +MSSQL / MySQL / PyODBC specific - charset in config file - invalid + Skip If $DB_MODULE not in ["pymssql", "pymysql", "pyodbc"] + Run Keyword And Expect Error OperationalError: (20002, b'Unknown error') + ... Connect Using Config File ${DB_MODULE}/charset_invalid + +SQlite specific - connection params as custom keyword args + [Setup] Skip If $DB_MODULE != "sqlite3" + Remove File ${DBName}.db + Connect To Database + ... dbapiModuleName=${DB_MODULE} + ... database=./${DBName}.db + ... isolation_level=${EMPTY} + +SQlite specific - custom connection params in config file + [Setup] Skip If $DB_MODULE != "sqlite3" + Remove File ${DBName}.db + Connect Using Config File ${DB_MODULE}/simple_default_alias From 455ddcee064b455975964f9932cb5c625ba263be Mon Sep 17 00:00:00 2001 From: amochin Date: Tue, 24 Sep 2024 19:47:52 +0200 Subject: [PATCH 02/30] Connection params tests - fix expected error messages for pymysql --- test/tests/common_tests/connection_params.robot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot index c0514f5..e4d3b7e 100644 --- a/test/tests/common_tests/connection_params.robot +++ b/test/tests/common_tests/connection_params.robot @@ -18,8 +18,8 @@ Test Teardown Disconnect From Database ... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* ... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' &{Errors pymysql} -... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* -... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' +... missing basic params=OperationalError: (1045, "Access denied* +... invalid custom param=TypeError: __init__() got an unexpected keyword argument 'blah' &{Errors pyodbc} ... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* ... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' From b543fb9a5c4bb6dab307d817c09d49d5b9ef49dd Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 08:06:25 +0200 Subject: [PATCH 03/30] minor docs improvements --- src/DatabaseLibrary/connection_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 5463305..a8ddf8b 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -201,9 +201,9 @@ def connect_to_database( - _thick,lib_dir=_ Examples - | Connect To Database | psycopg2 | my_db | postgres | s3cr3t | tiger.foobar.com | 5432 | - | Connect To Database | psycopg2 | my_db | postgres | s3cr3t | tiger.foobar.com | 5432 | my_custom_param=value | - | Connect To Database | psycopg2 | my_db | postgres | s3cr3t | tiger.foobar.com | 5432 | alias=my_alias | + | Connect To Database | psycopg2 | my_db | user | pass | tiger.foobar.com | 5432 | + | Connect To Database | psycopg2 | my_db | user | pass | tiger.foobar.com | 5432 | my_custom_param=value | + | Connect To Database | psycopg2 | my_db | user | pass | tiger.foobar.com | 5432 | alias=my_alias | | Connect To Database | dbConfigFile=my_db_params.cfg | """ config = ConfigReader(dbConfigFile, alias) From e7096e6093c4d7758314a8a54cc2cad4aedd78e0 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 10:07:05 +0200 Subject: [PATCH 04/30] Stop using '{SQL Server}' as fallback value for pyodbc driver --- src/DatabaseLibrary/connection_manager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index a8ddf8b..eb5636c 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -311,9 +311,13 @@ def _arg_or_config(arg_value, param_name, mandatory=False): elif dbapiModuleName in ["pyodbc", "pypyodbc"]: dbPort = dbPort or 1433 dbCharset = dbCharset or "utf8mb4" - dbDriver = dbDriver or "{SQL Server}" - con_str = f"DRIVER={dbDriver};DATABASE={dbName};UID={dbUsername};PWD={dbPassword};charset={dbCharset};" - if "mysql" in dbDriver.lower(): + if dbDriver: + con_str = f"DRIVER={dbDriver};" + else: + logger.info("No ODBC driver specified") + logger.info(f"List of installed ODBC drivers: {db_api_2.drivers()}") + con_str += f"DATABASE={dbName};UID={dbUsername};PWD={dbPassword};charset={dbCharset};" + if dbDriver and "mysql" in dbDriver.lower(): con_str += f"SERVER={dbHost}:{dbPort}" else: con_str += f"SERVER={dbHost},{dbPort}" From c78e1e0c236b66c8c78bf2eefff0cc6b0df073b5 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 10:23:25 +0200 Subject: [PATCH 05/30] Small bugfix in logging - avoid losing the last bracket after hiding the password, if it was the last parameter --- src/DatabaseLibrary/connection_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index eb5636c..131985a 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -230,11 +230,11 @@ def _log_all_connection_params(*, connection_object=None, connection_string=None msg += f"'{param_value}'" else: msg += f"{param_value}" - msg += ")" if dbPassword: msg = msg.replace(f"'{dbPassword}'", "***") msg = self._hide_password_values(msg) msg = msg.replace("connect(, ", "connect(") + msg += ")" logger.info(msg) def _arg_or_config(arg_value, param_name, mandatory=False): From 14759410f74c40f2e7b644b4483044cb5f972316 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 10:26:08 +0200 Subject: [PATCH 06/30] Connection tests - MySQL has own error message when charset is invalid --- test/tests/common_tests/connection_params.robot | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot index e4d3b7e..5aecd3e 100644 --- a/test/tests/common_tests/connection_params.robot +++ b/test/tests/common_tests/connection_params.robot @@ -134,11 +134,17 @@ MSSQL / MySQL / PyODBC specific - charset as keyword argument ... dbPort=${DB_PORT} ... dbCharset=LATIN1 -MSSQL / MySQL / PyODBC specific - charset in config file - invalid - Skip If $DB_MODULE not in ["pymssql", "pymysql", "pyodbc"] +MSSQL / PyODBC specific - charset in config file - invalid + Skip If $DB_MODULE not in ["pymssql", "pyodbc"] Run Keyword And Expect Error OperationalError: (20002, b'Unknown error') ... Connect Using Config File ${DB_MODULE}/charset_invalid +MySQL specific - charset in config file - invalid + Skip If $DB_MODULE not in ["pymysql"] + Run Keyword And Expect Error AttributeError: 'NoneType' object has no attribute 'encoding'' + ... Connect Using Config File ${DB_MODULE}/charset_invalid + + SQlite specific - connection params as custom keyword args [Setup] Skip If $DB_MODULE != "sqlite3" Remove File ${DBName}.db From ff97767197233ca9105f09ccc3bf385d2919a8cf Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 10:29:01 +0200 Subject: [PATCH 07/30] Fix setting the con_str for pyodbc --- src/DatabaseLibrary/connection_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 131985a..652fb9d 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -314,6 +314,7 @@ def _arg_or_config(arg_value, param_name, mandatory=False): if dbDriver: con_str = f"DRIVER={dbDriver};" else: + con_str = "" logger.info("No ODBC driver specified") logger.info(f"List of installed ODBC drivers: {db_api_2.drivers()}") con_str += f"DATABASE={dbName};UID={dbUsername};PWD={dbPassword};charset={dbCharset};" From dcdc5b4789c2a5c12f4bfd5dbb9059f4e362ed70 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 10:54:14 +0200 Subject: [PATCH 08/30] Fix connection logging for excel --- src/DatabaseLibrary/connection_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 652fb9d..51ff022 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -337,7 +337,7 @@ def _arg_or_config(arg_value, param_name, mandatory=False): elif dbapiModuleName == "excelrw": con_str += "0;" con_str += 'Extended Properties="Excel 8.0;HDR=YES";)' - logger.info(f"Connecting using : {dbapiModuleName}.connect({con_str}, autocommit=True)") + logger.info(f"Connecting using : {db_api_module_name}.connect({con_str}, autocommit=True)") db_connection = db_api_2.connect(con_str, autocommit=True) elif dbapiModuleName in ["ibm_db", "ibm_db_dbi"]: From 1dfb77feb3247ad0d8b94319077b9a1349d2f2bf Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 10:55:31 +0200 Subject: [PATCH 09/30] Make connection params optional for pyodbc as well (using connection string) --- src/DatabaseLibrary/connection_manager.py | 31 ++++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 51ff022..b37c31d 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -186,10 +186,11 @@ def connect_to_database( they are normally not passed to the Python DB module at all, except: - _dbPort_ - commonly used port number for known databases is set as fallback - _dbCharset_ - _UTF8_ is used as fallback for _pymysql_, _pymssql_ and _pyodbc_ - - _dbDriver_ - _{SQL Server}_ is used as fallback for _pyodbc_ - _driverMode_ - _thin_ is used as fallback for _oracledb_ - Other params are passed to the Python DB module module as provided. + Other custom params from keyword arguments and config file are passed to the Python DB module as provided - + normally as arguments for the _connect()_ function. However, when using *pyodbc*, the connection is established + using a connection string - so all the custom params are added into it instead of function arguments. Optional ``alias`` parameter can be used for creating multiple open connections, even for different databases. If the same alias is given twice then previous connection will be overridden. @@ -311,21 +312,33 @@ def _arg_or_config(arg_value, param_name, mandatory=False): elif dbapiModuleName in ["pyodbc", "pypyodbc"]: dbPort = dbPort or 1433 dbCharset = dbCharset or "utf8mb4" + if dbDriver: con_str = f"DRIVER={dbDriver};" else: con_str = "" logger.info("No ODBC driver specified") logger.info(f"List of installed ODBC drivers: {db_api_2.drivers()}") - con_str += f"DATABASE={dbName};UID={dbUsername};PWD={dbPassword};charset={dbCharset};" - if dbDriver and "mysql" in dbDriver.lower(): - con_str += f"SERVER={dbHost}:{dbPort}" - else: - con_str += f"SERVER={dbHost},{dbPort}" + if dbName: + con_str += f"DATABASE={dbName};" + if dbUsername: + con_str += f"UID={dbUsername};" + if dbPassword: + con_str += f"PWD={dbPassword};" + if dbCharset: + con_str += f"charset={dbCharset};" + if dbHost and dbPort: + if dbDriver and "mysql" in dbDriver.lower(): + con_str += f"SERVER={dbHost}:{dbPort};" + else: + con_str += f"SERVER={dbHost},{dbPort};" + for param_name, param_value in custom_connection_params.items(): - con_str += f";{param_name}={param_value}" + con_str += f"{param_name}={param_value};" + for param_name, param_value in other_config_file_params.items(): - con_str += f";{param_name}={param_value}" + con_str += f"{param_name}={param_value};" + _log_all_connection_params(connection_string=con_str) db_connection = db_api_2.connect(con_str) From 501164be8e80c8fcd66780a4ffb81841105863ac Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 11:05:11 +0200 Subject: [PATCH 10/30] Tests - wrong comma in expected error message --- test/tests/common_tests/connection_params.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot index 5aecd3e..ff8db1a 100644 --- a/test/tests/common_tests/connection_params.robot +++ b/test/tests/common_tests/connection_params.robot @@ -141,7 +141,7 @@ MSSQL / PyODBC specific - charset in config file - invalid MySQL specific - charset in config file - invalid Skip If $DB_MODULE not in ["pymysql"] - Run Keyword And Expect Error AttributeError: 'NoneType' object has no attribute 'encoding'' + Run Keyword And Expect Error AttributeError: 'NoneType' object has no attribute 'encoding' ... Connect Using Config File ${DB_MODULE}/charset_invalid From 36e25503d4a64bbb1ed4e393eca90fc3cca614f5 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 11:25:31 +0200 Subject: [PATCH 11/30] Fix hiding pass value in logging connection string --- src/DatabaseLibrary/connection_manager.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index b37c31d..7ed7660 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -142,14 +142,14 @@ def __init__(self): self.connection_store: ConnectionStore = ConnectionStore() @staticmethod - def _hide_password_values(string_with_pass): + def _hide_password_values(string_with_pass, params_separator=","): string_with_hidden_pass = string_with_pass for pass_param_name in ["pass", "passwd", "password", "pwd", "PWD"]: pass_param_name += "=" splitted = string_with_hidden_pass.split(pass_param_name) if len(splitted) < 2: continue - splitted = splitted[1].split(",") + splitted = splitted[1].split(params_separator) value_to_hide = splitted[0] string_with_hidden_pass = string_with_hidden_pass.replace( f"{pass_param_name}{value_to_hide}", f"{pass_param_name}***" @@ -223,8 +223,10 @@ def _build_connection_params(custom_params=True, **basic_params): def _log_all_connection_params(*, connection_object=None, connection_string=None, **connection_params): connection_object = connection_object or dbapiModuleName msg = f"Connect to DB using : {connection_object}.connect(" + params_separator = "," if connection_string: msg += f'"{connection_string}"' + params_separator = ";" for param_name, param_value in connection_params.items(): msg += f", {param_name}=" if isinstance(param_value, str): @@ -233,7 +235,7 @@ def _log_all_connection_params(*, connection_object=None, connection_string=None msg += f"{param_value}" if dbPassword: msg = msg.replace(f"'{dbPassword}'", "***") - msg = self._hide_password_values(msg) + msg = self._hide_password_values(msg, params_separator) msg = msg.replace("connect(, ", "connect(") msg += ")" logger.info(msg) From 946c2592de78e9e9f501c31e96c5a56651e0512a Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 11:39:50 +0200 Subject: [PATCH 12/30] Make connection params tests work also for pyodbc --- test/resources/config_files/pyodbc/charset_invalid.cfg | 1 + .../config_files/pyodbc/custom_param_password.cfg | 5 +++-- .../config_files/pyodbc/invalid_custom_params.cfg | 1 + .../resources/config_files/pyodbc/simple_default_alias.cfg | 3 ++- .../config_files/pyodbc/some_basic_params_missing.cfg | 3 ++- test/resources/config_files/pyodbc/valid_custom_params.cfg | 7 ++++--- test/resources/config_files/pyodbc/wrong_password.cfg | 3 ++- test/tests/common_tests/connection_params.robot | 6 +++++- 8 files changed, 20 insertions(+), 9 deletions(-) diff --git a/test/resources/config_files/pyodbc/charset_invalid.cfg b/test/resources/config_files/pyodbc/charset_invalid.cfg index 3e7b0d7..73a0be8 100644 --- a/test/resources/config_files/pyodbc/charset_invalid.cfg +++ b/test/resources/config_files/pyodbc/charset_invalid.cfg @@ -5,4 +5,5 @@ user=db_user password=pass dbHost=127.0.0.1 dbPort=3306 +dbDriver={MySQL ODBC 8.0 ANSI Driver} dbCharset=wrong \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/custom_param_password.cfg b/test/resources/config_files/pyodbc/custom_param_password.cfg index 6989a2d..6021e0a 100644 --- a/test/resources/config_files/pyodbc/custom_param_password.cfg +++ b/test/resources/config_files/pyodbc/custom_param_password.cfg @@ -1,6 +1,7 @@ [default] dbapiModuleName=pyodbc dbName=db -password=pass +PWD=pass dbHost=127.0.0.1 -dbPort=3306 \ No newline at end of file +dbPort=3306 +dbDriver={MySQL ODBC 8.0 ANSI Driver} \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/invalid_custom_params.cfg b/test/resources/config_files/pyodbc/invalid_custom_params.cfg index e7001c5..7fe6682 100644 --- a/test/resources/config_files/pyodbc/invalid_custom_params.cfg +++ b/test/resources/config_files/pyodbc/invalid_custom_params.cfg @@ -5,4 +5,5 @@ dbUsername=db_user dbPassword=pass dbHost=127.0.0.1 dbPort=3306 +dbDriver={MySQL ODBC 8.0 ANSI Driver} blah=blah \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/simple_default_alias.cfg b/test/resources/config_files/pyodbc/simple_default_alias.cfg index 91b1569..d7cf249 100644 --- a/test/resources/config_files/pyodbc/simple_default_alias.cfg +++ b/test/resources/config_files/pyodbc/simple_default_alias.cfg @@ -4,4 +4,5 @@ dbName=db dbUsername=db_user dbPassword=pass dbHost=127.0.0.1 -dbPort=3306 \ No newline at end of file +dbPort=3306 +dbDriver={MySQL ODBC 8.0 ANSI Driver} \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/some_basic_params_missing.cfg b/test/resources/config_files/pyodbc/some_basic_params_missing.cfg index 436ebc9..99e69f6 100644 --- a/test/resources/config_files/pyodbc/some_basic_params_missing.cfg +++ b/test/resources/config_files/pyodbc/some_basic_params_missing.cfg @@ -1,3 +1,4 @@ [default] dbapiModuleName=pyodbc -dbName=db \ No newline at end of file +dbName=db +dbDriver={MySQL ODBC 8.0 ANSI Driver} \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/valid_custom_params.cfg b/test/resources/config_files/pyodbc/valid_custom_params.cfg index eb8d71f..eab93c8 100644 --- a/test/resources/config_files/pyodbc/valid_custom_params.cfg +++ b/test/resources/config_files/pyodbc/valid_custom_params.cfg @@ -1,7 +1,8 @@ [default] dbapiModuleName=pyodbc dbName=db -user=db_user -password=pass +UID=db_user +PWD=pass dbHost=127.0.0.1 -dbPort=3306 \ No newline at end of file +dbPort=3306 +dbDriver={MySQL ODBC 8.0 ANSI Driver} \ No newline at end of file diff --git a/test/resources/config_files/pyodbc/wrong_password.cfg b/test/resources/config_files/pyodbc/wrong_password.cfg index 622e1d8..ac7a25c 100644 --- a/test/resources/config_files/pyodbc/wrong_password.cfg +++ b/test/resources/config_files/pyodbc/wrong_password.cfg @@ -4,4 +4,5 @@ dbName=db dbUsername=db_user dbPassword=wrong dbHost=127.0.0.1 -dbPort=3306 \ No newline at end of file +dbPort=3306 +dbDriver={MySQL ODBC 8.0 ANSI Driver} \ No newline at end of file diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot index ff8db1a..91c6dc4 100644 --- a/test/tests/common_tests/connection_params.robot +++ b/test/tests/common_tests/connection_params.robot @@ -21,7 +21,7 @@ Test Teardown Disconnect From Database ... missing basic params=OperationalError: (1045, "Access denied* ... invalid custom param=TypeError: __init__() got an unexpected keyword argument 'blah' &{Errors pyodbc} -... missing basic params=OperationalError: (20002, b'DB-Lib error message 20002, severity 9* +... missing basic params=REGEXP: InterfaceError.*Data source name not found and no default driver specified.* ... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' &{Errors} @@ -46,6 +46,7 @@ All basic params, no config file ... dbPassword=${DB_PASS} ... dbHost=${DB_HOST} ... dbPort=${DB_PORT} + ... dbDriver=${DB_DRIVER} Missing basic params are accepted, error from Python DB module Run Keyword And Expect Error @@ -59,6 +60,7 @@ Custom params as keyword args - valid ... dbName=${DB_NAME} ... dbHost=${DB_HOST} ... dbPort=${DB_PORT} + ... dbDriver=${DB_DRIVER} ... user=${DB_USER} ... password=${DB_PASS} @@ -72,6 +74,7 @@ Custom params as keyword args - invalid, error from Python DB module ... dbPort=${DB_PORT} ... dbUsername=${DB_USER} ... dbPassword=${DB_PASS} + ... dbDriver=${DB_DRIVER} ... blah=blah All basic params in config file @@ -132,6 +135,7 @@ MSSQL / MySQL / PyODBC specific - charset as keyword argument ... dbPassword=${DB_PASS} ... dbHost=${DB_HOST} ... dbPort=${DB_PORT} + ... dbDriver=${DB_DRIVER} ... dbCharset=LATIN1 MSSQL / PyODBC specific - charset in config file - invalid From 1bb5ecc15732a69756d252eb965a117731eea4c6 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 11:59:59 +0200 Subject: [PATCH 13/30] Fix connection params tests for pyodbc --- .../config_files/pyodbc/some_basic_params_missing.cfg | 3 +-- test/tests/common_tests/connection_params.robot | 11 +++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/resources/config_files/pyodbc/some_basic_params_missing.cfg b/test/resources/config_files/pyodbc/some_basic_params_missing.cfg index 99e69f6..436ebc9 100644 --- a/test/resources/config_files/pyodbc/some_basic_params_missing.cfg +++ b/test/resources/config_files/pyodbc/some_basic_params_missing.cfg @@ -1,4 +1,3 @@ [default] dbapiModuleName=pyodbc -dbName=db -dbDriver={MySQL ODBC 8.0 ANSI Driver} \ No newline at end of file +dbName=db \ No newline at end of file diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot index 91c6dc4..9846ac9 100644 --- a/test/tests/common_tests/connection_params.robot +++ b/test/tests/common_tests/connection_params.robot @@ -22,7 +22,7 @@ Test Teardown Disconnect From Database ... invalid custom param=TypeError: __init__() got an unexpected keyword argument 'blah' &{Errors pyodbc} ... missing basic params=REGEXP: InterfaceError.*Data source name not found and no default driver specified.* -... invalid custom param=TypeError: connect() got an unexpected keyword argument 'blah' +... invalid custom param= &{Errors} ... psycopg2=${Errors psycopg2} @@ -65,6 +65,8 @@ Custom params as keyword args - valid ... password=${DB_PASS} Custom params as keyword args - invalid, error from Python DB module + Skip If $DB_MODULE != "pyodbc" + ... pyodbc doesn't always throw an error if some wrong parameter was provided Run Keyword And Expect Error ... ${Errors}[${DB_MODULE}][invalid custom param] ... Connect To Database @@ -139,7 +141,7 @@ MSSQL / MySQL / PyODBC specific - charset as keyword argument ... dbCharset=LATIN1 MSSQL / PyODBC specific - charset in config file - invalid - Skip If $DB_MODULE not in ["pymssql", "pyodbc"] + Skip If $DB_MODULE not in ["pymssql"] Run Keyword And Expect Error OperationalError: (20002, b'Unknown error') ... Connect Using Config File ${DB_MODULE}/charset_invalid @@ -148,6 +150,11 @@ MySQL specific - charset in config file - invalid Run Keyword And Expect Error AttributeError: 'NoneType' object has no attribute 'encoding' ... Connect Using Config File ${DB_MODULE}/charset_invalid +PyODBC specific - charset in config file - invalid + Skip If $DB_MODULE not in ["pyodbc"] + Run Keyword And Expect Error REGEXP: .*Can't initialize character set wrong.* + ... Connect Using Config File ${DB_MODULE}/charset_invalid + SQlite specific - connection params as custom keyword args [Setup] Skip If $DB_MODULE != "sqlite3" From 1c45681508e11a132a2541bfa022e645f0018437 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 12:06:33 +0200 Subject: [PATCH 14/30] Fix connection params tests for pyodbc again --- test/tests/common_tests/connection_params.robot | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/tests/common_tests/connection_params.robot b/test/tests/common_tests/connection_params.robot index 9846ac9..5a7f0de 100644 --- a/test/tests/common_tests/connection_params.robot +++ b/test/tests/common_tests/connection_params.robot @@ -22,7 +22,6 @@ Test Teardown Disconnect From Database ... invalid custom param=TypeError: __init__() got an unexpected keyword argument 'blah' &{Errors pyodbc} ... missing basic params=REGEXP: InterfaceError.*Data source name not found and no default driver specified.* -... invalid custom param= &{Errors} ... psycopg2=${Errors psycopg2} @@ -65,7 +64,7 @@ Custom params as keyword args - valid ... password=${DB_PASS} Custom params as keyword args - invalid, error from Python DB module - Skip If $DB_MODULE != "pyodbc" + Skip If $DB_MODULE == "pyodbc" ... pyodbc doesn't always throw an error if some wrong parameter was provided Run Keyword And Expect Error ... ${Errors}[${DB_MODULE}][invalid custom param] @@ -92,6 +91,8 @@ Custom params from config file - valid Connect Using Config File ${DB_MODULE}/valid_custom_params Custom params from config file - invalid, error from Python DB module + Skip If $DB_MODULE == "pyodbc" + ... pyodbc doesn't always throw an error if some wrong parameter was provided Run Keyword And Expect Error ... ${Errors}[${DB_MODULE}][invalid custom param] ... Connect Using Config File ${DB_MODULE}/invalid_custom_params @@ -140,7 +141,7 @@ MSSQL / MySQL / PyODBC specific - charset as keyword argument ... dbDriver=${DB_DRIVER} ... dbCharset=LATIN1 -MSSQL / PyODBC specific - charset in config file - invalid +MSSQL specific - charset in config file - invalid Skip If $DB_MODULE not in ["pymssql"] Run Keyword And Expect Error OperationalError: (20002, b'Unknown error') ... Connect Using Config File ${DB_MODULE}/charset_invalid From 32b91b8dd437ba663a60fc71f7d9aed9e62a4e07 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 12:26:30 +0200 Subject: [PATCH 15/30] optimize module import condition for "excel" --- src/DatabaseLibrary/connection_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index 7ed7660..ceae00b 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -280,10 +280,9 @@ def _arg_or_config(arg_value, param_name, mandatory=False): if dbapiModuleName == "excel" or dbapiModuleName == "excelrw": db_api_module_name = "pyodbc" - db_api_2 = importlib.import_module("pyodbc") else: db_api_module_name = dbapiModuleName - db_api_2 = importlib.import_module(dbapiModuleName) + db_api_2 = importlib.import_module(db_api_module_name) if dbapiModuleName in ["MySQLdb", "pymysql"]: dbPort = dbPort or 3306 From f93f1cd54f73768d5ff6a5afae261d475cb697ed Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 12:52:14 +0200 Subject: [PATCH 16/30] Update unit tests --- test/tests/utests/test_connection_manager.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/test/tests/utests/test_connection_manager.py b/test/tests/utests/test_connection_manager.py index bbc9ea2..1f55c45 100644 --- a/test/tests/utests/test_connection_manager.py +++ b/test/tests/utests/test_connection_manager.py @@ -13,22 +13,11 @@ class TestConnectWithConfigFile: def test_connect_with_empty_config(self): conn_manager = ConnectionManager() config_path = str(TEST_DATA / "empty.cfg") - with pytest.raises(ValueError, match=re.escape("Configuration file does not have [default] section.")): - conn_manager.connect_to_database("my_client", dbConfigFile=config_path) - - def test_connect_no_params_no_config(self): - conn_manager = ConnectionManager() - with pytest.raises(ValueError, match="Required 'dbName' parameter was not provided in keyword arguments."): - conn_manager.connect_to_database("my_client") - - def test_connect_missing_option(self): - conn_manager = ConnectionManager() - config_path = str(TEST_DATA / "no_option.cfg") with pytest.raises( ValueError, - match="Required 'dbPassword' parameter missing in both keyword arguments and configuration file.", + match="Required parameter 'dbapiModuleName' was not provided - neither in keyword arguments nor in config file", ): - conn_manager.connect_to_database("my_client", dbConfigFile=config_path) + conn_manager.connect_to_database(dbConfigFile=config_path) def test_aliased_section(self): conn_manager = ConnectionManager() From 166befad434ec71d30274ee3ff3802baeb975f76 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 12:52:45 +0200 Subject: [PATCH 17/30] Check "if not None" for not set args instead of bool way is more precise --- src/DatabaseLibrary/connection_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/DatabaseLibrary/connection_manager.py b/src/DatabaseLibrary/connection_manager.py index ceae00b..6ee8e84 100644 --- a/src/DatabaseLibrary/connection_manager.py +++ b/src/DatabaseLibrary/connection_manager.py @@ -242,9 +242,9 @@ def _log_all_connection_params(*, connection_object=None, connection_string=None def _arg_or_config(arg_value, param_name, mandatory=False): val_from_config = config.pop(param_name) - if arg_value: + if arg_value is not None: final_value = arg_value - if val_from_config: + if val_from_config is not None: logger.info( f"Parameter '{param_name}' set both as keyword argument and in config file, " "but keyword arguments take precedence" @@ -266,7 +266,7 @@ def _arg_or_config(arg_value, param_name, mandatory=False): dbPassword = _arg_or_config(dbPassword, "dbPassword") dbHost = _arg_or_config(dbHost, "dbHost") dbPort = _arg_or_config(dbPort, "dbPort") - if dbPort: + if dbPort is not None: dbPort = int(dbPort) dbCharset = _arg_or_config(dbCharset, "dbCharset") dbDriver = _arg_or_config(dbDriver, "dbDriver") From 21343dbb428e83a91c3ece439b58c43bd7235b57 Mon Sep 17 00:00:00 2001 From: amochin Date: Wed, 25 Sep 2024 13:02:09 +0200 Subject: [PATCH 18/30] Update docs --- doc/index.html | 51 +++++++++++++++++++++++---------- src/DatabaseLibrary/__init__.py | 4 +++ 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/doc/index.html b/doc/index.html index 3e685f3..bed2160 100644 --- a/doc/index.html +++ b/doc/index.html @@ -6,9 +6,9 @@ - + - - - - -