From 00f0515d36cf226421d25ea62cf91461fa009836 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Tue, 14 Jun 2022 15:01:28 +0200 Subject: [PATCH 1/4] Remove redundant configuration This statement was a no-op, given the value defined in BASE_CONFIG --- irctest/controllers/ergo.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/irctest/controllers/ergo.py b/irctest/controllers/ergo.py index 3878e7b9..08034fca 100644 --- a/irctest/controllers/ergo.py +++ b/irctest/controllers/ergo.py @@ -272,6 +272,7 @@ def addMysqlToConfig(self, config: Optional[Dict] = None) -> Dict: config = self.baseConfig() if not mysql_password: return config + config["datastore"]["mysql"] = { "enabled": True, "host": "localhost", @@ -280,11 +281,7 @@ def addMysqlToConfig(self, config: Optional[Dict] = None) -> Dict: "history-database": "ergo_history", "timeout": "3s", } - config["accounts"]["multiclient"] = { - "enabled": True, - "allowed-by-default": True, - "always-on": "disabled", - } + config["history"]["persistent"] = { "enabled": True, "unregistered-channels": True, From 8bd102a39106318e669471e4447b257df46821ed Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Tue, 14 Jun 2022 10:50:16 +0200 Subject: [PATCH 2/4] ergo: Create MySQL subprocess instead of using external DB This starts each test with a clean database, so we can remove chan/nick randomization from stateful tests (chathistory and roleplay). It will also allow testing Ergo with a MySQL backend for the KV store instead of buntdb. Additionally, this makes it much easier to run these tests, than having to manually configure such a database. --- irctest/controllers/ergo.py | 77 ++++++++++++++++++++++++----- irctest/server_tests/chathistory.py | 2 - 2 files changed, 66 insertions(+), 13 deletions(-) diff --git a/irctest/controllers/ergo.py b/irctest/controllers/ergo.py index 08034fca..431ad0cf 100644 --- a/irctest/controllers/ergo.py +++ b/irctest/controllers/ergo.py @@ -3,7 +3,7 @@ import os import shutil import subprocess -from typing import Any, Dict, Optional, Set, Type, Union +from typing import Any, Dict, List, Optional, Set, Type, Union from irctest.basecontrollers import ( BaseServerController, @@ -139,6 +139,7 @@ class ErgoController(BaseServerController, DirectoryBasedController): supported_sasl_mechanisms = {"PLAIN", "SCRAM-SHA-256"} supports_sts = True extban_mute_char = "m" + mysql_proc: Optional[subprocess.Popen] = None def create_config(self) -> None: super().create_config() @@ -215,6 +216,16 @@ def run( [*faketime_cmd, "ergo", "run", "--conf", self._config_path, "--quiet"] ) + def terminate(self) -> None: + if self.mysql_proc is not None: + self.mysql_proc.terminate() + super().terminate() + + def kill(self) -> None: + if self.mysql_proc is not None: + self.mysql_proc.kill() + super().kill() + def wait_for_services(self) -> None: # Nothing to wait for, they start at the same time as Ergo. pass @@ -266,18 +277,12 @@ def addLoggingToConfig(self, config: Optional[Dict] = None) -> Dict: config.update(LOGGING_CONFIG) return config - def addMysqlToConfig(self, config: Optional[Dict] = None) -> Dict: - mysql_password = os.getenv("MYSQL_PASSWORD") - if config is None: - config = self.baseConfig() - if not mysql_password: - return config - + def addMysqlToConfig(self, config: Dict) -> Dict: + socket_path = self.startMysql() + self.createMysqlDatabase(socket_path, "ergo_history") config["datastore"]["mysql"] = { "enabled": True, - "host": "localhost", - "user": "ergo", - "password": mysql_password, + "socket-path": socket_path, "history-database": "ergo_history", "timeout": "3s", } @@ -290,6 +295,56 @@ def addMysqlToConfig(self, config: Optional[Dict] = None) -> Dict: } return config + def startMysql(self) -> str: + """Starts a new MySQL server listening on a UNIX socket, returns the socket + path""" + assert self.directory + mysql_dir = os.path.join(self.directory, "mysql") + socket_path = os.path.join(mysql_dir, "mysql.socket") + os.mkdir(mysql_dir) + + print("Starting MySQL...") + self.mysql_proc = subprocess.Popen( + [ + "mysqld", + "--no-defaults", + "--tmpdir=" + mysql_dir, + "--datadir=" + mysql_dir, + "--socket=" + socket_path, + "--skip-networking", + "--skip-grant-tables", + ], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + mysql_stdout = self.mysql_proc.stdout + assert mysql_stdout is not None # for mypy... + lines: List[bytes] = [] + while self.mysql_proc.returncode is None: + line = mysql_stdout.readline() + lines.append(lines) + if b"mysqld: ready for connections." in line: + break + assert self.mysql_proc.returncode is None, ( + "MySQL unexpected stopped: " + b"\n".join(lines).decode() + ) + print("MySQL started") + + return socket_path + + def createMysqlDatabase(self, socket_path: str, database_name: str) -> None: + subprocess.check_call( + [ + "mysql", + "--no-defaults", + "-S", + socket_path, + "-e", + f"CREATE DATABASE {database_name};", + ] + ) + def rehash(self, case: BaseServerTestCase, config: Dict) -> None: self._config = config self._write_config() diff --git a/irctest/server_tests/chathistory.py b/irctest/server_tests/chathistory.py index 28a201a4..8da89b8b 100644 --- a/irctest/server_tests/chathistory.py +++ b/irctest/server_tests/chathistory.py @@ -18,8 +18,6 @@ # Keep this in sync with validate_chathistory() SUBCOMMANDS = ["LATEST", "BEFORE", "AFTER", "BETWEEN", "AROUND"] -MYSQL_PASSWORD = "" - def validate_chathistory_batch(msgs): batch_tag = None From d3e2a3eab55f63712885e99293953f0b175cb327 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Tue, 14 Jun 2022 15:09:46 +0200 Subject: [PATCH 3/4] ergo: Add $ERGO_HISTORY_BACKEND to opt-in to history mysql backend --- irctest/controllers/ergo.py | 44 ++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/irctest/controllers/ergo.py b/irctest/controllers/ergo.py index 431ad0cf..7af665f6 100644 --- a/irctest/controllers/ergo.py +++ b/irctest/controllers/ergo.py @@ -174,7 +174,7 @@ def run( enable_chathistory = self.test_config.chathistory enable_roleplay = self.test_config.ergo_roleplay if enable_chathistory or enable_roleplay: - config = self.addMysqlToConfig(config) + self.addDatabaseToConfig(config) if enable_roleplay: config["roleplay"] = {"enabled": True} @@ -277,23 +277,31 @@ def addLoggingToConfig(self, config: Optional[Dict] = None) -> Dict: config.update(LOGGING_CONFIG) return config - def addMysqlToConfig(self, config: Dict) -> Dict: - socket_path = self.startMysql() - self.createMysqlDatabase(socket_path, "ergo_history") - config["datastore"]["mysql"] = { - "enabled": True, - "socket-path": socket_path, - "history-database": "ergo_history", - "timeout": "3s", - } - - config["history"]["persistent"] = { - "enabled": True, - "unregistered-channels": True, - "registered-channels": "opt-out", - "direct-messages": "opt-out", - } - return config + def addDatabaseToConfig(self, config: Dict) -> None: + history_backend = os.environ.get("ERGO_HISTORY_BACKEND", "memory") + if history_backend == "memory": + # nothing to do, this is the default + pass + elif history_backend == "mysql": + socket_path = self.startMysql() + self.createMysqlDatabase(socket_path, "ergo_history") + config["datastore"]["mysql"] = { + "enabled": True, + "socket-path": socket_path, + "history-database": "ergo_history", + "timeout": "3s", + } + config["history"]["persistent"] = { + "enabled": True, + "unregistered-channels": True, + "registered-channels": "opt-out", + "direct-messages": "opt-out", + } + else: + raise ValueError( + f"Invalid $ERGO_HISTORY_BACKEND value: {history_backend}. " + f"It should be 'memory' (the default) or 'mysql'" + ) def startMysql(self) -> str: """Starts a new MySQL server listening on a UNIX socket, returns the socket From 686e0a1055239502036dd143a60f39b101094509 Mon Sep 17 00:00:00 2001 From: Valentin Lorentz Date: Tue, 14 Jun 2022 15:29:10 +0200 Subject: [PATCH 4/4] Initialize MySQL --- irctest/controllers/ergo.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/irctest/controllers/ergo.py b/irctest/controllers/ergo.py index 7af665f6..7d421be4 100644 --- a/irctest/controllers/ergo.py +++ b/irctest/controllers/ergo.py @@ -306,12 +306,38 @@ def addDatabaseToConfig(self, config: Dict) -> None: def startMysql(self) -> str: """Starts a new MySQL server listening on a UNIX socket, returns the socket path""" + # Function based on pifpaf's MySQL driver: + # https://github.com/jd/pifpaf/blob/3.1.5/pifpaf/drivers/mysql.py assert self.directory mysql_dir = os.path.join(self.directory, "mysql") socket_path = os.path.join(mysql_dir, "mysql.socket") os.mkdir(mysql_dir) print("Starting MySQL...") + try: + subprocess.check_call( + [ + "mysqld", + "--no-defaults", + "--tmpdir=" + mysql_dir, + "--initialize-insecure", + "--datadir=" + mysql_dir, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + # Initialize the old way + subprocess.check_call( + [ + "mysql_install_db", + "--no-defaults", + "--tmpdir=" + mysql_dir, + "--datadir=" + mysql_dir, + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) self.mysql_proc = subprocess.Popen( [ "mysqld",