diff --git a/mlos_bench/mlos_bench/__init__.py b/mlos_bench/mlos_bench/__init__.py index 519ee74ba6..681faed56b 100644 --- a/mlos_bench/mlos_bench/__init__.py +++ b/mlos_bench/mlos_bench/__init__.py @@ -150,7 +150,6 @@ The entry point for these configs can be found `here `_. ->>> from subprocess import run >>> # Note: we show the command wrapped in python here for testing purposes. >>> # Alternatively replace test-cli-local-env-bench.jsonc with >>> # test-cli-local-env-opt.jsonc for one that does an optimization loop. @@ -158,10 +157,13 @@ ... --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc \ ... --globals experiment_test_local.jsonc \ ... --tunable_values tunable-values/tunable-values-local.jsonc" + >>> print(f"Here's the shell command you'd actually run:\n# {cmd}") Here's the shell command you'd actually run: # mlos_bench --config mlos_bench/mlos_bench/tests/config/cli/test-cli-local-env-bench.jsonc --globals experiment_test_local.jsonc --tunable_values tunable-values/tunable-values-local.jsonc + >>> # Now we run the command and check the output. +>>> from subprocess import run >>> result = run(cmd, shell=True, capture_output=True, text=True, check=True) >>> assert result.returncode == 0 >>> lines = result.stderr.splitlines() diff --git a/mlos_bench/mlos_bench/config/__init__.py b/mlos_bench/mlos_bench/config/__init__.py index 508d09c7a4..455f94ed02 100644 --- a/mlos_bench/mlos_bench/config/__init__.py +++ b/mlos_bench/mlos_bench/config/__init__.py @@ -178,7 +178,7 @@ for some examples of CLI configs. Globals and Variable Substitution -+++++++++++++++++++++++++++++++++ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :py:attr:`Globals ` are basically just key-value variables that can be used in other configs using @@ -208,14 +208,15 @@ Here is a list of some well known variables that are provided or required by the system and may be used in the config files: -- ``$experiment_id``: A unique identifier for the ``Experiment``. +- ``$experiment_id`` : A unique identifier for the ``Experiment``. Typically provided in globals. -- ``$trial_id``: A unique identifier for the ``Trial`` currently being executed. +- ``$trial_id`` : A unique identifier for the ``Trial`` currently being executed. This can be useful in the configs for :py:mod:`mlos_bench.environments` for instance (e.g., when writing scripts). -- ``$trial_runner_id``: A unique identifier for the ``TrialRunner``. - This can be useful when running multiple trials in parallel (e.g., to - provision a numbered VM per worker). +- ``$trial_runner_id`` : A unique identifier for the + :py:class:`~mlos_bench.schedulers.trial_runner.TrialRunner`. This can be + useful when running multiple trials in parallel (e.g., to provision a + numbered VM per worker). Tunable Configs ^^^^^^^^^^^^^^^ @@ -311,7 +312,7 @@ expected structure of the ``config`` section. In certain cases (e.g., script command execution) the variable substitution rules -take on slightly different behavior +take on slightly different behavior. See various documentation in :py:mod:`mlos_bench.environments` for more details. Config Processing diff --git a/mlos_bench/mlos_bench/services/__init__.py b/mlos_bench/mlos_bench/services/__init__.py index 65ffc8e8d8..5e249c4a3d 100644 --- a/mlos_bench/mlos_bench/services/__init__.py +++ b/mlos_bench/mlos_bench/services/__init__.py @@ -6,6 +6,18 @@ Services for implementing Environments for mlos_bench. TODO: Improve documentation here. + +Overview +-------- +TODO: Explain Service mix-ins and how they get used with Environments. + +Config +------ +TODO: Explain how to configure Services. + +See Also +-------- +TODO: Provide references to different related classes. """ from mlos_bench.services.base_fileshare import FileShareService diff --git a/mlos_bench/mlos_bench/services/config_persistence.py b/mlos_bench/mlos_bench/services/config_persistence.py index 5ecf68a042..e3f237230c 100644 --- a/mlos_bench/mlos_bench/services/config_persistence.py +++ b/mlos_bench/mlos_bench/services/config_persistence.py @@ -7,9 +7,51 @@ benchmark :py:class:`.Environment`, :py:mod:`~mlos_bench.tunables`, :py:class:`.Service` functions, etc from JSON configuration files and strings. +Typically the :py:class:`.ConfigPersistenceService` is provided automatically by +the :py:mod:`mlos_bench.launcher`. + +It's ``config_path`` parameter is a list of directories to search for the +configuration files referenced in other JSON config files or +:py:mod:`mlos_bench.run` ``--cli-options``. + +That value can itself be adjusted with the ``--config-path`` CLI option or set +in a ``--config`` CLI options file. + +Regardless of the values there, the service will always search the included +config files from the :py:mod:`mlos_bench` package. + See Also -------- mlos_bench.config : Overview of the configuration system. +mlos_bench.run : CLI options for the ``mlos_bench`` command. + +Examples +-------- +>>> # Create a new instance of the ConfigPersistenceService. +>>> # This will search for config files in the current directory's config subdir. +>>> # It will also search in the built-in config files that come with the package. +>>> # And the current working directory. +>>> service = ConfigPersistenceService(config={"config_path": ["./config"]}) + +>>> # One of the things the ConfigLoaderType does is find and load config files +>>> # referenced in other JSON files. +>>> # For instance: +>>> config_file_path = "optimizers/mlos_core_default_opt.jsonc" +>>> resolved_file = service.resolve_path(config_file_path) + +>>> # The resolved file should be the same as the expected file. +>>> # That is, the resolved file should be the same as the built-in file. +>>> from importlib.resources import files +>>> from os.path import abspath, samefile +>>> expected_file = abspath(files("mlos_bench.config").joinpath(config_file_path)) +>>> assert samefile(resolved_file, expected_file) + +>>> # Create an Optimizer from that file. +>>> from mlos_bench.config.schemas.config_schemas import ConfigSchema +>>> config = service.load_config(config_file_path, schema_type=ConfigSchema.OPTIMIZER) +>>> optimizer = service.build_optimizer(tunables=TunableGroups(), service=service, config=config) +>>> from mlos_bench.optimizers.base_optimizer import Optimizer +>>> assert isinstance(optimizer, Optimizer) """ import logging diff --git a/mlos_bench/mlos_bench/services/types/__init__.py b/mlos_bench/mlos_bench/services/types/__init__.py index e2d0cb55b5..32c443b15c 100644 --- a/mlos_bench/mlos_bench/services/types/__init__.py +++ b/mlos_bench/mlos_bench/services/types/__init__.py @@ -2,8 +2,53 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Service types for implementing declaring Service behavior for Environments to use in -mlos_bench. +""" +Service types (i.e., :external:py:class:`~typing.Protocol`) for declaring implementation +:py:class:`~mlos_bench.services.base_service.Service` behavior for +:py:mod:`~mlos_bench.environments` to use in :py:mod:`mlos_bench`. + +Overview +-------- +Service loading in ``mlos_bench`` uses a +`mix-in `_ approach to combine the +functionality of multiple classes specified at runtime through config files into +a single class that the :py:mod:`~mlos_bench.environments` can use to invoke the +actions that they were configured to perform (e.g., provisioning a VM, deploying +a network, running a script, etc.). + +Since Services are loaded at runtime and can be swapped out by referencing a +different set of config files via the ``--services`` :py:mod:`mlos_bench.run` +CLI option, this can make it difficult to do type and config checking. + +To address this we define :external:py:func:`~typing.runtime_checkable` decorated +`Protocols `_ (e.g., *interfaces* in other +languages) to declare the expected behavior of the ``Services`` that are loaded +at runtime in the ``Environments`` that use them. + +For example, the :py:class:`.SupportsFileShareOps` Protocol declares the +expected behavior of a Service that can +:py:meth:`~.SupportsFileShareOps.download` and +:py:meth:`~.SupportsFileShareOps.upload` files to and from a remote file share. + +But we can have more than one Service that implements that Protocol (e.g., one +for Azure, one for AWS, one for a remote SSH server, etc.). + +This allows us to define the expected behavior of the Service that the +Environment will need, but not the specific implementation details. + +It also allows users to define Environment configs that are more reusable so +that we can swap out the Service implementations at runtime without having to +change the Environment config. + +That way we can run Experiments on more than one platform rather easily. + +See the classes below for an overview of the types of Services that are +currently available for Environments. + +Notes +----- +If you find that there are missing types or that you need to add a new Service +type, please `submit a PR `_ to add it here. """ from mlos_bench.services.types.authenticator_type import SupportsAuth diff --git a/mlos_bench/mlos_bench/services/types/bound_method.py b/mlos_bench/mlos_bench/services/types/bound_method.py index 8e6179cffe..10039064aa 100644 --- a/mlos_bench/mlos_bench/services/types/bound_method.py +++ b/mlos_bench/mlos_bench/services/types/bound_method.py @@ -2,7 +2,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol representing a bound method.""" +""" +Protocol representing a bound method. + +Notes +----- +Mostly just used for type checking of +:py:class:`~mlos_bench.services.base_service.Service` mix-ins. +""" from typing import Any, Protocol, runtime_checkable diff --git a/mlos_bench/mlos_bench/services/types/config_loader_type.py b/mlos_bench/mlos_bench/services/types/config_loader_type.py index 94d52ed41a..e33e129d4e 100644 --- a/mlos_bench/mlos_bench/services/types/config_loader_type.py +++ b/mlos_bench/mlos_bench/services/types/config_loader_type.py @@ -2,7 +2,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol interface for helper functions to lookup and load configs.""" +""" +Protocol interface for helper functions to lookup and load configs. + +See Also +-------- +:py:class:`~mlos_bench.services.config_persistence.ConfigPersistenceService` +""" from __future__ import annotations diff --git a/mlos_bench/mlos_bench/services/types/remote_config_type.py b/mlos_bench/mlos_bench/services/types/remote_config_type.py index 09a1e8c20f..777655eb21 100644 --- a/mlos_bench/mlos_bench/services/types/remote_config_type.py +++ b/mlos_bench/mlos_bench/services/types/remote_config_type.py @@ -2,7 +2,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # -"""Protocol interface for configuring cloud services.""" +"""Protocol interface for configuring cloud (Saas) services.""" from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable