diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc583b0..08f3ca9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,13 +11,13 @@ build:docker: script: - docker build --tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH} . tags: - - docker + - docker-image-builder build:precommit: stage: build image: python:3.11 before_script: - - pip3 install -r requirements-dev.txt + - pip install .[dev] script: - pre-commit run --all-files @@ -27,8 +27,7 @@ build:test: services: - rabbitmq:latest before_script: - - pip3 install -r requirements.txt - - pip3 install -r requirements-dev.txt + - pip3 install .[dev] script: - pytest -v @@ -41,7 +40,7 @@ build:dist: paths: - dist/ tags: - - docker + - docker-image-builder # Stage: deploy ############################################################################## diff --git a/Dockerfile b/Dockerfile index 69d0784..c657ff2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,19 @@ -FROM python:3.11 +# build stage +FROM python:3.11 AS builder -COPY requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt +WORKDIR /build +COPY . . -COPY . /tmp/controller -RUN cd /tmp/controller && \ - python3 setup.py sdist && \ - pip3 install dist/*.tar.gz && \ - rm -rf /tmp/controller +RUN pip install . -ENTRYPOINT [ "villas-controller" ] +RUN python3 setup.py sdist && \ + pip install dist/*.tar.gz --target /install + +# minimal runtime image +FROM python:3.11-slim AS runtime + +COPY --from=builder /install /usr/local/lib/python3.11/site-packages +COPY etc/*.json etc/*.yaml /etc/villas/controller/ +COPY villas-controller.service /etc/systemd/system/ + +ENTRYPOINT ["/usr/local/lib/python3.11/site-packages/bin/villas-controller"] diff --git a/etc/config_simplekub.yaml b/etc/config_simplekub.yaml new file mode 100644 index 0000000..6b4f5dd --- /dev/null +++ b/etc/config_simplekub.yaml @@ -0,0 +1,17 @@ +--- +broker: + url: amqp://admin:vieQuoo2sieDahHee8ohM5aThaibiPei@villas-broker:5672/ + +components: +- type: generic + category: manager + name: Generic Manager + location: VM Iris + uuid: eddb51a0-557b-4848-ac7a-faccc7c51fa3 + +- category: manager + type: kubernetes-simple + name: Simple Kubernetes Manager + location: VM Iris + uuid: 4f8fb73e-7e74-11eb-8f63-f3ccc3ab82f6 + namespace: villas-controller diff --git a/etc/params_k8s_dpsim.yaml b/etc/params_k8s_dpsim.yaml index 5ce3b84..fa0ed9e 100644 --- a/etc/params_k8s_dpsim.yaml +++ b/etc/params_k8s_dpsim.yaml @@ -13,9 +13,9 @@ properties: name: dpsim spec: suspend: true - activeDeadlineSeconds: 120 # kill the Job after 1h + activeDeadlineSeconds: 3600 # kill the Job after 1h backoffLimit: 0 # only try to run pod once, no retries - ttlSecondsAfterFinished: 120 # delete the Job resources 1h after completion + ttlSecondsAfterFinished: 3600 # delete the Job resources 1h after completion template: spec: restartPolicy: Never diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..799ee97 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "villas-controller" +dynamic = ["version"] +description = "A controller/orchestration API for real-time power system simulators" +readme = "README.md" +requires-python = ">=3.7" +authors = [ + { name="Steffen Vogel", email="acs-software@eonerc.rwth-aachen.de" } +] +dependencies = [ + "dotmap", + "kombu", + "termcolor", + "psutil", + "requests", + "villas-node>=0.10.2", + "kubernetes", + "xdg", + "PyYAML", + "tornado", + "jsonschema>=4.1.0", + "pyusb" +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pre-commit" +] + +[project.license] +text = "Apache-2.0" + +[tool.setuptools.dynamic] +version = { attr = "villas.controller.__version__" } + +[tool.setuptools.packages.find] +include = ["villas*"] + +[project.scripts] +villas-controller = "villas.controller.main:main" +villas-ctl = "villas.controller.main:main" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 51f1982..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,2 +0,0 @@ -pre-commit -pytest diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d0d9c97..0000000 --- a/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -kombu -termcolor -psutil -requests -villas-node>=0.10.2 -kubernetes -xdg -dotmap -PyYAML -tornado -jsonschema>=4.1.0 -psutil -pyusb diff --git a/setup.py b/setup.py index 4f5b5f2..a6e30fd 100644 --- a/setup.py +++ b/setup.py @@ -1,69 +1,9 @@ -from setuptools import setup, find_namespace_packages +from setuptools import setup from glob import glob -import os -import re - - -def get_version(): - here = os.path.abspath(os.path.dirname(__file__)) - init_file = os.path.join(here, "villas", "controller", "__init__.py") - - with open(init_file, "r") as f: - content = f.read() - - match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", content, re.M) - if match: - return match.group(1) - - raise RuntimeError("Version not found") - - -with open('README.md') as f: - long_description = f.read() - setup( - name='villas-controller', - version=get_version(), - description='A controller/orchestration API for real-time ' - 'power system simulators', - long_description=long_description, - long_description_content_type='text/markdown', - url='https://www.fein-aachen.org/projects/villas-controller/', - author='Steffen Vogel', - author_email='acs-software@eonerc.rwth-aachen.de', - license='Apache License 2.0', - keywords='simulation controller villas', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3' - ], - packages=find_namespace_packages(include=['villas.*']), - install_requires=[ - 'dotmap', - 'kombu', - 'termcolor', - 'psutil', - 'requests', - 'villas-node>=0.10.2', - 'kubernetes', - 'xdg', - 'PyYAML', - 'tornado', - 'jsonschema>=4.1.0', - 'psutil', - 'pyusb' - ], data_files=[ ('/etc/villas/controller', glob('etc/*.{json,yaml}')), ('/etc/systemd/system', ['villas-controller.service']) - ], - entry_points={ - 'console_scripts': [ - 'villas-ctl=villas.controller.main:main', - 'villas-controller=villas.controller.main:main' - ], - }, - include_package_data=True + ] ) diff --git a/villas/controller/__init__.py b/villas/controller/__init__.py index 6a9beea..3d26edf 100644 --- a/villas/controller/__init__.py +++ b/villas/controller/__init__.py @@ -1 +1 @@ -__version__ = "0.4.0" +__version__ = "0.4.1" diff --git a/villas/controller/component.py b/villas/controller/component.py index a121f09..4a086f8 100644 --- a/villas/controller/component.py +++ b/villas/controller/component.py @@ -107,7 +107,11 @@ def load_schema(self): fo = resources.open_text(pkg, res) loadedschema = yaml.load(fo, yaml.SafeLoader) - schema[name] = Draft202012Validator(loadedschema) + try: + Draft202012Validator.check_schema(loadedschema) + schema[name] = loadedschema + except jsonschema.exceptions.SchemaError: + self.logger.warning("Schema is invalid!") return schema @@ -148,7 +152,7 @@ def status(self): **self.headers }, 'schema': { - name: v.schema for name, v in self.schema.items() + name: v for name, v in self.schema.items() } } @@ -277,12 +281,15 @@ def from_dict(dict): def publish_status(self): if not self.mixin: + self.logger.warn('No mixin!') return self.mixin.publish(self.status, headers=self.headers) def publish_status_periodically(self): - self.logger.info('Start state publish thread') + self.logger.info('Start state publish thread, initial status: %s', + self.status) + self.publish_status() # publish the first update immediately while not self.publish_status_thread_stop.wait( self.publish_status_interval): diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py index 1f04ff7..42f364d 100644 --- a/villas/controller/components/manager.py +++ b/villas/controller/components/manager.py @@ -28,6 +28,9 @@ def from_dict(dict): if type == 'kubernetes': from villas.controller.components.managers import kubernetes return kubernetes.KubernetesManager(**dict) + if type == 'kubernetes-simple': + from villas.controller.components.managers import kubernetes_simple + return kubernetes_simple.KubernetesManagerSimple(**dict) if type == 'villas-node': from villas.controller.components.managers import villas_node # noqa E501 return villas_node.VILLASnodeManager(**dict) @@ -43,7 +46,6 @@ def from_dict(dict): def add_component(self, comp): if comp.uuid in self.mixin.components: existing_comp = self.mixin.components[comp.uuid] - raise SimulationException(self, 'Component with same UUID ' + 'already exists!', component=existing_comp) diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py index c50d26e..db701c2 100644 --- a/villas/controller/components/managers/kubernetes.py +++ b/villas/controller/components/managers/kubernetes.py @@ -22,42 +22,36 @@ class KubernetesManager(Manager): def __init__(self, **args): super().__init__(**args) - self.thread_stop = threading.Event() - - self.pod_watcher_thread = threading.Thread( - target=self._run_pod_watcher) - self.job_watcher_thread = threading.Thread( - target=self._run_job_watcher) - self.event_watcher_thread = threading.Thread( - target=self._run_event_watcher) - if os.environ.get('KUBECONFIG'): k8s.config.load_kube_config() else: k8s.config.load_incluster_config() - self.namespace = args.get('namespace', 'default') + # the namespace in which to create the jobs + # and to watch for events + self.namespace = os.environ.get('NAMESPACE') + self.namespace = ''.join([self.namespace, '-controller']) - self.my_namespace = os.environ.get('NAMESPACE') + # name and UID of the pod in which this controller is running + # used in kubernetes simulator to set the owner reference self.my_pod_name = os.environ.get('POD_NAME') self.my_pod_uid = os.environ.get('POD_UID') - self._check_namespace(self.namespace) + self.thread_stop = threading.Event() + + self.pod_watcher_thread = threading.Thread( + target=self._run_pod_watcher) + self.job_watcher_thread = threading.Thread( + target=self._run_job_watcher) + self.event_watcher_thread = threading.Thread( + target=self._run_event_watcher) - # self.pod_watcher_thread.start() - # self.job_watcher_thread.start() self.event_watcher_thread.setDaemon(True) self.event_watcher_thread.start() - def _check_namespace(self, ns): - c = k8s.client.CoreV1Api() - - namespaces = c.list_namespace() - for namespace in namespaces.items: - if namespace.metadata.name == ns: - return - - raise RuntimeError(f'Namespace {ns} does not exist') + # Not used yet, can support more complex logic + # self.pod_watcher_thread.start() + # self.job_watcher_thread.start() def _run_pod_watcher(self): w = k8s.watch.Watch() @@ -107,6 +101,10 @@ def _run_event_watcher(self): if _match(comp.job.metadata.name, eo.involved_object.name): + if comp._state == 'stopping': + # incoming events are old repetitions + continue + if eo.reason == 'Completed': comp.change_state('stopping', True) elif eo.reason == 'Started': diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py new file mode 100644 index 0000000..f8e706d --- /dev/null +++ b/villas/controller/components/managers/kubernetes_simple.py @@ -0,0 +1,82 @@ +from villas.controller.components.managers.kubernetes import KubernetesManager +from villas.controller.components.simulators.kubernetes import KubernetesJob + +parameters_simple = { + 'type': 'kubernetes', + 'category': 'simulator', + 'uuid': None, + 'name': '', + 'properties': { + 'job': { + 'apiVersion': 'batch/v1', + 'kind': 'Job', + 'metadata': { + 'name': '' + }, + 'spec': { + 'activeDeadlineSeconds': 3600, + 'backoffLimit': 2, + 'template': { + 'spec': { + 'restartPolicy': 'Never', + 'containers': [ + { + 'image': '', + 'imagePullPolicy': 'Always', + 'name': 'jobcontainer', + 'securityContext': { + 'privileged': False + } + } + ] + } + } + } + } + } +} + + +class KubernetesManagerSimple(KubernetesManager): + + def __init__(self, **args): + super().__init__(**args) + + def create(self, payload): + params = payload.get('parameters', {}) + sim_name = payload.get('name', 'Kubernetes Simulator') + jobname = params.get('jobname', 'noname') + adls = params.get('activeDeadlineSeconds', 3600) + if type(adls) is str: + adls = int(adls) + image = params.get('image') + name = params.get('name') + privileged = params.get('privileged', False) + uuid = params.get('uuid') + self.logger.info('uuid:') + self.logger.info(uuid) + + if image is None: + self.logger.error('No image given, will try super.create') + super().create(payload) + return + + parameters = parameters_simple + parameters['name'] = sim_name + job = parameters['properties']['job'] + job['metadata']['name'] = jobname + job['spec']['activeDeadlineSeconds'] = adls + job_container = job['spec']['template']['spec']['containers'][0] + job_container['image'] = image + job_container['securityContext']['privileged'] = privileged + + parameters['job'] = job + + if name: + parameters['name'] = name + + if uuid: + parameters['uuid'] = uuid + + comp = KubernetesJob(self, **parameters) + self.add_component(comp) diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py index 5da40b0..82fa682 100644 --- a/villas/controller/components/simulators/kubernetes.py +++ b/villas/controller/components/simulators/kubernetes.py @@ -2,6 +2,7 @@ from typing import TYPE_CHECKING import json import signal +import time import kubernetes as k8s @@ -28,6 +29,7 @@ def __init__(self, manager: KubernetesManager, **args): self.job = None self.pods = set() + self.cm_name = '' self.custom_schema = props.get('schema', {}) @@ -65,6 +67,7 @@ def _owner(self): def _prepare_job(self, job, payload): # Create config map cm = self._create_config_map(payload) + self.cm_name = cm.metadata.name # Create volumes v = k8s.client.V1Volume( @@ -173,6 +176,9 @@ def _delete_job(self): self.job = None self.properties['job_name'] = None self.properties['pod_names'] = [] + # job isn't immediately deleted + # let the user see that something is happening + time.sleep(7) def start(self, payload): # Delete prior job @@ -194,9 +200,9 @@ def start(self, payload): self.properties['job_name'] = self.job.metadata.name self.properties['namespace'] = self.manager.namespace - def stop(self, payload): + def stop(self, message): + self.change_state('stopping', True) self._delete_job() - self.change_state('idle') def _send_signal(self, sig): @@ -227,6 +233,8 @@ def resume(self, payload): self.change_state('running') def reset(self, payload): + self.change_state('resetting', True) + self.mixin.drain_publish_queue() self._delete_job() super().reset(payload) diff --git a/villas/controller/controller.py b/villas/controller/controller.py index 177f851..0bdcc98 100644 --- a/villas/controller/controller.py +++ b/villas/controller/controller.py @@ -72,7 +72,7 @@ def add_managers(self): def publish(self, body, **kwargs): self.publish_queue.put((body, kwargs)) - def _drain_publish_queue(self): + def drain_publish_queue(self): try: while msg := self.publish_queue.get(False): body = msg[0] @@ -84,10 +84,12 @@ def _drain_publish_queue(self): self.producer.publish(body, **kwargs) except queue.Empty: pass + except TimeoutError: + LOGGER.warn('TimeoutError, let kombu reconnect..') def on_iteration(self): # Drain publish queue - self._drain_publish_queue() + self.drain_publish_queue() # Update components added = self.components.keys() - self.active_components.keys() @@ -136,7 +138,7 @@ def shutdown(self): c.on_shutdown() # Publish last status updates before shutdown - self._drain_publish_queue() + self.drain_publish_queue() self.should_terminate = True @property diff --git a/villas/controller/schemas/manager/kubernetes-simple/__init__.py b/villas/controller/schemas/manager/kubernetes-simple/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/villas/controller/schemas/manager/kubernetes-simple/create.yaml b/villas/controller/schemas/manager/kubernetes-simple/create.yaml new file mode 100644 index 0000000..474a9a4 --- /dev/null +++ b/villas/controller/schemas/manager/kubernetes-simple/create.yaml @@ -0,0 +1,33 @@ +--- +$schema: http://json-schema.org/draft-04/schema# + +type: object +title: 'Simple Kubernetes Job' +required: + - image +properties: + name: + type: string + title: 'Simulator Name' + default: 'Kubernetes Simulator' + uuid: + type: string + title: UUID + default: 8dfd03b2-1c78-11ec-9621-0242ac130002 + jobname: + type: string + title: 'Jobname' + default: myjob + activeDeadlineSeconds: + type: number + title: activeDeadlineSeconds + default: 3600 + image: + type: string + title: Image + default: perl + privileged: + type: boolean + title: Privileged + default: false + description: 'WARNING: If true, the container has root privileges on the host'