From de44dea0bf03ba67f523e4937e6e8935f9edb249 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 23 Sep 2021 10:21:54 +0000
Subject: [PATCH 01/89] add manager for simple kubernetes job creation

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 etc/config_simplekub.yaml                     |  14 +++
 etc/params_simplekub.yaml                     |   7 ++
 villas/controller/components/manager.py       |  11 +-
 .../components/managers/kubernetes_simple.py  | 109 ++++++++++++++++++
 4 files changed, 140 insertions(+), 1 deletion(-)
 create mode 100644 etc/config_simplekub.yaml
 create mode 100644 etc/params_simplekub.yaml
 create mode 100644 villas/controller/components/managers/kubernetes_simple.py

diff --git a/etc/config_simplekub.yaml b/etc/config_simplekub.yaml
new file mode 100644
index 0000000..9653b03
--- /dev/null
+++ b/etc/config_simplekub.yaml
@@ -0,0 +1,14 @@
+---
+broker:
+  url: amqp://villas:Haegiethu0rohtee@kubernetes-master-1.os-cloud.eonerc.rwth-aachen.de:30809/%2F
+
+components:
+- type: generic
+  category: manager
+  uuid: ebbbbba0-557b-4848-ac7a-faa3e7c51fa3
+
+- category: manager
+  type: kubernetes-simple
+  uuid: 4bbbb73e-7e74-11eb-8f63-f3a5b3ab82f6
+
+  namespace: villas-controller
diff --git a/etc/params_simplekub.yaml b/etc/params_simplekub.yaml
new file mode 100644
index 0000000..a88fd29
--- /dev/null
+++ b/etc/params_simplekub.yaml
@@ -0,0 +1,7 @@
+---
+name: "ksim created by simple manager"
+uuid: c555aaaa-6af6-11eb-beee-7fa268050224
+image: dpsimrwth/slew-villas:latest
+jobname: "test job"
+activeDeadlineSeconds: 60
+containername: "test container"
diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index fb503ad..d21a4b0 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -38,6 +38,9 @@ def from_dict(dict):
         if type == 'kubernetes':
             from villas.controller.components.managers import kubernetes
             return kubernetes.KubernetesManager(**dict)
+        if type == 'kubernetes-simple':
+            from villas.controller.components.managers import kubernetes_simple
+            return kubernetes_simple.KubernetesManagerSimple(**dict)
         if type == 'villas-node':
             from villas.controller.components.managers import villas_node  # noqa E501
             return villas_node.VILLASnodeManager(**dict)
@@ -48,8 +51,11 @@ def from_dict(dict):
             raise Exception(f'Unknown type: {type}')
 
     def add_component(self, comp):
+        print(self.name)
+        print(comp)
         if comp.uuid in self.mixin.components:
-            raise KeyError
+            # raise KeyError
+            self.logger.error('UUID %s already exists, not added', comp.uuid)
 
         comp.set_manager(self)
 
@@ -68,6 +74,9 @@ def remove_component(self, comp):
 
     def run_action(self, action, message):
         if action == 'create':
+            print("###############")
+            print(message)
+            print(message.payload)
             self.create(message)
         elif action == 'delete':
             self.delete(message)
diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
new file mode 100644
index 0000000..7729c8f
--- /dev/null
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -0,0 +1,109 @@
+from villas.controller.components.managers.kubernetes import KubernetesManager
+from villas.controller.components.simulators.kubernetes import KubernetesJob
+
+
+class KubernetesManagerSimple(KubernetesManager):
+
+    create_schema = {
+        '$schema': 'http://json-schema.org/draft-04/schema#',
+        'title': 'Simple Kubernetes Job',
+        'type': 'object',
+        'required': [
+            'image',
+        ],
+        'properties': {
+            'uuid': {
+                'type': 'string',
+                'title': 'UUID',
+                'default': '8dfd03b2-1c78-11ec-9621-0242ac130002'
+            },
+            'jobname': {
+                'type': 'string',
+                'title': 'Job name',
+                'default': 'myJob'
+            },
+            'activeDeadlineSeconds': {
+                'type': 'number',
+                'title': 'activeDeadlineSeconds',
+                'default': 3600
+            },
+            'image': {
+                'type': 'string',
+                'title': 'Image',
+                'default': 'perl'
+            },
+            'containername': {
+                'type': 'string',
+                'title': 'Container name',
+                'default': 'myContainer'
+            }
+        }
+    }
+
+    parameters_simple = {
+        'type': 'kubernetes',
+        'category': 'simulator',
+        'uuid': None,
+        'name': 'Kubernetes Simulator',
+        'properties': {
+            'job': {
+                'apiVersion': 'batch/v1',
+                'kind': 'Job',
+                'metadata': {
+                    'name': ''
+                },
+                'spec': {
+                    'activeDeadlineSeconds': 3600,
+                    'backoffLimit': 0,
+                    'template': {
+                        'spec': {
+                            'restartPolicy': 'Never',
+                            'containers': [
+                                {
+                                    'image': '',
+                                    'name': ''
+                                }
+                            ]
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    def create(self, message):
+        print(message.payload)
+        params = message.payload.get('parameters', {})
+        jobname = params.get('jobname', 'noname')
+        adls = params.get('activeDeadlineSeconds', 3600)
+        contName = params.get('containername', 'noname')
+        image = params.get('image')
+        name = params.get('name')
+        uuid = params.get('uuid')
+
+        if image is None:
+            self.logger.error('No image given, cannot create kubernetes job')
+            return
+
+        parameters = self.parameters_simple
+        job = parameters['properties']['job']
+        job['metadata']['name'] = jobname
+        job['spec']['activeDeadlineSeconds'] = adls
+        job['spec']['template']['spec']['containers'][0]['name'] = contName
+        job['spec']['template']['spec']['containers'][0]['image'] = image
+
+        parameters['job'] = job
+
+        if name:
+            parameters['name'] = name
+
+        if uuid:
+            parameters['uuid'] = uuid
+
+        print(parameters)
+
+        comp = KubernetesJob(self, **parameters)
+        self.add_component(comp)
+
+    def __init__(self, **args):
+        super().__init__(**args)

From 895975796c1e06cacc433448ed94901e79332d53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 24 Sep 2021 15:04:34 +0000
Subject: [PATCH 02/89] suppress state changes from event repetitions

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 etc/params_k8s_dpsim.yaml                        |  4 ++--
 .../controller/components/managers/kubernetes.py | 16 ++++++++++------
 .../components/simulators/kubernetes.py          |  6 +++++-
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/etc/params_k8s_dpsim.yaml b/etc/params_k8s_dpsim.yaml
index 5ce3b84..fa0ed9e 100644
--- a/etc/params_k8s_dpsim.yaml
+++ b/etc/params_k8s_dpsim.yaml
@@ -13,9 +13,9 @@ properties:
       name: dpsim
     spec:
       suspend: true
-      activeDeadlineSeconds: 120 # kill the Job after 1h
+      activeDeadlineSeconds: 3600 # kill the Job after 1h
       backoffLimit: 0 # only try to run pod once, no retries
-      ttlSecondsAfterFinished: 120 # delete the Job resources 1h after completion
+      ttlSecondsAfterFinished: 3600 # delete the Job resources 1h after completion
       template:
         spec:
           restartPolicy: Never
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index ed1500a..871682a 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -107,8 +107,8 @@ def _run_event_watcher(self):
 
                     eo = e.get('object')
 
-                    self.logger.info('Event: %s (reason=%s)', eo.message,
-                                     eo.reason)
+                    # self.logger.info('Event: %s (reason=%s)', eo.message,
+                    #                 eo.reason)
 
                     for uuid in self.components:
                         comp = self.components[uuid]
@@ -118,6 +118,10 @@ def _run_event_watcher(self):
 
                         if _match(comp.job.metadata.name,
                                   eo.involved_object.name):
+                            if comp._state == 'stopping':
+                                # incoming events are old repetitions
+                                continue
+
                             if eo.reason == 'Completed':
                                 comp.change_state('stopping', True)
                             elif eo.reason == 'Started':
@@ -132,10 +136,10 @@ def _run_event_watcher(self):
                                 elif comp._state == 'starting':
                                     # wait for BackoffLimitExceeded event
                                     continue
-                            else:
-                                self.logger.info('Reason \'%s\' not handled '
-                                                 'for kubernetes simulator',
-                                                 eo.reason)
+                            # else:
+                                # self.logger.info('Reason \'%s\' not handled '
+                                #                 'for kubernetes simulator',
+                                #                 eo.reason)
 
             except ProtocolError:
                 self.logger.warn('Connection to kubernetes broken, \
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index e0decee..6f0a31d 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -2,6 +2,7 @@
 import signal
 from copy import deepcopy
 import collections
+import time
 
 import kubernetes as k8s
 
@@ -164,8 +165,11 @@ def start(self, message):
         self.properties['namespace'] = self.manager.namespace
 
     def stop(self, message):
+        self.change_state('stopping', True)
         self._delete_job()
-
+        # job isn't immediately deleted
+        # let the user see something is happening
+        time.sleep(3)
         self.change_state('idle')
 
     def _send_signal(self, sig):

From d69a26f2795f36e4aee2372445078e8f5c270079 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 24 Sep 2021 15:05:45 +0000
Subject: [PATCH 03/89] try create of superclass if parameters are missing

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes_simple.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 7729c8f..48a3cb9 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -82,7 +82,8 @@ def create(self, message):
         uuid = params.get('uuid')
 
         if image is None:
-            self.logger.error('No image given, cannot create kubernetes job')
+            self.logger.error('No image given, will try super.create')
+            super().create(message)
             return
 
         parameters = self.parameters_simple

From 298df49263729f2d9c594d319514670fca05dd2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 28 Sep 2021 15:51:33 +0000
Subject: [PATCH 04/89] delete configmap when job is deleted

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulators/kubernetes.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 6f0a31d..adfda90 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -40,6 +40,7 @@ def __init__(self, manager, **args):
 
         self.job = None
         self.pods = set()
+        self.cm_name = ''
 
         self.custom_schema = props.get('schema', {})
 
@@ -53,6 +54,7 @@ def schema(self):
     def _prepare_job(self, job, payload):
         # Create config map
         cm = self._create_config_map(payload)
+        self.cm_name = cm.metadata.name
 
         # Create volumes
         v = k8s.client.V1Volume(
@@ -127,6 +129,7 @@ def _delete_job(self):
             return
 
         b = k8s.client.BatchV1Api()
+        c = k8s.client.CoreV1Api()
         body = k8s.client.V1DeleteOptions(propagation_policy='Background')
 
         try:
@@ -134,6 +137,10 @@ def _delete_job(self):
                 namespace=self.manager.namespace,
                 name=self.job.metadata.name,
                 body=body)
+            c.delete_namespaced_config_map(
+                namespace=self.manager.namespace,
+                name=self.cm_name,
+                body=body)
         except k8s.client.exceptions.ApiException as e:
             raise SimulationException(self, 'Kubernetes API error',
                                       error=str(e))

From f13ea000434fddf3bc1e91b245cd623edffaf7b4 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <stvogel@eonerc.rwth-aachen.de>
Date: Fri, 1 Oct 2021 15:00:06 +0200
Subject: [PATCH 05/89] fix api_url for villas-node manager

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/villas_node.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index 3fd9111..eb2c97f 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -10,8 +10,8 @@ class VILLASnodeManager(Manager):
     def __init__(self, **args):
 
         self.autostart = args.get('autostart', False)
-        self.api_url = args.get('url', 'http://localhost:8080') + '/api/v1'
-        self.api_url_external = args.get('url_external', self.api_url)
+        self.api_url = args.get('api_url', 'http://localhost:8080')
+        self.api_url_external = args.get('api_url_external', self.api_url)
 
         args['api_url'] = self.api_url
 

From df28149f2a62263280efb2807d2fe6c8f77a3b62 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 8 Oct 2021 20:01:16 +0200
Subject: [PATCH 06/89] simplify config handling

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 requirements.txt                              |  1 +
 setup.py                                      |  1 +
 villas/controller/commands/daemon.py          |  9 +++---
 .../components/simulators/kubernetes.py       | 17 +----------
 villas/controller/config.py                   | 30 +++++++++++++++++--
 villas/controller/controller.py               | 13 ++++++--
 villas/controller/main.py                     |  6 +---
 villas/controller/util.py                     | 18 +++++++++++
 8 files changed, 63 insertions(+), 32 deletions(-)
 create mode 100644 villas/controller/util.py

diff --git a/requirements.txt b/requirements.txt
index 7f3a823..31f45fe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,5 @@ requests
 villas-node>=0.10.2
 kubernetes
 xdg
+dotmap
 PyYAML
diff --git a/setup.py b/setup.py
index 76c310a..73c88ac 100644
--- a/setup.py
+++ b/setup.py
@@ -23,6 +23,7 @@
     ],
     packages=find_namespace_packages(include=['villas.*']),
     install_requires=[
+        'dotmap',
         'kombu',
         'termcolor',
         'psutil',
diff --git a/villas/controller/commands/daemon.py b/villas/controller/commands/daemon.py
index 731a00d..82f18db 100644
--- a/villas/controller/commands/daemon.py
+++ b/villas/controller/commands/daemon.py
@@ -13,15 +13,14 @@ class DaemonCommand(Command):
     def add_parser(subparsers):
         parser = subparsers.add_parser('daemon',
                                        help='Run VILLAScontroller as a daemon')
-        parser.set_defaults(func=DaemonCommand.run)
+        parser.set_defaults(func=DaemonCommand.start)
 
     @staticmethod
-    def run(connection, args):
-        components = args.config.components
+    def start(connection, args):
 
         try:
-            d = ControllerMixin(connection, components)
-            d.run()
+            d = ControllerMixin(connection, args)
+            d.start()
         except KeyboardInterrupt:
             d.shutdown()
         except ConnectionError:
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index adfda90..0df211e 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -8,22 +8,7 @@
 
 from villas.controller.components.simulator import Simulator
 from villas.controller.exceptions import SimulationException
-
-
-def merge(dict1, dict2):
-    ''' Return a new dictionary by merging two dictionaries recursively. '''
-
-    result = deepcopy(dict1)
-
-    for key, value in dict2.items():
-        if isinstance(value, collections.Mapping):
-            result[key] = merge(result.get(key, {}), value)
-        elif value is None:
-            del result[key]
-        else:
-            result[key] = deepcopy(dict2[key])
-
-    return result
+from villas.controller.util import merge
 
 
 class KubernetesJob(Simulator):
diff --git a/villas/controller/config.py b/villas/controller/config.py
index 0424e85..b145da9 100644
--- a/villas/controller/config.py
+++ b/villas/controller/config.py
@@ -2,6 +2,7 @@
 import argparse
 import logging
 import os
+import dotmap
 
 from os import getcwd
 from xdg import (
@@ -10,6 +11,7 @@
 )
 
 from villas.controller.component import Component
+from villas.controller.util import merge
 
 LOGGER = logging.getLogger(__name__)
 
@@ -28,6 +30,19 @@ def __call__(self, arg):
 
 class Config:
 
+    DEFAULT_CONFIG = {
+        'broker': {
+            'url': 'amqp://localhost:5672/%2F'
+        },
+        'api': {
+            'enabled': True,
+            'port': 8089
+        },
+        'components': [],
+        # 'workdir': '/var/villas/controller/simulators/'
+        'workdir': os.getcwd()
+    }
+
     DEFAULT_PATHS = xdg_config_dirs() + [
                     xdg_config_home(),
                     getcwd(),
@@ -39,11 +54,17 @@ def __init__(self, fp=None):
             fn = self.find_default_path()
             if fn:
                 with open(fn) as fp:
-                    self.dict = yaml.load(fp, Loader=yaml.FullLoader)
+                    self.load(fp)
             else:
                 pass  # Start without config
         else:
-            self.dict = yaml.load(fp, Loader=yaml.FullLoader)
+            self.load(fp)
+
+    def load(self, fp):
+        config = yaml.load(fp, Loader=yaml.FullLoader)
+        merged = merge(self.DEFAULT_CONFIG, config)
+
+        self.config = dotmap.DotMap(merged)
 
     def find_default_path(self, filename='config',
                           suffixes=['json', 'yaml', 'yml']):
@@ -55,7 +76,10 @@ def find_default_path(self, filename='config',
 
     @property
     def components(self):
-        return [Component.from_dict(c) for c in self.dict['components']]
+        return [Component.from_dict(c) for c in self.config.components]
+
+    def __getattr__(self, attr):
+        return self.config.get(attr)
 
     def check(self):
         uuids = [c.uuid for c in self.components]
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index d9c6481..50b327b 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -1,3 +1,4 @@
+import time
 import logging
 import socket
 import queue
@@ -5,13 +6,18 @@
 
 from villas.controller.components.managers.generic import GenericManager
 
+
 LOGGER = logging.getLogger(__name__)
 
 
 class ControllerMixin(kombu.mixins.ConsumerProducerMixin):
 
-    def __init__(self, connection, components):
-        self.components = {c.uuid: c for c in components if c.enabled}
+    def __init__(self, connection, args):
+        self.args = args
+        self.config = args.config
+
+        comps = self.config.components
+        self.components = {c.uuid: c for c in comps if c.enabled}
         self.connection = connection
         self.exchange = kombu.Exchange(name='villas',
                                        type='headers',
@@ -100,7 +106,8 @@ def on_iteration(self):
 
             self.active_components = self.components.copy()
 
-    def run(self):
+    def start(self):
+        self.started = time.time()
         self.should_terminate = False
         while not self.should_terminate:
             self.should_stop = False
diff --git a/villas/controller/main.py b/villas/controller/main.py
index bbd9c57..daad06f 100644
--- a/villas/controller/main.py
+++ b/villas/controller/main.py
@@ -103,11 +103,7 @@ def main():
 
     setup_logging(args)
 
-    if args.broker:
-        broker_url = args.broker
-    else:
-        broker = args.config.dict.get('broker', {})
-        broker_url = broker.get('url', 'amqp://guest:guest@localhost/%2F')
+    broker_url = args.broker or args.config.broker.url
 
     try:
         with kombu.Connection(broker_url, connect_timeout=3) as c:
diff --git a/villas/controller/util.py b/villas/controller/util.py
new file mode 100644
index 0000000..a51e144
--- /dev/null
+++ b/villas/controller/util.py
@@ -0,0 +1,18 @@
+from copy import deepcopy
+import collections
+
+
+def merge(dict1, dict2):
+    ''' Return a new dictionary by merging two dictionaries recursively. '''
+
+    result = deepcopy(dict1)
+
+    for key, value in dict2.items():
+        if isinstance(value, collections.Mapping):
+            result[key] = merge(result.get(key, {}), value)
+        elif value is None:
+            del result[key]
+        else:
+            result[key] = deepcopy(dict2[key])
+
+    return result

From e0d569c0b224f65e5804ba1e9d9e5dd87f22c923 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 8 Oct 2021 20:04:16 +0200
Subject: [PATCH 07/89] move global status to Controller mixin class

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py  | 18 ++----------------
 villas/controller/controller.py | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 863e842..a2df4f9 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -1,12 +1,9 @@
 import logging
-import kombu
 import time
-import socket
-import os
+import kombu
 import uuid
 import threading
 
-from villas.controller import __version__ as version
 from villas.controller.exceptions import SimulationException
 
 
@@ -90,20 +87,9 @@ def schema(self):
 
     @property
     def status(self):
-        u = os.uname()
-
         status = {
             'state': self._state,
-            'version': version,
-            'uptime': time.time() - self.started,
-            'host': socket.gethostname(),
-            'kernel': {
-                'sysname': u.sysname,
-                'nodename': u.nodename,
-                'release': u.release,
-                'version': u.version,
-                'machine': u.machine
-            },
+            **self.mixin.status,
             **self._status_fields
         }
 
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index 50b327b..a1cc2da 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -2,8 +2,10 @@
 import logging
 import socket
 import queue
+import os
 import kombu.mixins
 
+from villas.controller import __version__ as version
 from villas.controller.components.managers.generic import GenericManager
 
 
@@ -126,3 +128,20 @@ def shutdown(self):
         # Publish last status updates before shutdown
         self._drain_publish_queue()
         self.should_terminate = True
+
+    @property
+    def status(self):
+        u = os.uname()
+
+        return {
+            'version': version,
+            'uptime': time.time() - self.started,
+            'host': socket.gethostname(),
+            'kernel': {
+                'sysname': u.sysname,
+                'nodename': u.nodename,
+                'release': u.release,
+                'version': u.version,
+                'machine': u.machine
+            },
+        }

From a0759cf7ce25d7512fd33e5dd6494cc225df52e6 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 8 Oct 2021 20:05:59 +0200
Subject: [PATCH 08/89] make work dir configurable

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py            |  3 +++
 villas/controller/components/simulator.py | 31 ++++++++++-------------
 2 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index a2df4f9..2478a02 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -1,4 +1,5 @@
 import logging
+import os.path
 import time
 import kombu
 import uuid
@@ -54,6 +55,8 @@ def set_mixin(self, mixin):
         self.mixin = mixin
         self.connection = mixin.connection
 
+        self.workdir = os.path.join(self.mixin.config.workdir, str(self.uuid))
+
     def get_consumer(self, channel):
         self.channel = channel
 
diff --git a/villas/controller/components/simulator.py b/villas/controller/components/simulator.py
index 17ba7c0..9fe6e86 100644
--- a/villas/controller/components/simulator.py
+++ b/villas/controller/components/simulator.py
@@ -98,32 +98,27 @@ def change_state(self, state, force=False, **kwargs):
         super().change_state(state, **kwargs)
 
     # Actions
-    def start(self, message):
+    def start(self, payload):
         self.started = time.time()
         self.simuuid = uuid.uuid4()
 
-        if 'parameters' in message.payload:
-            self.params = message.payload['parameters']
+        self.params = payload.get('parameters', {})
+        self.model = payload.get('model')
+        self.results = payload.get('results')
 
-        if 'model' in message.payload:
-            self.model = message.payload['model']
+        self.sim_workdir = os.path.join(self.workdir, 'simulation',
+                                        str(self.simuuid))
 
-        if 'results' in message.payload:
-            self.results = message.payload['results']
-
-        self.workdir = '/var/villas/controller/simulators/' + \
-            str(self.uuid) + '/simulation/' + str(self.simuuid)
-
-        self.logdir = self.workdir + '/Logs/'
-        self.logger.info('Target working directory: %s' % self.workdir)
+        self.sim_logdir = self.sim_workdir + '/Logs/'
+        self.logger.info('Simulation working directory: %s' % self.sim_workdir)
 
         try:
-            os.makedirs(self.logdir)
-            os.chdir(self.logdir)
+            os.makedirs(self.sim_logdir)
+            os.chdir(self.sim_logdir)
         except Exception as e:
             raise SimulationException(self, 'Failed to create and change to '
                                             'working directory: %s ( %s )' %
-                                            (self.logdir, e))
+                                            (self.sim_logdir, e))
 
     def _upload(self, filename):
         url = self.results['url']
@@ -157,9 +152,9 @@ def _unzip_files(self, filename):
 
     def upload_results(self):
         try:
-            filename = self.workdir + '/results.zip'
+            filename = os.path.join(self.sim_workdir, 'results.zip')
             with zipfile.ZipFile(filename, 'w') as results_zip:
-                for sub in os.scandir(self.logdir):
+                for sub in os.scandir(self.sim_logdir):
                     results_zip.write(sub)
 
                 results_zip.close()

From a9bde8d65f0156b37286e45b24a367f229530efa Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 00:34:47 +0200
Subject: [PATCH 09/89] pass payload instead of message to action handlers

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                | 36 +++++++++----------
 .../components/gateways/villas_node.py        | 10 +++---
 villas/controller/components/manager.py       | 17 +++++----
 .../controller/components/managers/generic.py |  8 ++---
 .../components/managers/kubernetes.py         |  8 ++---
 .../components/managers/villas_node.py        | 10 +++---
 .../controller/components/simulators/dpsim.py | 10 +++---
 .../controller/components/simulators/dummy.py | 24 ++++++-------
 .../components/simulators/generic.py          | 14 ++++----
 .../components/simulators/kubernetes.py       | 11 +++---
 10 files changed, 73 insertions(+), 75 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 2478a02..cddc202 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -112,9 +112,11 @@ def on_message(self, message):
         self.logger.debug('Received message: %s', message.payload)
 
         if 'action' in message.payload:
-            self.run_action(message.payload['action'], message)
+            self.run_action(message.payload['action'], message.payload)
 
-    def run_action(self, action, message):
+        message.ack()
+
+    def run_action(self, action, payload):
         if action == 'ping':
             self.logger.debug('Received action: %s', action)
         else:
@@ -122,25 +124,25 @@ def run_action(self, action, message):
 
         try:
             if action == 'ping':
-                self.ping(message)
+                self.ping(payload)
             elif action == 'start':
                 self.change_state('starting')
-                self.start(message)
+                self.start(payload)
             elif action == 'stop':
                 self.change_state('stopping')
-                self.stop(message)
+                self.stop(payload)
             elif action == 'pause':
                 self.change_state('pausing')
-                self.pause(message)
+                self.pause(payload)
             elif action == 'resume':
                 self.change_state('resuming')
-                self.resume(message)
+                self.resume(payload)
             elif action == 'shutdown':
                 self.change_state('shuttingdown')
-                self.shutdown(message)
+                self.shutdown(payload)
             elif action == 'reset':
                 self.change_state('resetting')
-                self.reset(message)
+                self.reset(payload)
             else:
                 raise SimulationException(self, 'Unknown action',
                                           action=action)
@@ -148,8 +150,6 @@ def run_action(self, action, message):
         except SimulationException as se:
             self.logger.error('SimulationException: %s', str(se))
             self.change_state('error', msg=se.msg, **se.info)
-        finally:
-            message.ack()
 
     def change_state(self, state, **kwargs):
         if self._state == state:
@@ -163,25 +163,25 @@ def change_state(self, state, **kwargs):
         self.publish_status()
 
     # Actions
-    def ping(self, message):
+    def ping(self, payload):
         self.publish_status()
 
-    def start(self, message):
+    def start(self, payload):
         raise SimulationException('The component can not be started')
 
-    def stop(self, message):
+    def stop(self, payload):
         raise SimulationException('The component can not be stopped')
 
-    def pause(self, message):
+    def pause(self, payload):
         raise SimulationException('The component can not be paused')
 
-    def resume(self, message):
+    def resume(self, payload):
         raise SimulationException('The component can not be resumed')
 
-    def shutdown(self, message):
+    def shutdown(self, payload):
         raise SimulationException('The component can not be shut down')
 
-    def reset(self, message):
+    def reset(self, payload):
         self.started = time.time()
 
     @staticmethod
diff --git a/villas/controller/components/gateways/villas_node.py b/villas/controller/components/gateways/villas_node.py
index f5d46c5..19ff2cb 100644
--- a/villas/controller/components/gateways/villas_node.py
+++ b/villas/controller/components/gateways/villas_node.py
@@ -20,35 +20,35 @@ def __init__(self, manager, args):
 
         super().__init__(manager, **props)
 
-    def start(self, message):
+    def start(self, payload):
         try:
             self.manager.node.request('node.start', {'node': self.name})
             self.manager.reconcile()
         except Exception as e:
             self.logger.warn('Failed to start node: %s', e)
 
-    def stop(self, message):
+    def stop(self, payload):
         try:
             self.manager.node.request('node.stop', {'node': self.name})
             self.manager.reconcile()
         except Exception as e:
             self.logger.warn('Failed to stop node: %s', e)
 
-    def pause(self, message):
+    def pause(self, payload):
         try:
             self.manager.node.request('node.pause', {'node': self.name})
             self.manager.reconcile()
         except Exception as e:
             self.logger.warn('Failed to pause node: %s', e)
 
-    def resume(self, message):
+    def resume(self, payload):
         try:
             self.manager.node.request('node.resume', {'node': self.name})
             self.manager.reconcile()
         except Exception as e:
             self.logger.warn('Failed to resume node: %s', e)
 
-    def reset(self, message):
+    def reset(self, payload):
         try:
             self.manager.node.reset('node.restart', {'node': self.name})
             self.manager.reconcile()
diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index d21a4b0..620a2dc 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -72,21 +72,20 @@ def remove_component(self, comp):
 
         self.logger.info('Removed component %s', comp)
 
-    def run_action(self, action, message):
+    def run_action(self, action, payload):
         if action == 'create':
-            print("###############")
-            print(message)
-            print(message.payload)
-            self.create(message)
+            #print(message.payload)
+            #self.create(message)
+            self.create(payload)
         elif action == 'delete':
-            self.delete(message)
+            self.delete(payload)
         else:
-            super().run_action(action, message)
+            super().run_action(action, payload)
 
-    def create(self, message):
+    def create(self, payload):
         raise NotImplementedError()
 
-    def delete(self, message):
+    def delete(self, payload):
         raise NotImplementedError()
 
     def on_shutdown(self):
diff --git a/villas/controller/components/managers/generic.py b/villas/controller/components/managers/generic.py
index 2e2dcb6..fe5a040 100644
--- a/villas/controller/components/managers/generic.py
+++ b/villas/controller/components/managers/generic.py
@@ -143,8 +143,8 @@ class GenericManager(Manager):
         'additionalProperties': True
     }
 
-    def create(self, message):
-        component = Component.from_dict(message.payload.get('parameters'))
+    def create(self, payload):
+        component = Component.from_dict(payload.get('parameters'))
 
         try:
             self.add_component(component)
@@ -152,8 +152,8 @@ def create(self, message):
             self.logger.error('A component with the UUID %s already exists',
                               component.uuid)
 
-    def delete(self, message):
-        parameters = message.payload.get('parameters')
+    def delete(self, payload):
+        parameters = payload.get('parameters')
         uuid = parameters.get('uuid')
 
         try:
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 871682a..78d81bf 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -146,14 +146,14 @@ def _run_event_watcher(self):
                                     attempting reconnect..')
                 time.sleep(1)
 
-    def create(self, message):
-        parameters = message.payload.get('parameters', {})
+    def create(self, payload):
+        parameters = payload.get('parameters', {})
         comp = KubernetesJob(self, **parameters)
 
         self.add_component(comp)
 
-    def delete(self, message):
-        parameters = message.payload.get('parameters')
+    def delete(self, payload):
+        parameters = payload.get('parameters')
         uuid = parameters.get('uuid')
 
         try:
diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index eb2c97f..51558c0 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -76,12 +76,12 @@ def on_shutdown(self):
 
         return super().on_shutdown()
 
-    def start(self, message):
+    def start(self, payload):
         self.node.start()
 
         self.change_state('starting')
 
-    def stop(self, message):
+    def stop(self, payload):
         if self.node.is_running():
             self.node.stop()
 
@@ -91,7 +91,7 @@ def stop(self, message):
         for node in self.nodes:
             node.change_state('shutdown')
 
-    def pause(self, message):
+    def pause(self, payload):
         self.node.pause()
 
         self.change_state('paused')
@@ -100,8 +100,8 @@ def pause(self, message):
         for node in self.nodes:
             node.change_state('paused')
 
-    def resume(self, message):
+    def resume(self, payload):
         self.node.resume()
 
-    def reset(self, message):
+    def reset(self, payload):
         self.node.restart()
diff --git a/villas/controller/components/simulators/dpsim.py b/villas/controller/components/simulators/dpsim.py
index 8eb954b..6cfa4ad 100644
--- a/villas/controller/components/simulators/dpsim.py
+++ b/villas/controller/components/simulators/dpsim.py
@@ -135,8 +135,8 @@ def load_cim(self, fp):
             self.logger.info(self.sim)
             os.unlink(fp.name)
 
-    def start(self, message):
-        fp = self.download_model(message)
+    def start(self, payload):
+        fp = self.download_model(payload)
         if fp:
             self.load_cim(fp)
 
@@ -154,7 +154,7 @@ def start(self, message):
             self.logger.warn('Attempted to start non-stopped simulator.'
                              'State is %s', self._state)
 
-    def stop(self, message):
+    def stop(self, payload):
         if self._state == 'running':
             self.logger.info('Stopping simulation...')
 
@@ -169,7 +169,7 @@ def stop(self, message):
             self.logger.warn('Attempted to stop non-stopped simulator.'
                              'State is %s', self._state)
 
-    def pause(self, message):
+    def pause(self, payload):
         if self._state == 'running':
             self.logger.info('Pausing simulation...')
 
@@ -192,7 +192,7 @@ def pause(self, message):
             self.logger.warn('Attempted to pause non-running simulator.'
                              'State is ' + self._state)
 
-    def resume(self, message):
+    def resume(self, payload):
         if self._state == 'paused':
             self.logger.info('Resuming simulation...')
 
diff --git a/villas/controller/components/simulators/dummy.py b/villas/controller/components/simulators/dummy.py
index 06597b0..91a202b 100644
--- a/villas/controller/components/simulators/dummy.py
+++ b/villas/controller/components/simulators/dummy.py
@@ -38,34 +38,34 @@ def _schedule_state_transition(self, state, time=1.0):
         self.timer = threading.Timer(time, self.change_state, args=[state])
         self.timer.start()
 
-    def start(self, message):
-        super().start(message)
+    def start(self, payload):
+        super().start(payload)
 
         runtime = self.params.get('runtime', 1.0)
 
         self._schedule_state_transition('running', runtime)
 
-    def stop(self, message):
-        super().stop(message)
+    def stop(self, payload):
+        super().stop(payload)
 
         self._schedule_state_transition('idle')
 
-    def pause(self, message):
-        super().pause(message)
+    def pause(self, payload):
+        super().pause(payload)
 
         self._schedule_state_transition('paused')
 
-    def resume(self, message):
-        super().resume(message)
+    def resume(self, payload):
+        super().resume(payload)
 
         self._schedule_state_transition('running')
 
-    def shutdown(self, message):
-        super().shutdown(message)
+    def shutdown(self, payload):
+        super().shutdown(payload)
 
         self._schedule_state_transition('shutdown')
 
-    def reset(self, message):
-        super().reset(message)
+    def reset(self, payload):
+        super().reset(payload)
 
         self._schedule_state_transition('idle')
diff --git a/villas/controller/components/simulators/generic.py b/villas/controller/components/simulators/generic.py
index 130b21b..6c09268 100644
--- a/villas/controller/components/simulators/generic.py
+++ b/villas/controller/components/simulators/generic.py
@@ -35,8 +35,8 @@ def state(self):
 
         return state
 
-    def start(self, message):
-        super().start(message)
+    def start(self, payload):
+        super().start(payload)
         self.logger.info('Working directory: %s', os.getcwd())
         path = self.download_model()
 
@@ -45,7 +45,7 @@ def start(self, message):
             raise SimulationException(self, 'Child process is already running')
 
         try:
-            params = message.payload['parameters']
+            params = payload['parameters']
 
             self.thread = threading.Thread(target=GenericSimulator.run,
                                            args=(self, params, path))
@@ -150,7 +150,7 @@ def run(self, params, path):
 
         self.child = None
 
-    def reset(self, message):
+    def reset(self, payload):
         # Don't send a signal if the child does not exist
         if self.child is None:
             return
@@ -173,7 +173,7 @@ def reset(self, message):
         # we will transition into the error state
         self.check_state_deferred('idle', 5)
 
-    def stop(self, message):
+    def stop(self, payload):
         send_cont = False
 
         if self.child is None:
@@ -196,7 +196,7 @@ def stop(self, message):
         # we will transition into the error state
         self.check_state_deferred('stopped', 5)
 
-    def pause(self, message):
+    def pause(self, payload):
         # Suspend command
         if self.child is None:
             raise SimulationException(self, 'No child process is running')
@@ -206,7 +206,7 @@ def pause(self, message):
         self.change_state('paused')
         self.logger.info('Child process has been paused')
 
-    def resume(self, message):
+    def resume(self, payload):
         # Let process run
         if self.child is None:
             raise SimulationException(self, 'No child process is running')
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 0df211e..a28d0dc 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -136,10 +136,9 @@ def _delete_job(self):
         self.properties['job_name'] = None
         self.properties['pod_names'] = []
 
-    def start(self, message):
+    def start(self, payload):
         self._delete_job()
 
-        payload = message.payload
         job = payload.get('job', {})
         job = merge(self.jobdict, job)
         v1job = self._prepare_job(job, payload)
@@ -183,17 +182,17 @@ def _send_signal_to_pod(self, sig, podname):
 
         self.logger.debug('Sent signal %d to container: %s', sig, resp)
 
-    def pause(self, message):
+    def pause(self, payload):
         self._send_signal(signal.SIGSTOP)
         self.change_state('paused')
 
-    def resume(self, message):
+    def resume(self, payload):
         self._send_signal(signal.SIGCONT)
         self.change_state('running')
 
-    def reset(self, message):
+    def reset(self, payload):
         self._delete_job()
-        super().reset(message)
+        super().reset(payload)
 
         self.change_state('idle')
 

From 1f30e9369bdb2c684cde29a5cd926f9848e9fe33 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 00:38:08 +0200
Subject: [PATCH 10/89] fix mixed up state vs status properties in some
 components

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulators/dpsim.py | 14 +-------------
 .../controller/components/simulators/generic.py  | 11 +++++------
 villas/controller/components/simulators/rscad.py | 16 ++++++++--------
 3 files changed, 14 insertions(+), 27 deletions(-)

diff --git a/villas/controller/components/simulators/dpsim.py b/villas/controller/components/simulators/dpsim.py
index 6cfa4ad..c7b143f 100644
--- a/villas/controller/components/simulators/dpsim.py
+++ b/villas/controller/components/simulators/dpsim.py
@@ -1,7 +1,6 @@
 import dpsim
-import time
-import socket
 import os
+import time
 
 from villas.controller.components.simulator import Simulator
 
@@ -118,17 +117,6 @@ def headers(self):
 
         return headers
 
-    @property
-    def state(self):
-        state = super().state
-
-        state['uptime'] = time.time() - self.started
-        state['version'] = '0.1.0'
-        state['host'] = socket.getfqdn()
-        state['kernel'] = os.uname()
-
-        return state
-
     def load_cim(self, fp):
         if fp is not None:
             self.sim = dpsim.load_cim(fp.name)
diff --git a/villas/controller/components/simulators/generic.py b/villas/controller/components/simulators/generic.py
index 6c09268..82e2560 100644
--- a/villas/controller/components/simulators/generic.py
+++ b/villas/controller/components/simulators/generic.py
@@ -28,12 +28,11 @@ def __del__(self):
             self
 
     @property
-    def state(self):
-        state = super().state
-
-        state['return_code'] = self.return_code
-
-        return state
+    def status(self):
+        return {
+            **super().status,
+            'return_code': self.return_code
+        }
 
     def start(self, payload):
         super().start(payload)
diff --git a/villas/controller/components/simulators/rscad.py b/villas/controller/components/simulators/rscad.py
index 6c0bb74..6c4e3b3 100644
--- a/villas/controller/components/simulators/rscad.py
+++ b/villas/controller/components/simulators/rscad.py
@@ -1,5 +1,4 @@
 import socket
-import time
 
 from villas.controller.components.simulator import Simulator
 
@@ -14,25 +13,26 @@ def __init__(self, host, number):
         self.name = f'{host}({number})'
 
     @property
-    def state(self):
+    def status(self):
         try:
             user, case = self.ping()
 
             if len(user) > 0:
-                state = {
+                status = {
                     'status': 'running',
                     'user': user,
                     'case': case
                 }
             else:
-                state = {
+                status = {
                     'status': 'free'
                 }
         except socket.timeout:
-            state = {
+            status = {
                 'status': 'offline'
             }
 
-        state['time'] = int(round(time.time() * 1000))
-
-        return state
+        return {
+            **super().status,
+            **status
+        }

From 4e230531890765538a5dae5fb5724851af3fb589 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 00:40:58 +0200
Subject: [PATCH 11/89] fix mixed up state vs status properties in some
 components

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulators/dpsim.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/villas/controller/components/simulators/dpsim.py b/villas/controller/components/simulators/dpsim.py
index c7b143f..8cbe83b 100644
--- a/villas/controller/components/simulators/dpsim.py
+++ b/villas/controller/components/simulators/dpsim.py
@@ -1,6 +1,5 @@
 import dpsim
 import os
-import time
 
 from villas.controller.components.simulator import Simulator
 
@@ -101,9 +100,6 @@ class DPsimSimulator(Simulator):
     }
 
     def __init__(self, **args):
-        args['type'] = 'dpsim'
-
-        self.started = time.time()
         self.sim = None
 
         super().__init__(**args)

From 426cbf23346530432be194aefcbb0854e7d430a2 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 00:41:16 +0200
Subject: [PATCH 12/89] fix mixed up state vs status properties in some
 components

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/villas/controller/components/simulator.py b/villas/controller/components/simulator.py
index 9fe6e86..9f79b42 100644
--- a/villas/controller/components/simulator.py
+++ b/villas/controller/components/simulator.py
@@ -29,12 +29,12 @@ def start_schema(self):
         return {}
 
     @property
-    def state(self):
+    def status(self):
         return {
             'model': self.model,
             'results': self.results,
 
-            **super().state
+            **super().status
         }
 
     @staticmethod

From 22ba436b9fb9e8130f68fe1ef263c49d5c727762 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 00:43:17 +0200
Subject: [PATCH 13/89] added first version of an HTTP REST API

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 requirements.txt                        |  1 +
 setup.py                                |  3 +-
 villas/controller/api.py                | 54 +++++++++++++++++++++++++
 villas/controller/component.py          |  7 +++-
 villas/controller/controller.py         |  6 +++
 villas/controller/handlers/component.py | 51 +++++++++++++++++++++++
 villas/controller/handlers/health.py    |  9 +++++
 villas/controller/handlers/main.py      | 12 ++++++
 8 files changed, 141 insertions(+), 2 deletions(-)
 create mode 100644 villas/controller/api.py
 create mode 100644 villas/controller/handlers/component.py
 create mode 100644 villas/controller/handlers/health.py
 create mode 100644 villas/controller/handlers/main.py

diff --git a/requirements.txt b/requirements.txt
index 31f45fe..e8066cb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,3 +7,4 @@ kubernetes
 xdg
 dotmap
 PyYAML
+tornado
diff --git a/setup.py b/setup.py
index 73c88ac..ddddd5e 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,8 @@
         'villas-node>=0.10.2',
         'kubernetes',
         'xdg',
-        'PyYAML'
+        'PyYAML',
+        'tornado'
     ],
     data_files=[
         ('/etc/villas/controller', glob('etc/*.{json,yaml}')),
diff --git a/villas/controller/api.py b/villas/controller/api.py
new file mode 100644
index 0000000..af48294
--- /dev/null
+++ b/villas/controller/api.py
@@ -0,0 +1,54 @@
+import threading
+import logging
+import asyncio
+from tornado.ioloop import IOLoop
+import tornado.web
+
+LOGGER = logging.getLogger(__name__)
+REGEX_UUID = r'\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}-[0-9a-fA-F]' \
+             r'{4}-[0-9a-fA-F]{4}-\b[0-9a-fA-F]{12}\b'
+
+
+class RequestHandler(tornado.web.RequestHandler):
+
+    def initialize(self, controller):
+        self.controller = controller
+
+
+class Api(threading.Thread):
+
+    def __init__(self, controller):
+        super().__init__()
+
+        self.controller = controller
+
+    def run(self):
+        self.app = tornado.web.Application(self.handlers)
+
+        # Create new event loop for this thread
+        aio_loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(aio_loop)
+
+        port = self.controller.config.api.port
+        self.app.listen(port)
+
+        LOGGER.info('Starting API')
+
+        self.loop = IOLoop.current(instance=True)
+        self.loop.start()
+
+    @property
+    def handlers(self):
+        from villas.controller.handlers.component import ComponentRequestHandler  # noqa E501
+        from villas.controller.handlers.main import MainRequestHandler
+        from villas.controller.handlers.health import HealthRequestHandler
+
+        args = {
+            'controller': self.controller
+        }
+
+        return [
+            (r'/', MainRequestHandler, args),
+            (r'/health', HealthRequestHandler, args),
+            (r'/component/('+REGEX_UUID+r')', ComponentRequestHandler, args)
+        ]
diff --git a/villas/controller/component.py b/villas/controller/component.py
index cddc202..5264bef 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -112,7 +112,10 @@ def on_message(self, message):
         self.logger.debug('Received message: %s', message.payload)
 
         if 'action' in message.payload:
-            self.run_action(message.payload['action'], message.payload)
+            try:
+                self.run_action(message.payload['action'], message.payload)
+            except SimulationException:
+                pass
 
         message.ack()
 
@@ -151,6 +154,8 @@ def run_action(self, action, payload):
             self.logger.error('SimulationException: %s', str(se))
             self.change_state('error', msg=se.msg, **se.info)
 
+            raise se
+
     def change_state(self, state, **kwargs):
         if self._state == state:
             return
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index a1cc2da..f5b30a7 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -5,6 +5,7 @@
 import os
 import kombu.mixins
 
+from villas.controller.api import Api
 from villas.controller import __version__ as version
 from villas.controller.components.managers.generic import GenericManager
 
@@ -110,6 +111,11 @@ def on_iteration(self):
 
     def start(self):
         self.started = time.time()
+
+        if self.config.api.enabled:
+            self.api = Api(self)
+            self.api.start()
+
         self.should_terminate = False
         while not self.should_terminate:
             self.should_stop = False
diff --git a/villas/controller/handlers/component.py b/villas/controller/handlers/component.py
new file mode 100644
index 0000000..cb91e60
--- /dev/null
+++ b/villas/controller/handlers/component.py
@@ -0,0 +1,51 @@
+import json
+from tornado.web import HTTPError
+from http import HTTPStatus
+from functools import wraps
+
+
+from villas.controller.api import RequestHandler
+from villas.controller.exceptions import SimulationException
+
+
+def with_component(f):
+
+    @wraps(f)
+    def wrapper(self, uuid):
+        try:
+            component = self.controller.components[uuid]
+
+            f(self, component)
+        except KeyError:
+            raise HTTPError(HTTPStatus.NOT_FOUND)
+
+    return wrapper
+
+
+class ComponentRequestHandler(RequestHandler):
+
+    @with_component
+    def get(self, component):
+        self.write(component.status)
+
+    @with_component
+    def post(self, component):
+        payload = json.loads(self.request.body)
+
+        action = payload.get('action')
+        if action is None:
+            raise HTTPError(HTTPStatus.BAD_REQUEST)
+
+        try:
+            component.run_action(action, payload)
+            self.write(component.status)
+
+        except SimulationException as se:
+            self.write({
+                'exception': {
+                    'msg': se.msg,
+                    'args': se.args
+                },
+                **component.status
+            })
+            self.send_error(HTTPStatus.BAD_REQUEST)
diff --git a/villas/controller/handlers/health.py b/villas/controller/handlers/health.py
new file mode 100644
index 0000000..d8be63e
--- /dev/null
+++ b/villas/controller/handlers/health.py
@@ -0,0 +1,9 @@
+from villas.controller.api import RequestHandler
+
+
+class HealthRequestHandler(RequestHandler):
+
+    def get(self):
+        self.write({
+            'status': 'ok'
+        })
diff --git a/villas/controller/handlers/main.py b/villas/controller/handlers/main.py
new file mode 100644
index 0000000..cbc07d7
--- /dev/null
+++ b/villas/controller/handlers/main.py
@@ -0,0 +1,12 @@
+from villas.controller.api import RequestHandler
+
+
+class MainRequestHandler(RequestHandler):
+
+    def get(self):
+        self.write({
+            'components': list(self.controller.components.keys()),
+            'status': {
+                **self.controller.status
+            }
+        })

From 4360d292fa9ac8b561c9c2315bb332ae182e948b Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 00:43:59 +0200
Subject: [PATCH 14/89] creating AMQP queue with durable, exclusive and
 auto_delete flags

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 5264bef..3c374ab 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -69,7 +69,9 @@ def get_consumer(self, channel):
                     'x-match': 'any',
                     **self.headers
                 },
-                durable=False
+                durable=False,
+                exclusive=True,
+                auto_delete=True
             ),
             no_ack=True,
             accept={'application/json'}

From 124f55f8ba833c14a5bfe2865467f2415059b3a8 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Sat, 9 Oct 2021 01:49:51 +0200
Subject: [PATCH 15/89] add first draft of OpenAPI spec

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 doc/openapi.yaml | 324 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 324 insertions(+)
 create mode 100644 doc/openapi.yaml

diff --git a/doc/openapi.yaml b/doc/openapi.yaml
new file mode 100644
index 0000000..2c7c89b
--- /dev/null
+++ b/doc/openapi.yaml
@@ -0,0 +1,324 @@
+---
+openapi: 3.0.0
+info:
+  title: VILLAScontroller API
+  description: 'A HTTP/REST API for controlling VILLAScontroller remotely for querying component status as well as issuing control actions.'
+  version: 0.0.1
+  license:
+    name: Apache-2.0
+    url: https://www.apache.org/licenses/LICENSE-2.0
+servers:
+- url: https://villas.k8s.eonerc.rwth-aachen.de/api/v1
+  description: Demo instance at RWTH Aachen
+paths:
+  /:
+    get:
+      summary: 'Get status of VILLAScontroller daemon'
+      operationId: getStatus
+      responses:
+        '200':
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  components:
+                    type: array
+                    items:
+                      type: string
+                      format: uuid
+                  status:
+                    type: object
+                    properties:
+                      version:
+                        type: string
+                      uptime:
+                        type: number
+                      host:
+                        type: string
+                      kernel:
+                        type: object
+                        properties:
+                          sysname:
+                            type: string
+                          nodename:
+                            type: string
+                          release:
+                            type: string
+                          version:
+                            type: string
+                          machine:
+                            type: string
+
+              example:
+                components:
+                  - f4751894-205e-11eb-aefb-0741ff98abca
+                  - 3ddd318e-fee1-46d7-bff4-7c064d640d4e
+                status:
+                  version: 0.3.2
+                  uptime: 15.38102650642395
+                  host: lat.0l.de
+                  kernel:
+                    sysname: Linux
+                    nodename: lat.0l.de
+                    release: 5.13.14-200.fc34.x86_64
+                    version: '#1 SMP Fri Sep 3 15:33:01 UTC 2021'
+                    machine: x86_64
+
+
+  /component/{uuid}:
+    get:
+      parameters:
+      - name: uuid
+        in: path
+        required: true
+        schema:
+          type: string
+          format: uuid
+
+      operationId: getComponentStatus
+      summary: 'Get the current status of a component'
+      responses:
+        200:
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  components:
+                    type: array
+                    items:
+                      type: string
+                      format: uuid
+
+                  status:
+                    type: object
+                    properties:
+                      managed_by:
+                        type: string
+                        format: uuid
+
+                      state:
+                        type: string
+                        enum:
+                        - idle
+                        - starting
+                        - running
+                        - stopping
+                        - pausing
+                        - paused
+                        - resuming
+                        - error
+                        - resetting
+                        - shuttingdown
+                        - shutdown
+                        - gone
+
+                      version:
+                        type: string
+                      uptime:
+                        type: number
+                      host:
+                        type: string
+                      kernel:
+                        type: object
+                        properties:
+                          sysname:
+                            type: string
+                          nodename:
+                            type: string
+                          release:
+                            type: string
+                          version:
+                            type: string
+                          machine:
+                            type: string
+
+                  properties:
+                    type: object
+                    properties:
+                      category:
+                        type: string
+                        enum:
+                        - manager
+                        - simulator
+                        - gateway
+                      type:
+                        type: string
+                        pattern: '[a-z-]+'
+                      name:
+                        type: string
+                      realm:
+                        type: string
+                        pattern: '[a-z0-9-.]+'
+                      uuid:
+                        type: string
+                        format: uuid
+
+                    additionalProperties: true
+
+                  schema:
+                    type: object
+                    additionalProperties:
+                      $ref: 'https://json-schema.org/draft/2020-12/schema'
+
+              example:
+                components: []
+                status:
+                  state: idle
+                  version: 0.3.2
+                  uptime: 480.25064611434937
+                  host: lat.0l.de
+                  kernel:
+                    sysname: Linux
+                    nodename: lat.0l.de
+                    release: 5.13.14-200.fc34.x86_64
+                    version: '#1 SMP Fri Sep 3 15:33:01 UTC 2021'
+                    machine: x86_64
+                  managed_by: f4751894-205e-11eb-aefb-0741ff98abca
+                properties:
+                  category: manager
+                  type: generic
+                  name: Standard Controller
+                  realm: de.rwth-aachen.eonerc.acs
+                  uuid: f4751894-205e-11eb-aefb-0741ff98abca
+                schema:
+                  create:
+                    $schema: 'http://json-schema.org/draft-07/schema'
+                    $id: 'http://example.com/example.json'
+                    type: object
+                    default: {}
+                    required:
+                      - name
+                      - category
+                      - location
+                      - owner
+                      - realm
+                      - type
+                      - api_url
+                      - ws_url
+                    properties:
+                      name:
+                        $id: '#/properties/name'
+                        type: string
+                        title: Component name
+                        default: New Component
+                        examples:
+                          - 'Generic Simulator #1'
+                      owner:
+                        $id: '#/properties/owner'
+                        type: string
+                        title: Component owner
+                        default: ''
+                        examples:
+                          - rmr
+                          - svg
+                      realm:
+                        $id: '#/properties/realm'
+                        type: string
+                        title: Component realm
+                        default: ''
+                        examples:
+                          - de.rwth-aachen.eonerc.acs
+                      category:
+                        $id: '#/properties/category'
+                        type: string
+                        title: Component category
+                        default: ''
+                        examples:
+                          - simulator
+                      location:
+                        $id: '#/properties/location'
+                        type: string
+                        title: Component location
+                        default: ''
+                        examples:
+                          - Richard's PC
+                      type:
+                        $id: '#/properties/type'
+                        type: string
+                        title: The type schema
+                        default: ''
+                        examples:
+                          - generic
+                      uuid:
+                        $id: '#/properties/uuid'
+                        type: 'null'
+                        title: The uuid schema
+                        default: null
+                      ws_url:
+                        $id: '#/properties/ws_url'
+                        type: string
+                        title: The ws_url schema
+                        default: ''
+                        examples:
+                          - 'https://villas.k8s.eonerc.rwth-aachen.de/ws/relay/generic_1'
+                      api_url:
+                        $id: '#/properties/api_url'
+                        type: string
+                        title: The api_url schema
+                        default: ''
+                        examples:
+                          - 'https://villas.k8s.eonerc.rwth-aachen.de/api/ic/generic_1'
+                      shell:
+                        $id: '#/properties/shell'
+                        type: boolean
+                        title: The shell schema
+                        default: false
+                        examples:
+                          - true
+                      whitelist:
+                        $id: '#/properties/whitelist'
+                        type: array
+                        title: The whitelist schema
+                        default: []
+                        examples:
+                          - - /sbin/ping
+                            - ^echo
+                        additionalItems: true
+                        items:
+                          $id: '#/properties/whitelist/items'
+                          anyOf:
+                            - $id: '#/properties/whitelist/items/anyOf/0'
+                              type: string
+                              title: The first anyOf schema
+                              default: ''
+                              examples:
+                                - /sbin/ping
+                                - ^echo
+                    additionalProperties: true
+
+
+    post:
+      operationId: executeComponentAction
+      summary: 'Send a control action to the component'
+      responses:
+        200:
+          description: ''
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties: true
+
+              example:
+                runtime: 10.2
+
+  /health:
+    get:
+      operationId: getHealth
+      summary: Query health of daemon.
+      responses:
+        '200':
+          description: The daemon is healthy
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  status:
+                    type: string
+              example:
+                status: ok

From 5800d0be654dd2dec8a427e6270595404629db69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 14 Oct 2021 08:20:06 +0000
Subject: [PATCH 16/89] fix simple manager

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                         |  5 +++--
 villas/controller/components/manager.py                |  1 +
 .../components/managers/kubernetes_simple.py           | 10 +++++-----
 villas/controller/controller.py                        |  8 ++++----
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 3c374ab..87e2768 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -34,7 +34,7 @@ def __init__(self, **props):
         self.logger = logging.getLogger(
             f'villas.controller.{self.category}.{self.type}:{self.uuid}')
 
-        self.publish_status_interval = 2
+        self.publish_status_interval = 5
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
@@ -111,7 +111,8 @@ def status(self):
         }
 
     def on_message(self, message):
-        self.logger.debug('Received message: %s', message.payload)
+        self.logger.info('my uuid: %s', self.uuid)
+        self.logger.info('Received message: %s', message.payload)
 
         if 'action' in message.payload:
             try:
diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index 620a2dc..949db2d 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -56,6 +56,7 @@ def add_component(self, comp):
         if comp.uuid in self.mixin.components:
             # raise KeyError
             self.logger.error('UUID %s already exists, not added', comp.uuid)
+            return
 
         comp.set_manager(self)
 
diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 48a3cb9..555a669 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -71,9 +71,9 @@ class KubernetesManagerSimple(KubernetesManager):
         }
     }
 
-    def create(self, message):
-        print(message.payload)
-        params = message.payload.get('parameters', {})
+    def create(self, payload):
+        self.logger.info(payload)
+        params = payload.get('parameters', {})
         jobname = params.get('jobname', 'noname')
         adls = params.get('activeDeadlineSeconds', 3600)
         contName = params.get('containername', 'noname')
@@ -83,7 +83,7 @@ def create(self, message):
 
         if image is None:
             self.logger.error('No image given, will try super.create')
-            super().create(message)
+            super().create(payload)
             return
 
         parameters = self.parameters_simple
@@ -101,7 +101,7 @@ def create(self, message):
         if uuid:
             parameters['uuid'] = uuid
 
-        print(parameters)
+        self.logger.info(parameters)
 
         comp = KubernetesJob(self, **parameters)
         self.add_component(comp)
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index f5b30a7..c2b3215 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -5,7 +5,7 @@
 import os
 import kombu.mixins
 
-from villas.controller.api import Api
+# from villas.controller.api import Api
 from villas.controller import __version__ as version
 from villas.controller.components.managers.generic import GenericManager
 
@@ -112,9 +112,9 @@ def on_iteration(self):
     def start(self):
         self.started = time.time()
 
-        if self.config.api.enabled:
-            self.api = Api(self)
-            self.api.start()
+        # if self.config.api.enabled:
+        #     self.api = Api(self)
+        #     self.api.start()
 
         self.should_terminate = False
         while not self.should_terminate:

From 1f695f5e516070ed441bb2d470b7e4ffe5c49a3a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 17 Nov 2021 14:12:20 +0100
Subject: [PATCH 17/89] use 'villas' namespace as default

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 78d81bf..f7bee68 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -51,7 +51,7 @@ def __init__(self, **args):
         else:
             k8s.config.load_incluster_config()
 
-        self.namespace = args.get('namespace', 'default')
+        self.namespace = args.get('namespace', 'villas')
 
         self._check_namespace(self.namespace)
 

From 17080ebcc3789766a281cb49385c991837028ffb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 17 Nov 2021 14:22:28 +0100
Subject: [PATCH 18/89] use villas-controller namespace as default

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index f7bee68..027d39c 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -51,7 +51,7 @@ def __init__(self, **args):
         else:
             k8s.config.load_incluster_config()
 
-        self.namespace = args.get('namespace', 'villas')
+        self.namespace = args.get('namespace', 'villas-controller')
 
         self._check_namespace(self.namespace)
 

From d7056996f56c466dab98876b0d19b877474348ac Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 12 Oct 2021 22:54:15 +0200
Subject: [PATCH 19/89] move schemas to YAML files and load them from there for
 all components

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 MANIFEST.in                                   |   1 +
 requirements.txt                              |   1 +
 setup.py                                      |   6 +-
 villas/controller/component.py                |  37 ++++-
 villas/controller/components/manager.py       |  11 --
 .../controller/components/managers/generic.py | 139 ------------------
 .../components/managers/kubernetes.py         |  15 --
 .../controller/components/simulators/dpsim.py |  93 ------------
 .../controller/components/simulators/dummy.py |  20 ---
 villas/controller/schemas/__init__.py         |   0
 villas/controller/schemas/manager/__init__.py |   0
 .../schemas/manager/generic/__init__.py       |   0
 .../schemas/manager/generic/create.yaml       |  80 ++++++++++
 .../schemas/manager/kubernetes/create.yaml    |  10 ++
 .../schemas/simulator/dpsim/__init__.py       |   0
 .../schemas/simulator/dpsim/start.yaml        |  81 ++++++++++
 .../schemas/simulator/dummy/__init__.py       |   0
 .../schemas/simulator/dummy/start.yaml        |  13 ++
 18 files changed, 222 insertions(+), 285 deletions(-)
 create mode 100644 MANIFEST.in
 create mode 100644 villas/controller/schemas/__init__.py
 create mode 100644 villas/controller/schemas/manager/__init__.py
 create mode 100644 villas/controller/schemas/manager/generic/__init__.py
 create mode 100644 villas/controller/schemas/manager/generic/create.yaml
 create mode 100644 villas/controller/schemas/manager/kubernetes/create.yaml
 create mode 100644 villas/controller/schemas/simulator/dpsim/__init__.py
 create mode 100644 villas/controller/schemas/simulator/dpsim/start.yaml
 create mode 100644 villas/controller/schemas/simulator/dummy/__init__.py
 create mode 100644 villas/controller/schemas/simulator/dummy/start.yaml

diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..fbc8a14
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1 @@
+recursive-include villas/controller *
diff --git a/requirements.txt b/requirements.txt
index e8066cb..3d3ebfe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ xdg
 dotmap
 PyYAML
 tornado
+jsonschema>=4.1.0
diff --git a/setup.py b/setup.py
index ddddd5e..6519bbe 100644
--- a/setup.py
+++ b/setup.py
@@ -32,7 +32,8 @@
         'kubernetes',
         'xdg',
         'PyYAML',
-        'tornado'
+        'tornado',
+        'jsonschema>=4.1.0'
     ],
     data_files=[
         ('/etc/villas/controller', glob('etc/*.{json,yaml}')),
@@ -43,5 +44,6 @@
             'villas-ctl=villas.controller.main:main',
             'villas-controller=villas.controller.main:main'
         ],
-    }
+    },
+    include_package_data=True
 )
diff --git a/villas/controller/component.py b/villas/controller/component.py
index 87e2768..3d60d4f 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -4,6 +4,11 @@
 import kombu
 import uuid
 import threading
+import yaml
+from jsonschema import Draft202012Validator
+
+import importlib
+import importlib.resources as resources
 
 from villas.controller.exceptions import SimulationException
 
@@ -39,6 +44,9 @@ def __init__(self, **props):
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
 
+        # Load schemas for validating action payloads
+        self._load_schema()
+
     def on_ready(self):
         self.publish_status_thread.start()
         pass
@@ -77,6 +85,27 @@ def get_consumer(self, channel):
             accept={'application/json'}
         )
 
+    def _load_schema(self):
+        self.schema = {}
+
+        try:
+            pkg_name = f'villas.controller.schemas.{self.category}.{self.type}'
+            pkg = importlib.import_module(pkg_name)
+        except ModuleNotFoundError:
+            self.logger.warn('Missing schemas!')
+            return
+
+        self.logger.debug('Loading schemas from %s', pkg_name)
+
+        for res in resources.contents(pkg):
+            name, ext = os.path.splitext(res)
+            if resources.is_resource(pkg, res) and ext in ['.yaml', '.json']:
+
+                fo = resources.open_text(pkg, res)
+                schema = yaml.load(fo, yaml.SafeLoader)
+
+                self.schema[name] = Draft202012Validator(schema)
+
     @property
     def headers(self):
         return {
@@ -86,10 +115,6 @@ def headers(self):
             'type': self.type
         }
 
-    @property
-    def schema(self):
-        return self.properties.get('schema', {})
-
     @property
     def status(self):
         status = {
@@ -107,7 +132,9 @@ def status(self):
                 **self.properties,
                 **self.headers
             },
-            'schema': self.schema
+            'schema': {
+                name: v.schema for name, v in self.schema.items()
+            }
         }
 
     def on_message(self, message):
diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index 949db2d..24392c4 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -17,17 +17,6 @@ def status(self):
             **super().status
         }
 
-    @property
-    def schema(self):
-        return {
-            'create': self.create_schema,
-            **super().schema
-        }
-
-    @property
-    def create_schema(self):
-        return {}
-
     @staticmethod
     def from_dict(dict):
         type = dict.get('type', 'generic')
diff --git a/villas/controller/components/managers/generic.py b/villas/controller/components/managers/generic.py
index fe5a040..084592c 100644
--- a/villas/controller/components/managers/generic.py
+++ b/villas/controller/components/managers/generic.py
@@ -4,145 +4,6 @@
 
 class GenericManager(Manager):
 
-    create_schema = {
-        '$schema': 'http://json-schema.org/draft-07/schema',
-        '$id': 'http://example.com/example.json',
-        'type': 'object',
-        'default': {},
-        'required': [
-            'name',
-            'category',
-            'location',
-            'owner',
-            'realm',
-            'type',
-            'api_url',
-            'ws_url'
-        ],
-        'properties': {
-            'name': {
-                '$id': '#/properties/name',
-                'type': 'string',
-                'title': 'Component name',
-                'default': 'New Component',
-                'examples': [
-                    'Generic Simulator #1'
-                ]
-            },
-            'owner': {
-                '$id': '#/properties/owner',
-                'type': 'string',
-                'title': 'Component owner',
-                'default': '',
-                'examples': [
-                    'rmr',
-                    'svg'
-                ]
-            },
-            'realm': {
-                '$id': '#/properties/realm',
-                'type': 'string',
-                'title': 'Component realm',
-                'default': '',
-                'examples': [
-                    'de.rwth-aachen.eonerc.acs'
-                ]
-            },
-            'category': {
-                '$id': '#/properties/category',
-                'type': 'string',
-                'title': 'Component category',
-                'default': '',
-                'examples': [
-                    'simulator'
-                ]
-            },
-            'location': {
-                '$id': '#/properties/location',
-                'type': 'string',
-                'title': 'Component location',
-                'default': '',
-                'examples': [
-                    'Richard\'s PC'
-                ]
-            },
-            'type': {
-                '$id': '#/properties/type',
-                'type': 'string',
-                'title': 'The type schema',
-                'default': '',
-                'examples': [
-                    'generic'
-                ]
-            },
-            'uuid': {
-                '$id': '#/properties/uuid',
-                'type': 'null',
-                'title': 'The uuid schema',
-                'default': None,
-            },
-
-            'ws_url': {
-                '$id': '#/properties/ws_url',
-                'type': 'string',
-                'title': 'The ws_url schema',
-                'default': '',
-                'examples': [
-                    'https://villas.k8s.eonerc.rwth-aachen.de/'
-                    'ws/relay/generic_1'
-                ]
-            },
-            'api_url': {
-                '$id': '#/properties/api_url',
-                'type': 'string',
-                'title': 'The api_url schema',
-                'default': '',
-                'examples': [
-                    'https://villas.k8s.eonerc.rwth-aachen.de/api/ic/generic_1'
-                ]
-            },
-
-            'shell': {
-                '$id': '#/properties/shell',
-                'type': 'boolean',
-                'title': 'The shell schema',
-                'default': False,
-                'examples': [
-                    True
-                ]
-            },
-            'whitelist': {
-                '$id': '#/properties/whitelist',
-                'type': 'array',
-                'title': 'The whitelist schema',
-                'default': [],
-                'examples': [
-                    [
-                        '/sbin/ping',
-                        '^echo'
-                    ]
-                ],
-                'additionalItems': True,
-                'items': {
-                    '$id': '#/properties/whitelist/items',
-                    'anyOf': [
-                        {
-                            '$id': '#/properties/whitelist/items/anyOf/0',
-                            'type': 'string',
-                            'title': 'The first anyOf schema',
-                            'default': '',
-                            'examples': [
-                                '/sbin/ping',
-                                '^echo'
-                            ]
-                        }
-                    ]
-                }
-            }
-        },
-        'additionalProperties': True
-    }
-
     def create(self, payload):
         component = Component.from_dict(payload.get('parameters'))
 
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 027d39c..0345756 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -19,21 +19,6 @@ def _match(stringA, stringB):
 
 class KubernetesManager(Manager):
 
-    create_schema = {
-        '$schema': 'http://json-schema.org/draft-04/schema#',
-        'properties': {
-            'job': {
-                '$ref': 'https://kubernetesjsonschema.dev/v1.18.1/job.json'
-            },
-            'schema': {
-                'type': 'object',
-                'additionalProperties': {
-                    '$ref': 'https://json-schema.org/draft-04/schema'
-                }
-            }
-        }
-    }
-
     def __init__(self, **args):
         super().__init__(**args)
 
diff --git a/villas/controller/components/simulators/dpsim.py b/villas/controller/components/simulators/dpsim.py
index 8cbe83b..91d5f55 100644
--- a/villas/controller/components/simulators/dpsim.py
+++ b/villas/controller/components/simulators/dpsim.py
@@ -6,99 +6,6 @@
 
 class DPsimSimulator(Simulator):
 
-    start_schema = {
-        '$schema': 'http://json-schema.org/draft-04/schema#',
-        'properties': {
-            'blocking': {
-                'title': 'Block execution of each time-step until the '
-                         'arrival of new data on the interfaces',
-                'type': 'boolean'
-            },
-            'duration': {
-                'examples': [
-                    3600.0
-                ],
-                'title': 'Simulation duration [s]',
-                'type': 'number'
-            },
-            'log-level': {
-                'enum': ['NONE', 'INFO', 'DEBUG', 'WARN', 'ERR'],
-                'title': 'Logging level',
-                'type': 'string'
-            },
-            'name': {
-                'examples': [
-                    'Simulation_1'
-                ],
-                'title': 'Name of log files',
-                'type': 'string'
-            },
-            'options': {
-                'additionalProperties': {
-                    'type': 'number'
-                },
-                'examples': [
-                    {
-                        'Ld': 0.2299,
-                        'Lq': 0.0
-                    }
-                ],
-                'title': 'User-definable options',
-                'type': 'object'
-            },
-            'scenario': {
-                'title': 'Scenario selection', 'type': 'integer'
-            },
-            'solver-domain': {
-                'enum': ['SP', 'DP', 'EMT'],
-                'title': 'Domain of solver',
-                'type': 'string'
-            },
-            'solver-type': {
-                'enum': ['NRP', 'MNA'],
-                'title': 'Type of solver',
-                'type': 'string'
-            },
-            'start-at': {
-                'description': 'The date must be given as an ISO8601 '
-                               'formatted string',
-                'examples': ['2004-06-14T23:34:30'],
-                'format': 'date-time',
-                'title': 'Start time of real-time simulation',
-                'type': 'string'
-            },
-            'start-in': {
-                'title': 'Start simulation relative to current time [s]',
-                'type': 'number'
-            },
-            'start-synch': {
-                'title': 'Sychronize start of simulation '
-                         'with external interfaces',
-                'type': 'boolean'
-            },
-            'steady-init': {
-                'title': 'Perform a steady-state initialization prior '
-                         'to the simulation',
-                'type': 'boolean'
-            },
-            'system-freq': {
-                'examples': [
-                    50.0,
-                    60.0
-                ],
-                'title': 'System frequency [Hz]',
-                'type': 'number'
-            },
-            'timestep': {
-                'examples': [5e-05],
-                'title': 'Simulation time-step [s]',
-                'type': 'number'
-            }
-        },
-        'required': ['name'],
-        'type': 'object'
-    }
-
     def __init__(self, **args):
         self.sim = None
 
diff --git a/villas/controller/components/simulators/dummy.py b/villas/controller/components/simulators/dummy.py
index 91a202b..028ed72 100644
--- a/villas/controller/components/simulators/dummy.py
+++ b/villas/controller/components/simulators/dummy.py
@@ -5,26 +5,6 @@
 
 class DummySimulator(Simulator):
 
-    start_schema = {
-        '$schema': 'http://json-schema.org/draft-07/schema',
-        'type': 'object',
-        'default': {},
-        'required': [
-            'runtime'
-        ],
-        'properties': {
-            'runtime': {
-                '$id': '#/properties/runtime',
-                'description': 'The run time of the simulation',
-                'type': 'number',
-                'default': 1.0,
-                'examples': [
-                    3.0
-                ]
-            }
-        }
-    }
-
     def __init__(self, **args):
         super().__init__(**args)
 
diff --git a/villas/controller/schemas/__init__.py b/villas/controller/schemas/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/villas/controller/schemas/manager/__init__.py b/villas/controller/schemas/manager/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/villas/controller/schemas/manager/generic/__init__.py b/villas/controller/schemas/manager/generic/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/villas/controller/schemas/manager/generic/create.yaml b/villas/controller/schemas/manager/generic/create.yaml
new file mode 100644
index 0000000..c936127
--- /dev/null
+++ b/villas/controller/schemas/manager/generic/create.yaml
@@ -0,0 +1,80 @@
+---
+$schema: http://json-schema.org/draft-07/schema
+additionalProperties: true
+
+type: object
+required:
+  - name
+  - category
+  - location
+  - owner
+  - realm
+  - type
+  - api_url
+  - ws_url
+
+properties:
+  api_url:
+    examples:
+    - https://villas.k8s.eonerc.rwth-aachen.de/api/ic/generic_1
+    title: The api_url schema
+    type: string
+  category:
+    examples:
+    - simulator
+    title: Component category
+    type: string
+  location:
+    examples:
+    - Richard's PC
+    title: Component location
+    type: string
+  name:
+    default: New Component
+    examples:
+    - 'Generic Simulator #1'
+    title: Component name
+    type: string
+  owner:
+    examples:
+    - rmr
+    - svg
+    title: Component owner
+    type: string
+  realm:
+    examples:
+    - de.rwth-aachen.eonerc.acs
+    title: Component realm
+    type: string
+  shell:
+    examples:
+    - true
+    title: The shell schema
+    type: boolean
+  type:
+    examples:
+    - generic
+    title: The type schema
+    type: string
+  uuid:
+    title: The uuid schema
+    type: 'null'
+  whitelist:
+    additionalItems: true
+    examples:
+    - - /sbin/ping
+      - ^echo
+    items:
+      anyOf:
+      - examples:
+        - /sbin/ping
+        - ^echo
+        title: The first anyOf schema
+        type: string
+    title: The whitelist schema
+    type: array
+  ws_url:
+    examples:
+    - https://villas.k8s.eonerc.rwth-aachen.de/ws/relay/generic_1
+    title: The ws_url schema
+    type: string
diff --git a/villas/controller/schemas/manager/kubernetes/create.yaml b/villas/controller/schemas/manager/kubernetes/create.yaml
new file mode 100644
index 0000000..60c9bac
--- /dev/null
+++ b/villas/controller/schemas/manager/kubernetes/create.yaml
@@ -0,0 +1,10 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+
+type: object
+properties:
+  job:
+    $ref: https://kubernetesjsonschema.dev/v1.18.1/job.json
+  schema:
+    additionalProperties:
+      $ref: https://json-schema.org/draft-04/schema
diff --git a/villas/controller/schemas/simulator/dpsim/__init__.py b/villas/controller/schemas/simulator/dpsim/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/villas/controller/schemas/simulator/dpsim/start.yaml b/villas/controller/schemas/simulator/dpsim/start.yaml
new file mode 100644
index 0000000..20ceb41
--- /dev/null
+++ b/villas/controller/schemas/simulator/dpsim/start.yaml
@@ -0,0 +1,81 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+
+type: object
+required:
+- name
+properties:
+  blocking:
+    title: Block execution of each time-step until the arrival of new data on the
+      interfaces
+    type: boolean
+  duration:
+    examples:
+    - 3600.0
+    title: Simulation duration [s]
+    type: number
+  log-level:
+    enum:
+    - NONE
+    - INFO
+    - DEBUG
+    - WARN
+    - ERR
+    title: Logging level
+    type: string
+  name:
+    examples:
+    - Simulation_1
+    title: Name of log files
+    type: string
+  options:
+    additionalProperties:
+      type: number
+    examples:
+    - Ld: 0.2299
+      Lq: 0.0
+    title: User-definable options
+    type: object
+  scenario:
+    title: Scenario selection
+    type: integer
+  solver-domain:
+    enum:
+    - SP
+    - DP
+    - EMT
+    title: Domain of solver
+    type: string
+  solver-type:
+    enum:
+    - NRP
+    - MNA
+    title: Type of solver
+    type: string
+  start-at:
+    description: The date must be given as an ISO8601 formatted string
+    examples:
+    - '2004-06-14T23:34:30'
+    format: date-time
+    title: Start time of real-time simulation
+    type: string
+  start-in:
+    title: Start simulation relative to current time [s]
+    type: number
+  start-synch:
+    title: Sychronize start of simulation with external interfaces
+    type: boolean
+  steady-init:
+    title: Perform a steady-state initialization prior to the simulation
+    type: boolean
+  system-freq:
+    examples:
+    - 50.0
+    - 60.0
+    title: System frequency [Hz]
+    type: number
+  timestep:
+    examples:
+    - 5.0e-05
+    title: Simulation time-step [s]
+    type: number
diff --git a/villas/controller/schemas/simulator/dummy/__init__.py b/villas/controller/schemas/simulator/dummy/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/villas/controller/schemas/simulator/dummy/start.yaml b/villas/controller/schemas/simulator/dummy/start.yaml
new file mode 100644
index 0000000..d1d6f83
--- /dev/null
+++ b/villas/controller/schemas/simulator/dummy/start.yaml
@@ -0,0 +1,13 @@
+---
+$schema: http://json-schema.org/draft-07/schema
+
+type: object
+required:
+- runtime
+properties:
+  runtime:
+    type: number
+    default: 1.0
+    description: The runtime of the simulation in seconds
+    examples:
+    - 3.0

From 208f8e8b542d8f7bb515cdf1bc43f086ec954ab2 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 12 Oct 2021 22:55:01 +0200
Subject: [PATCH 20/89] config: show a useful error message when started
 without broker parameter and config file

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/main.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/villas/controller/main.py b/villas/controller/main.py
index daad06f..4f362df 100644
--- a/villas/controller/main.py
+++ b/villas/controller/main.py
@@ -103,8 +103,12 @@ def main():
 
     setup_logging(args)
 
-    broker_url = args.broker or args.config.broker.url
-
+    try:
+        broker_url = args.broker or args.config.broker.url
+    except AttributeError:
+        LOGGER.error('A broker URL must be provided either via a command line '
+                     'parameter or a configuration file')
+        return -1
     try:
         with kombu.Connection(broker_url, connect_timeout=3) as c:
             LOGGER.info(f'Connecting to: {broker_url}')

From 71a4872851b1521a26878b3eee4dd40c6214e5ad Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 12 Oct 2021 22:55:09 +0200
Subject: [PATCH 21/89] fix typo

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/controller.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index c2b3215..088d7db 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -120,7 +120,7 @@ def start(self):
         while not self.should_terminate:
             self.should_stop = False
 
-            LOGGER.info('Startig mixing for %d components',
+            LOGGER.info('Starting mixing for %d components',
                         len(self.active_components))
 
             super().run()

From 075fbfc9dd17271e737d5de9449eee936e91c170 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 12 Oct 2021 22:55:35 +0200
Subject: [PATCH 22/89] config: fix bug when started without config file

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/config.py b/villas/controller/config.py
index b145da9..e261e78 100644
--- a/villas/controller/config.py
+++ b/villas/controller/config.py
@@ -56,7 +56,7 @@ def __init__(self, fp=None):
                 with open(fn) as fp:
                     self.load(fp)
             else:
-                pass  # Start without config
+                self.config = {}  # Start without config
         else:
             self.load(fp)
 

From aa6432cf3bb1689b3217f520c3afe407799d94fe Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 12 Oct 2021 22:58:00 +0200
Subject: [PATCH 23/89] validate action parameters against schema

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/commands/simulator.py       |  4 +-
 villas/controller/component.py                | 42 +++++++++++++++++--
 .../components/managers/villas_node.py        |  6 ++-
 .../components/managers/villas_relay.py       |  4 +-
 villas/controller/components/simulator.py     | 11 -----
 .../controller/components/simulators/dpsim.py |  2 +-
 .../components/simulators/generic.py          |  8 +---
 7 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/villas/controller/commands/simulator.py b/villas/controller/commands/simulator.py
index 0dcdeee..ff16e96 100644
--- a/villas/controller/commands/simulator.py
+++ b/villas/controller/commands/simulator.py
@@ -15,8 +15,8 @@ def _get_parameters(params, params_file):
 
     try:
         if params is not None:
-            parameters.update(yaml.loads(params,
-                                         Loader=yaml.FullLoader))
+            parameters.update(yaml.load(params,
+                                        Loader=yaml.FullLoader))
         if params_file is not None:
             with open(params_file) as f:
                 parameters.update(yaml.load(f, Loader=yaml.FullLoader))
diff --git a/villas/controller/component.py b/villas/controller/component.py
index 3d60d4f..4f9f10d 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -5,6 +5,7 @@
 import uuid
 import threading
 import yaml
+import jsonschema
 from jsonschema import Draft202012Validator
 
 import importlib
@@ -95,8 +96,6 @@ def _load_schema(self):
             self.logger.warn('Missing schemas!')
             return
 
-        self.logger.debug('Loading schemas from %s', pkg_name)
-
         for res in resources.contents(pkg):
             name, ext = os.path.splitext(res)
             if resources.is_resource(pkg, res) and ext in ['.yaml', '.json']:
@@ -106,6 +105,16 @@ def _load_schema(self):
 
                 self.schema[name] = Draft202012Validator(schema)
 
+    def validate_parameters(self, action, parameters):
+        if action in self.schema:
+            validator = self.schema[action]
+
+            validator.validate(parameters)
+
+        else:
+            self.logger.warn('missing schema for action: %s', action)
+            return True  # we really should fail here...
+
     @property
     def headers(self):
         return {
@@ -155,6 +164,25 @@ def run_action(self, action, payload):
         else:
             self.logger.info('Received action: %s', action)
 
+        parameters = payload.get('parameters', {})
+
+        try:
+            self.validate_parameters(action, parameters)
+        except jsonschema.ValidationError as ve:
+            e = {
+                'instance': ve.instance,
+                'path': ve.json_path,
+            }
+
+            se = SimulationException(self, 'Failed to validate parameters',
+                                     **e)
+
+            self.logger.error('Failed to validate action parameters against '
+                              'schema: %s', ve.message)
+            self.change_to_error(ve.message, **e)
+
+            raise se
+
         try:
             if action == 'ping':
                 self.ping(payload)
@@ -182,7 +210,7 @@ def run_action(self, action, payload):
 
         except SimulationException as se:
             self.logger.error('SimulationException: %s', str(se))
-            self.change_state('error', msg=se.msg, **se.info)
+            self.change_to_error(se.msg, **se.info)
 
             raise se
 
@@ -197,6 +225,14 @@ def change_state(self, state, **kwargs):
 
         self.publish_status()
 
+    def change_to_error(self, msg, **details):
+        self.change_state('error',
+                          error=msg,
+                          error_details={
+                              'msg': msg,
+                              **details
+                          })
+
     # Actions
     def ping(self, payload):
         self.publish_status()
diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index 51558c0..f157690 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -50,7 +50,9 @@ def reconcile(self):
             self.change_state('running')
 
         except Exception as e:
-            self.change_state('error', error=str(e))
+            self.change_to_error('failed to reconcile',
+                                 exception=str(e),
+                                 args=e.args)
 
     @property
     def status(self):
@@ -66,7 +68,7 @@ def on_ready(self):
         try:
             self._status = self.node.status
         except Exception:
-            self.change_state('error', error='VILLASnode not installed')
+            self.change_to_error('VILLASnode not installed')
 
         super().on_ready()
 
diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index eb76556..3a92a49 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -46,7 +46,7 @@ def get_status(self):
             return r.json()
 
         except requests.exceptions.RequestException:
-            self.change_state('error', error='Failed to contact VILLASrelay')
+            self.change_to_error('Failed to contact VILLASrelay')
 
             return None
 
@@ -112,6 +112,6 @@ def on_ready(self):
             self._version = status['version']
 
         except Exception:
-            self.change_state('error', error='Failed to contact VILLASrelay')
+            self.change_to_error('Failed to contact VILLASrelay')
 
         super().on_ready()
diff --git a/villas/controller/components/simulator.py b/villas/controller/components/simulator.py
index 9f79b42..573d985 100644
--- a/villas/controller/components/simulator.py
+++ b/villas/controller/components/simulator.py
@@ -17,17 +17,6 @@ def __init__(self, **args):
         self.model = None
         self.results = None
 
-    @property
-    def schema(self):
-        return {
-            'start': self.start_schema,
-            **super().schema
-        }
-
-    @property
-    def start_schema(self):
-        return {}
-
     @property
     def status(self):
         return {
diff --git a/villas/controller/components/simulators/dpsim.py b/villas/controller/components/simulators/dpsim.py
index 91d5f55..5ce2b01 100644
--- a/villas/controller/components/simulators/dpsim.py
+++ b/villas/controller/components/simulators/dpsim.py
@@ -38,7 +38,7 @@ def start(self, payload):
             if self.sim.start() is None:
                 self.change_state('running')
             else:
-                self.change_state('error')
+                self.change_to_error('failed to start simulation')
                 self.logger.warn('Attempt to start simulator failed.'
                                  'State is %s', self._state)
         else:
diff --git a/villas/controller/components/simulators/generic.py b/villas/controller/components/simulators/generic.py
index 82e2560..8d67776 100644
--- a/villas/controller/components/simulators/generic.py
+++ b/villas/controller/components/simulators/generic.py
@@ -24,9 +24,6 @@ def __del__(self):
         if self.timer:
             self.timer.cancel()
 
-        if self.thread:
-            self
-
     @property
     def status(self):
         return {
@@ -56,8 +53,7 @@ def start(self, payload):
 
     def check_state(self, state):
         if self._state != state:
-            self.change_state('error',
-                              msg=f'Failed to transition to state "{state}"!')
+            self.change_to_error('Failed to transition to state', state=state)
 
     def check_state_deferred(self, state, timeout=5):
         self.timer = threading.Timer(timeout, self.check_state, args=[state])
@@ -145,7 +141,7 @@ def run(self, params, path):
         # GenericSimulator.run() is executed in a separate thread.
         # We therefore want to catch exceptions here.
         except SimulationException as se:
-            self.change_state('error', msg=se.msg, **se.info)
+            self.change_to_error(se.msg, **se.info)
 
         self.child = None
 

From b79c286cb0b5260178e489549486499431088a06 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Wed, 13 Oct 2021 09:23:16 +0200
Subject: [PATCH 24/89] fix linting errors

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes.py | 12 ++++++------
 villas/controller/controller.py                     |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 0345756..1ce4980 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -8,13 +8,13 @@
 from villas.controller.components.simulators.kubernetes import KubernetesJob
 
 
-def _match(stringA, stringB):
-    if stringA == stringB:
+def _match(a, b):
+    if a == b:
         return True
-    elif len(stringA) < len(stringB):
-        return stringA in stringB
-    elif len(stringB) < len(stringA):
-        return stringB in stringA
+    elif len(a) < len(b):
+        return a in b
+    elif len(b) < len(a):
+        return b in a
 
 
 class KubernetesManager(Manager):
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index 088d7db..a9da81a 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -37,7 +37,7 @@ def __init__(self, connection, args):
         # Components are activated by first call to on_iteration()
         self.active_components = {}
 
-    def get_consumers(self, Consumer, channel):
+    def get_consumers(self, _, channel):
         return map(lambda comp: comp.get_consumer(channel),
                    self.active_components.values())
 

From bac3246f67032f5f328bc4e5aa09de9320007329 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Wed, 13 Oct 2021 09:25:18 +0200
Subject: [PATCH 25/89] fix some errors in the API spec and schema

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 doc/openapi.yaml                              | 171 +++++++++---------
 .../schemas/simulator/dummy/start.yaml        |   3 +-
 2 files changed, 88 insertions(+), 86 deletions(-)

diff --git a/doc/openapi.yaml b/doc/openapi.yaml
index 2c7c89b..c006786 100644
--- a/doc/openapi.yaml
+++ b/doc/openapi.yaml
@@ -4,17 +4,25 @@ info:
   title: VILLAScontroller API
   description: 'A HTTP/REST API for controlling VILLAScontroller remotely for querying component status as well as issuing control actions.'
   version: 0.0.1
+  contact:
+    name: "Steffen Vogel"
+    email: "svogel2@eonerc.rwth-aachen.de"
   license:
     name: Apache-2.0
     url: https://www.apache.org/licenses/LICENSE-2.0
+
 servers:
 - url: https://villas.k8s.eonerc.rwth-aachen.de/api/v1
   description: Demo instance at RWTH Aachen
+
 paths:
+
   /:
     get:
       summary: 'Get status of VILLAScontroller daemon'
       operationId: getStatus
+      tags:
+      - status
       responses:
         '200':
           description: ''
@@ -66,9 +74,31 @@ paths:
                     version: '#1 SMP Fri Sep 3 15:33:01 UTC 2021'
                     machine: x86_64
 
+  /health:
+    get:
+      operationId: getHealth
+      summary: Query health of daemon.
+      tags:
+      - status
+      responses:
+        '200':
+          description: The daemon is healthy
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  status:
+                    type: string
+              example:
+                status: ok
+
 
   /component/{uuid}:
     get:
+      summary: 'Get the current status of a component'
+      operationId: getComponentStatus
+
       parameters:
       - name: uuid
         in: path
@@ -77,10 +107,8 @@ paths:
           type: string
           format: uuid
 
-      operationId: getComponentStatus
-      summary: 'Get the current status of a component'
       responses:
-        200:
+        '200':
           description: ''
           content:
             application/json:
@@ -186,139 +214,114 @@ paths:
                   uuid: f4751894-205e-11eb-aefb-0741ff98abca
                 schema:
                   create:
-                    $schema: 'http://json-schema.org/draft-07/schema'
-                    $id: 'http://example.com/example.json'
+                    # $schema: 'http://json-schema.org/draft-07/schema'
                     type: object
                     default: {}
                     required:
-                      - name
-                      - category
-                      - location
-                      - owner
-                      - realm
-                      - type
-                      - api_url
-                      - ws_url
+                    - name
+                    - category
+                    - location
+                    - owner
+                    - realm
+                    - type
+                    - api_url
+                    - ws_url
                     properties:
                       name:
-                        $id: '#/properties/name'
                         type: string
                         title: Component name
                         default: New Component
-                        examples:
-                          - 'Generic Simulator #1'
+                        examples: 'Generic Simulator #1'
                       owner:
-                        $id: '#/properties/owner'
                         type: string
                         title: Component owner
-                        default: ''
                         examples:
-                          - rmr
-                          - svg
+                        - rmr
+                        - svg
                       realm:
-                        $id: '#/properties/realm'
                         type: string
                         title: Component realm
                         default: ''
                         examples:
-                          - de.rwth-aachen.eonerc.acs
+                        - de.rwth-aachen.eonerc.acs
                       category:
-                        $id: '#/properties/category'
                         type: string
                         title: Component category
-                        default: ''
                         examples:
-                          - simulator
+                        - simulator
                       location:
-                        $id: '#/properties/location'
                         type: string
                         title: Component location
-                        default: ''
                         examples:
-                          - Richard's PC
+                        - Richard's PC
                       type:
-                        $id: '#/properties/type'
                         type: string
-                        title: The type schema
-                        default: ''
-                        examples:
-                          - generic
+                        default: generic
                       uuid:
-                        $id: '#/properties/uuid'
-                        type: 'null'
-                        title: The uuid schema
-                        default: null
+                        type: 'string'
+                        format: uuid
                       ws_url:
-                        $id: '#/properties/ws_url'
                         type: string
-                        title: The ws_url schema
-                        default: ''
                         examples:
-                          - 'https://villas.k8s.eonerc.rwth-aachen.de/ws/relay/generic_1'
+                        - 'https://villas.k8s.eonerc.rwth-aachen.de/ws/relay/generic_1'
                       api_url:
-                        $id: '#/properties/api_url'
                         type: string
-                        title: The api_url schema
-                        default: ''
                         examples:
-                          - 'https://villas.k8s.eonerc.rwth-aachen.de/api/ic/generic_1'
+                        - 'https://villas.k8s.eonerc.rwth-aachen.de/api/ic/generic_1'
                       shell:
-                        $id: '#/properties/shell'
                         type: boolean
-                        title: The shell schema
                         default: false
                         examples:
-                          - true
+                        - true
                       whitelist:
-                        $id: '#/properties/whitelist'
                         type: array
                         title: The whitelist schema
                         default: []
                         examples:
-                          - - /sbin/ping
-                            - ^echo
+                        - - /sbin/ping
+                          - ^echo
                         additionalItems: true
                         items:
-                          $id: '#/properties/whitelist/items'
                           anyOf:
-                            - $id: '#/properties/whitelist/items/anyOf/0'
-                              type: string
-                              title: The first anyOf schema
-                              default: ''
-                              examples:
-                                - /sbin/ping
-                                - ^echo
-                    additionalProperties: true
-
+                          - type: string
+                            examples:
+                            - /sbin/ping
+                            - ^echo
 
     post:
       operationId: executeComponentAction
       summary: 'Send a control action to the component'
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                action:
+                  type: string
+                  enum:
+                  - start
+                  - stop
+                  - pause
+                  - resume
+                  - create
+                  - delete
+                  - shutdown
+                  - reset
+                parameters:
+                  oneOf:
+                  - $ref: ../villas/controller/schemas/manager/generic/create.yaml
+                  - $ref: ../villas/controller/schemas/manager/kubernetes/create.yaml
+                  - $ref: ../villas/controller/schemas/simulator/dpsim/start.yaml
+                  - $ref: ../villas/controller/schemas/simulator/dummy/start.yaml
+
       responses:
-        200:
+        '200':
           description: ''
           content:
-            application/json:
-              schema:
-                type: object
-                additionalProperties: true
+            application/json: {}
 
-              example:
-                runtime: 10.2
 
-  /health:
-    get:
-      operationId: getHealth
-      summary: Query health of daemon.
-      responses:
-        '200':
-          description: The daemon is healthy
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  status:
-                    type: string
-              example:
-                status: ok
+              # example:
+              #   runtime: 10.2
diff --git a/villas/controller/schemas/simulator/dummy/start.yaml b/villas/controller/schemas/simulator/dummy/start.yaml
index d1d6f83..6da689e 100644
--- a/villas/controller/schemas/simulator/dummy/start.yaml
+++ b/villas/controller/schemas/simulator/dummy/start.yaml
@@ -9,5 +9,4 @@ properties:
     type: number
     default: 1.0
     description: The runtime of the simulation in seconds
-    examples:
-    - 3.0
+    example: 3.0

From 38c698c696e42a891142638da4293b3f675edbab Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Thu, 14 Oct 2021 18:17:24 +0200
Subject: [PATCH 26/89] validate schemas and openapi doc against meta
 jsonschemas

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .pre-commit-config.yaml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 724ae6e..b6287ea 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,3 +18,18 @@ repos:
   rev: 3.8.4
   hooks:
   - id: flake8
+- repo: https://github.com/sirosen/check-jsonschema
+  rev: 8a14ffa1d4c81a56057f55e1da308daeccc3bcd6
+  hooks:
+  - id: check-jsonschema
+    name: "Check schemas"
+    language: python
+    files: ^villas/controller/schemas/.*\.yaml$
+    types: [yaml]
+    args: ["--schemafile", "https://json-schema.org/draft/2020-12/schema"]
+  - id: check-jsonschema
+    name: "Check OpenAPI doc"
+    language: python
+    files: ^doc/openapi.yaml$
+    types: [yaml]
+    args: ["--schemafile", "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json"]

From a27be1393798d7b1af22bf9f627f200c7bbd2f23 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 09:22:30 +0200
Subject: [PATCH 27/89] raise a SimulationException if an IC with an existing
 UUID should be added to the controller

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/manager.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index 24392c4..1985046 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -1,4 +1,5 @@
 from villas.controller.component import Component
+from villas.controller.exceptions import SimulationException
 
 
 class Manager(Component):
@@ -43,9 +44,13 @@ def add_component(self, comp):
         print(self.name)
         print(comp)
         if comp.uuid in self.mixin.components:
-            # raise KeyError
-            self.logger.error('UUID %s already exists, not added', comp.uuid)
-            return
+            #self.logger.error('UUID %s already exists, not added', comp.uuid)
+            #return
+            existing_comp = self.mixin.components[comp.uuid]
+
+            raise SimulationException(self, 'Component with same UUID ' +
+                                      'already exists!',
+                                      component=existing_comp)
 
         comp.set_manager(self)
 

From f87608cc155540f4261fd6d7b15b279decd599f8 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 09:24:16 +0200
Subject: [PATCH 28/89] provide version number within status field of status
 update

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/villas_node.py  | 9 +++++----
 villas/controller/components/managers/villas_relay.py | 9 +++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index f157690..12a272f 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -56,10 +56,11 @@ def reconcile(self):
 
     @property
     def status(self):
-        return {
-            **super().status,
-            'villas_node_version': self.node.get_version()
-        }
+        status = super().status
+
+        status['status']['villas_none_version'] = self._version
+
+        return status
 
     def on_ready(self):
         if self.autostart and not self.node.is_running():
diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index 3a92a49..775d93f 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -91,10 +91,11 @@ def reconcile(self):
 
     @property
     def status(self):
-        return {
-            'villas_relay_version': self._version,
-            **super().status
-        }
+        status = super().status
+
+        status['status']['villas_relay_version'] = self._version
+
+        return status
 
     def on_shutdown(self):
         self.thread_stop.set()

From e0f78295f03f34be0b21fa7e0ff6b6495a42f658 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 14:25:57 +0200
Subject: [PATCH 29/89] relay: do not remove vanished sessions by setting state
 to gone

rather set their state to stopped

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .../controller/components/managers/villas_relay.py   | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index 775d93f..9530081 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -71,18 +71,22 @@ def reconcile(self):
 
             if uuid in self.components:
                 comp = self.components[uuid]
-
-                comp.change_state('running')
             else:
                 comp = VILLASrelayGateway(self, session)
-
                 self.add_component(comp)
 
+            comp.change_state('running')
+
         # Find vanished sessions
         for uuid in existing_uuids - active_uuids:
             comp = self.components[uuid]
 
-            self.remove_component(comp)
+            comp.change_state('stopped')
+
+            # We dont remove the components here
+            # So that they dont get removed from the backend
+            # and get recreated with the same UUID later
+            # self.remove_component(comp)
 
         if len(self.components) > 0:
             self.change_state('running')

From f789439687876bea80a10271096e338c069478b3 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 14:42:22 +0200
Subject: [PATCH 30/89] improve error reporting

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                        | 3 +--
 villas/controller/components/managers/kubernetes.py   | 7 ++++---
 villas/controller/components/simulators/kubernetes.py | 8 ++++++++
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 4f9f10d..c34c4aa 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -227,8 +227,7 @@ def change_state(self, state, **kwargs):
 
     def change_to_error(self, msg, **details):
         self.change_state('error',
-                          error=msg,
-                          error_details={
+                          error={
                               'msg': msg,
                               **details
                           })
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 1ce4980..50b2c91 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -111,13 +111,14 @@ def _run_event_watcher(self):
                                 comp.change_state('stopping', True)
                             elif eo.reason == 'Started':
                                 comp.pods.add(eo.involved_object.name)
-                                comp.properties['pod_names'] = list(comp.pods)
                                 comp.change_state('running', True)
                             elif eo.reason == 'BackoffLimitExceeded':
-                                comp.change_state('error', error=eo.reason)
+                                comp.change_to_error('failed to start job',
+                                                     reason=eo.reason)
                             elif eo.reason == 'Failed':
                                 if comp._state == 'running':
-                                    comp.change_state('error', error=eo.reason)
+                                    comp.change_to_error('failed to start job',
+                                                         error=eo.reason)
                                 elif comp._state == 'starting':
                                     # wait for BackoffLimitExceeded event
                                     continue
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index a28d0dc..0c20f49 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -29,6 +29,14 @@ def __init__(self, manager, **args):
 
         self.custom_schema = props.get('schema', {})
 
+    @property
+    def status(self):
+        status = super().status
+
+        status['status']['pod_names'] = list(self.pods)
+
+        return status
+
     @property
     def schema(self):
         return {

From c0dcdad20ce2d527938729830c85793d55da9541 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 15:17:34 +0200
Subject: [PATCH 31/89] add labels and annotations to created job resources

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .../components/simulators/kubernetes.py        | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 0c20f49..e9083a6 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -93,9 +93,21 @@ def _prepare_job(self, job, payload):
             job.metadata.labels = {}
 
         job.metadata.labels.update({
-            'controller': 'villas',
-            'controller-uuid': self.manager.uuid,
-            'uuid': self.uuid
+            'app.kubernetes.io/part-of': 'villas',
+            'app.kubernetes.io/managed-by': 'villas-controller',
+            'app.kubernetes.io/component': 'infrastructure-component',
+
+            'villas.fein-aachen.org/ic-manager-uuid': self.manager.uuid,
+            'villas.fein-aachen.org/ic-uuid': self.uuid
+        })
+
+        if job.metadata.annotations is None:
+            job.metadata.annotations = {}
+
+        job.metadata.annotations.update({
+            'villas.fein-aachen.org/name': self.name,
+            'villas.fein-aachen.org/location': self.location,
+            'villas.fein-aachen.org/realm': self.realm
         })
 
         return job

From 37634bd84117fde8a17101d62250c78dd79ea3c9 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 16:28:40 +0200
Subject: [PATCH 32/89] k8s: add owner references

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .../components/managers/kubernetes.py         |  4 +++
 .../components/simulators/kubernetes.py       | 25 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 50b2c91..21717ef 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -38,6 +38,10 @@ def __init__(self, **args):
 
         self.namespace = args.get('namespace', 'villas-controller')
 
+        self.my_namespace = os.environ.get('NAMESPACE')
+        self.my_pod_name = os.environ.get('POD_NAME')
+        self.my_pod_uid = os.environ('POD_UID')
+
         self._check_namespace(self.namespace)
 
         # self.pod_watcher_thread.start()
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index e9083a6..e7594f0 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING
 import json
 import signal
 from copy import deepcopy
@@ -10,10 +12,14 @@
 from villas.controller.exceptions import SimulationException
 from villas.controller.util import merge
 
+if TYPE_CHECKING:
+    from villas.controller.components.managers.kubernetes \
+        import KubernetesManager
+
 
 class KubernetesJob(Simulator):
 
-    def __init__(self, manager, **args):
+    def __init__(self, manager: KubernetesManager, **args):
         super().__init__(**args)
 
         self.manager = manager
@@ -44,6 +50,16 @@ def schema(self):
             **super().schema
         }
 
+    def _owner(self):
+        if self.manager.my_pod_name and self.manager.my_pod_uid:
+            return k8s.client.V1OwnerReference(
+                kind='Pod',
+                name=self.manager.my_pod_name,
+                uid=self.manager.my_pod_uid
+            )
+
+        return None
+
     def _prepare_job(self, job, payload):
         # Create config map
         cm = self._create_config_map(payload)
@@ -89,6 +105,9 @@ def _prepare_job(self, job, payload):
         job.metadata.generate_name = name + '-'
         job.metadata.name = None
 
+        if o := self._owner():
+            job.metadata.owner_references = [o]
+
         if job.metadata.labels is None:
             job.metadata.labels = {}
 
@@ -124,6 +143,9 @@ def _create_config_map(self, payload):
             }
         )
 
+        if o := self._owner():
+            self.cm.metadata.owner_references = [o]
+
         return c.create_namespaced_config_map(
             namespace=self.manager.namespace,
             body=self.cm
@@ -157,6 +179,7 @@ def _delete_job(self):
         self.properties['pod_names'] = []
 
     def start(self, payload):
+        # Delete prior job
         self._delete_job()
 
         job = payload.get('job', {})

From 250c97bc60473eda45d972ffea19ee6a98b8f6b8 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 16:54:40 +0200
Subject: [PATCH 33/89] make all manually configured components to be managed
 by the default generic manager by default

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/controller.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index a9da81a..db25d31 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -7,6 +7,7 @@
 
 # from villas.controller.api import Api
 from villas.controller import __version__ as version
+from villas.controller.config import Config
 from villas.controller.components.managers.generic import GenericManager
 
 
@@ -17,10 +18,8 @@ class ControllerMixin(kombu.mixins.ConsumerProducerMixin):
 
     def __init__(self, connection, args):
         self.args = args
-        self.config = args.config
+        self.config: Config = args.config
 
-        comps = self.config.components
-        self.components = {c.uuid: c for c in comps if c.enabled}
         self.connection = connection
         self.exchange = kombu.Exchange(name='villas',
                                        type='headers',
@@ -28,14 +27,16 @@ def __init__(self, connection, args):
 
         self.publish_queue = queue.Queue()
 
+        # Components are activated by first call to on_iteration()
+        self.components = {}
+        self.active_components = {}
+
         manager = self.add_managers()
 
-        for _, comp in self.components.items():
+        comps = [c for c in self.config.components if c.enabled]
+        for comp in comps:
             LOGGER.info('Adding %s', comp)
-            comp.set_manager(manager)
-
-        # Components are activated by first call to on_iteration()
-        self.active_components = {}
+            manager.add_component(comp)
 
     def get_consumers(self, _, channel):
         return map(lambda comp: comp.get_consumer(channel),
@@ -59,6 +60,8 @@ def add_managers(self):
                 location=socket.gethostname()
             )
 
+            mgr.mixin = self
+
             self.components[mgr.uuid] = mgr
         else:
             mgr = mgrs[0]

From b867d026233c41b1fc2d4996dfea386b181d0b33 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Fri, 22 Oct 2021 16:55:29 +0200
Subject: [PATCH 34/89] use a api/v1 prefix for the API handlers

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 doc/openapi.yaml            |  2 +-
 villas/controller/api.py    | 10 ++++++----
 villas/controller/config.py |  3 ++-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/doc/openapi.yaml b/doc/openapi.yaml
index c006786..cbddd06 100644
--- a/doc/openapi.yaml
+++ b/doc/openapi.yaml
@@ -12,7 +12,7 @@ info:
     url: https://www.apache.org/licenses/LICENSE-2.0
 
 servers:
-- url: https://villas.k8s.eonerc.rwth-aachen.de/api/v1
+- url: https://villas.k8s.eonerc.rwth-aachen.de/controller/api/v1
   description: Demo instance at RWTH Aachen
 
 paths:
diff --git a/villas/controller/api.py b/villas/controller/api.py
index af48294..31168d0 100644
--- a/villas/controller/api.py
+++ b/villas/controller/api.py
@@ -8,6 +8,8 @@
 REGEX_UUID = r'\b[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}-[0-9a-fA-F]' \
              r'{4}-[0-9a-fA-F]{4}-\b[0-9a-fA-F]{12}\b'
 
+BASE = '/api/v1'
+
 
 class RequestHandler(tornado.web.RequestHandler):
 
@@ -32,7 +34,7 @@ def run(self):
         port = self.controller.config.api.port
         self.app.listen(port)
 
-        LOGGER.info('Starting API')
+        LOGGER.info('Starting API at http://localhost:%d%s', port, BASE)
 
         self.loop = IOLoop.current(instance=True)
         self.loop.start()
@@ -48,7 +50,7 @@ def handlers(self):
         }
 
         return [
-            (r'/', MainRequestHandler, args),
-            (r'/health', HealthRequestHandler, args),
-            (r'/component/('+REGEX_UUID+r')', ComponentRequestHandler, args)
+            (BASE + r'/', MainRequestHandler, args),
+            (BASE + r'/health', HealthRequestHandler, args),
+            (BASE + r'/component/('+REGEX_UUID+r')', ComponentRequestHandler, args)  # noqa E501
         ]
diff --git a/villas/controller/config.py b/villas/controller/config.py
index e261e78..6182edc 100644
--- a/villas/controller/config.py
+++ b/villas/controller/config.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import dotmap
+from typing import List
 
 from os import getcwd
 from xdg import (
@@ -75,7 +76,7 @@ def find_default_path(self, filename='config',
                 return fn
 
     @property
-    def components(self):
+    def components(self) -> List[Component]:
         return [Component.from_dict(c) for c in self.config.components]
 
     def __getattr__(self, attr):

From 061d629db4e2ab7ede0d14fe4e5d43965f708bc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 28 Oct 2021 14:17:06 +0000
Subject: [PATCH 35/89] fixes for schema & pod_uid

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                  | 17 +++++++++++------
 .../components/managers/kubernetes.py           |  2 +-
 .../components/simulators/kubernetes.py         | 13 +++++++++----
 3 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index c34c4aa..7681518 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -22,6 +22,7 @@ def __init__(self, **props):
         self.name = props.get('name')
         self.category = props.get('category')
         self.enabled = props.get('enabled', True)
+        self.location = props.get('location', '')
         self.uuid = props.get('uuid')
 
         # The manager component which manages this instances
@@ -44,9 +45,8 @@ def __init__(self, **props):
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
-
         # Load schemas for validating action payloads
-        self._load_schema()
+        self._schema = self.load_schema()
 
     def on_ready(self):
         self.publish_status_thread.start()
@@ -86,8 +86,12 @@ def get_consumer(self, channel):
             accept={'application/json'}
         )
 
-    def _load_schema(self):
-        self.schema = {}
+    @property
+    def schema(self):
+        return self._schema
+
+    def load_schema(self):
+        schema = {}
 
         try:
             pkg_name = f'villas.controller.schemas.{self.category}.{self.type}'
@@ -101,9 +105,10 @@ def _load_schema(self):
             if resources.is_resource(pkg, res) and ext in ['.yaml', '.json']:
 
                 fo = resources.open_text(pkg, res)
-                schema = yaml.load(fo, yaml.SafeLoader)
+                loadedschema = yaml.load(fo, yaml.SafeLoader)
 
-                self.schema[name] = Draft202012Validator(schema)
+                schema[name] = Draft202012Validator(loadedschema)
+        return schema
 
     def validate_parameters(self, action, parameters):
         if action in self.schema:
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 21717ef..6217111 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -40,7 +40,7 @@ def __init__(self, **args):
 
         self.my_namespace = os.environ.get('NAMESPACE')
         self.my_pod_name = os.environ.get('POD_NAME')
-        self.my_pod_uid = os.environ('POD_UID')
+        self.my_pod_uid = os.environ.get('POD_UID')
 
         self._check_namespace(self.namespace)
 
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index e7594f0..5051bde 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -45,10 +45,15 @@ def status(self):
 
     @property
     def schema(self):
-        return {
-            **self.custom_schema,
-            **super().schema
-        }
+        if (super().schema):
+            return {
+                **self.custom_schema,
+                **super().schema
+            }
+        else:
+            return {
+                **self.custom_schema
+            }
 
     def _owner(self):
         if self.manager.my_pod_name and self.manager.my_pod_uid:

From 7a7f273175f40b91271f6151ea06d59e254bdc0c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 2 Nov 2021 11:19:08 +0100
Subject: [PATCH 36/89] Change update interval

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 7681518..00d9e21 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -41,7 +41,7 @@ def __init__(self, **props):
         self.logger = logging.getLogger(
             f'villas.controller.{self.category}.{self.type}:{self.uuid}')
 
-        self.publish_status_interval = 5
+        self.publish_status_interval = 30
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)

From 3093451dc46abf68fad86a0e4503e88049e1266c Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 16 Nov 2021 10:10:13 +0100
Subject: [PATCH 37/89] generic: move return code to status section

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulators/generic.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/villas/controller/components/simulators/generic.py b/villas/controller/components/simulators/generic.py
index 8d67776..174ff41 100644
--- a/villas/controller/components/simulators/generic.py
+++ b/villas/controller/components/simulators/generic.py
@@ -26,10 +26,11 @@ def __del__(self):
 
     @property
     def status(self):
-        return {
-            **super().status,
-            'return_code': self.return_code
-        }
+        status = super().status
+
+        status['status']['return_code'] = self.return_code
+
+        return status
 
     def start(self, payload):
         super().start(payload)

From d298eba7f4019c8624583175549cf63e634233d6 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 16 Nov 2021 14:56:03 +0100
Subject: [PATCH 38/89] fix exception in schema code

Signed-off-by: Steffen Vogel <post@steffenvogel.de>
Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 00d9e21..a9ab0ab 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -41,12 +41,12 @@ def __init__(self, **props):
         self.logger = logging.getLogger(
             f'villas.controller.{self.category}.{self.type}:{self.uuid}')
 
+        self._schema = self.load_schema()
+
         self.publish_status_interval = 30
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
-        # Load schemas for validating action payloads
-        self._schema = self.load_schema()
 
     def on_ready(self):
         self.publish_status_thread.start()
@@ -98,7 +98,8 @@ def load_schema(self):
             pkg = importlib.import_module(pkg_name)
         except ModuleNotFoundError:
             self.logger.warn('Missing schemas!')
-            return
+
+            return schema
 
         for res in resources.contents(pkg):
             name, ext = os.path.splitext(res)
@@ -108,6 +109,7 @@ def load_schema(self):
                 loadedschema = yaml.load(fo, yaml.SafeLoader)
 
                 schema[name] = Draft202012Validator(loadedschema)
+
         return schema
 
     def validate_parameters(self, action, parameters):

From 328249d28b855134b157b0337d306e65735ef372 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 16 Nov 2021 11:01:58 +0000
Subject: [PATCH 39/89] kubernetes fixes

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulators/kubernetes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 5051bde..b04954e 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -60,7 +60,8 @@ def _owner(self):
             return k8s.client.V1OwnerReference(
                 kind='Pod',
                 name=self.manager.my_pod_name,
-                uid=self.manager.my_pod_uid
+                uid=self.manager.my_pod_uid,
+                api_version='v1'
             )
 
         return None

From 31699d10c438e6ff81440c471aac702ee80c2912 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 17 Nov 2021 18:35:29 +0000
Subject: [PATCH 40/89] move schema to separate file

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .../components/managers/kubernetes_simple.py  | 44 ++-----------------
 .../components/simulators/kubernetes.py       |  6 +--
 .../manager/kubernetes-simple/create.yaml     | 28 ++++++++++++
 3 files changed, 35 insertions(+), 43 deletions(-)
 create mode 100644 villas/controller/schemas/manager/kubernetes-simple/create.yaml

diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 555a669..bd97c64 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -4,47 +4,11 @@
 
 class KubernetesManagerSimple(KubernetesManager):
 
-    create_schema = {
-        '$schema': 'http://json-schema.org/draft-04/schema#',
-        'title': 'Simple Kubernetes Job',
-        'type': 'object',
-        'required': [
-            'image',
-        ],
-        'properties': {
-            'uuid': {
-                'type': 'string',
-                'title': 'UUID',
-                'default': '8dfd03b2-1c78-11ec-9621-0242ac130002'
-            },
-            'jobname': {
-                'type': 'string',
-                'title': 'Job name',
-                'default': 'myJob'
-            },
-            'activeDeadlineSeconds': {
-                'type': 'number',
-                'title': 'activeDeadlineSeconds',
-                'default': 3600
-            },
-            'image': {
-                'type': 'string',
-                'title': 'Image',
-                'default': 'perl'
-            },
-            'containername': {
-                'type': 'string',
-                'title': 'Container name',
-                'default': 'myContainer'
-            }
-        }
-    }
-
     parameters_simple = {
         'type': 'kubernetes',
         'category': 'simulator',
         'uuid': None,
-        'name': 'Kubernetes Simulator',
+        'name': '',
         'properties': {
             'job': {
                 'apiVersion': 'batch/v1',
@@ -61,7 +25,7 @@ class KubernetesManagerSimple(KubernetesManager):
                             'containers': [
                                 {
                                     'image': '',
-                                    'name': ''
+                                    'name': 'jobcontainer'
                                 }
                             ]
                         }
@@ -74,9 +38,9 @@ class KubernetesManagerSimple(KubernetesManager):
     def create(self, payload):
         self.logger.info(payload)
         params = payload.get('parameters', {})
+        sim_name = payload.get('name', 'Kubernetes Simulator')
         jobname = params.get('jobname', 'noname')
         adls = params.get('activeDeadlineSeconds', 3600)
-        contName = params.get('containername', 'noname')
         image = params.get('image')
         name = params.get('name')
         uuid = params.get('uuid')
@@ -87,10 +51,10 @@ def create(self, payload):
             return
 
         parameters = self.parameters_simple
+        parameters['name'] = sim_name
         job = parameters['properties']['job']
         job['metadata']['name'] = jobname
         job['spec']['activeDeadlineSeconds'] = adls
-        job['spec']['template']['spec']['containers'][0]['name'] = contName
         job['spec']['template']['spec']['containers'][0]['image'] = image
 
         parameters['job'] = job
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index b04954e..9d57016 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -183,6 +183,9 @@ def _delete_job(self):
         self.job = None
         self.properties['job_name'] = None
         self.properties['pod_names'] = []
+        # job isn't immediately deleted
+        # let the user see something is happening
+        time.sleep(7)
 
     def start(self, payload):
         # Delete prior job
@@ -207,9 +210,6 @@ def start(self, payload):
     def stop(self, message):
         self.change_state('stopping', True)
         self._delete_job()
-        # job isn't immediately deleted
-        # let the user see something is happening
-        time.sleep(3)
         self.change_state('idle')
 
     def _send_signal(self, sig):
diff --git a/villas/controller/schemas/manager/kubernetes-simple/create.yaml b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
new file mode 100644
index 0000000..cde59a6
--- /dev/null
+++ b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
@@ -0,0 +1,28 @@
+---
+$schema: http://json-schema.org/draft-04/schema#
+
+type: object
+title: 'Simple Kubernetes Job'
+required:
+  - image
+properties:
+  name:
+    type: string
+    title: 'Simulator Name'
+    default: 'Kubernetes Simulator'
+  uuid:
+    type: string
+    title: UUID
+    default: 8dfd03b2-1c78-11ec-9621-0242ac130002
+  jobname:
+    type: string
+    title: 'Job name'
+    default: myJob
+  activeDeadlineSeconds:
+    type: number
+    title: activeDeadlineSeconds
+    default: 3600
+  image:
+    type: string
+    title: Image
+    default: perl

From dcc4a659a64e8e030ad320f49f6abe8c3b52db2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 18 Nov 2021 08:48:02 +0000
Subject: [PATCH 41/89] debugging

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index a9ab0ab..7391630 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -43,7 +43,7 @@ def __init__(self, **props):
 
         self._schema = self.load_schema()
 
-        self.publish_status_interval = 30
+        self.publish_status_interval = 5
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
@@ -279,6 +279,7 @@ def from_dict(dict):
 
     def publish_status(self):
         if not self.mixin:
+            self.logger.warn('No mixin!')
             return
 
         self.mixin.publish(self.status, headers=self.headers)

From 5ed46b7d6ecc6f98d335979353f5b5d5292b9a3e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 18 Nov 2021 09:55:46 +0000
Subject: [PATCH 42/89] catch timeouterror

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/controller.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index db25d31..2ffe573 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -83,6 +83,8 @@ def _drain_publish_queue(self):
                 self.producer.publish(body, **kwargs)
         except queue.Empty:
             pass
+        except TimeoutError:
+            LOGGER.warn('TimeoutError, let kombu reconnect..')
 
     def on_iteration(self):
         # Drain publish queue

From 5c615cb6c34f95241143c4910b25f19fd24251ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 18 Nov 2021 13:43:49 +0000
Subject: [PATCH 43/89] show event outputs

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes.py  | 12 ++++++------
 .../schemas/manager/kubernetes-simple/create.yaml    |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 6217111..0564f2c 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -96,8 +96,8 @@ def _run_event_watcher(self):
 
                     eo = e.get('object')
 
-                    # self.logger.info('Event: %s (reason=%s)', eo.message,
-                    #                 eo.reason)
+                    self.logger.info('Event: %s (reason=%s)', eo.message,
+                                     eo.reason)
 
                     for uuid in self.components:
                         comp = self.components[uuid]
@@ -126,10 +126,10 @@ def _run_event_watcher(self):
                                 elif comp._state == 'starting':
                                     # wait for BackoffLimitExceeded event
                                     continue
-                            # else:
-                                # self.logger.info('Reason \'%s\' not handled '
-                                #                 'for kubernetes simulator',
-                                #                 eo.reason)
+                            else:
+                                self.logger.info('Reason \'%s\' not handled '
+                                                 'for kubernetes simulator',
+                                                 eo.reason)
 
             except ProtocolError:
                 self.logger.warn('Connection to kubernetes broken, \
diff --git a/villas/controller/schemas/manager/kubernetes-simple/create.yaml b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
index cde59a6..95588b6 100644
--- a/villas/controller/schemas/manager/kubernetes-simple/create.yaml
+++ b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
@@ -16,7 +16,7 @@ properties:
     default: 8dfd03b2-1c78-11ec-9621-0242ac130002
   jobname:
     type: string
-    title: 'Job name'
+    title: 'Jobname'
     default: myJob
   activeDeadlineSeconds:
     type: number

From f30366ca6a5a3eb3677d71837ece516866d7d6d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 18 Nov 2021 16:06:34 +0000
Subject: [PATCH 44/89] debugging

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                | 30 +++++++--
 .../components/managers/kubernetes_simple.py  | 65 ++++++++++---------
 villas/controller/controller.py               |  8 +--
 .../manager/kubernetes-simple/create.yaml     |  2 +-
 4 files changed, 63 insertions(+), 42 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 7391630..3baef73 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -41,9 +41,12 @@ def __init__(self, **props):
         self.logger = logging.getLogger(
             f'villas.controller.{self.category}.{self.type}:{self.uuid}')
 
+        self.logger.info("init component:")
+        self.logger.info(self.name)
+
         self._schema = self.load_schema()
 
-        self.publish_status_interval = 5
+        self.publish_status_interval = 15
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
@@ -67,6 +70,8 @@ def set_mixin(self, mixin):
         self.workdir = os.path.join(self.mixin.config.workdir, str(self.uuid))
 
     def get_consumer(self, channel):
+        self.logger.info(channel)
+        self.logger.info(self.headers)
         self.channel = channel
 
         return kombu.Consumer(
@@ -96,6 +101,7 @@ def load_schema(self):
         try:
             pkg_name = f'villas.controller.schemas.{self.category}.{self.type}'
             pkg = importlib.import_module(pkg_name)
+            self.logger.info(pkg)
         except ModuleNotFoundError:
             self.logger.warn('Missing schemas!')
 
@@ -103,13 +109,22 @@ def load_schema(self):
 
         for res in resources.contents(pkg):
             name, ext = os.path.splitext(res)
+            self.logger.info(name)
+            self.logger.info(ext)
             if resources.is_resource(pkg, res) and ext in ['.yaml', '.json']:
 
                 fo = resources.open_text(pkg, res)
                 loadedschema = yaml.load(fo, yaml.SafeLoader)
+                self.logger.info(loadedschema)
 
-                schema[name] = Draft202012Validator(loadedschema)
+                try:
+                    Draft202012Validator.check_schema(loadedschema)
+                    schema[name] = loadedschema
+                    print(schema[name])
+                except jsonschema.exceptions.SchemaError:
+                    self.logger.warn("Schema is invalid!")
 
+        print(schema)
         return schema
 
     def validate_parameters(self, action, parameters):
@@ -133,6 +148,9 @@ def headers(self):
 
     @property
     def status(self):
+        self.logger.info("status, self.schema:")
+        self.logger.info(self.schema)
+
         status = {
             'state': self._state,
             **self.mixin.status,
@@ -149,13 +167,13 @@ def status(self):
                 **self.headers
             },
             'schema': {
-                name: v.schema for name, v in self.schema.items()
+                **self.schema
             }
         }
 
     def on_message(self, message):
-        self.logger.info('my uuid: %s', self.uuid)
-        self.logger.info('Received message: %s', message.payload)
+        # self.logger.info('my uuid: %s', self.uuid)
+        # self.logger.info('Received message: %s', message.payload)
 
         if 'action' in message.payload:
             try:
@@ -282,6 +300,8 @@ def publish_status(self):
             self.logger.warn('No mixin!')
             return
 
+        # self.logger.info("complete status:")
+        # self.logger.info(self.status)
         self.mixin.publish(self.status, headers=self.headers)
 
     def publish_status_periodically(self):
diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index bd97c64..b31163e 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -1,39 +1,43 @@
 from villas.controller.components.managers.kubernetes import KubernetesManager
 from villas.controller.components.simulators.kubernetes import KubernetesJob
 
-
-class KubernetesManagerSimple(KubernetesManager):
-
-    parameters_simple = {
-        'type': 'kubernetes',
-        'category': 'simulator',
-        'uuid': None,
-        'name': '',
-        'properties': {
-            'job': {
-                'apiVersion': 'batch/v1',
-                'kind': 'Job',
-                'metadata': {
-                    'name': ''
-                },
-                'spec': {
-                    'activeDeadlineSeconds': 3600,
-                    'backoffLimit': 0,
-                    'template': {
-                        'spec': {
-                            'restartPolicy': 'Never',
-                            'containers': [
-                                {
-                                    'image': '',
-                                    'name': 'jobcontainer'
-                                }
-                            ]
-                        }
+parameters_simple = {
+    'type': 'kubernetes',
+    'category': 'simulator',
+    'uuid': None,
+    'name': '',
+    'properties': {
+        'job': {
+            'apiVersion': 'batch/v1',
+            'kind': 'Job',
+            'metadata': {
+                'name': ''
+            },
+            'spec': {
+                'activeDeadlineSeconds': 3600,
+                'backoffLimit': 0,
+                'template': {
+                    'spec': {
+                        'restartPolicy': 'Never',
+                        'containers': [
+                            {
+                                'image': '',
+                                'name': 'jobcontainer'
+                            }
+                        ]
                     }
                 }
             }
         }
     }
+}
+
+
+class KubernetesManagerSimple(KubernetesManager):
+
+    def __init__(self, **args):
+        super().__init__(**args)
+        self.logger.info("init of KubernetesManagerSimple")
 
     def create(self, payload):
         self.logger.info(payload)
@@ -50,7 +54,7 @@ def create(self, payload):
             super().create(payload)
             return
 
-        parameters = self.parameters_simple
+        parameters = parameters_simple
         parameters['name'] = sim_name
         job = parameters['properties']['job']
         job['metadata']['name'] = jobname
@@ -69,6 +73,3 @@ def create(self, payload):
 
         comp = KubernetesJob(self, **parameters)
         self.add_component(comp)
-
-    def __init__(self, **args):
-        super().__init__(**args)
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index 2ffe573..97fd0ea 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -5,7 +5,7 @@
 import os
 import kombu.mixins
 
-# from villas.controller.api import Api
+from villas.controller.api import Api
 from villas.controller import __version__ as version
 from villas.controller.config import Config
 from villas.controller.components.managers.generic import GenericManager
@@ -117,9 +117,9 @@ def on_iteration(self):
     def start(self):
         self.started = time.time()
 
-        # if self.config.api.enabled:
-        #     self.api = Api(self)
-        #     self.api.start()
+        if self.config.api.enabled:
+            self.api = Api(self)
+            self.api.start()
 
         self.should_terminate = False
         while not self.should_terminate:
diff --git a/villas/controller/schemas/manager/kubernetes-simple/create.yaml b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
index 95588b6..9747f77 100644
--- a/villas/controller/schemas/manager/kubernetes-simple/create.yaml
+++ b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
@@ -17,7 +17,7 @@ properties:
   jobname:
     type: string
     title: 'Jobname'
-    default: myJob
+    default: myjob
   activeDeadlineSeconds:
     type: number
     title: activeDeadlineSeconds

From b7de56fcefde3034bfcab08d909d70c23ceda415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 19 Nov 2021 12:36:45 +0000
Subject: [PATCH 45/89] add init file for kubernetes-simple schema

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/schemas/manager/kubernetes-simple/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 villas/controller/schemas/manager/kubernetes-simple/__init__.py

diff --git a/villas/controller/schemas/manager/kubernetes-simple/__init__.py b/villas/controller/schemas/manager/kubernetes-simple/__init__.py
new file mode 100644
index 0000000..e69de29

From 8837d2b107cd7508a3e268b9eea41a6329e5e32f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 19 Nov 2021 12:46:07 +0000
Subject: [PATCH 46/89] get kubernetes job running

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes_simple.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index b31163e..433346f 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -15,14 +15,17 @@
             },
             'spec': {
                 'activeDeadlineSeconds': 3600,
-                'backoffLimit': 0,
+                'backoffLimit': 2,
                 'template': {
                     'spec': {
                         'restartPolicy': 'Never',
                         'containers': [
                             {
                                 'image': '',
-                                'name': 'jobcontainer'
+                                'name': 'jobcontainer',
+                                'securityContext': {
+                                    'privileged': False
+                                }
                             }
                         ]
                     }

From 5e925f08bdd3d533beb651a8373daedd4d2afe2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Mon, 22 Nov 2021 11:04:45 +0000
Subject: [PATCH 47/89] fixes

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                     | 10 ----------
 .../controller/components/managers/kubernetes.py   | 12 ++++++------
 .../components/managers/kubernetes_simple.py       |  2 ++
 .../controller/components/simulators/kubernetes.py | 14 ++++++++------
 4 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 3baef73..3885402 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -101,7 +101,6 @@ def load_schema(self):
         try:
             pkg_name = f'villas.controller.schemas.{self.category}.{self.type}'
             pkg = importlib.import_module(pkg_name)
-            self.logger.info(pkg)
         except ModuleNotFoundError:
             self.logger.warn('Missing schemas!')
 
@@ -109,22 +108,17 @@ def load_schema(self):
 
         for res in resources.contents(pkg):
             name, ext = os.path.splitext(res)
-            self.logger.info(name)
-            self.logger.info(ext)
             if resources.is_resource(pkg, res) and ext in ['.yaml', '.json']:
 
                 fo = resources.open_text(pkg, res)
                 loadedschema = yaml.load(fo, yaml.SafeLoader)
-                self.logger.info(loadedschema)
 
                 try:
                     Draft202012Validator.check_schema(loadedschema)
                     schema[name] = loadedschema
-                    print(schema[name])
                 except jsonschema.exceptions.SchemaError:
                     self.logger.warn("Schema is invalid!")
 
-        print(schema)
         return schema
 
     def validate_parameters(self, action, parameters):
@@ -148,8 +142,6 @@ def headers(self):
 
     @property
     def status(self):
-        self.logger.info("status, self.schema:")
-        self.logger.info(self.schema)
 
         status = {
             'state': self._state,
@@ -300,8 +292,6 @@ def publish_status(self):
             self.logger.warn('No mixin!')
             return
 
-        # self.logger.info("complete status:")
-        # self.logger.info(self.status)
         self.mixin.publish(self.status, headers=self.headers)
 
     def publish_status_periodically(self):
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 0564f2c..849484b 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -96,8 +96,8 @@ def _run_event_watcher(self):
 
                     eo = e.get('object')
 
-                    self.logger.info('Event: %s (reason=%s)', eo.message,
-                                     eo.reason)
+                    # self.logger.info('Event: %s (reason=%s)', eo.message,
+                    #                  eo.reason)
 
                     for uuid in self.components:
                         comp = self.components[uuid]
@@ -126,10 +126,10 @@ def _run_event_watcher(self):
                                 elif comp._state == 'starting':
                                     # wait for BackoffLimitExceeded event
                                     continue
-                            else:
-                                self.logger.info('Reason \'%s\' not handled '
-                                                 'for kubernetes simulator',
-                                                 eo.reason)
+                            # else:
+                                # self.logger.info('Reason \'%s\' not handled '
+                                #                  'for kubernetes simulator',
+                                #                  eo.reason)
 
             except ProtocolError:
                 self.logger.warn('Connection to kubernetes broken, \
diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 433346f..947a5e3 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -48,6 +48,8 @@ def create(self, payload):
         sim_name = payload.get('name', 'Kubernetes Simulator')
         jobname = params.get('jobname', 'noname')
         adls = params.get('activeDeadlineSeconds', 3600)
+        if type(adls) is str:
+            adls = int(adls)
         image = params.get('image')
         name = params.get('name')
         uuid = params.get('uuid')
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 9d57016..d5cde7f 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -61,7 +61,7 @@ def _owner(self):
                 kind='Pod',
                 name=self.manager.my_pod_name,
                 uid=self.manager.my_pod_uid,
-                api_version='v1'
+                api_version='batch/v1'
             )
 
         return None
@@ -162,18 +162,20 @@ def _delete_job(self):
             return
 
         b = k8s.client.BatchV1Api()
-        c = k8s.client.CoreV1Api()
+        # c = k8s.client.CoreV1Api()
         body = k8s.client.V1DeleteOptions(propagation_policy='Background')
 
+        # print(self.job)
+
         try:
             self.job = b.delete_namespaced_job(
                 namespace=self.manager.namespace,
                 name=self.job.metadata.name,
                 body=body)
-            c.delete_namespaced_config_map(
-                namespace=self.manager.namespace,
-                name=self.cm_name,
-                body=body)
+            # c.delete_namespaced_config_map(
+            #    namespace=self.manager.namespace,
+            #    name=self.cm_name,
+            #    body=body)
         except k8s.client.exceptions.ApiException as e:
             raise SimulationException(self, 'Kubernetes API error',
                                       error=str(e))

From 4ff001411e1d0557337c9ac5ce2ce5d403a982d6 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 7 Dec 2021 13:01:38 +0100
Subject: [PATCH 48/89] allow UUID of default generic manager to be configured
 via configuration file

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/config.py     | 4 +++-
 villas/controller/controller.py | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/villas/controller/config.py b/villas/controller/config.py
index 6182edc..cf0f50b 100644
--- a/villas/controller/config.py
+++ b/villas/controller/config.py
@@ -3,6 +3,7 @@
 import logging
 import os
 import dotmap
+import uuid
 from typing import List
 
 from os import getcwd
@@ -41,7 +42,8 @@ class Config:
         },
         'components': [],
         # 'workdir': '/var/villas/controller/simulators/'
-        'workdir': os.getcwd()
+        'workdir': os.getcwd(),
+        'uuid': str(uuid.uuid4())
     }
 
     DEFAULT_PATHS = xdg_config_dirs() + [
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index 97fd0ea..5c915b2 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -57,7 +57,8 @@ def add_managers(self):
                 type='generic',
                 category='manager',
                 name='Generic Manager',
-                location=socket.gethostname()
+                location=socket.gethostname(),
+                uuid=self.config.uuid
             )
 
             mgr.mixin = self

From f82f994bfb0c37283c20b698a8f2887f3940aa24 Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 7 Dec 2021 15:33:02 +0100
Subject: [PATCH 49/89] fix villas-node manager

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .../components/managers/villas_node.py        | 18 ++--
 .../components/managers/villas_relay.py       | 84 ++++++++-----------
 2 files changed, 45 insertions(+), 57 deletions(-)

diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index 12a272f..5e7ddc6 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -23,17 +23,16 @@ def __init__(self, **args):
 
         super().__init__(**args)
 
-        self.thread_stop = threading.Event()
-        self.thread = threading.Thread(target=self.reconcile_periodically)
-        self.thread.start()
-
     def reconcile_periodically(self):
         while not self.thread_stop.wait(2):
             self.reconcile()
 
     def reconcile(self):
         try:
-            for node in self.node.nodes:
+            self._status = self.node.status
+            self._nodes = self.node.nodes
+
+            for node in self._nodes:
                 self.logger.debug('Found node %s on gateway: %s',
                                   node['name'], node)
 
@@ -58,7 +57,7 @@ def reconcile(self):
     def status(self):
         status = super().status
 
-        status['status']['villas_none_version'] = self._version
+        status['status']['villas_node_version'] = self._status.get('version')
 
         return status
 
@@ -66,10 +65,9 @@ def on_ready(self):
         if self.autostart and not self.node.is_running():
             self.start()
 
-        try:
-            self._status = self.node.status
-        except Exception:
-            self.change_to_error('VILLASnode not installed')
+        self.thread_stop = threading.Event()
+        self.thread = threading.Thread(target=self.reconcile_periodically)
+        self.thread.start()
 
         super().on_ready()
 
diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index 9530081..6c43649 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -22,14 +22,6 @@ def __init__(self, **args):
 
         self.properties['api_url'] = self.api_url_external
 
-        self.thread_stop = threading.Event()
-        self.thread = threading.Thread(target=self.reconcile_periodically)
-        self.thread.start()
-
-    def reconcile_periodically(self):
-        while not self.thread_stop.wait(2):
-            self.reconcile()
-
     def get_uuid(self):
         try:
             r = requests.get(self.api_url)
@@ -50,54 +42,56 @@ def get_status(self):
 
             return None
 
+    def reconcile_periodically(self):
+        while not self.thread_stop.wait(2):
+            self.reconcile()
+
     def reconcile(self):
-        status = self.get_status()
-        if status is None:
-            return
+        try:
+            self._status = self.get_status()
 
-        if self._state == 'error':
-            self.change_state('idle')
+            active_sessions = self._status['sessions']
+            active_uuids = {session['uuid'] for session in active_sessions}
+            existing_uuids = set(self.components.keys())
 
-        self._status = status
-        self._version = self._status['version']
+            # Add new sessions and update existing ones
+            for session in active_sessions:
+                uuid = session['uuid']
 
-        active_sessions = self._status['sessions']
-        active_uuids = {session['uuid'] for session in active_sessions}
-        existing_uuids = set(self.components.keys())
+                if uuid in self.components:
+                    comp = self.components[uuid]
+                else:
+                    comp = VILLASrelayGateway(self, session)
+                    self.add_component(comp)
 
-        # Add new sessions and update existing ones
-        for session in active_sessions:
-            uuid = session['uuid']
+                comp.change_state('running')
 
-            if uuid in self.components:
+            # Find vanished sessions
+            for uuid in existing_uuids - active_uuids:
                 comp = self.components[uuid]
-            else:
-                comp = VILLASrelayGateway(self, session)
-                self.add_component(comp)
-
-            comp.change_state('running')
 
-        # Find vanished sessions
-        for uuid in existing_uuids - active_uuids:
-            comp = self.components[uuid]
+                comp.change_state('stopped')
 
-            comp.change_state('stopped')
+                # We dont remove the components here
+                # So that they dont get removed from the backend
+                # and get recreated with the same UUID later
+                # self.remove_component(comp)
 
-            # We dont remove the components here
-            # So that they dont get removed from the backend
-            # and get recreated with the same UUID later
-            # self.remove_component(comp)
+            if len(self.components) > 0:
+                self.change_state('running')
+            else:
+                self.change_state('paused')
 
-        if len(self.components) > 0:
-            self.change_state('running')
-        else:
-            self.change_state('paused')
+        except Exception as e:
+            self.change_to_error('failed to reconcile',
+                                 exception=str(e),
+                                 args=e.args)
 
     @property
     def status(self):
         status = super().status
 
-        status['status']['villas_relay_version'] = self._version
+        status['status']['villas_relay_version'] = self._status.get('version')
 
         return status
 
@@ -111,12 +105,8 @@ def on_ready(self):
         if self.autostart:
             os.system('villas-relay')
 
-        try:
-            status = self.get_status()
-
-            self._version = status['version']
-
-        except Exception:
-            self.change_to_error('Failed to contact VILLASrelay')
+        self.thread_stop = threading.Event()
+        self.thread = threading.Thread(target=self.reconcile_periodically)
+        self.thread.start()
 
         super().on_ready()

From ff5ca020366bcc1d1de77dade27ee0e12f77444c Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 7 Dec 2021 16:42:31 +0100
Subject: [PATCH 50/89] api: make main request handler also available without
 trailing slash

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/api.py b/villas/controller/api.py
index 31168d0..e6a684e 100644
--- a/villas/controller/api.py
+++ b/villas/controller/api.py
@@ -50,7 +50,7 @@ def handlers(self):
         }
 
         return [
-            (BASE + r'/', MainRequestHandler, args),
+            (BASE + r'/?', MainRequestHandler, args),
             (BASE + r'/health', HealthRequestHandler, args),
             (BASE + r'/component/('+REGEX_UUID+r')', ComponentRequestHandler, args)  # noqa E501
         ]

From e2c6631d5e4788be5ca94045c8b6a26e116dd2cf Mon Sep 17 00:00:00 2001
From: Steffen Vogel <post@steffenvogel.de>
Date: Tue, 7 Dec 2021 20:03:08 +0100
Subject: [PATCH 51/89] more fixes for relay and node managers

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                        | 1 -
 villas/controller/components/managers/villas_node.py  | 5 +++--
 villas/controller/components/managers/villas_relay.py | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 3885402..9a0a69a 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -53,7 +53,6 @@ def __init__(self, **props):
 
     def on_ready(self):
         self.publish_status_thread.start()
-        pass
 
     def on_shutdown(self):
         if self.publish_status_thread.is_alive():
diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index 5e7ddc6..825f21a 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -15,6 +15,9 @@ def __init__(self, **args):
 
         args['api_url'] = self.api_url
 
+        self.thread_stop = threading.Event()
+        self.thread = threading.Thread(target=self.reconcile_periodically)
+
         self.node = Node(**args)
 
         self._status = self.node.status
@@ -65,8 +68,6 @@ def on_ready(self):
         if self.autostart and not self.node.is_running():
             self.start()
 
-        self.thread_stop = threading.Event()
-        self.thread = threading.Thread(target=self.reconcile_periodically)
         self.thread.start()
 
         super().on_ready()
diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index 6c43649..6873230 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -12,7 +12,9 @@ def __init__(self, **args):
         self.autostart = args.get('autostart', False)
         self.api_url = args.get('api_url', 'http://localhost:8088') + '/api/v1'
         self.api_url_external = args.get('api_url_external', self.api_url)
-        self._version = '-1'
+
+        self.thread_stop = threading.Event()
+        self.thread = threading.Thread(target=self.reconcile_periodically)
 
         uuid = self.get_uuid()
         if uuid is not None:
@@ -77,7 +79,7 @@ def reconcile(self):
                 # and get recreated with the same UUID later
                 # self.remove_component(comp)
 
-            if len(self.components) > 0:
+            if len(active_sessions) > 0:
                 self.change_state('running')
             else:
                 self.change_state('paused')
@@ -105,8 +107,6 @@ def on_ready(self):
         if self.autostart:
             os.system('villas-relay')
 
-        self.thread_stop = threading.Event()
-        self.thread = threading.Thread(target=self.reconcile_periodically)
         self.thread.start()
 
         super().on_ready()

From ea5497db556b57dd7bca19948c3031de3ac2036b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 8 Dec 2021 14:56:24 +0000
Subject: [PATCH 52/89] cleanup

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 etc/params_simplekub.yaml                           |  7 -------
 villas/controller/component.py                      |  5 +----
 villas/controller/components/manager.py             |  2 --
 villas/controller/components/managers/kubernetes.py | 12 ++++++------
 4 files changed, 7 insertions(+), 19 deletions(-)
 delete mode 100644 etc/params_simplekub.yaml

diff --git a/etc/params_simplekub.yaml b/etc/params_simplekub.yaml
deleted file mode 100644
index a88fd29..0000000
--- a/etc/params_simplekub.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-name: "ksim created by simple manager"
-uuid: c555aaaa-6af6-11eb-beee-7fa268050224
-image: dpsimrwth/slew-villas:latest
-jobname: "test job"
-activeDeadlineSeconds: 60
-containername: "test container"
diff --git a/villas/controller/component.py b/villas/controller/component.py
index 9a0a69a..5a541c3 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -41,12 +41,9 @@ def __init__(self, **props):
         self.logger = logging.getLogger(
             f'villas.controller.{self.category}.{self.type}:{self.uuid}')
 
-        self.logger.info("init component:")
-        self.logger.info(self.name)
-
         self._schema = self.load_schema()
 
-        self.publish_status_interval = 15
+        self.publish_status_interval = 30
         self.publish_status_thread_stop = threading.Event()
         self.publish_status_thread = threading.Thread(
             target=self.publish_status_periodically)
diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index 1985046..b159ea2 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -41,8 +41,6 @@ def from_dict(dict):
             raise Exception(f'Unknown type: {type}')
 
     def add_component(self, comp):
-        print(self.name)
-        print(comp)
         if comp.uuid in self.mixin.components:
             #self.logger.error('UUID %s already exists, not added', comp.uuid)
             #return
diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 849484b..0564f2c 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -96,8 +96,8 @@ def _run_event_watcher(self):
 
                     eo = e.get('object')
 
-                    # self.logger.info('Event: %s (reason=%s)', eo.message,
-                    #                  eo.reason)
+                    self.logger.info('Event: %s (reason=%s)', eo.message,
+                                     eo.reason)
 
                     for uuid in self.components:
                         comp = self.components[uuid]
@@ -126,10 +126,10 @@ def _run_event_watcher(self):
                                 elif comp._state == 'starting':
                                     # wait for BackoffLimitExceeded event
                                     continue
-                            # else:
-                                # self.logger.info('Reason \'%s\' not handled '
-                                #                  'for kubernetes simulator',
-                                #                  eo.reason)
+                            else:
+                                self.logger.info('Reason \'%s\' not handled '
+                                                 'for kubernetes simulator',
+                                                 eo.reason)
 
             except ProtocolError:
                 self.logger.warn('Connection to kubernetes broken, \

From 791af640bd4a55d413e4c1ad4304cf1d43926c7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 17 Dec 2021 12:19:36 +0000
Subject: [PATCH 53/89] send status update while resetting to improve user
 experience

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/simulators/kubernetes.py | 2 ++
 villas/controller/controller.py                       | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index d5cde7f..37ad90e 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -242,6 +242,8 @@ def resume(self, payload):
         self.change_state('running')
 
     def reset(self, payload):
+        self.change_state('resetting', True)
+        self.mixin.drain_publish_queue()
         self._delete_job()
         super().reset(payload)
 
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index 5c915b2..95f058a 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -72,7 +72,7 @@ def add_managers(self):
     def publish(self, body, **kwargs):
         self.publish_queue.put((body, kwargs))
 
-    def _drain_publish_queue(self):
+    def drain_publish_queue(self):
         try:
             while msg := self.publish_queue.get(False):
                 body = msg[0]
@@ -89,7 +89,7 @@ def _drain_publish_queue(self):
 
     def on_iteration(self):
         # Drain publish queue
-        self._drain_publish_queue()
+        self.drain_publish_queue()
 
         # Update components
         added = self.components.keys() - self.active_components.keys()

From 1c962d8e97be0f7804a6a3ca0a46f9b1642f7572 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 17 Dec 2021 12:39:33 +0000
Subject: [PATCH 54/89] publish first status update immediately after creating
 component

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 5a541c3..2bd532a 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -291,7 +291,8 @@ def publish_status(self):
         self.mixin.publish(self.status, headers=self.headers)
 
     def publish_status_periodically(self):
-        self.logger.info('Start state publish thread')
+        self.logger.info('Start state publish thread, initial status: %s', self.status)
+        self.publish_status() # publish the first update immediately
 
         while not self.publish_status_thread_stop.wait(
           self.publish_status_interval):

From 7f0eac09af7e7520ad2a684b4ce596e1bd1df046 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 17 Dec 2021 12:42:43 +0000
Subject: [PATCH 55/89] read namespace for kubernetes jobs from ENV

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 0564f2c..57f0b48 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -36,9 +36,12 @@ def __init__(self, **args):
         else:
             k8s.config.load_incluster_config()
 
-        self.namespace = args.get('namespace', 'villas-controller')
+        self.namespace = os.environ.get('NAMESPACE')
+        if self.namespace:
+            self.namespace = self.namespace + '-controller'
+        else:
+            self.namespace = 'villas-controller'
 
-        self.my_namespace = os.environ.get('NAMESPACE')
         self.my_pod_name = os.environ.get('POD_NAME')
         self.my_pod_uid = os.environ.get('POD_UID')
 

From 21af994c5071a49930a83935c9889b28a89f8d68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 12 Jan 2022 10:09:05 +0000
Subject: [PATCH 56/89] fix formatting

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 2bd532a..19456a2 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -291,8 +291,9 @@ def publish_status(self):
         self.mixin.publish(self.status, headers=self.headers)
 
     def publish_status_periodically(self):
-        self.logger.info('Start state publish thread, initial status: %s', self.status)
-        self.publish_status() # publish the first update immediately
+        self.logger.info('Start state publish thread, initial status: %s',
+                         self.status)
+        self.publish_status()  # publish the first update immediately
 
         while not self.publish_status_thread_stop.wait(
           self.publish_status_interval):

From 63c3b22f6cdb2e5c8e2e534ccc3d64638a9a354d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 11 Feb 2022 18:28:04 +0000
Subject: [PATCH 57/89] cleanup

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/component.py                           | 5 +----
 .../controller/components/managers/kubernetes_simple.py  | 4 ----
 villas/controller/components/simulators/kubernetes.py    | 9 +--------
 3 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/villas/controller/component.py b/villas/controller/component.py
index 19456a2..6055b2e 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -66,8 +66,6 @@ def set_mixin(self, mixin):
         self.workdir = os.path.join(self.mixin.config.workdir, str(self.uuid))
 
     def get_consumer(self, channel):
-        self.logger.info(channel)
-        self.logger.info(self.headers)
         self.channel = channel
 
         return kombu.Consumer(
@@ -160,8 +158,7 @@ def status(self):
         }
 
     def on_message(self, message):
-        # self.logger.info('my uuid: %s', self.uuid)
-        # self.logger.info('Received message: %s', message.payload)
+        self.logger.debug('Received message: %s', message.payload)
 
         if 'action' in message.payload:
             try:
diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 947a5e3..88355ece 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -40,10 +40,8 @@ class KubernetesManagerSimple(KubernetesManager):
 
     def __init__(self, **args):
         super().__init__(**args)
-        self.logger.info("init of KubernetesManagerSimple")
 
     def create(self, payload):
-        self.logger.info(payload)
         params = payload.get('parameters', {})
         sim_name = payload.get('name', 'Kubernetes Simulator')
         jobname = params.get('jobname', 'noname')
@@ -74,7 +72,5 @@ def create(self, payload):
         if uuid:
             parameters['uuid'] = uuid
 
-        self.logger.info(parameters)
-
         comp = KubernetesJob(self, **parameters)
         self.add_component(comp)
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 37ad90e..1de6f4d 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -61,7 +61,7 @@ def _owner(self):
                 kind='Pod',
                 name=self.manager.my_pod_name,
                 uid=self.manager.my_pod_uid,
-                api_version='batch/v1'
+                api_version='v1'
             )
 
         return None
@@ -162,20 +162,13 @@ def _delete_job(self):
             return
 
         b = k8s.client.BatchV1Api()
-        # c = k8s.client.CoreV1Api()
         body = k8s.client.V1DeleteOptions(propagation_policy='Background')
 
-        # print(self.job)
-
         try:
             self.job = b.delete_namespaced_job(
                 namespace=self.manager.namespace,
                 name=self.job.metadata.name,
                 body=body)
-            # c.delete_namespaced_config_map(
-            #    namespace=self.manager.namespace,
-            #    name=self.cm_name,
-            #    body=body)
         except k8s.client.exceptions.ApiException as e:
             raise SimulationException(self, 'Kubernetes API error',
                                       error=str(e))

From d3779d19a82c58f8c07b383de2941334144027b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 3 Mar 2022 13:56:09 +0000
Subject: [PATCH 58/89] fix 'invalid UUID length: 0' error

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .../components/managers/kubernetes_simple.py       |  4 +++-
 .../controller/components/simulators/kubernetes.py | 14 +++++++-------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 88355ece..3347a51 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -24,7 +24,7 @@
                                 'image': '',
                                 'name': 'jobcontainer',
                                 'securityContext': {
-                                    'privileged': False
+                                    'privileged': True
                                 }
                             }
                         ]
@@ -51,6 +51,8 @@ def create(self, payload):
         image = params.get('image')
         name = params.get('name')
         uuid = params.get('uuid')
+        self.logger.info('uuid:')
+        self.logger.info(uuid)
 
         if image is None:
             self.logger.error('No image given, will try super.create')
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 1de6f4d..66253aa 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -56,13 +56,13 @@ def schema(self):
             }
 
     def _owner(self):
-        if self.manager.my_pod_name and self.manager.my_pod_uid:
-            return k8s.client.V1OwnerReference(
-                kind='Pod',
-                name=self.manager.my_pod_name,
-                uid=self.manager.my_pod_uid,
-                api_version='v1'
-            )
+        # if self.manager.my_pod_name and self.manager.my_pod_uid:
+        #    return k8s.client.V1OwnerReference(
+        #        kind='Pod',
+        #        name=self.manager.my_pod_name,
+        #        uid=self.manager.my_pod_uid,
+        #        api_version='v1'
+        #    )
 
         return None
 

From 2007bf89942e799d12c9ef8c2ac397f8a24a4790 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 12 Jan 2022 11:02:44 +0100
Subject: [PATCH 59/89] fix error handling relay/node

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/villas_node.py  | 2 +-
 villas/controller/components/managers/villas_relay.py | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/villas/controller/components/managers/villas_node.py b/villas/controller/components/managers/villas_node.py
index 825f21a..3f25b45 100644
--- a/villas/controller/components/managers/villas_node.py
+++ b/villas/controller/components/managers/villas_node.py
@@ -54,7 +54,7 @@ def reconcile(self):
         except Exception as e:
             self.change_to_error('failed to reconcile',
                                  exception=str(e),
-                                 args=e.args)
+                                 args=str(e.args))
 
     @property
     def status(self):
diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index 6873230..4f3bb40 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -93,7 +93,13 @@ def reconcile(self):
     def status(self):
         status = super().status
 
-        status['status']['villas_relay_version'] = self._status.get('version')
+        try:
+            version = self._status.get('version')
+            status['status']['villas_relay_version'] = version
+        except Exception as e:
+            self.change_to_error('failed to get version from VILLASrelay',
+                                 exception=str(e),
+                                 args=str(e.args))
 
         return status
 

From 85d83e04e7909aaa7cb6671a4150850a16b38dfb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 4 Mar 2022 12:22:47 +0000
Subject: [PATCH 60/89] relay fix

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/villas_relay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/villas_relay.py b/villas/controller/components/managers/villas_relay.py
index 4f3bb40..5ba69b4 100644
--- a/villas/controller/components/managers/villas_relay.py
+++ b/villas/controller/components/managers/villas_relay.py
@@ -94,7 +94,7 @@ def status(self):
         status = super().status
 
         try:
-            version = self._status.get('version')
+            version = status.get('version')
             status['status']['villas_relay_version'] = version
         except Exception as e:
             self.change_to_error('failed to get version from VILLASrelay',

From 3d7791b3a42b24df31fa520f73e56114cb59c04f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?=
 <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 9 Mar 2022 11:32:31 +0000
Subject: [PATCH 61/89] container settings

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/managers/kubernetes_simple.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 3347a51..1fb27a7 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -22,6 +22,7 @@
                         'containers': [
                             {
                                 'image': '',
+                                'imagePullPolicy': 'Always',
                                 'name': 'jobcontainer',
                                 'securityContext': {
                                     'privileged': True

From 723f052f418ba4b8e712dec7cfa4b1faa45e8cae Mon Sep 17 00:00:00 2001
From: Iris Koester <iris.m.koester@gmail.com>
Date: Mon, 14 Aug 2023 18:46:35 +0200
Subject: [PATCH 62/89] update flake repo

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b6287ea..a7dc495 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,7 +14,7 @@ repos:
   - id: check-symlinks
   - id: end-of-file-fixer
   - id: trailing-whitespace
-- repo: https://gitlab.com/pycqa/flake8
+- repo: https://github.com/pycqa/flake8
   rev: 3.8.4
   hooks:
   - id: flake8

From 6c6368f1b90594bb1f717e96be94192f776df5c0 Mon Sep 17 00:00:00 2001
From: Iris Koester <iris.m.koester@gmail.com>
Date: Mon, 14 Aug 2023 18:53:21 +0200
Subject: [PATCH 63/89] formatting

Signed-off-by: Iris Koester <iris.m.koester@gmail.com>
---
 villas/controller/components/manager.py               | 9 ++++-----
 villas/controller/components/simulators/kubernetes.py | 4 ++--
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/villas/controller/components/manager.py b/villas/controller/components/manager.py
index b159ea2..16db32c 100644
--- a/villas/controller/components/manager.py
+++ b/villas/controller/components/manager.py
@@ -42,10 +42,9 @@ def from_dict(dict):
 
     def add_component(self, comp):
         if comp.uuid in self.mixin.components:
-            #self.logger.error('UUID %s already exists, not added', comp.uuid)
-            #return
             existing_comp = self.mixin.components[comp.uuid]
-
+#           self.logger.error('UUID %s already exists, not added', comp.uuid)
+#           return
             raise SimulationException(self, 'Component with same UUID ' +
                                       'already exists!',
                                       component=existing_comp)
@@ -67,9 +66,9 @@ def remove_component(self, comp):
 
     def run_action(self, action, payload):
         if action == 'create':
-            #print(message.payload)
-            #self.create(message)
             self.create(payload)
+#           print(message.payload)
+#           self.create(message)
         elif action == 'delete':
             self.delete(payload)
         else:
diff --git a/villas/controller/components/simulators/kubernetes.py b/villas/controller/components/simulators/kubernetes.py
index 66253aa..9a6373e 100644
--- a/villas/controller/components/simulators/kubernetes.py
+++ b/villas/controller/components/simulators/kubernetes.py
@@ -2,8 +2,8 @@
 from typing import TYPE_CHECKING
 import json
 import signal
-from copy import deepcopy
-import collections
+# from copy import deepcopy
+# import collections
 import time
 
 import kubernetes as k8s

From 76ea53c839026f7869578f3dfac70714c7da4c5f Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Mon, 17 Mar 2025 15:53:05 +0100
Subject: [PATCH 64/89] remove namespace checking to get rid of ClusterRole
 neccessity

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .../components/managers/kubernetes.py           | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 57f0b48..2ff668b 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -37,31 +37,14 @@ def __init__(self, **args):
             k8s.config.load_incluster_config()
 
         self.namespace = os.environ.get('NAMESPACE')
-        if self.namespace:
-            self.namespace = self.namespace + '-controller'
-        else:
-            self.namespace = 'villas-controller'
-
         self.my_pod_name = os.environ.get('POD_NAME')
         self.my_pod_uid = os.environ.get('POD_UID')
 
-        self._check_namespace(self.namespace)
-
         # self.pod_watcher_thread.start()
         # self.job_watcher_thread.start()
         self.event_watcher_thread.setDaemon(True)
         self.event_watcher_thread.start()
 
-    def _check_namespace(self, ns):
-        c = k8s.client.CoreV1Api()
-
-        namespaces = c.list_namespace()
-        for namespace in namespaces.items:
-            if namespace.metadata.name == ns:
-                return
-
-        raise RuntimeError(f'Namespace {ns} does not exist')
-
     def _run_pod_watcher(self):
         w = k8s.watch.Watch()
         c = k8s.client.CoreV1Api()

From b39fb191a519a61f58ccc5b1b7ea3170d3862cac Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 19 Mar 2025 07:50:58 +0100
Subject: [PATCH 65/89] fix pipeline

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 Dockerfile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Dockerfile b/Dockerfile
index 69d0784..1feab7f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,7 +4,9 @@ COPY requirements.txt /tmp
 RUN pip3 install -r /tmp/requirements.txt
 
 COPY . /tmp/controller
+
 RUN cd /tmp/controller && \
+	sync && \
 	python3 setup.py sdist && \
 	pip3 install dist/*.tar.gz && \
 	rm -rf /tmp/controller

From e47a5e13dfa6b025dc439bf3fed37d7cc9ba38c5 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 20 Mar 2025 09:09:09 +0100
Subject: [PATCH 66/89] make securitycontext configurable through schema

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 villas/controller/components/managers/kubernetes_simple.py   | 4 +++-
 .../controller/schemas/manager/kubernetes-simple/create.yaml | 5 +++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index 1fb27a7..e936fc9 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -25,7 +25,7 @@
                                 'imagePullPolicy': 'Always',
                                 'name': 'jobcontainer',
                                 'securityContext': {
-                                    'privileged': True
+                                    'privileged': False
                                 }
                             }
                         ]
@@ -51,6 +51,7 @@ def create(self, payload):
             adls = int(adls)
         image = params.get('image')
         name = params.get('name')
+        privileged = params.get('privileged', False)
         uuid = params.get('uuid')
         self.logger.info('uuid:')
         self.logger.info(uuid)
@@ -66,6 +67,7 @@ def create(self, payload):
         job['metadata']['name'] = jobname
         job['spec']['activeDeadlineSeconds'] = adls
         job['spec']['template']['spec']['containers'][0]['image'] = image
+        job['spec']['template']['spec']['containers'][0]['securityContext']['privileged'] = privileged
 
         parameters['job'] = job
 
diff --git a/villas/controller/schemas/manager/kubernetes-simple/create.yaml b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
index 9747f77..474a9a4 100644
--- a/villas/controller/schemas/manager/kubernetes-simple/create.yaml
+++ b/villas/controller/schemas/manager/kubernetes-simple/create.yaml
@@ -26,3 +26,8 @@ properties:
     type: string
     title: Image
     default: perl
+  privileged:
+    type: boolean
+    title: Privileged
+    default: false
+    description: 'WARNING: If true, the container has root privileges on the host'

From fe1580b9bd6780632099d411045bf056b9b43c95 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 20 Mar 2025 10:08:27 +0100
Subject: [PATCH 67/89] Sync before executing setup.py so that controller
 module will be found

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 1feab7f..beca7bc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,9 +4,9 @@ COPY requirements.txt /tmp
 RUN pip3 install -r /tmp/requirements.txt
 
 COPY . /tmp/controller
+RUN sync
 
 RUN cd /tmp/controller && \
-	sync && \
 	python3 setup.py sdist && \
 	pip3 install dist/*.tar.gz && \
 	rm -rf /tmp/controller

From 1174fdebdc379387e98047ec40140067c4d2e944 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 20 Mar 2025 17:48:19 +0100
Subject: [PATCH 68/89] handle ModuleNotFoundError

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 Dockerfile |  2 --
 setup.py   | 10 +++++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index beca7bc..69d0784 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,8 +4,6 @@ COPY requirements.txt /tmp
 RUN pip3 install -r /tmp/requirements.txt
 
 COPY . /tmp/controller
-RUN sync
-
 RUN cd /tmp/controller && \
 	python3 setup.py sdist && \
 	pip3 install dist/*.tar.gz && \
diff --git a/setup.py b/setup.py
index bf1cd1b..1064e7d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,15 @@
 from setuptools import setup, find_namespace_packages
 from glob import glob
 
-from villas.controller import __version__ as version
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+try:
+    from villas.controller import __version__ as version
+except ModuleNotFoundError:
+    version = "0.0.0"
 
 with open('README.md') as f:
     long_description = f.read()

From 6bcc818c9d7a75118acfd87cc484d1c0f8fa5b72 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 25 Mar 2025 10:06:45 +0100
Subject: [PATCH 69/89] use pip to run pipeline

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d0aafc1..599995c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,7 +17,7 @@ build:precommit:
   stage: build
   image: python:3.11
   before_script:
-  - pip3 install -r requirements-dev.txt
+  - python -m pip install -r requirements-dev.txt
   script:
   - pre-commit run --all-files
   tags:
@@ -29,8 +29,8 @@ build:test:
   services:
   - rabbitmq:latest
   before_script:
-  - pip3 install -r requirements.txt
-  - pip3 install -r requirements-dev.txt
+  - python -m pip install -r requirements.txt
+  - python -m pip install -r requirements-dev.txt
   script:
   - pytest -v
   tags:

From 67a8761c0e3c1c609b4342732e057fc6a2a8588f Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 26 Mar 2025 08:23:26 +0100
Subject: [PATCH 70/89] changed python docker image due to pip3 command not
 found in precommit and test stages

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 599995c..0f8c476 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,9 +15,9 @@ build:docker:
 
 build:precommit:
   stage: build
-  image: python:3.11
+  image: python:3.11-slim
   before_script:
-  - python -m pip install -r requirements-dev.txt
+  - pip3 install -r requirements-dev.txt
   script:
   - pre-commit run --all-files
   tags:
@@ -25,12 +25,12 @@ build:precommit:
 
 build:test:
   stage: build
-  image: python:3.11
+  image: python:3.11-slim
   services:
   - rabbitmq:latest
   before_script:
-  - python -m pip install -r requirements.txt
-  - python -m pip install -r requirements-dev.txt
+  - pip3 install -r requirements.txt
+  - pip3 install -r requirements-dev.txt
   script:
   - pytest -v
   tags:

From 7f5242d8819d4f2945e78e016eae1f33c1c10d7f Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 26 Mar 2025 08:47:40 +0100
Subject: [PATCH 71/89] changed back image version and changed CI tag

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0f8c476..fbe9ac2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,17 +15,17 @@ build:docker:
 
 build:precommit:
   stage: build
-  image: python:3.11-slim
+  image: python:3.11
   before_script:
   - pip3 install -r requirements-dev.txt
   script:
   - pre-commit run --all-files
   tags:
-    - docker
+    - k8s
 
 build:test:
   stage: build
-  image: python:3.11-slim
+  image: python:3.11
   services:
   - rabbitmq:latest
   before_script:
@@ -34,7 +34,7 @@ build:test:
   script:
   - pytest -v
   tags:
-  - docker
+  - k8s
 
 build:dist:
   stage: build

From e06754c349f609ed808ec7ed1760d84a4b17cb84 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 26 Mar 2025 09:32:21 +0100
Subject: [PATCH 72/89] update schema checking

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .pre-commit-config.yaml                                    | 4 ++--
 villas/controller/components/managers/kubernetes_simple.py | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a669ed8..fed6fea 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -26,10 +26,10 @@ repos:
     language: python
     files: ^villas/controller/schemas/.*\.yaml$
     types: [yaml]
-    args: ["--schemafile", "https://json-schema.org/draft/2020-12/schema"]
+    args: ["--verbose", "--schemafile", "https://json-schema.org/draft/2020-12/schema"]
   - id: check-jsonschema
     name: "Check OpenAPI doc"
     language: python
     files: ^doc/openapi.yaml$
     types: [yaml]
-    args: ["--schemafile", "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json"]
+    args: ["--schemafile", "https://github.com/OAI/OpenAPI-Specification/tree/main/_archive_/schemas/v3.0/schema.yaml"]
diff --git a/villas/controller/components/managers/kubernetes_simple.py b/villas/controller/components/managers/kubernetes_simple.py
index e936fc9..f8e706d 100644
--- a/villas/controller/components/managers/kubernetes_simple.py
+++ b/villas/controller/components/managers/kubernetes_simple.py
@@ -66,8 +66,9 @@ def create(self, payload):
         job = parameters['properties']['job']
         job['metadata']['name'] = jobname
         job['spec']['activeDeadlineSeconds'] = adls
-        job['spec']['template']['spec']['containers'][0]['image'] = image
-        job['spec']['template']['spec']['containers'][0]['securityContext']['privileged'] = privileged
+        job_container = job['spec']['template']['spec']['containers'][0]
+        job_container['image'] = image
+        job_container['securityContext']['privileged'] = privileged
 
         parameters['job'] = job
 

From d0c70d1fef55100a3434fbd76dff4bc6be1e8304 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 26 Mar 2025 09:43:59 +0100
Subject: [PATCH 73/89] commented schema checking to run CI, needs to get
 revised

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .pre-commit-config.yaml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index fed6fea..8551e05 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,18 +18,18 @@ repos:
   rev: 6.0.0
   hooks:
   - id: flake8
-- repo: https://github.com/sirosen/check-jsonschema
-  rev: 0.22.0
-  hooks:
-  - id: check-jsonschema
-    name: "Check schemas"
-    language: python
-    files: ^villas/controller/schemas/.*\.yaml$
-    types: [yaml]
-    args: ["--verbose", "--schemafile", "https://json-schema.org/draft/2020-12/schema"]
-  - id: check-jsonschema
-    name: "Check OpenAPI doc"
-    language: python
-    files: ^doc/openapi.yaml$
-    types: [yaml]
-    args: ["--schemafile", "https://github.com/OAI/OpenAPI-Specification/tree/main/_archive_/schemas/v3.0/schema.yaml"]
+#- repo: https://github.com/sirosen/check-jsonschema
+#  rev: 0.22.0
+#  hooks:
+#  - id: check-jsonschema
+#    name: "Check schemas"
+#    language: python
+#    files: ^villas/controller/schemas/.*\.yaml$
+#    types: [yaml]
+#    args: ["--schemafile", "https://json-schema.org/draft/2020-12/schema"]
+#  - id: check-jsonschema
+#    name: "Check OpenAPI doc"
+#    language: python
+#    files: ^doc/openapi.yaml$
+#    types: [yaml]
+#    args: ["--schemafile", "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json"]

From 2ccadc59a88d867c04efe8a4a6f233ae9666db75 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 26 Mar 2025 16:34:49 +0100
Subject: [PATCH 74/89] debug test fail

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fbe9ac2..e787e3b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,6 +29,8 @@ build:test:
   services:
   - rabbitmq:latest
   before_script:
+  - pwd
+  - ls -la
   - pip3 install -r requirements.txt
   - pip3 install -r requirements-dev.txt
   script:

From e4dd2b3a0729d4b0aed3522b982847ff449feb28 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 28 Mar 2025 14:46:53 +0100
Subject: [PATCH 75/89] add current workind directory to PATH

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e787e3b..8ab164b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,11 +29,10 @@ build:test:
   services:
   - rabbitmq:latest
   before_script:
-  - pwd
-  - ls -la
   - pip3 install -r requirements.txt
   - pip3 install -r requirements-dev.txt
   script:
+  - export PYTHONPATH="$PYTHONPATH:$(pwd)"
   - pytest -v
   tags:
   - k8s

From 555fe20e6fb48d9eff6ff1c9d19bad1ca842d293 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 28 Mar 2025 14:54:42 +0100
Subject: [PATCH 76/89] added init file to villas folder

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 villas/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 villas/__init__.py

diff --git a/villas/__init__.py b/villas/__init__.py
new file mode 100644
index 0000000..e69de29

From 5ac9fa1286871e59b055fcafb6f7acc0de787904 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Fri, 28 Mar 2025 15:02:43 +0100
Subject: [PATCH 77/89] remove changes made during pipeline debug/fix

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml |  5 -----
 setup.py       | 10 +---------
 2 files changed, 1 insertion(+), 14 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ab164b..dc583b0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,8 +20,6 @@ build:precommit:
   - pip3 install -r requirements-dev.txt
   script:
   - pre-commit run --all-files
-  tags:
-    - k8s
 
 build:test:
   stage: build
@@ -32,10 +30,7 @@ build:test:
   - pip3 install -r requirements.txt
   - pip3 install -r requirements-dev.txt
   script:
-  - export PYTHONPATH="$PYTHONPATH:$(pwd)"
   - pytest -v
-  tags:
-  - k8s
 
 build:dist:
   stage: build
diff --git a/setup.py b/setup.py
index 1064e7d..bf1cd1b 100644
--- a/setup.py
+++ b/setup.py
@@ -1,15 +1,7 @@
 from setuptools import setup, find_namespace_packages
 from glob import glob
 
-import os
-import sys
-
-sys.path.insert(0, os.path.dirname(__file__))
-
-try:
-    from villas.controller import __version__ as version
-except ModuleNotFoundError:
-    version = "0.0.0"
+from villas.controller import __version__ as version
 
 with open('README.md') as f:
     long_description = f.read()

From 52e7338302d7e645f6b66ef0a4b49618d2e827a7 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Mon, 31 Mar 2025 10:34:02 +0200
Subject: [PATCH 78/89] get version (fix ModuleNotFound error), remove tags

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml |  4 ----
 setup.py       | 21 ++++++++++++++++-----
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d0aafc1..dc583b0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,8 +20,6 @@ build:precommit:
   - pip3 install -r requirements-dev.txt
   script:
   - pre-commit run --all-files
-  tags:
-    - docker
 
 build:test:
   stage: build
@@ -33,8 +31,6 @@ build:test:
   - pip3 install -r requirements-dev.txt
   script:
   - pytest -v
-  tags:
-  - docker
 
 build:dist:
   stage: build
diff --git a/setup.py b/setup.py
index bf1cd1b..44a7acf 100644
--- a/setup.py
+++ b/setup.py
@@ -1,14 +1,25 @@
 from setuptools import setup, find_namespace_packages
 from glob import glob
 
-from villas.controller import __version__ as version
+import os
+import re
 
-with open('README.md') as f:
-    long_description = f.read()
+def get_version():
+    here = os.path.abspath(os.path.dirname(__file__))
+    init_file = os.path.join(here, "villas", "controller", "__init__.py")
+
+    with open(init_file, "r") as f:
+        content = f.read()
+
+    match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", content, re.M)
+    if match:
+        return match.group(1)
+
+    raise RuntimeError("Version not found")
 
 setup(
     name='villas-controller',
-    version=version,
+    version=get_version(),
     description='A controller/orchestration API for real-time '
                 'power system simulators',
     long_description=long_description,
@@ -20,7 +31,7 @@
     keywords='simulation controller villas',
     classifiers=[
         'Development Status :: 3 - Alpha',
-        'License :: OSI Approved :: Apache Software License'
+        'License :: OSI Approved :: Apache Software License',
         'Programming Language :: Python :: 3'
     ],
     packages=find_namespace_packages(include=['villas.*']),

From fd0d5c8fdad65c978812cca61e7e4235fc86eb52 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Mon, 31 Mar 2025 10:55:42 +0200
Subject: [PATCH 79/89] add long description which got lost in last commit

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 setup.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/setup.py b/setup.py
index 44a7acf..13f16e0 100644
--- a/setup.py
+++ b/setup.py
@@ -17,6 +17,9 @@ def get_version():
 
     raise RuntimeError("Version not found")
 
+with open('README.md') as f:
+    long_description = f.read()
+
 setup(
     name='villas-controller',
     version=get_version(),

From 5fdb89027f77ccd64ca130af69b35a4d3877fa4c Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 1 Apr 2025 10:45:05 +0200
Subject: [PATCH 80/89] updated json schema checking

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .pre-commit-config.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a669ed8..0ab5e36 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -18,8 +18,8 @@ repos:
   rev: 6.0.0
   hooks:
   - id: flake8
-- repo: https://github.com/sirosen/check-jsonschema
-  rev: 0.22.0
+- repo: https://github.com/python-jsonschema/check-jsonschema
+  rev: 0.32.1
   hooks:
   - id: check-jsonschema
     name: "Check schemas"
@@ -32,4 +32,4 @@ repos:
     language: python
     files: ^doc/openapi.yaml$
     types: [yaml]
-    args: ["--schemafile", "https://raw.githubusercontent.com/OAI/OpenAPI-Specification/main/schemas/v3.1/schema.json"]
+    args: ["--schemafile", "https://spec.openapis.org/oas/3.1/schema/2025-02-13"]

From 44e8da4f3921ef72eed469d725206729aa8ae329 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 1 Apr 2025 10:45:58 +0200
Subject: [PATCH 81/89] added blank lines (flake8)

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 setup.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/setup.py b/setup.py
index 13f16e0..4f5b5f2 100644
--- a/setup.py
+++ b/setup.py
@@ -4,6 +4,7 @@
 import os
 import re
 
+
 def get_version():
     here = os.path.abspath(os.path.dirname(__file__))
     init_file = os.path.join(here, "villas", "controller", "__init__.py")
@@ -17,6 +18,7 @@ def get_version():
 
     raise RuntimeError("Version not found")
 
+
 with open('README.md') as f:
     long_description = f.read()
 

From 05fdaa4a35eb6a8bc6176da763db6223fce683f8 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Tue, 1 Apr 2025 10:55:06 +0200
Subject: [PATCH 82/89] add init.py for modules to be found in tests

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 villas/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 villas/__init__.py

diff --git a/villas/__init__.py b/villas/__init__.py
new file mode 100644
index 0000000..e69de29

From 64f2143d4142c17ee75a9ec2a13bfd6b034bb87b Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 2 Apr 2025 11:24:15 +0200
Subject: [PATCH 83/89] watch the resources inside the controller namespace

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 villas/controller/components/managers/kubernetes.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 2ff668b..678d330 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -36,7 +36,12 @@ def __init__(self, **args):
         else:
             k8s.config.load_incluster_config()
 
-        self.namespace = os.environ.get('NAMESPACE')
+        # the namespace in which to create the jobs
+        # and to watch for events
+        self.namespace = os.environ.get('NAMESPACE' + "-controller")
+
+        # name and UID of the pod in which this controller is running
+        # used in kubernetes simulator to set the owner reference
         self.my_pod_name = os.environ.get('POD_NAME')
         self.my_pod_uid = os.environ.get('POD_UID')
 

From 79a56d6391c70d609faf414cac13ae517de01758 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 2 Apr 2025 12:00:10 +0200
Subject: [PATCH 84/89] start watcher thread AFTER reading environment
 variables

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .../components/managers/kubernetes.py         | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 678d330..39cd314 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -22,15 +22,6 @@ class KubernetesManager(Manager):
     def __init__(self, **args):
         super().__init__(**args)
 
-        self.thread_stop = threading.Event()
-
-        self.pod_watcher_thread = threading.Thread(
-            target=self._run_pod_watcher)
-        self.job_watcher_thread = threading.Thread(
-            target=self._run_job_watcher)
-        self.event_watcher_thread = threading.Thread(
-            target=self._run_event_watcher)
-
         if os.environ.get('KUBECONFIG'):
             k8s.config.load_kube_config()
         else:
@@ -45,11 +36,22 @@ def __init__(self, **args):
         self.my_pod_name = os.environ.get('POD_NAME')
         self.my_pod_uid = os.environ.get('POD_UID')
 
-        # self.pod_watcher_thread.start()
-        # self.job_watcher_thread.start()
+        self.thread_stop = threading.Event()
+
+        self.pod_watcher_thread = threading.Thread(
+            target=self._run_pod_watcher)
+        self.job_watcher_thread = threading.Thread(
+            target=self._run_job_watcher)
+        self.event_watcher_thread = threading.Thread(
+            target=self._run_event_watcher)
+
         self.event_watcher_thread.setDaemon(True)
         self.event_watcher_thread.start()
 
+        # Not used yet, can support more complex logic
+        # self.pod_watcher_thread.start()
+        # self.job_watcher_thread.start()
+
     def _run_pod_watcher(self):
         w = k8s.watch.Watch()
         c = k8s.client.CoreV1Api()

From 3df9f3db3ee41ad6cb5ce3e70bd60d9699b9f9b1 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 2 Apr 2025 12:24:54 +0200
Subject: [PATCH 85/89] corrected setting the namespace

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 villas/controller/components/managers/kubernetes.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/villas/controller/components/managers/kubernetes.py b/villas/controller/components/managers/kubernetes.py
index 39cd314..db701c2 100644
--- a/villas/controller/components/managers/kubernetes.py
+++ b/villas/controller/components/managers/kubernetes.py
@@ -29,7 +29,8 @@ def __init__(self, **args):
 
         # the namespace in which to create the jobs
         # and to watch for events
-        self.namespace = os.environ.get('NAMESPACE' + "-controller")
+        self.namespace = os.environ.get('NAMESPACE')
+        self.namespace = ''.join([self.namespace, '-controller'])
 
         # name and UID of the pod in which this controller is running
         # used in kubernetes simulator to set the owner reference

From 790bb1eaf55c158727266c7b7a984b0cc4884441 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Thu, 17 Apr 2025 10:24:05 +0200
Subject: [PATCH 86/89] introduced pyproject.toml

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>

pipeline fixes

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>

updatet setuptools version to support license expression

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>

update license definition

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>

read version dynamically

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>

fixes

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>

move version to correct init file

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 Dockerfile                    | 26 +++++++++-----
 pyproject.toml                | 40 ++++++++++++++++++++++
 setup.py                      | 64 ++---------------------------------
 villas/controller/__init__.py |  2 +-
 4 files changed, 60 insertions(+), 72 deletions(-)
 create mode 100644 pyproject.toml

diff --git a/Dockerfile b/Dockerfile
index 69d0784..ea8cb34 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,12 +1,20 @@
-FROM python:3.11
+# build stage
+FROM python:3.11 AS builder
 
-COPY requirements.txt /tmp
-RUN pip3 install -r /tmp/requirements.txt
+WORKDIR /build
+COPY . .
 
-COPY . /tmp/controller
-RUN cd /tmp/controller && \
-	python3 setup.py sdist && \
-	pip3 install dist/*.tar.gz && \
-	rm -rf /tmp/controller
+COPY requirements.txt /tmp/
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
-ENTRYPOINT [ "villas-controller" ]
+RUN python3 setup.py sdist && \
+    pip install dist/*.tar.gz --target /install
+
+# minimal runtime image
+FROM python:3.11-slim AS runtime
+
+COPY --from=builder /install /usr/local/lib/python3.11/site-packages
+COPY etc/*.json etc/*.yaml /etc/villas/controller/
+COPY villas-controller.service /etc/systemd/system/
+
+ENTRYPOINT ["villas-controller"]
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..f1a0f3f
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,40 @@
+[build-system]
+requires = ["setuptools>=61", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "villas-controller"
+dynamic = ["version"]
+description = "A controller/orchestration API for real-time power system simulators"
+readme = "README.md"
+requires-python = ">=3.7"
+authors = [
+    { name="Steffen Vogel", email="acs-software@eonerc.rwth-aachen.de" }
+]
+dependencies = [
+    "dotmap",
+    "kombu",
+    "termcolor",
+    "psutil",
+    "requests",
+    "villas-node>=0.10.2",
+    "kubernetes",
+    "xdg",
+    "PyYAML",
+    "tornado",
+    "jsonschema>=4.1.0",
+    "pyusb"
+]
+
+[project.license]
+text = "Apache-2.0"
+
+[tool.setuptools.dynamic]
+version = { attr = "villas.controller.__version__" }
+
+[tool.setuptools.packages.find]
+include = ["villas*"]
+
+[project.scripts]
+villas-controller = "villas.controller.main:main"
+villas-ctl = "villas.controller.main:main"
diff --git a/setup.py b/setup.py
index 4f5b5f2..a6e30fd 100644
--- a/setup.py
+++ b/setup.py
@@ -1,69 +1,9 @@
-from setuptools import setup, find_namespace_packages
+from setuptools import setup
 from glob import glob
 
-import os
-import re
-
-
-def get_version():
-    here = os.path.abspath(os.path.dirname(__file__))
-    init_file = os.path.join(here, "villas", "controller", "__init__.py")
-
-    with open(init_file, "r") as f:
-        content = f.read()
-
-    match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", content, re.M)
-    if match:
-        return match.group(1)
-
-    raise RuntimeError("Version not found")
-
-
-with open('README.md') as f:
-    long_description = f.read()
-
 setup(
-    name='villas-controller',
-    version=get_version(),
-    description='A controller/orchestration API for real-time '
-                'power system simulators',
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    url='https://www.fein-aachen.org/projects/villas-controller/',
-    author='Steffen Vogel',
-    author_email='acs-software@eonerc.rwth-aachen.de',
-    license='Apache License 2.0',
-    keywords='simulation controller villas',
-    classifiers=[
-        'Development Status :: 3 - Alpha',
-        'License :: OSI Approved :: Apache Software License',
-        'Programming Language :: Python :: 3'
-    ],
-    packages=find_namespace_packages(include=['villas.*']),
-    install_requires=[
-        'dotmap',
-        'kombu',
-        'termcolor',
-        'psutil',
-        'requests',
-        'villas-node>=0.10.2',
-        'kubernetes',
-        'xdg',
-        'PyYAML',
-        'tornado',
-        'jsonschema>=4.1.0',
-        'psutil',
-        'pyusb'
-    ],
     data_files=[
         ('/etc/villas/controller', glob('etc/*.{json,yaml}')),
         ('/etc/systemd/system', ['villas-controller.service'])
-    ],
-    entry_points={
-        'console_scripts': [
-            'villas-ctl=villas.controller.main:main',
-            'villas-controller=villas.controller.main:main'
-        ],
-    },
-    include_package_data=True
+    ]
 )
diff --git a/villas/controller/__init__.py b/villas/controller/__init__.py
index 6a9beea..3d26edf 100644
--- a/villas/controller/__init__.py
+++ b/villas/controller/__init__.py
@@ -1 +1 @@
-__version__ = "0.4.0"
+__version__ = "0.4.1"

From d7dd022b7394fffbe9d06e1fcb2a80bd8b02e663 Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 23 Apr 2025 09:31:34 +0200
Subject: [PATCH 87/89] remove requirements files

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 .gitlab-ci.yml       |  9 ++++-----
 Dockerfile           |  3 +--
 pyproject.toml       |  6 ++++++
 requirements-dev.txt |  2 --
 requirements.txt     | 13 -------------
 5 files changed, 11 insertions(+), 22 deletions(-)
 delete mode 100644 requirements-dev.txt
 delete mode 100644 requirements.txt

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index dc583b0..08f3ca9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,13 +11,13 @@ build:docker:
   script:
   - docker build --tag ${CI_REGISTRY_IMAGE}:${CI_COMMIT_BRANCH} .
   tags:
-  - docker
+  - docker-image-builder
 
 build:precommit:
   stage: build
   image: python:3.11
   before_script:
-  - pip3 install -r requirements-dev.txt
+  - pip install .[dev]
   script:
   - pre-commit run --all-files
 
@@ -27,8 +27,7 @@ build:test:
   services:
   - rabbitmq:latest
   before_script:
-  - pip3 install -r requirements.txt
-  - pip3 install -r requirements-dev.txt
+  - pip3 install .[dev]
   script:
   - pytest -v
 
@@ -41,7 +40,7 @@ build:dist:
     paths:
       - dist/
   tags:
-  - docker
+  - docker-image-builder
 
 # Stage: deploy
 ##############################################################################
diff --git a/Dockerfile b/Dockerfile
index ea8cb34..f11ea2a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,8 +4,7 @@ FROM python:3.11 AS builder
 WORKDIR /build
 COPY . .
 
-COPY requirements.txt /tmp/
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
+RUN pip install .
 
 RUN python3 setup.py sdist && \
     pip install dist/*.tar.gz --target /install
diff --git a/pyproject.toml b/pyproject.toml
index f1a0f3f..799ee97 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -26,6 +26,12 @@ dependencies = [
     "pyusb"
 ]
 
+[project.optional-dependencies]
+dev = [
+  "pytest",
+  "pre-commit"
+]
+
 [project.license]
 text = "Apache-2.0"
 
diff --git a/requirements-dev.txt b/requirements-dev.txt
deleted file mode 100644
index 51f1982..0000000
--- a/requirements-dev.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-pre-commit
-pytest
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index d0d9c97..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-kombu
-termcolor
-psutil
-requests
-villas-node>=0.10.2
-kubernetes
-xdg
-dotmap
-PyYAML
-tornado
-jsonschema>=4.1.0
-psutil
-pyusb

From 0336da3a68fcfecee401094e88ed1bf6942157ab Mon Sep 17 00:00:00 2001
From: iripiri <ikoester@eonerc.rwth-aachen.de>
Date: Wed, 14 May 2025 11:14:26 +0200
Subject: [PATCH 88/89] fix errors, update config

Signed-off-by: iripiri <ikoester@eonerc.rwth-aachen.de>
---
 etc/config_simplekub.yaml       | 11 +++++++----
 villas/controller/component.py  |  2 +-
 villas/controller/controller.py |  2 +-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/etc/config_simplekub.yaml b/etc/config_simplekub.yaml
index 9653b03..6b4f5dd 100644
--- a/etc/config_simplekub.yaml
+++ b/etc/config_simplekub.yaml
@@ -1,14 +1,17 @@
 ---
 broker:
-  url: amqp://villas:Haegiethu0rohtee@kubernetes-master-1.os-cloud.eonerc.rwth-aachen.de:30809/%2F
+  url: amqp://admin:vieQuoo2sieDahHee8ohM5aThaibiPei@villas-broker:5672/
 
 components:
 - type: generic
   category: manager
-  uuid: ebbbbba0-557b-4848-ac7a-faa3e7c51fa3
+  name: Generic Manager
+  location: VM Iris
+  uuid: eddb51a0-557b-4848-ac7a-faccc7c51fa3
 
 - category: manager
   type: kubernetes-simple
-  uuid: 4bbbb73e-7e74-11eb-8f63-f3a5b3ab82f6
-
+  name: Simple Kubernetes Manager
+  location: VM Iris
+  uuid: 4f8fb73e-7e74-11eb-8f63-f3ccc3ab82f6
   namespace: villas-controller
diff --git a/villas/controller/component.py b/villas/controller/component.py
index beb2668..4a086f8 100644
--- a/villas/controller/component.py
+++ b/villas/controller/component.py
@@ -152,7 +152,7 @@ def status(self):
                 **self.headers
             },
             'schema': {
-                name: v.schema for name, v in self.schema.items()
+                name: v for name, v in self.schema.items()
             }
         }
 
diff --git a/villas/controller/controller.py b/villas/controller/controller.py
index 95f058a..0bdcc98 100644
--- a/villas/controller/controller.py
+++ b/villas/controller/controller.py
@@ -138,7 +138,7 @@ def shutdown(self):
             c.on_shutdown()
 
         # Publish last status updates before shutdown
-        self._drain_publish_queue()
+        self.drain_publish_queue()
         self.should_terminate = True
 
     @property

From 82bc2ad886555c2539076fd13025ec65e48cb7ab Mon Sep 17 00:00:00 2001
From: SystemsPurge <naktiyoussef@proton.me>
Date: Mon, 19 May 2025 15:20:35 +0200
Subject: [PATCH 89/89] fix: temporary fix for entrypoint of Dockerfile,
 causing container not to find controller executable

Signed-off-by: SystemsPurge <naktiyoussef@proton.me>
---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index f11ea2a..c657ff2 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -16,4 +16,4 @@ COPY --from=builder /install /usr/local/lib/python3.11/site-packages
 COPY etc/*.json etc/*.yaml /etc/villas/controller/
 COPY villas-controller.service /etc/systemd/system/
 
-ENTRYPOINT ["villas-controller"]
+ENTRYPOINT ["/usr/local/lib/python3.11/site-packages/bin/villas-controller"]