From abbbd8b7e08750307162717d670f3773274cdd3c Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Thu, 20 Jul 2023 13:31:59 -0500 Subject: [PATCH 01/10] Move getId() to NameProvider --- .../java/org/myrobotlab/framework/interfaces/NameProvider.java | 2 ++ .../org/myrobotlab/framework/interfaces/ServiceInterface.java | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java b/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java index 94a02c237e..4e7bca7e67 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java @@ -4,4 +4,6 @@ public interface NameProvider { public String getName(); + String getId(); + } diff --git a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java index 1cc357418e..457872a1db 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java @@ -189,8 +189,6 @@ public interface ServiceInterface extends ServiceQueue, LoggingSink, NameTypePro */ void setOrder(int creationCount); - String getId(); - String getFullName(); void loadLocalizations(); From 5469cabd13f79c4363566b0c8d60aab6862c6066 Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Thu, 20 Jul 2023 13:39:06 -0500 Subject: [PATCH 02/10] Add python_services resource directory and auto-extract --- src/main/java/org/myrobotlab/io/FileIO.java | 17 ++++++++++ src/main/python/README.md | 25 +++++++++++++++ .../mrl/services/TestService.py | 5 +++ .../mrl/services/meta/TestServiceMeta.py | 11 +++++++ .../mrl/services/meta/__init__.py | 32 +++++++++++++++++++ 5 files changed, 90 insertions(+) create mode 100644 src/main/python/README.md create mode 100644 src/main/python/python_services/mrl/services/TestService.py create mode 100644 src/main/python/python_services/mrl/services/meta/TestServiceMeta.py create mode 100644 src/main/python/python_services/mrl/services/meta/__init__.py diff --git a/src/main/java/org/myrobotlab/io/FileIO.java b/src/main/java/org/myrobotlab/io/FileIO.java index 80d665e101..ea642737f9 100644 --- a/src/main/java/org/myrobotlab/io/FileIO.java +++ b/src/main/java/org/myrobotlab/io/FileIO.java @@ -392,6 +392,23 @@ static public final boolean extractResources() { return false; } + /** + * extractPythonServices extracts the entire python_services + * resource directory to the root directory, needed for all + * services written in Python. + * + * @return true if successful, false otherwise. + */ + static public boolean extractPythonServices() { + try { + extract(getRoot(), "python_services", null, false); + return true; + } catch (Exception e) { + Logging.logError(e); + } + return false; + } + /** * get configuration directory * diff --git a/src/main/python/README.md b/src/main/python/README.md new file mode 100644 index 0000000000..216d250448 --- /dev/null +++ b/src/main/python/README.md @@ -0,0 +1,25 @@ +## How Runtime starts a Python service +When the Java-side Runtime is instantiated, it extracts this python_services +directory, just like with resource. + +During install() when installing all, the virtual environment +is setup and configured, installing mrlpy and the python_services modules. + +A config option in Runtime's config sets whether to also start the python-side +runtime. If true, it first checks if the virtual environment is configured, +if so it starts a Python subprocess that execute's mrlpy's Runtime. + + +TODO replace explicit serviceRunners field with just finding services that implement +the interface when needed + +This Runtime connects to the Java-side and registers itself as a "ServiceRunner," +a new interface that denotes services able to start other services. + +When creating a service (calling Java-side Runtime.createService()) with +a foreign type (like py:mrlpy.services.TestService), Runtime will look +through all registered ServiceRunners to find any that can +start services with that language key. If it finds them, it will choose the +first one matching the desired ID to call createService() on. The service runner creates the service +on its side and then returns the constructed service, which Runtime then continues +to configure or start. diff --git a/src/main/python/python_services/mrl/services/TestService.py b/src/main/python/python_services/mrl/services/TestService.py new file mode 100644 index 0000000000..4357b1e7d9 --- /dev/null +++ b/src/main/python/python_services/mrl/services/TestService.py @@ -0,0 +1,5 @@ +from mrlpy.framework import Service + +class TestService(Service): + def __init__(self, name=""): + super(TestService, self).__init__(name) \ No newline at end of file diff --git a/src/main/python/python_services/mrl/services/meta/TestServiceMeta.py b/src/main/python/python_services/mrl/services/meta/TestServiceMeta.py new file mode 100644 index 0000000000..fec18313e4 --- /dev/null +++ b/src/main/python/python_services/mrl/services/meta/TestServiceMeta.py @@ -0,0 +1,11 @@ +MetaData( + service_type="TestService", + available=False, + dependencies=( + "llamacpp-python=1.0.0", + "guidance=2.0.0" + + ), + description="A test service not for actual use", + is_cloud_service=True +) \ No newline at end of file diff --git a/src/main/python/python_services/mrl/services/meta/__init__.py b/src/main/python/python_services/mrl/services/meta/__init__.py new file mode 100644 index 0000000000..fd9e73498d --- /dev/null +++ b/src/main/python/python_services/mrl/services/meta/__init__.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass +from typing import Tuple + + +""" +Should be in mrlpy. +Need to add Runtime.install() +and an auto-import of mrlpy.services.meta.* +in the Runtime initializer +""" + +metadata_instances = dict() + +@dataclass +class MetaData(): + service_type: str + available: bool = True + dependencies: Tuple = () + description: str = "" + is_cloud_service: bool = False + + def __post_init__(self): + if self.service_type is None or self.service_type == "": + raise ValueError("Cannot have empty service type field") + if self.service_type in metadata_instances: + raise ValueError(f"Cannot redefine {self.service_type}'s metadata") + + metadata_instances.update({self.service_type: self}) + + + + From 5ad1b24e736c91215914898265d9611a6d0c6de7 Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Thu, 20 Jul 2023 13:43:13 -0500 Subject: [PATCH 03/10] Factor out embedded Python management, add ServiceRunner interface and basic logic --- .../myrobotlab/ext/python/PythonUtils.java | 80 ++++++++++++ .../process/SubprocessException.java | 8 ++ .../java/org/myrobotlab/service/Py4j.java | 65 ++-------- .../java/org/myrobotlab/service/Runtime.java | 121 ++++++++++++------ .../service/interfaces/ServiceRunner.java | 28 ++++ .../myrobotlab/service/meta/RuntimeMeta.java | 7 +- 6 files changed, 215 insertions(+), 94 deletions(-) create mode 100644 src/main/java/org/myrobotlab/ext/python/PythonUtils.java create mode 100644 src/main/java/org/myrobotlab/process/SubprocessException.java create mode 100644 src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java diff --git a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java new file mode 100644 index 0000000000..79e52f6cad --- /dev/null +++ b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java @@ -0,0 +1,80 @@ +package org.myrobotlab.ext.python; + +import org.bytedeco.javacpp.Loader; +import org.myrobotlab.framework.Platform; +import org.myrobotlab.io.FileIO; +import org.myrobotlab.logging.LoggerFactory; +import org.myrobotlab.process.SubprocessException; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.myrobotlab.framework.Status.error; +import static org.myrobotlab.io.FileIO.fs; + +public class PythonUtils { + private static Logger logger = LoggerFactory.getLogger(PythonUtils.class); + + public static final String SYSTEM_PYTHON_COMMAND = "python3"; + + + /** + * Ensures a virtual environment is setup at the + * desired location, creating it if it doesn't exist, + * and returning the absolute file path to the + * virtual environment Python executable. + * + * @param venv The location to setup the virtual environment + * @param useBundledPython Whether to use the bundled Python + * or the system python for setting up + * the virtual environment. + * @return The location of the virtual environment interpreter + */ + public static String setupVenv(String venv, boolean useBundledPython, List packages) throws IOException, InterruptedException { + String pythonCommand = (Platform.getLocalInstance().isWindows()) ? venv + fs + "Scripts" + fs + "python.exe" : venv + fs + "bin" + fs + "python"; + if (!FileIO.checkDir(venv)) { + // We don't have an initialized virtual environment, so lets make one + // and install our required packages + String python; + ProcessBuilder installProcess; + int ret; + if (useBundledPython) { + python = Loader.load(org.bytedeco.cpython.python.class); + String venvLib = new File(python).getParent() + fs + "lib" + fs + "venv" + fs + "scripts" + fs + "nt"; + if (Platform.getLocalInstance().isWindows()) { + // Super hacky workaround, venv works differently on Windows and requires these two + // files, but they are not distributed in bare-bones Python or in any pip packages. + // So we copy them where it expects, and it seems to work now + String containingDir = new File(python).getParent(); + FileIO.copy(containingDir + fs + "python.exe", venvLib + fs + "python.exe"); + FileIO.copy(containingDir + fs + "pythonw.exe", venvLib + fs + "pythonw.exe"); + } + + installProcess = new ProcessBuilder(python, "-m", "venv", venv); + ret = installProcess.inheritIO().start().waitFor(); + if (ret != 0) { + String message = String.format("Could not create virtual environment, subprocess returned %s. If on Windows, make sure there is a python.exe file in %s", ret, venvLib); + error(message); + throw new SubprocessException(message); + } + } else { + python = SYSTEM_PYTHON_COMMAND; + } + + List command = new ArrayList<>(List.of(python, "-m", "pip", "install")); + command.addAll(packages); + installProcess = new ProcessBuilder(command.toArray(new String[0])); + ret = installProcess.inheritIO().start().waitFor(); + if (ret != 0) { + String message = String.format("Could not install desired packages (%s)", packages); + error(message); + throw new SubprocessException(message); + } + + } + return pythonCommand; + } +} diff --git a/src/main/java/org/myrobotlab/process/SubprocessException.java b/src/main/java/org/myrobotlab/process/SubprocessException.java new file mode 100644 index 0000000000..a1f79e455b --- /dev/null +++ b/src/main/java/org/myrobotlab/process/SubprocessException.java @@ -0,0 +1,8 @@ +package org.myrobotlab.process; + +public class SubprocessException extends RuntimeException { + public SubprocessException(String message) { + super(message); + } + +} diff --git a/src/main/java/org/myrobotlab/service/Py4j.java b/src/main/java/org/myrobotlab/service/Py4j.java index 89002d8ee6..a4a98772cb 100644 --- a/src/main/java/org/myrobotlab/service/Py4j.java +++ b/src/main/java/org/myrobotlab/service/Py4j.java @@ -1,21 +1,8 @@ package org.myrobotlab.service; -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.bytedeco.javacpp.Loader; import org.myrobotlab.codec.CodecUtils; +import org.myrobotlab.ext.python.PythonUtils; import org.myrobotlab.framework.Message; -import org.myrobotlab.framework.Platform; import org.myrobotlab.framework.Service; import org.myrobotlab.io.FileIO; import org.myrobotlab.io.StreamGobbler; @@ -26,11 +13,16 @@ import org.myrobotlab.service.data.Script; import org.myrobotlab.service.interfaces.Executor; import org.slf4j.Logger; - import py4j.GatewayServer; import py4j.GatewayServerListener; import py4j.Py4JServerConnection; +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.*; + /** * * @@ -110,6 +102,7 @@ public void run() { try { exitCode = process.waitFor(); } catch (InterruptedException e) { + warn("Waiting for process was interrupted. Exit code cannot be known"); } warn("process %s terminated with exit code %d", process.toString(), exitCode); } @@ -141,7 +134,7 @@ public void run() { * executed or saved to the file system, or updatd in memory which the js * client does */ - protected HashMap openedScripts = new HashMap(); + protected HashMap openedScripts = new HashMap<>(); /** * client process and connectivity reference @@ -423,45 +416,11 @@ public void startPythonProcess() { // Script requires full name as first command line argument String[] pythonArgs = {getFullName()}; - // Build the command to start the Python process - ProcessBuilder processBuilder; - if (((Py4jConfig) config).useBundledPython) { - String venv = getDataDir() + fs + "venv"; - pythonCommand = (Platform.getLocalInstance().isWindows()) ? venv + fs + "Scripts" + fs + "python.exe" : venv + fs + "bin" + fs + "python"; - if (!FileIO.checkDir(venv)) { - // We don't have an initialized virtual environment, so lets make one - // and install our required packages - String python = Loader.load(org.bytedeco.cpython.python.class); - String venvLib = new File(python).getParent() + fs + "lib" + fs + "venv" + fs + "scripts" + fs + "nt"; - if (Platform.getLocalInstance().isWindows()) { - // Super hacky workaround, venv works differently on Windows and requires these two - // files, but they are not distributed in bare-bones Python or in any pip packages. - // So we copy them where it expects, and it seems to work now - FileIO.copy(getResourceDir() + fs + "python.exe", venvLib + fs + "python.exe"); - FileIO.copy(getResourceDir() + fs + "pythonw.exe", venvLib + fs + "pythonw.exe"); - } - ProcessBuilder installProcess = new ProcessBuilder(python, "-m", "venv", venv); - int ret = installProcess.inheritIO().start().waitFor(); - if (ret != 0) { - error("Could not create virtual environment, subprocess returned {}. If on Windows, make sure there is a python.exe file in {}", ret, venvLib); - return; - } - - installProcess = new ProcessBuilder(pythonCommand, "-m", "pip", "install", "py4j"); - ret = installProcess.inheritIO().start().waitFor(); - if (ret != 0) { - error("Could not install package, subprocess returned " + ret); - return; - } - } + String venv = getDataDir() + fs + "venv"; + pythonCommand = PythonUtils.setupVenv(venv, ((Py4jConfig) config).useBundledPython, List.of("py4j")); - // Virtual environment should exist, so lets use that python - } else { - // Just use the system python - pythonCommand = "python"; - } - processBuilder = new ProcessBuilder(pythonCommand, pythonScript); + ProcessBuilder processBuilder = new ProcessBuilder(pythonCommand, pythonScript); processBuilder.redirectErrorStream(true); processBuilder.command().addAll(List.of(pythonArgs)); diff --git a/src/main/java/org/myrobotlab/service/Runtime.java b/src/main/java/org/myrobotlab/service/Runtime.java index 5aaec4a951..09af1a4396 100644 --- a/src/main/java/org/myrobotlab/service/Runtime.java +++ b/src/main/java/org/myrobotlab/service/Runtime.java @@ -20,47 +20,17 @@ import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.Random; -import java.util.Set; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; +import boofcv.alg.fiducial.qrcode.PackedBits8; import org.myrobotlab.codec.ClassUtil; import org.myrobotlab.codec.CodecUtils; import org.myrobotlab.codec.CodecUtils.ApiDescription; import org.myrobotlab.codec.ForeignProcessUtils; -import org.myrobotlab.framework.CmdConfig; -import org.myrobotlab.framework.CmdOptions; -import org.myrobotlab.framework.DescribeQuery; -import org.myrobotlab.framework.DescribeResults; -import org.myrobotlab.framework.Instantiator; -import org.myrobotlab.framework.MRLListener; -import org.myrobotlab.framework.Message; -import org.myrobotlab.framework.MethodCache; -import org.myrobotlab.framework.MethodEntry; -import org.myrobotlab.framework.NameGenerator; -import org.myrobotlab.framework.Peer; -import org.myrobotlab.framework.Plan; -import org.myrobotlab.framework.Platform; -import org.myrobotlab.framework.ProxyFactory; -import org.myrobotlab.framework.Registration; -import org.myrobotlab.framework.Service; -import org.myrobotlab.framework.ServiceReservation; -import org.myrobotlab.framework.Status; +import org.myrobotlab.framework.*; import org.myrobotlab.framework.interfaces.MessageListener; import org.myrobotlab.framework.interfaces.NameProvider; import org.myrobotlab.framework.interfaces.ServiceInterface; @@ -87,11 +57,7 @@ import org.myrobotlab.service.config.ServiceConfig.Listener; import org.myrobotlab.service.data.Locale; import org.myrobotlab.service.data.ServiceTypeNameResults; -import org.myrobotlab.service.interfaces.ConnectionManager; -import org.myrobotlab.service.interfaces.Gateway; -import org.myrobotlab.service.interfaces.LocaleProvider; -import org.myrobotlab.service.interfaces.RemoteMessageHandler; -import org.myrobotlab.service.interfaces.ServiceLifeCyclePublisher; +import org.myrobotlab.service.interfaces.*; import org.myrobotlab.service.meta.abstracts.MetaData; import org.myrobotlab.string.StringUtil; import org.slf4j.Logger; @@ -133,6 +99,12 @@ public class Runtime extends Service implements MessageListener, ServiceLifeCycl */ static private final Map registry = new TreeMap<>(); + /** + * List of all services capable of running other services, + * even those outside of Java-land. + */ + private static final List serviceRunners = new CopyOnWriteArrayList<>(); + /** * A plan is a request to runtime to change the system. Typically its to ask * to start and configure new services. The master plan is an accumulation of @@ -690,7 +662,7 @@ static private synchronized ServiceInterface createService(String name, String t } String fullTypeName; - if (type.contains(".")) { + if (ForeignProcessUtils.isForeignTypeKey(type) || type.contains(".")) { fullTypeName = type; } else { fullTypeName = String.format("org.myrobotlab.service.%s", type); @@ -721,6 +693,32 @@ static private synchronized ServiceInterface createService(String name, String t return sw; } + if (ForeignProcessUtils.isForeignTypeKey(type)) { + String languageKey = ForeignProcessUtils.getLanguageId(type); + // Needed cause of lambda requiring effectively-final variables + String finalType = type; + List possibleRunners = serviceRunners.stream() + .filter(runner -> runner.getSupportedLanguageKeys().contains(languageKey)) + .filter(runner -> runner.getAvailableServiceTypes().contains(finalType)) + .collect(Collectors.toList()); + if (possibleRunners.isEmpty()) { + log.error("Cannot find a service runner to start service with type {}", type); + return null; + } + + if (inId == null) { + return possibleRunners.get(0).createService(name, type, null); + } else { + Optional maybeRunner = possibleRunners.stream().filter(runner -> inId.equals(runner.getId())).findFirst(); + if (maybeRunner.isEmpty()) { + log.error("Cannot find compatible service runner with ID {}", inId); + return null; + } + return maybeRunner.get().createService(name, type, inId); + + } + } + try { if (log.isDebugEnabled()) { @@ -907,6 +905,7 @@ public static Runtime getInstance() { Security.getInstance(); runtime.getRepo().addStatusPublisher(runtime); FileIO.extractResources(); + FileIO.extractPythonServices(); // protected services we don't want to remove when releasing a config runtime.startingServices.add("runtime"); runtime.startingServices.add("security"); @@ -1788,6 +1787,48 @@ public static synchronized Registration register(Registration registration) { } registry.put(fullname, registration.service); + if (registration.interfaces.contains(ServiceRunner.class.getName())) { + if (Runtime.class.isAssignableFrom(registration.service.getClass())) { + + // Might not be needed, I'm just not sure how calling these methods + // on an emulated Runtime like mrlpy's would work + serviceRunners.add(new ServiceRunner() { + @Override + public String getName() { + return null; + } + + @Override + public String getId() { + return null; + } + + @Override + public List getSupportedLanguageKeys() { + try { + return (List) Runtime.get().sendBlocking(registration.getFullName(), "getSupportedLanguageKeys"); + } catch (TimeoutException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public List getAvailableServiceTypes() { + try { + return (List) Runtime.get().sendBlocking(registration.getFullName(), "getAvailableServiceTypes"); + } catch (TimeoutException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + @Override + public ServiceInterface createService(String name, String type, String inId) { + return null; + } + }); + } + serviceRunners.add((ServiceRunner) registration.service); + } if (runtime != null) { diff --git a/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java new file mode 100644 index 0000000000..70e1d74c6b --- /dev/null +++ b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java @@ -0,0 +1,28 @@ +package org.myrobotlab.service.interfaces; +import org.myrobotlab.framework.interfaces.NameProvider; +import org.myrobotlab.framework.interfaces.ServiceInterface; + +import java.util.List; + +public interface ServiceRunner extends NameProvider { + + /** + * Get the runner's supported service programming language + * keys, such as {@code py} or {@code kt}. Used to route + * service creation to a Runtime that supports the language + * the service is written in. + * + * @return The language type keys the runner supports + */ + List getSupportedLanguageKeys(); + + /** + * Gets the list of service classes the runner + * can start and manage on its own. + * + * @return The list of available service types + */ + List getAvailableServiceTypes(); + + ServiceInterface createService(String name, String type, String inId); +} diff --git a/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java b/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java index 8b21241562..00df0a4055 100644 --- a/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java +++ b/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java @@ -53,7 +53,12 @@ public RuntimeMeta() { addDependency("com.squareup.okhttp3", "okhttp", "3.9.0"); // force correct version of netty - needed for Vertx but not for Runtime ? - addDependency("io.netty", "netty-all", "4.1.82.Final"); + addDependency("io.netty", "netty-all", "4.1.82.Final"); + + // Used just as a Python exe redistributable. + // ABSOLUTELY NO JNI/JNA IS USED + addDependency("org.bytedeco", "cpython-platform", "3.10.8-1.5.8"); + addDependency("org.bytedeco", "cpython", "3.10.8-1.5.8"); } From 63eaed9e1da0b5b4d5f586be40472903a840c9db Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Sun, 23 Jul 2023 18:56:54 -0500 Subject: [PATCH 04/10] Autoextract and initialize embedded python for python_services --- .gitignore | 1 + .../myrobotlab/ext/python/PythonUtils.java | 74 +++++++++---- .../process/SubprocessException.java | 8 ++ .../java/org/myrobotlab/service/Runtime.java | 101 +++++++++++------- .../resource/framework/pom.xml.template | 7 ++ 5 files changed, 129 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index fb07e725f0..33f2a45200 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ build/ /data /resource +/python_services /bin /dist /repo diff --git a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java index 79e52f6cad..01f8cca659 100644 --- a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java +++ b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java @@ -38,33 +38,29 @@ public static String setupVenv(String venv, boolean useBundledPython, List command = new ArrayList<>(List.of(python, "-m", "pip", "install")); + + List command = new ArrayList<>(List.of(pythonCommand, "-m", "pip", "install")); command.addAll(packages); installProcess = new ProcessBuilder(command.toArray(new String[0])); ret = installProcess.inheritIO().start().waitFor(); @@ -77,4 +73,38 @@ public static String setupVenv(String venv, boolean useBundledPython, List

+ * TODO add process gobbler to echo on logging system + * + * @param packages The list of packages to install. Must be findable by Pip + * @throws SubprocessException If an I/O error occurs running Pip. + */ + public static int installPipPackages(String python, List packages) { + ProcessBuilder builder = new ProcessBuilder(python, "-m", "pip", "install"); + List currCommand = builder.command(); + currCommand.addAll(packages); + try { + return builder.inheritIO().start().waitFor(); + } catch (InterruptedException | IOException e) { + throw new SubprocessException("Unable to install packages " + packages + " with Python command " + python, e); + } + + } + + public static int runPythonScript(String python, File workingDirectory, String script, String... args) { + ProcessBuilder builder = new ProcessBuilder(python, script); + List currCommand = builder.command(); + currCommand.addAll(List.of(args)); + try { + return builder.inheritIO().directory(workingDirectory).start().waitFor(); + } catch (InterruptedException | IOException e) { + throw new SubprocessException("Unable to run script " + script + " with Python command " + python, e); + } + + } } diff --git a/src/main/java/org/myrobotlab/process/SubprocessException.java b/src/main/java/org/myrobotlab/process/SubprocessException.java index a1f79e455b..64fdd78b17 100644 --- a/src/main/java/org/myrobotlab/process/SubprocessException.java +++ b/src/main/java/org/myrobotlab/process/SubprocessException.java @@ -5,4 +5,12 @@ public SubprocessException(String message) { super(message); } + public SubprocessException(Throwable throwable) { + super(throwable); + } + + public SubprocessException(String message, Throwable throwable) { + super(message, throwable); + } + } diff --git a/src/main/java/org/myrobotlab/service/Runtime.java b/src/main/java/org/myrobotlab/service/Runtime.java index 09af1a4396..e88d74acbf 100644 --- a/src/main/java/org/myrobotlab/service/Runtime.java +++ b/src/main/java/org/myrobotlab/service/Runtime.java @@ -30,14 +30,12 @@ import org.myrobotlab.codec.CodecUtils; import org.myrobotlab.codec.CodecUtils.ApiDescription; import org.myrobotlab.codec.ForeignProcessUtils; +import org.myrobotlab.ext.python.PythonUtils; import org.myrobotlab.framework.*; import org.myrobotlab.framework.interfaces.MessageListener; import org.myrobotlab.framework.interfaces.NameProvider; import org.myrobotlab.framework.interfaces.ServiceInterface; -import org.myrobotlab.framework.repo.IvyWrapper; -import org.myrobotlab.framework.repo.Repo; -import org.myrobotlab.framework.repo.ServiceData; -import org.myrobotlab.framework.repo.ServiceDependency; +import org.myrobotlab.framework.repo.*; import org.myrobotlab.io.FileIO; import org.myrobotlab.logging.AppenderType; import org.myrobotlab.logging.LoggerFactory; @@ -214,7 +212,7 @@ public class Runtime extends Service implements MessageListener, ServiceLifeCycl */ transient private IvyWrapper repo = null; // was transient abstract Repo - transient private ServiceData serviceData = ServiceData.getLocalInstance(); + final transient private ServiceData serviceData = ServiceData.getLocalInstance(); /** * command line options @@ -285,6 +283,10 @@ public class Runtime extends Service implements MessageListener, ServiceLifeCycl */ protected Set startingServices = new HashSet<>(); + private static final String PYTHON_VENV_PATH ="python_services/venv"; + + private String pythonCommand; + /** * Wraps {@link java.lang.Runtime#availableProcessors()}. * @@ -906,6 +908,9 @@ public static Runtime getInstance() { runtime.getRepo().addStatusPublisher(runtime); FileIO.extractResources(); FileIO.extractPythonServices(); + + runtime.pythonCommand = PythonUtils.setupVenv(PYTHON_VENV_PATH, true, List.of("mrlpy")); + // protected services we don't want to remove when releasing a config runtime.startingServices.add("runtime"); runtime.startingServices.add("security"); @@ -1515,7 +1520,7 @@ static public void install(String serviceType) { * License - should be appropriately accepted or rejected by user * * @param serviceType - * the service tyype to install + * the service type to install * @param blocking * if this should block until done. * @@ -1533,8 +1538,24 @@ public void run() { try { if (serviceType == null) { r.getRepo().install(); + int returnCode = PythonUtils.runPythonScript( + r.pythonCommand, + new File("python_services"), + "python_services" + fs + "setup.py", + "install"); + if (returnCode != 0) { + r.error("Cannot install Python services, subprocess returned " + returnCode); + } } else { r.getRepo().install(serviceType); + int returnCode = PythonUtils.runPythonScript( + r.pythonCommand, + new File("python_services"), + "python_services" + fs + "setup.py", + "install", serviceType); + if (returnCode != 0) { + r.error("Cannot install Python services, subprocess returned " + returnCode); + } } } catch (Exception e) { r.error(e); @@ -1792,40 +1813,40 @@ public static synchronized Registration register(Registration registration) { // Might not be needed, I'm just not sure how calling these methods // on an emulated Runtime like mrlpy's would work - serviceRunners.add(new ServiceRunner() { - @Override - public String getName() { - return null; - } - - @Override - public String getId() { - return null; - } - - @Override - public List getSupportedLanguageKeys() { - try { - return (List) Runtime.get().sendBlocking(registration.getFullName(), "getSupportedLanguageKeys"); - } catch (TimeoutException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public List getAvailableServiceTypes() { - try { - return (List) Runtime.get().sendBlocking(registration.getFullName(), "getAvailableServiceTypes"); - } catch (TimeoutException | InterruptedException e) { - throw new RuntimeException(e); - } - } - - @Override - public ServiceInterface createService(String name, String type, String inId) { - return null; - } - }); +// serviceRunners.add(new ServiceRunner() { +// @Override +// public String getName() { +// return null; +// } +// +// @Override +// public String getId() { +// return null; +// } +// +// @Override +// public List getSupportedLanguageKeys() { +// try { +// return (List) Runtime.get().sendBlocking(registration.getFullName(), "getSupportedLanguageKeys"); +// } catch (TimeoutException | InterruptedException e) { +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public List getAvailableServiceTypes() { +// try { +// return (List) Runtime.get().sendBlocking(registration.getFullName(), "getAvailableServiceTypes"); +// } catch (TimeoutException | InterruptedException e) { +// throw new RuntimeException(e); +// } +// } +// +// @Override +// public ServiceInterface createService(String name, String type, String inId) { +// return null; +// } +// }); } serviceRunners.add((ServiceRunner) registration.service); } diff --git a/src/main/resources/resource/framework/pom.xml.template b/src/main/resources/resource/framework/pom.xml.template index 8a72e3c9ef..929ff378e7 100644 --- a/src/main/resources/resource/framework/pom.xml.template +++ b/src/main/resources/resource/framework/pom.xml.template @@ -125,6 +125,13 @@ **/*.java + + false + src/main/python + + ** + + From 25603f6805843ae7e68a85df47121014d2093642 Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Tue, 22 Aug 2023 14:02:35 -0500 Subject: [PATCH 05/10] Add cpython to runtime deps, remove maryspeech conflicting deps --- pom.xml | 444 ++++++++++++++++-- .../service/meta/MarySpeechMeta.java | 23 +- .../myrobotlab/service/meta/RuntimeMeta.java | 4 + 3 files changed, 433 insertions(+), 38 deletions(-) diff --git a/pom.xml b/pom.xml index 044205c9e9..0505c1b71b 100644 --- a/pom.xml +++ b/pom.xml @@ -163,10 +163,6 @@ - - - - org.boofcv @@ -216,12 +212,7 @@ - - org.bytedeco - javacpp - 1.5.7 - provided - + org.deeplearning4j deeplearning4j-core @@ -625,6 +616,30 @@ log4j log4j + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -641,6 +656,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -657,6 +696,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -673,6 +736,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -689,6 +776,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -713,6 +824,30 @@ log4j log4j + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -729,6 +864,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -745,6 +904,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -761,6 +944,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -777,6 +984,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -793,6 +1024,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -809,6 +1064,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -825,6 +1104,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -841,6 +1144,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -857,6 +1184,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -873,6 +1224,30 @@ org.apache.httpcomponents httpclient + + commons-io + commons-io + + + log4j + log4j + + + commons-lang + commons-lang + + + com.google.guava + guava + + + org.apache.opennlp + opennlp-tools + + + org.apache.opennlp + opennlp-maxent + @@ -889,10 +1264,6 @@ org.apache.httpcomponents httpclient - - org.slf4j - slf4j-api - commons-io commons-io @@ -917,10 +1288,6 @@ org.apache.opennlp opennlp-maxent - - org.slf4j - slf4j-log4j12 - @@ -1234,18 +1601,8 @@ 0.10.9.7 provided - - org.bytedeco - cpython-platform - 3.11.3-1.5.9 - provided - - - org.bytedeco - cpython - 3.11.3-1.5.9 - provided - + + @@ -1357,6 +1714,26 @@ 3.9.0 + + org.bytedeco + cpython-platform + 3.10.8-1.5.8 + + + org.bytedeco + cpython + 3.10.8-1.5.8 + + + org.bytedeco + javacpp + 1.5.8 + + + org.bytedeco + javacpp-platform + 1.5.8 + @@ -1730,6 +2107,13 @@ **/*.java + + false + src/main/python + + ** + + diff --git a/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java b/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java index bae28c5088..e06d87aa05 100644 --- a/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java +++ b/src/main/java/org/myrobotlab/service/meta/MarySpeechMeta.java @@ -20,6 +20,14 @@ public MarySpeechMeta() { addDescription("Speech synthesis based on MaryTTS"); addDependency("de.dfki.mary", "marytts", "5.2", "pom"); +// exclude("org.slf4j", "slf4j-api"); +// exclude("commons-io", "commons-io"); +// exclude("log4j", "log4j"); +// exclude("commons-lang", "commons-lang"); +// exclude("com.google.guava", "guava"); +// exclude("org.apache.opennlp", "opennlp-tools"); +// exclude("org.apache.opennlp", "opennlp-maxent"); +// exclude("org.slf4j", "slf4j-log4j12"); // FIXME - use the following config file to generate the needed data for // loadVoice() // main config for voices @@ -39,15 +47,14 @@ public MarySpeechMeta() { exclude("org.slf4j", "slf4j-log4j12"); exclude("log4j", "log4j"); } + exclude("commons-io", "commons-io"); + exclude("log4j", "log4j"); + exclude("commons-lang", "commons-lang"); + exclude("com.google.guava", "guava"); + exclude("org.apache.opennlp", "opennlp-tools"); + exclude("org.apache.opennlp", "opennlp-maxent"); } - exclude("org.slf4j", "slf4j-api"); - exclude("commons-io", "commons-io"); - exclude("log4j", "log4j"); - exclude("commons-lang", "commons-lang"); - exclude("com.google.guava", "guava"); - exclude("org.apache.opennlp", "opennlp-tools"); - exclude("org.apache.opennlp", "opennlp-maxent"); - exclude("org.slf4j", "slf4j-log4j12"); + addDependency("org.apache.logging.log4j", "log4j-1.2-api", "2.12.1"); addDependency("org.apache.logging.log4j", "log4j-api", "2.12.1"); diff --git a/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java b/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java index 00df0a4055..4fe0dafb01 100644 --- a/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java +++ b/src/main/java/org/myrobotlab/service/meta/RuntimeMeta.java @@ -59,6 +59,10 @@ public RuntimeMeta() { // ABSOLUTELY NO JNI/JNA IS USED addDependency("org.bytedeco", "cpython-platform", "3.10.8-1.5.8"); addDependency("org.bytedeco", "cpython", "3.10.8-1.5.8"); + addDependency("org.bytedeco", "javacpp", "1.5.8"); + addDependency("org.bytedeco", "javacpp-platform", "1.5.8"); + +// addDependency("org.apache.commons", "commons-lang3", "3.3.2"); } From 7b17b9f473ebc05dbb04f30ce2d139f132a7679a Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Tue, 22 Aug 2023 14:03:39 -0500 Subject: [PATCH 06/10] Move getId back to ServiceInterface --- .../java/org/myrobotlab/framework/interfaces/NameProvider.java | 1 - .../org/myrobotlab/framework/interfaces/ServiceInterface.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java b/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java index 4e7bca7e67..bb4b6cfce8 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/NameProvider.java @@ -4,6 +4,5 @@ public interface NameProvider { public String getName(); - String getId(); } diff --git a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java index 457872a1db..e31a01ec45 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java @@ -191,6 +191,8 @@ public interface ServiceInterface extends ServiceQueue, LoggingSink, NameTypePro String getFullName(); + String getId(); + void loadLocalizations(); void setLocale(String code); From 1f2cb73d19e50cf389ae1d02c19b28a2def9f227 Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Tue, 22 Aug 2023 14:07:15 -0500 Subject: [PATCH 07/10] Make ServiceRunner extend ServiceInterface instead of NameProvider --- .../java/org/myrobotlab/service/interfaces/ServiceRunner.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java index 70e1d74c6b..d540e05caf 100644 --- a/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java +++ b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java @@ -1,10 +1,9 @@ package org.myrobotlab.service.interfaces; -import org.myrobotlab.framework.interfaces.NameProvider; import org.myrobotlab.framework.interfaces.ServiceInterface; import java.util.List; -public interface ServiceRunner extends NameProvider { +public interface ServiceRunner extends ServiceInterface { /** * Get the runner's supported service programming language From ee4a94b7ae3340f0537f5697ca5b77ce23039079 Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Thu, 14 Sep 2023 11:36:16 -0500 Subject: [PATCH 08/10] Add ability to start the python runtime --- .../org/myrobotlab/ext/python/PythonUtils.java | 14 +++++++++++--- src/main/java/org/myrobotlab/service/Runtime.java | 13 ++++++++++++- src/main/python/python_services/mrl/bootstrap.py | 11 +++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 src/main/python/python_services/mrl/bootstrap.py diff --git a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java index 01f8cca659..f1dcf78b2c 100644 --- a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java +++ b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java @@ -95,14 +95,22 @@ public static int installPipPackages(String python, List packages) { } } - public static int runPythonScript(String python, File workingDirectory, String script, String... args) { + try { + return runPythonScriptAsync(python, workingDirectory, script, args).waitFor(); + } catch (InterruptedException e) { + throw new SubprocessException("Unable to run script " + script + " with Python command " + python, e); + } + } + + + public static Process runPythonScriptAsync(String python, File workingDirectory, String script, String... args) { ProcessBuilder builder = new ProcessBuilder(python, script); List currCommand = builder.command(); currCommand.addAll(List.of(args)); try { - return builder.inheritIO().directory(workingDirectory).start().waitFor(); - } catch (InterruptedException | IOException e) { + return builder.inheritIO().directory(workingDirectory).start(); + } catch (IOException e) { throw new SubprocessException("Unable to run script " + script + " with Python command " + python, e); } diff --git a/src/main/java/org/myrobotlab/service/Runtime.java b/src/main/java/org/myrobotlab/service/Runtime.java index 2373f0e873..75f68fb218 100644 --- a/src/main/java/org/myrobotlab/service/Runtime.java +++ b/src/main/java/org/myrobotlab/service/Runtime.java @@ -321,7 +321,9 @@ public class Runtime extends Service implements MessageListener, */ protected Set startingServices = new HashSet<>(); - private static final String PYTHON_VENV_PATH ="python_services/venv"; + private static final String PYTHON_SERVICES_PATH = "python_services"; + + private static final String PYTHON_VENV_PATH = PYTHON_SERVICES_PATH + fs + "venv"; private String pythonCommand; @@ -638,6 +640,15 @@ public void setAutoStart(boolean autoStart) throws IOException { invoke("getStartYml"); } + public void startPythonRuntime() { + PythonUtils.runPythonScriptAsync( + pythonCommand, + new File("."), + PYTHON_SERVICES_PATH + fs + "mrl" + fs + "bootstrap.py", + Platform.getLocalInstance().getId() + "-python" + ); + } + /** * Framework owned method - core of creating a new service. This method will * create a service with the given name and of the given type. If the type diff --git a/src/main/python/python_services/mrl/bootstrap.py b/src/main/python/python_services/mrl/bootstrap.py new file mode 100644 index 0000000000..220a17b92b --- /dev/null +++ b/src/main/python/python_services/mrl/bootstrap.py @@ -0,0 +1,11 @@ +import logging +import sys + +from mrlpy import mcommand +from mrlpy.framework import runtime +from mrlpy.framework.runtime import Runtime +runtime.runtime_id = sys.argv[1] + +Runtime.init_runtime() +logging.basicConfig(level=logging.INFO, force=True) +mcommand.connect(id=sys.argv[1], daemon=False) From 08cd599746bc619468e31c8736bcb48b976c840f Mon Sep 17 00:00:00 2001 From: Branden Butler Date: Thu, 14 Sep 2023 11:41:01 -0500 Subject: [PATCH 09/10] Allow working directory to be switched --- src/main/java/org/myrobotlab/ext/python/PythonUtils.java | 2 +- src/main/java/org/myrobotlab/service/Runtime.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java index f1dcf78b2c..536793fece 100644 --- a/src/main/java/org/myrobotlab/ext/python/PythonUtils.java +++ b/src/main/java/org/myrobotlab/ext/python/PythonUtils.java @@ -71,7 +71,7 @@ public static String setupVenv(String venv, boolean useBundledPython, List Date: Thu, 14 Sep 2023 15:11:30 -0500 Subject: [PATCH 10/10] Add getSerializedState and make Runtime a ServiceRunner --- .../myrobotlab/framework/Registration.java | 2 +- .../interfaces/ServiceInterface.java | 7 +++++ .../java/org/myrobotlab/service/Runtime.java | 30 ++++++++++++------- .../service/interfaces/ServiceRunner.java | 2 +- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/myrobotlab/framework/Registration.java b/src/main/java/org/myrobotlab/framework/Registration.java index 4f0b3070dc..5f43e9ffe7 100644 --- a/src/main/java/org/myrobotlab/framework/Registration.java +++ b/src/main/java/org/myrobotlab/framework/Registration.java @@ -72,7 +72,7 @@ public Registration(ServiceInterface service) { this.typeKey = service.getTypeKey(); // when this registration is re-broadcasted to remotes it will use this // serialization to init state - this.state = CodecUtils.toJson(service); + this.state = service.getSerializedState(); // if this is a local registration - need reference to service this.service = service; } diff --git a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java index e51bdf2d60..e3f2316fad 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/ServiceInterface.java @@ -6,6 +6,8 @@ import java.util.Map; import java.util.Set; +import com.fasterxml.jackson.annotation.JsonIgnore; +import org.myrobotlab.codec.CodecUtils; import org.myrobotlab.framework.Inbox; import org.myrobotlab.framework.MRLListener; import org.myrobotlab.framework.MethodCache; @@ -226,4 +228,9 @@ public interface ServiceInterface extends ServiceQueue, LoggingSink, NameTypePro * get all the subscriptions to this service */ public Map> getNotifyList(); + + @JsonIgnore + default String getSerializedState() { + return CodecUtils.toJson(this); + } } diff --git a/src/main/java/org/myrobotlab/service/Runtime.java b/src/main/java/org/myrobotlab/service/Runtime.java index 9ea1b3c954..59c44397c0 100644 --- a/src/main/java/org/myrobotlab/service/Runtime.java +++ b/src/main/java/org/myrobotlab/service/Runtime.java @@ -123,7 +123,7 @@ * VAR OF RUNTIME ! * */ -public class Runtime extends Service implements MessageListener, ServiceLifeCyclePublisher, RemoteMessageHandler, ConnectionManager, Gateway, LocaleProvider { +public class Runtime extends Service implements MessageListener, ServiceLifeCyclePublisher, RemoteMessageHandler, ConnectionManager, Gateway, LocaleProvider, ServiceRunner { final static private long serialVersionUID = 1L; // FIXME - AVOID STATIC FIELDS !!! use .getInstance() to get the singleton @@ -649,6 +649,21 @@ public void startPythonRuntime() { ); } + @Override + public List getSupportedLanguageKeys() { + return null; + } + + @Override + public List getAvailableServiceTypes() { + return Arrays.asList(getServiceNames()); + } + + @Override + public ServiceInterface startService(String name, String type) { + return Runtime.start(name, type); + } + /** * Framework owned method - core of creating a new service. This method will * create a service with the given name and of the given type. If the type @@ -755,19 +770,19 @@ static private synchronized ServiceInterface createService(String name, String t .filter(runner -> runner.getAvailableServiceTypes().contains(finalType)) .collect(Collectors.toList()); if (possibleRunners.isEmpty()) { - log.error("Cannot find a service runner to start service with type {}", type); + log.error("Cannot find a service runner to start service with type {}, all known runners: {}", type, serviceRunners); return null; } if (inId == null) { - return possibleRunners.get(0).createService(name, type, null); + return possibleRunners.get(0).startService(name, type); } else { Optional maybeRunner = possibleRunners.stream().filter(runner -> inId.equals(runner.getId())).findFirst(); if (maybeRunner.isEmpty()) { log.error("Cannot find compatible service runner with ID {}", inId); return null; } - return maybeRunner.get().createService(name, type, inId); + return maybeRunner.get().startService(name, type); } } @@ -1199,7 +1214,7 @@ public static Map getMethodMap(String inName) { * @return list of registrations */ synchronized public List getServiceList() { - return registry.values().stream().map(si -> new Registration(si.getId(), si.getName(), si.getTypeKey())).collect(Collectors.toList()); + return registry.values().stream().map(Registration::new).collect(Collectors.toList()); } // FIXME - scary function - returns private data @@ -1757,11 +1772,6 @@ public void onState(ServiceInterface updatedService) { registry.put(String.format("%s@%s", updatedService.getName(), updatedService.getId()), updatedService); } - public static synchronized Registration register(String id, String name, String typeKey, ArrayList interfaces) { - Registration proxy = new Registration(id, name, typeKey, interfaces); - register(proxy); - return proxy; - } /** * Registration is the process where a remote system sends detailed info diff --git a/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java index d540e05caf..33b09b6cc6 100644 --- a/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java +++ b/src/main/java/org/myrobotlab/service/interfaces/ServiceRunner.java @@ -23,5 +23,5 @@ public interface ServiceRunner extends ServiceInterface { */ List getAvailableServiceTypes(); - ServiceInterface createService(String name, String type, String inId); + ServiceInterface startService(String name, String type); }