feat: improve llama server logging

Author: carlrobertoh

src/main/java/ee/carlrobert/codegpt/completions/llama/LlamaServerAgent.java

@@ -1,5 +1,7 @@
 package ee.carlrobert.codegpt.completions.llama;
 
+import static java.lang.String.format;
+
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.intellij.execution.ExecutionException;
 import com.intellij.execution.configurations.GeneralCommandLine;
@@ -9,18 +11,21 @@
 import com.intellij.execution.process.ProcessListener;
 import com.intellij.execution.process.ProcessOutputType;
 import com.intellij.icons.AllIcons.Actions;
+import com.intellij.notification.NotificationType;
 import com.intellij.openapi.Disposable;
 import com.intellij.openapi.application.ApplicationManager;
 import com.intellij.openapi.components.Service;
 import com.intellij.openapi.diagnostic.Logger;
 import com.intellij.openapi.util.Key;
 import com.intellij.ui.components.JBLabel;
+import ee.carlrobert.codegpt.CodeGPTBundle;
 import ee.carlrobert.codegpt.CodeGPTPlugin;
-import ee.carlrobert.codegpt.settings.service.LlamaServiceSelectionForm;
 import ee.carlrobert.codegpt.settings.service.ServerProgressPanel;
 import ee.carlrobert.codegpt.settings.state.LlamaSettingsState;
+import ee.carlrobert.codegpt.util.OverlayUtil;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
 import javax.swing.SwingConstants;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -30,19 +35,20 @@ public final class LlamaServerAgent implements Disposable {
 
   private static final Logger LOG = Logger.getInstance(LlamaServerAgent.class);
 
-  private static @Nullable OSProcessHandler makeProcessHandler;
-  private static @Nullable OSProcessHandler startServerProcessHandler;
+  private @Nullable OSProcessHandler makeProcessHandler;
+  private @Nullable OSProcessHandler startServerProcessHandler;
 
   public void startAgent(
-      LlamaServiceSelectionForm llamaServiceSelectionForm,
+      LlamaServerStartupParams params,
       ServerProgressPanel serverProgressPanel,
       Runnable onSuccess) {
     ApplicationManager.getApplication().invokeLater(() -> {
       try {
-        serverProgressPanel.updateText("Building llama.cpp...");
+        serverProgressPanel.updateText(
+            CodeGPTBundle.get("llamaServerAgent.buildingProject.description"));
         makeProcessHandler = new OSProcessHandler(getMakeCommandLinde());
         makeProcessHandler.addProcessListener(
-            getMakeProcessListener(llamaServiceSelectionForm, serverProgressPanel, onSuccess));
+            getMakeProcessListener(params, serverProgressPanel, onSuccess));
         makeProcessHandler.startNotify();
       } catch (ExecutionException e) {
         throw new RuntimeException(e);
@@ -63,9 +69,11 @@ public boolean isServerRunning() {
   }
 
   private ProcessListener getMakeProcessListener(
-      LlamaServiceSelectionForm serviceSelectionForm,
+      LlamaServerStartupParams params,
       ServerProgressPanel serverProgressPanel,
       Runnable onSuccess) {
+    LOG.info("Building llama project");
+
     return new ProcessAdapter() {
       @Override
       public void onTextAvailable(@NotNull ProcessEvent event, @NotNull Key outputType) {
@@ -75,21 +83,16 @@ public void onTextAvailable(@NotNull ProcessEvent event, @NotNull Key outputType
       @Override
       public void processTerminated(@NotNull ProcessEvent event) {
         try {
-          serverProgressPanel.updateText("Booting up server...");
-          startServerProcessHandler = new OSProcessHandler.Silent(
-              getServerCommandLine(
-                  serviceSelectionForm.getLlamaModelPreferencesForm().getActualModelPath(),
-                  serviceSelectionForm.getContextSize(),
-                  serviceSelectionForm.getThreads(),
-                  serviceSelectionForm.getServerPort(),
-                  serviceSelectionForm.getListOfAdditionalParameters()));
-          startServerProcessHandler.addProcessListener(getProcessListener(
-              serviceSelectionForm.getServerPort(),
-              serverProgressPanel,
-              onSuccess));
+          LOG.info("Booting up llama server");
+
+          serverProgressPanel.updateText(
+              CodeGPTBundle.get("llamaServerAgent.serverBootup.description"));
+          startServerProcessHandler = new OSProcessHandler.Silent(getServerCommandLine(params));
+          startServerProcessHandler.addProcessListener(
+              getProcessListener(params.getPort(), serverProgressPanel, onSuccess));
           startServerProcessHandler.startNotify();
         } catch (ExecutionException ex) {
-          LOG.error("Unable to start the server", ex);
+          LOG.error("Unable to start llama server", ex);
           throw new RuntimeException(ex);
         }
       }
@@ -102,9 +105,18 @@ private ProcessListener getProcessListener(
       Runnable onSuccess) {
     return new ProcessAdapter() {
       private final ObjectMapper objectMapper = new ObjectMapper();
+      private final List<String> errorLines = new CopyOnWriteArrayList<>();
 
       @Override
       public void processTerminated(@NotNull ProcessEvent event) {
+        if (errorLines.isEmpty()) {
+          LOG.info(format("Server terminated with code %d", event.getExitCode()));
+        } else {
+          var error = String.join("", errorLines);
+          OverlayUtil.showNotification(error, NotificationType.ERROR);
+          LOG.error(error);
+        }
+
         serverProgressPanel.displayComponent(new JBLabel(
             "Server terminated",
             Actions.Cancel,
@@ -113,12 +125,19 @@ public void processTerminated(@NotNull ProcessEvent event) {
 
       @Override
       public void onTextAvailable(@NotNull ProcessEvent event, @NotNull Key outputType) {
-        LOG.debug(event.getText());
+        if (ProcessOutputType.isStderr(outputType)) {
+          errorLines.add(event.getText());
+          return;
+        }
+
+        if (ProcessOutputType.isStdout(outputType)) {
+          LOG.info(event.getText());
 
-        if (outputType == ProcessOutputType.STDOUT) {
           try {
             var serverMessage = objectMapper.readValue(event.getText(), LlamaServerMessage.class);
             if ("HTTP server listening".equals(serverMessage.getMessage())) {
+              LOG.info("Server up and running!");
+
               LlamaSettingsState.getInstance().setServerPort(port);
               onSuccess.run();
             }
@@ -139,21 +158,16 @@ private static GeneralCommandLine getMakeCommandLinde() {
     return commandLine;
   }
 
-  private GeneralCommandLine getServerCommandLine(
-      String modelPath,
-      int contextLength,
-      int threads,
-      int port,
-      List<String> additionalParameters) {
+  private GeneralCommandLine getServerCommandLine(LlamaServerStartupParams params) {
     GeneralCommandLine commandLine = new GeneralCommandLine().withCharset(StandardCharsets.UTF_8);
     commandLine.setExePath("./server");
     commandLine.withWorkDirectory(CodeGPTPlugin.getLlamaSourcePath());
     commandLine.addParameters(
-        "-m", modelPath,
-        "-c", String.valueOf(contextLength),
-        "--port", String.valueOf(port),
-        "-t", String.valueOf(threads));
-    commandLine.addParameters(additionalParameters);
+        "-m", params.getModelPath(),
+        "-c", String.valueOf(params.getContextLength()),
+        "--port", String.valueOf(params.getPort()),
+        "-t", String.valueOf(params.getThreads()));
+    commandLine.addParameters(params.getAdditionalParameters());
     commandLine.setRedirectErrorStream(false);
     return commandLine;
   }

src/main/java/ee/carlrobert/codegpt/completions/llama/LlamaServerStartupParams.java

@@ -0,0 +1,45 @@
+package ee.carlrobert.codegpt.completions.llama;
+
+import java.util.List;
+
+public class LlamaServerStartupParams {
+
+  private final String modelPath;
+  private final int contextLength;
+  private final int threads;
+  private final int port;
+  private final List<String> additionalParameters;
+
+  public LlamaServerStartupParams(
+      String modelPath,
+      int contextLength,
+      int threads,
+      int port,
+      List<String> additionalParameters) {
+    this.modelPath = modelPath;
+    this.contextLength = contextLength;
+    this.threads = threads;
+    this.port = port;
+    this.additionalParameters = additionalParameters;
+  }
+
+  public String getModelPath() {
+    return modelPath;
+  }
+
+  public int getContextLength() {
+    return contextLength;
+  }
+
+  public int getThreads() {
+    return threads;
+  }
+
+  public int getPort() {
+    return port;
+  }
+
+  public List<String> getAdditionalParameters() {
+    return additionalParameters;
+  }
+}

src/main/java/ee/carlrobert/codegpt/settings/service/LlamaServiceSelectionForm.java

@@ -18,6 +18,7 @@
 import ee.carlrobert.codegpt.CodeGPTPlugin;
 import ee.carlrobert.codegpt.completions.HuggingFaceModel;
 import ee.carlrobert.codegpt.completions.llama.LlamaServerAgent;
+import ee.carlrobert.codegpt.completions.llama.LlamaServerStartupParams;
 import ee.carlrobert.codegpt.settings.state.LlamaSettingsState;
 import ee.carlrobert.codegpt.util.OverlayUtil;
 import java.awt.BorderLayout;
@@ -179,11 +180,21 @@ private JButton getServerButton(
         llamaServerAgent.stopAgent();
       } else {
         disableForm(serverButton, serverProgressPanel);
-        llamaServerAgent.startAgent(this, serverProgressPanel, () -> {
-          setFormEnabled(false);
-          serverProgressPanel.displayComponent(
-              new JBLabel("Server running", Actions.Checked, SwingConstants.LEADING));
-        });
+        llamaServerAgent.startAgent(
+            new LlamaServerStartupParams(
+                llamaModelPreferencesForm.getActualModelPath(),
+                getContextSize(),
+                getThreads(),
+                getServerPort(),
+                getListOfAdditionalParameters()),
+            serverProgressPanel,
+            () -> {
+              setFormEnabled(false);
+              serverProgressPanel.displayComponent(new JBLabel(
+                  CodeGPTBundle.get("settingsConfigurable.service.llama.progress.serverRunning"),
+                  Actions.Checked,
+                  SwingConstants.LEADING));
+            });
       }
     });
     return serverButton;

src/main/resources/messages/codegpt.properties

@@ -43,11 +43,12 @@ settingsConfigurable.service.llama.contextSize.label=Prompt context size:
 settingsConfigurable.service.llama.contextSize.comment=The size of the prompt context. LLaMA models were built with a context of 2048, which will provide better results for longer input/inference.
 settingsConfigurable.service.llama.threads.label=Threads:
 settingsConfigurable.service.llama.threads.comment=The number of threads available to execute the model. It is not recommended to specify a number greater than the number of processor cores.
-settingsConfigurable.service.llama.additionalParameters.label=Additional parameters
+settingsConfigurable.service.llama.additionalParameters.label=Additional parameters:
 settingsConfigurable.service.llama.additionalParameters.comment=<html>Additional command-line parameters for the server startup process, separated by commas. See the full <a href="https://github.com/ggerganov/llama.cpp/blob/master/examples/server/README.md">list of options</a>.<p><i>Example: "--n-gpu-layers, 1,  --no-mmap, --mlock"</i></p></html>
 settingsConfigurable.service.llama.port.label=Port:
 settingsConfigurable.service.llama.startServer.label=Start server
 settingsConfigurable.service.llama.stopServer.label=Stop server
+settingsConfigurable.service.llama.progress.serverRunning=Server running
 settingsConfigurable.service.llama.progress.stoppingServer=Stopping a server...
 settingsConfigurable.service.llama.progress.startingServer=Starting a server...
 settingsConfigurable.service.llama.progress.downloadingModel.title=Downloading Model
@@ -134,3 +135,5 @@ checkForUpdatesTask.title=Checking for CodeGPT update...
 checkForUpdatesTask.notification.message=An update for CodeGPT is available.
 checkForUpdatesTask.notification.installButton=Install update
 checkForUpdatesTask.notification.hideButton=Do not show again
+llamaServerAgent.buildingProject.description=Building llama.cpp...
+llamaServerAgent.serverBootup.description=Booting up server...