nixd/test: specify textDocument/didOpen via nix code blocks (#679)

This makes the test cases less verbose and enables syntax highlighting
but most importantly makes multiline documents readable so that test
definitions don't contain lines such as:

"text":"{\n x = 1;\n anonymousLambda = { a }: a;\n namedLambda = a: a;\n
numbers = 1 + 2.0;\n bool = true;\n bool2= false;\n string = \"1\";\n
string2 = \"${builtins.foo}\";\n undefined = x;\n list = [ 1 2 3
];\n}\n"

Before multiline documents were duplicated in another nix code block
(that was ignored by the test runner) for readability but these
duplicated code blocks could (and did) of course get out of sync with
the actual test code.

Author: inclyc

nixd/lspserver/include/lspserver/Connection.h

@@ -40,9 +40,13 @@ class InboundPort {
 
   JSONStreamStyle StreamStyle = JSONStreamStyle::Standard;
 
-  bool readStandardMessage(std::string &JSONString);
+  /// Read one message as specified in the LSP standard.
+  /// A Language Server Protocol message starts with a set of
+  /// HTTP headers, delimited  by \r\n, and terminated by an empty line (\r\n).
+  llvm::Expected<llvm::json::Value> readStandardMessage(std::string &Buffer);
 
-  bool readDelimitedMessage(std::string &JSONString);
+  /// Read one message, expecting the input to be one of our Markdown lit-tests.
+  llvm::Expected<llvm::json::Value> readDelimitedMessage(std::string &Buffer);
 
   /// \brief Notify the inbound port to close the connection
   void close() { Close = true; }
@@ -51,11 +55,9 @@ class InboundPort {
               JSONStreamStyle StreamStyle = JSONStreamStyle::Standard)
       : Close(false), In(In), StreamStyle(StreamStyle) {};
 
-  /// Read messages specified in LSP standard, and collect standard json string
-  /// into \p JSONString.
-  /// A Language Server Protocol message starts with a set of
-  /// HTTP headers, delimited  by \r\n, and terminated by an empty line (\r\n).
-  bool readMessage(std::string &JSONString);
+  /// Read one LSP message depending on the configured StreamStyle.
+  /// The parsed value is returned. The given \p Buffer is only used internally.
+  llvm::Expected<llvm::json::Value> readMessage(std::string &Buffer);
 
   /// Dispatch messages to on{Notify,Call,Reply} ( \p Handlers)
   /// Return values should be forwarded from \p Handlers

nixd/lspserver/src/Connection.cpp

@@ -24,6 +24,28 @@ llvm::cl::opt<int> ClientProcessID{
         "Client process ID, if this PID died, the server should exit."),
     llvm::cl::init(getppid())};
 
+std::string jsonToString(llvm::json::Value &Message) {
+  std::string Result;
+  llvm::raw_string_ostream OS(Result);
+  OS << Message;
+  return Result;
+}
+
+class ReadEOF : public llvm::ErrorInfo<ReadEOF> {
+public:
+  static char ID;
+  ReadEOF() = default;
+
+  /// No need to implement this, we don't need to log this error.
+  void log(llvm::raw_ostream &OS) const override { __builtin_unreachable(); }
+
+  std::error_code convertToErrorCode() const override {
+    return std::make_error_code(std::errc::io_error);
+  }
+};
+
+char ReadEOF::ID;
+
 } // namespace
 
 namespace lspserver {
@@ -184,12 +206,13 @@ bool readLine(int fd, const std::atomic<bool> &Close,
   }
 }
 
-bool InboundPort::readStandardMessage(std::string &JSONString) {
+llvm::Expected<llvm::json::Value>
+InboundPort::readStandardMessage(std::string &Buffer) {
   unsigned long long ContentLength = 0;
   llvm::SmallString<128> Line;
   while (true) {
     if (!readLine(In, Close, Line))
-      return false;
+      return llvm::make_error<ReadEOF>(); // EOF
 
     llvm::StringRef LineRef = Line;
 
@@ -205,76 +228,112 @@ bool InboundPort::readStandardMessage(std::string &JSONString) {
     // It's another header, ignore it.
   }
 
-  JSONString.resize(ContentLength);
+  Buffer.resize(ContentLength);
   for (size_t Pos = 0, Read; Pos < ContentLength; Pos += Read) {
 
-    Read = read(In, JSONString.data() + Pos, ContentLength - Pos);
+    Read = read(In, Buffer.data() + Pos, ContentLength - Pos);
 
     if (Read == 0) {
       elog("Input was aborted. Read only {0} bytes of expected {1}.", Pos,
            ContentLength);
-      return false;
+      return llvm::make_error<ReadEOF>();
     }
   }
-  return true;
+  return llvm::json::parse(Buffer);
 }
 
-bool InboundPort::readDelimitedMessage(std::string &JSONString) {
-  JSONString.clear();
+llvm::Expected<llvm::json::Value>
+InboundPort::readDelimitedMessage(std::string &Buffer) {
+  enum class State { Prose, JSONBlock, NixBlock };
+  State State = State::Prose;
+  Buffer.clear();
   llvm::SmallString<128> Line;
-  bool IsInputBlock = false;
+  std::string NixDocURI;
   while (readLine(In, Close, Line)) {
     auto LineRef = Line.str().trim();
-    if (IsInputBlock) {
-      // We are in input blocks, read lines and append JSONString.
+    if (State == State::Prose) {
+      if (LineRef.starts_with("```json"))
+        State = State::JSONBlock;
+      else if (LineRef.consume_front("```nix ")) {
+        State = State::NixBlock;
+        NixDocURI = LineRef.str();
+      }
+    } else if (State == State::JSONBlock) {
+      // We are in a JSON block, read lines and append JSONString.
       if (LineRef.starts_with("#")) // comment
         continue;
 
       // End of the block
       if (LineRef.starts_with("```")) {
-        IsInputBlock = false;
-        break;
+        return llvm::json::parse(Buffer);
       }
 
-      JSONString += Line;
+      Buffer += Line;
+    } else if (State == State::NixBlock) {
+      // We are in a Nix block. (This was implemented to make the .md test
+      // files more readable, particularly regarding multiline Nix documents,
+      // so that the newlines don't have to be \n escaped.)
+
+      if (LineRef.starts_with("```")) {
+        return llvm::json::Object{
+            {"jsonrpc", "2.0"},
+            {"method", "textDocument/didOpen"},
+            {
+                "params",
+                llvm::json::Object{
+                    {
+                        "textDocument",
+                        llvm::json::Object{
+                            {"uri", NixDocURI},
+                            {"languageId", "nix"},
+                            {"version", 1},
+                            {"text", llvm::StringRef(Buffer).rtrim().str()},
+                        },
+                    },
+                },
+            }};
+      }
+      Buffer += Line;
+      Buffer += "\n";
     } else {
-      if (LineRef.starts_with("```json"))
-        IsInputBlock = true;
+      assert(false && "unreachable");
     }
   }
-  return true; // Including at EOF
+  return llvm::make_error<ReadEOF>(); // EOF
 }
 
-bool InboundPort::readMessage(std::string &JSONString) {
+llvm::Expected<llvm::json::Value>
+InboundPort::readMessage(std::string &Buffer) {
   switch (StreamStyle) {
 
   case JSONStreamStyle::Standard:
-    return readStandardMessage(JSONString);
+    return readStandardMessage(Buffer);
   case JSONStreamStyle::Delimited:
-    return readDelimitedMessage(JSONString);
-    break;
+    return readDelimitedMessage(Buffer);
   }
   assert(false && "Invalid stream style");
   __builtin_unreachable();
 }
 
 void InboundPort::loop(MessageHandler &Handler) {
-  std::string JSONString;
-  llvm::SmallString<128> Line;
+  std::string Buffer;
 
   for (;;) {
-    if (readMessage(JSONString)) {
-      vlog("<<< {0}", JSONString);
-      if (auto ExpectedParsedJSON = llvm::json::parse(JSONString)) {
-        if (!dispatch(*ExpectedParsedJSON, Handler))
-          return;
-      } else {
-        auto Err = ExpectedParsedJSON.takeError();
-        elog("The received json cannot be parsed, reason: {0}", Err);
+    if (auto Message = readMessage(Buffer)) {
+      vlog("<<< {0}", jsonToString(*Message));
+      if (!dispatch(*Message, Handler))
         return;
-      }
     } else {
-      return;
+      // Handle error while reading message.
+      return [&]() {
+        auto Err = Message.takeError();
+        if (Err.isA<ReadEOF>())
+          return; // Stop reading.
+        else if (Err.isA<llvm::json::ParseError>())
+          elog("The received json cannot be parsed, reason: {0}", Err);
+        else
+          elog("Error reading message: {0}", llvm::toString(std::move(Err)));
+      }();
     }
   }
 }

nixd/tools/nixd/test/code-action/quick-fix.md

@@ -21,19 +21,8 @@
 
 <-- textDocument/didOpen
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///basic.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"/*"
-      }
-   }
-}
+```nix file:///basic.nix
+/*
 ```
 
 <-- textDocument/codeAction(2)

nixd/tools/nixd/test/completion/comment.md

@@ -23,19 +23,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"{ x = 1; y = /*   */2;}"
-      }
-   }
-}
+```nix file:///completion.nix
+{ x = 1; y = /*   */2;}
 ```
 
 ```json

nixd/tools/nixd/test/completion/dot-select.md

@@ -23,19 +23,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"{ bar = 1; foo = foo.foo.; }"
-      }
-   }
-}
+```nix file:///completion.nix
+{ bar = 1; foo = foo.foo.; }
 ```
 
 ```json

nixd/tools/nixd/test/completion/dot.md

@@ -23,19 +23,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"{ bar = 1; foo. }"
-      }
-   }
-}
+```nix file:///completion.nix
+{ bar = 1; foo. }
 ```
 
 ```json

nixd/tools/nixd/test/completion/nested.md

@@ -23,19 +23,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"{ bar = 1; foo = { ba = }; }"
-      }
-   }
-}
+```nix file:///completion.nix
+{ bar = 1; foo = { ba = }; }
 ```
 
 ```json

nixd/tools/nixd/test/completion/nixpkgs.md

@@ -21,19 +21,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"with pkgs; [ a ]"
-      }
-   }
-}
+```nix file:///completion.nix
+with pkgs; [ a ]
 ```
 
 ```json

nixd/tools/nixd/test/completion/option-stop.md

@@ -23,19 +23,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"{ bar = 1; foo.bar. }"
-      }
-   }
-}
+```nix file:///completion.nix
+{ bar = 1; foo.bar. }
 ```
 
 ```json

nixd/tools/nixd/test/completion/options-in-config.md

@@ -23,19 +23,8 @@
 <-- textDocument/didOpen
 
 
-```json
-{
-   "jsonrpc":"2.0",
-   "method":"textDocument/didOpen",
-   "params":{
-      "textDocument":{
-         "uri":"file:///completion.nix",
-         "languageId":"nix",
-         "version":1,
-         "text":"{ config = { fo }; }"
-      }
-   }
-}
+```nix file:///completion.nix
+{ config = { foo }; }
 ```
 
 ```json