diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e766c46a1..fe28a2657 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,3 +139,29 @@ jobs: apps: scala-steward - run: scala-steward validate-repo-config .scala-steward.conf + + build-parser: + name: Build parser + strategy: + matrix: + os: [ubuntu-latest, macos-latest, macos-13] + java: [temurin@11] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + steps: + - name: Checkout current branch (fast) + uses: actions/checkout@v4 + + - name: Setup Java (temurin@11) + id: setup-java-temurin-11 + if: matrix.java == 'temurin@11' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + + - name: Setup sbt + uses: sbt/setup-sbt@v1 + + - name: Parser tests + run: sbt treesitter/test diff --git a/.gitignore b/.gitignore index afc0453fe..8c95b2bd0 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ smithyql-log.txt result .version +build/smithy diff --git a/.mergify.yml b/.mergify.yml index 1ca240269..1a79a8d94 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -87,6 +87,14 @@ pull_request_rules: add: - parser remove: [] +- name: Label parser-gen PRs + conditions: + - files~=^modules/parser-gen/ + actions: + label: + add: + - parser-gen + remove: [] - name: Label plugin-core PRs conditions: - files~=^modules/plugin-core/ @@ -119,3 +127,11 @@ pull_request_rules: add: - source remove: [] +- name: Label treesitter PRs + conditions: + - files~=^modules/treesitter/ + actions: + label: + add: + - treesitter + remove: [] diff --git a/build.sbt b/build.sbt index 5d71c35e2..a73ffc9e3 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,5 @@ +import sbt.util.CacheImplicits._ + inThisBuild( List( organization := "com.kubukoz.playground", @@ -72,6 +74,28 @@ ThisBuild / githubWorkflowBuild := List( ThisBuild / githubWorkflowArtifactUpload := false ThisBuild / githubWorkflowGeneratedCI ~= (_.filterNot(_.id == "publish")) +ThisBuild / githubWorkflowGeneratedCI += WorkflowJob( + id = "build-parser", + name = "Build parser", + oses = List("ubuntu-latest", "macos-latest", "macos-13"), + scalas = Nil, + timeoutMinutes = Some(10), + steps = + List( + WorkflowStep.Checkout + ) ++ + WorkflowStep + .SetupJava(githubWorkflowJavaVersions.value.toList, enableCaching = false) + ++ List( + WorkflowStep.SetupSbt, + // intentionally not using WorkflowStep.Sbt so that we don't set up Nix + // as this should work without it (native libs shall be pre-built by contributors) + WorkflowStep.Run( + name = Some("Parser tests"), + commands = "sbt treesitter/test" :: Nil, + ), + ), +) ThisBuild / mergifyStewardConfig ~= (_.map(_.withMergeMinors(true))) @@ -128,6 +152,7 @@ val commonSettings = Seq( tlFatalWarnings := false, mimaPreviousArtifacts := Set.empty, mimaFailOnNoPrevious := false, + resolvers += "Sonatype S01 snapshots" at "https://s01.oss.sonatype.org/content/repositories/snapshots", ) def module( @@ -175,8 +200,59 @@ lazy val parser = module("parser") .dependsOn( ast % "test->test;compile->compile", source % "test->test;compile->compile", + treesitter % "test->compile", ) +lazy val treesitter = module("treesitter") + .settings( + libraryDependencies ++= Seq( + "org.polyvariant.treesitter4s" %% "core" % "0.4.0" + ), + Compile / sourceGenerators += { + Def.task { + Process(Seq("tree-sitter", "generate"), Some((os.pwd / "tree-sitter-smithyql").toIO)).!! + + val pargenClasspath = (parsergen / Compile / fullClasspath).value.files.map(os.Path(_)) + val targetDir = os.Path((Compile / sourceManaged).value) / "ts4s-parsergen" + + type Input = Seq[os.Path] + + def generate(input: Input) = { + val pargenClasspath = input + + val paths = + Process( + List( + "java", + "-cp", + pargenClasspath.mkString(":"), + (parsergen / Compile / discoveredMainClasses).value.head, + ), + None, + "CODEGEN_TARGET" -> targetDir.toString, + ).!!(streams.value.log).linesIterator.toList + + paths.map(os.Path(_)).map(_.toIO) + } + + generate(pargenClasspath) + }.taskValue + }, + ) + +lazy val parsergen = module("parser-gen") + .settings( + libraryDependencies ++= Seq( + "dev.optics" %% "monocle-core" % "3.3.0", + "com.disneystreaming.smithy4s" %% "smithy4s-json" % smithy4sVersion.value, + ("org.scalameta" %% "scalameta" % "4.11.0").cross(CrossVersion.for3Use2_13), + "org.polyvariant.treesitter4s" %% "core" % "0.4.0", + "com.lihaoyi" %% "os-lib" % "0.11.3", + ), + scalacOptions -= "-release:21", + ) + .enablePlugins(Smithy4sCodegenPlugin) + // Formatter for the SmithyQL language constructs lazy val formatter = module("formatter") .settings( @@ -231,6 +307,7 @@ lazy val core = module("core") examples % "test->compile", pluginCore, ast, + treesitter, source % "test->test;compile->compile", parser % "test->compile;test->test", formatter, @@ -291,6 +368,7 @@ lazy val e2e = module("e2e") parser / publishLocal, pluginCore / publishLocal, source / publishLocal, + treesitter / publishLocal, ast / publishLocal, formatter / publishLocal, protocol4s / publishLocal, @@ -320,6 +398,7 @@ lazy val root = project core, examples, parser, + parsergen, formatter, languageSupport, lspKernel, @@ -328,4 +407,5 @@ lazy val root = project pluginCore, pluginSample, e2e, + treesitter, ) diff --git a/flake.lock b/flake.lock index 0ef3c0b09..f448bae36 100644 --- a/flake.lock +++ b/flake.lock @@ -20,16 +20,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1730602179, - "narHash": "sha256-efgLzQAWSzJuCLiCaQUCDu4NudNlHdg2NzGLX5GYaEY=", + "lastModified": 1734875076, + "narHash": "sha256-Pzyb+YNG5u3zP79zoi8HXYMs15Q5dfjDgwCdUI5B0nY=", "owner": "nixos", "repo": "nixpkgs", - "rev": "3c2f1c4ca372622cb2f9de8016c9a0b1cbd0f37c", + "rev": "1807c2b91223227ad5599d7067a61665c52d1295", "type": "github" }, "original": { "owner": "nixos", - "ref": "nixos-24.05", + "ref": "nixos-24.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index c81ba4511..06149495b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,16 +1,13 @@ { inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05"; + nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11"; flake-utils.url = "github:numtide/flake-utils"; }; outputs = { self, nixpkgs, flake-utils, ... }: - flake-utils.lib.eachDefaultSystem ( - system: - let - pkgs = import nixpkgs { inherit system; }; - in - { + flake-utils.lib.eachDefaultSystem (system: + let pkgs = import nixpkgs { inherit system; }; + in { devShells.default = pkgs.mkShell { buildInputs = [ pkgs.yarn @@ -18,10 +15,33 @@ pkgs.sbt pkgs.jless pkgs.gnupg + (pkgs.tree-sitter.override { webUISupport = true; }) # temporary, while we don't download coursier ourselves pkgs.coursier ] ++ pkgs.lib.optionals pkgs.stdenv.isLinux [ pkgs.xvfb-run ]; }; - } - ); + packages.tree-sitter-smithyql = pkgs.stdenv.mkDerivation { + name = "tree-sitter-smithyql"; + src = ./tree-sitter-smithyql; + buildInputs = [ pkgs.tree-sitter pkgs.nodejs ]; + buildPhase = '' + tree-sitter generate + cc src/parser.c -shared -o $out + ''; + dontInstall = true; + }; + packages.tree-sitter-smithyql-all = pkgs.stdenv.mkDerivation { + name = "tree-sitter-smithyql-all"; + src = ./tree-sitter-smithyql; + dontBuild = true; + installPhase = '' + mkdir $out + cd $out + mkdir darwin-aarch64 && cp ${self.packages.aarch64-darwin.tree-sitter-smithyql} darwin-aarch64/libtree-sitter-smithyql.dylib + mkdir darwin-x86-64 && cp ${self.packages.x86_64-darwin.tree-sitter-smithyql} darwin-x86-64/libtree-sitter-smithyql.dylib + mkdir linux-aarch64 && cp ${self.packages.aarch64-linux.tree-sitter-smithyql} linux-aarch64/libtree-sitter-smithyql.so + mkdir linux-x86-64 && cp ${self.packages.x86_64-linux.tree-sitter-smithyql} linux-x86-64/libtree-sitter-smithyql.so + ''; + }; + }); } diff --git a/modules/core/src/main/scala/playground/ASTAdapter.scala b/modules/core/src/main/scala/playground/ASTAdapter.scala new file mode 100644 index 000000000..c7d6816f5 --- /dev/null +++ b/modules/core/src/main/scala/playground/ASTAdapter.scala @@ -0,0 +1,12 @@ +package playground + +import cats.syntax.all.* +import playground.smithyql.QualifiedIdentifier + +object ASTAdapter { + + def decodeQI(qi: playground.generated.nodes.QualifiedIdentifier): Option[QualifiedIdentifier] = + (qi.namespace.map(_.source).toNel, qi.selection.map(_.source)) + .mapN(QualifiedIdentifier.apply) + +} diff --git a/modules/core/src/main/scala/playground/MultiServiceResolver.scala b/modules/core/src/main/scala/playground/MultiServiceResolver.scala index 15c5fc7c6..ff35f20f0 100644 --- a/modules/core/src/main/scala/playground/MultiServiceResolver.scala +++ b/modules/core/src/main/scala/playground/MultiServiceResolver.scala @@ -9,6 +9,7 @@ import playground.smithyql.UseClause import playground.smithyql.WithSource object MultiServiceResolver { + import playground.smithyql.tsutils.* /** Determines which service should be used for a query. The rules are: * - If the operation name has a service identifier, there MUST be a service with that name @@ -37,6 +38,34 @@ object MultiServiceResolver { case None => resolveImplicit(queryOperationName.operationName, serviceIndex, useClauses) } + /** Determines which service should be used for a query. The rules are: + * - If the operation name has a service identifier, there MUST be a service with that name + * that contains the given operation. + * - If there's no service identifier, find all matching services that are included in the use + * clauses. MUST find exactly one entry. + * + * In other cases, such as when we can't find a unique entry, or the explicitly referenced + * service doesn't have an operation with a matching name, we fail. The latter might eventually + * be refactored to a separate piece of code. + * + * **Important**! + * + * This method assumes that all of the use clauses match the available service set. It does NOT + * perform a check on that. For the actual check, see PreludeCompiler. + */ + def resolveServiceTs( + queryOperationName: playground.generated.nodes.QueryOperationName, + serviceIndex: ServiceIndex, + useClauses: List[playground.generated.nodes.UseClause], + ): EitherNel[CompilationError, Option[QualifiedIdentifier]] = queryOperationName + .name + .flatTraverse { opName => + queryOperationName.service_identifier match { + case Some(explicitRef) => resolveExplicitTs(serviceIndex, explicitRef, opName) + case None => resolveImplicitTs(opName, serviceIndex, useClauses).map(_.some) + } + } + private def resolveExplicit( index: ServiceIndex, explicitRef: WithSource[QualifiedIdentifier], @@ -66,6 +95,39 @@ object MultiServiceResolver { case Some(_) => explicitRef.value.asRight } + private def resolveExplicitTs( + index: ServiceIndex, + explicitRef: playground.generated.nodes.QualifiedIdentifier, + operationName: playground.generated.nodes.OperationName, + ): EitherNel[CompilationError, Option[QualifiedIdentifier]] = ASTAdapter + .decodeQI(explicitRef) + .traverse { ref => + index.getService(ref) match { + // explicit reference exists, but the service doesn't + case None => + CompilationError + .error( + CompilationErrorDetails.UnknownService(index.serviceIds.toList), + explicitRef.range, + ) + .leftNel + + // the service exists, but doesn't have the requested operation + case Some(service) + if !service.operationNames.contains_(OperationName(operationName.source)) => + CompilationError + .error( + CompilationErrorDetails.OperationMissing(service.operationNames.toList), + operationName.range, + ) + .leftNel + + // all good + case Some(_) => ref.asRight + } + + } + private def resolveImplicit( operationName: WithSource[OperationName[WithSource]], index: ServiceIndex, @@ -90,4 +152,28 @@ object MultiServiceResolver { } } + private def resolveImplicitTs( + operationName: playground.generated.nodes.OperationName, + index: ServiceIndex, + useClauses: List[playground.generated.nodes.UseClause], + ): EitherNel[CompilationError, QualifiedIdentifier] = { + val matchingServices = index + .getServices(useClauses.flatMap(_.identifier).flatMap(ASTAdapter.decodeQI).toSet) + .filter(_.hasOperation(OperationName(operationName.source))) + + matchingServices match { + case one :: Nil => one.id.asRight + case _ => + CompilationError + .error( + CompilationErrorDetails + .AmbiguousService( + workspaceServices = index.serviceIds.toList + ), + operationName.range, + ) + .leftNel + } + } + } diff --git a/modules/core/src/main/scala/playground/smithyql/RangeIndex.scala b/modules/core/src/main/scala/playground/smithyql/RangeIndex.scala index e651290c6..144fd96ea 100644 --- a/modules/core/src/main/scala/playground/smithyql/RangeIndex.scala +++ b/modules/core/src/main/scala/playground/smithyql/RangeIndex.scala @@ -1,166 +1,96 @@ package playground.smithyql import cats.syntax.all.* +import tsutils.* +import util.chaining.* trait RangeIndex { def findAtPosition( pos: Position - ): NodeContext + ): Option[NodeContext] } object RangeIndex { - def build( - sf: SourceFile[WithSource] - ): RangeIndex = - new RangeIndex { - - private val allRanges: List[ContextRange] = { - val path = NodeContext.EmptyPath - - val preludeRanges: List[ContextRange] = sf - .prelude - .useClauses - .toNel - .foldMap { useClauses => - val newBase = path.inPrelude - - ContextRange(useClauses.map(_.range).reduceLeft(_.fakeUnion(_)), newBase) :: - sf.prelude - .useClauses - .mapWithIndex { - ( - uc, - i, - ) => - findInUseClause(uc, newBase.inUseClause(i)) - } - .combineAll - } + def build(parsed: playground.generated.nodes.SourceFile): RangeIndex = fromRanges { - val queryRanges = sf.queries(WithSource.unwrap).zipWithIndex.flatMap { case (rq, index) => - findInQuery(rq.query, path.inQuery(index)) - } + val root = NodeContext.EmptyPath - preludeRanges ++ queryRanges + val preludeRanges = parsed + .prelude + .toList + .flatMap { prelude => + val newBase = root.inPrelude + ContextRange(prelude.range, newBase) :: + prelude.use_clause.zipWithIndex.map { (useClause, i) => + ContextRange(useClause.range, newBase.inUseClause(i)) + } } - // Console - // .err - // .println( - // s"""Found ${allRanges.size} ranges for query ${q.operationName.value.text}: - // |${allRanges - // .map(_.render) - // .mkString("\n")}""".stripMargin - // ) - - def findAtPosition( - pos: Position - ): NodeContext = allRanges - .filter(_.range.contains(pos)) - .maxByOption(_.ctx.length) - .map(_.ctx) - // By default, we're on root level - .getOrElse(NodeContext.EmptyPath) - - } - - private def findInQuery( - q: WithSource[Query[WithSource]], - path: NodeContext, - ) = { - val qv = q.value - - List(ContextRange(q.range, path)) ++ - findInOperationName(qv.operationName, path.inOperationName) ++ - findInNode(qv.input, path.inOperationInput) - } - - private def findInUseClause( - useClause: WithSource[UseClause[WithSource]], - path: NodeContext, - ): List[ContextRange] = ContextRange(useClause.value.identifier.range, path) :: Nil - - private def findInOperationName( - operationName: WithSource[QueryOperationName[WithSource]], - path: NodeContext, - ): List[ContextRange] = - ContextRange( - operationName.value.operationName.range, - path, - ) :: Nil - - private def findInNode( - node: WithSource[InputNode[WithSource]], - ctx: NodeContext, - ): List[ContextRange] = { - def entireNode( - ctx: NodeContext - ) = ContextRange(node.range, ctx) - - val default = Function.const( - // Default case: can be triggered e.g. inside a string literal - // which would affect completions of enum values and timestamps. - entireNode(ctx) :: Nil + def inputNodeRanges(node: playground.generated.nodes.InputNode, base: NodeContext) + : List[ContextRange] = node.visit( + new playground.generated.nodes.InputNode.Visitor.Default[List[ContextRange]] { + def default: List[ContextRange] = Nil + + override def onString(node: playground.generated.nodes.String_): List[ContextRange] = + ContextRange(node.range.shrink1, base.inQuotes) :: Nil + + override def onList(node: playground.generated.nodes.List_): List[ContextRange] = + ContextRange(node.range.shrink1, base.inCollectionEntry(None)) :: + node.list_fields.zipWithIndex.flatMap { (inputNode, i) => + ContextRange(inputNode.range, base.inCollectionEntry(Some(i))) :: + inputNodeRanges(inputNode, base.inCollectionEntry(Some(i))) + } + + override def onStruct(node: playground.generated.nodes.Struct): List[ContextRange] = + ContextRange(node.range.shrink1, base.inStructBody) :: + node.bindings.toList.flatMap { binding => + (binding.key, binding.value).tupled.toList.flatMap { (key, value) => + ContextRange(value.range, base.inStructBody.inStructValue(key.source)) :: + inputNodeRanges(value, base.inStructBody.inStructValue(key.source)) + } + } + } ) - node - .value - .fold( - listed = l => entireNode(ctx) :: findInList(l, ctx), - struct = s => entireNode(ctx) :: findInStruct(s, ctx.inStructBody), - string = { _ => - val inQuotes = ContextRange( - node.range.shrink1, - ctx.inQuotes, - ) - - inQuotes :: entireNode(ctx) :: Nil - }, - int = default, - bool = default, - nul = default, - ) + val queryRanges = parsed.statements.zipWithIndex.flatMap { (stat, statementIndex) => + stat.run_query.toList.flatMap { runQuery => + ContextRange(runQuery.range, root.inQuery(statementIndex)) :: runQuery + .operation_name + .toList + .flatMap { operationName => + ContextRange(operationName.range, root.inQuery(statementIndex).inOperationName) :: Nil + } ++ + runQuery.input.toList.flatMap { input => + inputNodeRanges( + playground.generated.nodes.InputNode(input), + root.inQuery(statementIndex).inOperationInput, + ) - } + } + } - private def findInList( - list: Listed[WithSource], - ctx: NodeContext, - ): List[ContextRange] = { - val inItems = list - .values - .value - .zipWithIndex - .flatMap { case (entry, index) => findInNode(entry, ctx.inCollectionEntry(index.some)) } - - val inBody = ContextRange( - list - .values - .range, - ctx.inCollectionEntry(None), - ) + } - inBody :: inItems + preludeRanges ++ queryRanges } - private def findInStruct( - struct: Struct[WithSource], - ctx: NodeContext, - ): List[ContextRange] = { - // Struct fields that allow nesting in them - val inFields = struct - .fields - .value - .value - .flatMap { binding => - findInNode(binding.value, ctx.inStructValue(binding.identifier.value.text)) - } - - ContextRange(struct.fields.range, ctx) :: inFields - } + def fromRanges(allRanges: List[ContextRange]): RangeIndex = + pos => + allRanges + .filter(_.range.contains(pos)) + .tap { ranges => + // println() + // println("=======") + // println(s"all ranges: ${allRanges.map(_.render).mkString(", ")}") + // println(s"ranges for position ${pos.index}: ${ranges.map(_.render).mkString(", ")}") + // println("=======") + // println() + } + .maxByOption(_.ctx.length) + .map(_.ctx) } diff --git a/modules/core/src/main/scala/playground/smithyql/tsutils.scala b/modules/core/src/main/scala/playground/smithyql/tsutils.scala new file mode 100644 index 000000000..5cb55005c --- /dev/null +++ b/modules/core/src/main/scala/playground/smithyql/tsutils.scala @@ -0,0 +1,11 @@ +package playground.smithyql + +import org.polyvariant.treesitter4s.Node + +object tsutils { + + extension (node: Node) { + def range: SourceRange = SourceRange(Position(node.startByte), Position(node.endByte)) + } + +} diff --git a/modules/core/src/test/scala/playground/smithyql/AtPositionTests.scala b/modules/core/src/test/scala/playground/smithyql/AtPositionTests.scala index 1f162ef9e..afa738f31 100644 --- a/modules/core/src/test/scala/playground/smithyql/AtPositionTests.scala +++ b/modules/core/src/test/scala/playground/smithyql/AtPositionTests.scala @@ -1,5 +1,7 @@ package playground.smithyql +import cats.syntax.all.* +import org.polyvariant.treesitter4s.TreeSitterAPI import playground.Assertions.* import playground.Diffs.given import playground.TestTextUtils.* @@ -12,15 +14,16 @@ object AtPositionTests extends FunSuite { text: String ): NodeContext = { val (extracted, position) = extractCursor(text) - val parsed = - SourceParser[SourceFile] - .parse(extracted) - .toTry - .get + val parsedTs = playground + .generated + .nodes + .SourceFile + .unsafeApply(TreeSitterAPI.make("smithyql").parse(extracted).rootNode.get) RangeIndex - .build(parsed) + .build(parsedTs) .findAtPosition(position) + .getOrElse(NodeContext.EmptyPath) } // tests for before/after/between queries diff --git a/modules/language-support/src/main/scala/playground/language/CompletionProvider.scala b/modules/language-support/src/main/scala/playground/language/CompletionProvider.scala index 3fe420359..99adf2bcb 100644 --- a/modules/language-support/src/main/scala/playground/language/CompletionProvider.scala +++ b/modules/language-support/src/main/scala/playground/language/CompletionProvider.scala @@ -3,6 +3,8 @@ package playground.language import cats.Id import cats.kernel.Order.catsKernelOrderingForOrder import cats.syntax.all.* +import org.polyvariant.treesitter4s.TreeSitterAPI +import playground.ASTAdapter import playground.MultiServiceResolver import playground.ServiceIndex import playground.smithyql.NodeContext @@ -11,11 +13,7 @@ import playground.smithyql.NodeContext.^^: import playground.smithyql.OperationName import playground.smithyql.Position import playground.smithyql.QualifiedIdentifier -import playground.smithyql.Query import playground.smithyql.RangeIndex -import playground.smithyql.SourceFile -import playground.smithyql.WithSource -import playground.smithyql.parser.SourceParser import playground.smithyql.syntax.* import smithy.api.Examples import smithy4s.Hints @@ -77,14 +75,15 @@ object CompletionProvider { } def completeRootOperationName( - file: SourceFile[WithSource], + file: playground.generated.nodes.SourceFile, insertBodyStruct: CompletionItem.InsertBodyStruct, ) = { // double-check test coverage. // there's definitely a test missing for N>1 clauses. // https://github.com/kubukoz/smithy-playground/issues/161 - val presentServiceIds - : List[QualifiedIdentifier] = file.prelude.useClauses.map(_.value.identifier.value) + val presentServiceIds: List[QualifiedIdentifier] = file + .select(_.prelude.use_clause.identifier) + .flatMap(ASTAdapter.decodeQI) // for operations on root level we show: // - completions for ops from the service being used, which don't insert a use clause and don't show the service ID @@ -119,17 +118,18 @@ object CompletionProvider { // we're definitely in an existing query, so we don't insert a brace in either case. def completeOperationNameFor( - q: Query[WithSource], - sf: SourceFile[WithSource], + q: playground.generated.nodes.RunQuery, + sf: playground.generated.nodes.SourceFile, serviceId: Option[QualifiedIdentifier], ): List[CompletionItem] = serviceId match { case Some(serviceId) => // includes the current query's service reference // as it wouldn't result in ading a use clause - val presentServiceIdentifiers = - q.operationName.value.mapK(WithSource.unwrap).identifier.toList ++ - sf.prelude.useClauses.map(_.value.identifier.value) + val presentServiceIdentifiers = { + q.select(_.operation_name.service_identifier) ++ + sf.select(_.prelude.use_clause.identifier) + }.flatMap(ASTAdapter.decodeQI) completeOperationName( serviceId, @@ -141,18 +141,19 @@ object CompletionProvider { } def completeInQuery( - q: Query[WithSource], - sf: SourceFile[WithSource], + q: playground.generated.nodes.RunQuery, + sf: playground.generated.nodes.SourceFile, ctx: NodeContext, - ): List[CompletionItem] = { + ): List[CompletionItem] = q.operation_name.toList.flatMap { operationName => val resolvedServiceId = MultiServiceResolver - .resolveService( - q.operationName.value, + .resolveServiceTs( + operationName, serviceIndex, - sf.prelude.useClauses.map(_.value), + sf.select(_.prelude.use_clause), ) .toOption + .flatten ctx match { case NodeContext.PathEntry.AtOperationName ^^: EmptyPath => @@ -161,10 +162,12 @@ object CompletionProvider { case NodeContext.PathEntry.AtOperationInput ^^: ctx => resolvedServiceId match { case Some(serviceId) => - inputCompletions(serviceId)( - q.operationName.value.operationName.value.mapK(WithSource.unwrap) - ) - .getCompletions(ctx) + q.select(_.operation_name.name) + .map(id => OperationName[Id](id.source)) + .flatMap { + inputCompletions(serviceId)(_) + .getCompletions(ctx) + } case None => Nil } @@ -176,45 +179,48 @@ object CompletionProvider { ( doc, pos, - ) => - SourceParser[SourceFile].parse(doc) match { - case Left(_) => - // we can try to deal with this later - Nil - - case Right(sf) => - val matchingNode = RangeIndex - .build(sf) - .findAtPosition(pos) - - // System.err.println("matchingNode: " + matchingNode.render) - - matchingNode match { - case NodeContext.PathEntry.InQuery(n) ^^: rest => - val q = - sf - .queries(WithSource.unwrap) - .get(n.toLong) - .getOrElse(sys.error(s"Fatal error: no query at index $n")) - .query - .value - - completeInQuery(q, sf, rest) - - case NodeContext.PathEntry.AtPrelude ^^: - NodeContext.PathEntry.AtUseClause(_) ^^: - EmptyPath => - servicesById - .toList - .sortBy(_._1) - .map(CompletionItem.useServiceClause.tupled) - - case EmptyPath => completeRootOperationName(sf, CompletionItem.InsertBodyStruct.Yes) - - case _ => Nil - } + ) => { + val parsedTs = playground + .generated + .nodes + .SourceFile + .unsafeApply(TreeSitterAPI.make("smithyql").parse(doc).rootNode.get) + + val matchingNode = RangeIndex + .build(parsedTs) + .findAtPosition(pos) + .getOrElse(NodeContext.EmptyPath) + + // System.err.println("matchingNode: " + matchingNode.render) + + matchingNode match { + case NodeContext.PathEntry.InQuery(n) ^^: rest => + val q = parsedTs + .statements + .flatMap(_.run_query) + .get(n.toLong) + .getOrElse(sys.error(s"Fatal error: no query at index $n")) + + completeInQuery(q, parsedTs, rest) + + case NodeContext.PathEntry.AtPrelude ^^: + NodeContext.PathEntry.AtUseClause(_) ^^: + EmptyPath => + servicesById + .toList + .sortBy(_._1) + .map(CompletionItem.useServiceClause.tupled) + + case EmptyPath => + completeRootOperationName( + parsedTs, + CompletionItem.InsertBodyStruct.Yes, + ) + case _ => Nil } + + } } } diff --git a/modules/lsp-kernel/src/test/resources/test-workspaces/default/smithy-build.json b/modules/lsp-kernel/src/test/resources/test-workspaces/default/smithy-build.json index 87c0f2375..8f81b8282 100644 --- a/modules/lsp-kernel/src/test/resources/test-workspaces/default/smithy-build.json +++ b/modules/lsp-kernel/src/test/resources/test-workspaces/default/smithy-build.json @@ -3,6 +3,6 @@ "sources": ["./weather.smithy", "./no-runner.smithy"], "mavenDependencies": [ "com.disneystreaming.alloy:alloy-core:0.3.31", - "com.disneystreaming.smithy4s:smithy4s-protocol:0.18.37" + "com.disneystreaming.smithy4s:smithy4s-protocol:0.18.41" ] } diff --git a/modules/parser-gen/src/main/scala/playground/parsergen/IR.scala b/modules/parser-gen/src/main/scala/playground/parsergen/IR.scala new file mode 100644 index 000000000..e2c959d7c --- /dev/null +++ b/modules/parser-gen/src/main/scala/playground/parsergen/IR.scala @@ -0,0 +1,50 @@ +package playground.parsergen + +import cats.data.NonEmptyList +import treesittersmithy.FieldName +import treesittersmithy.NodeType +import treesittersmithy.TypeName + +enum Type { + case Union(name: TypeName, subtypes: NonEmptyList[Subtype]) + case Product(name: TypeName, fields: List[Field], children: Option[Children]) +} + +case class Field(name: FieldName, targetTypes: NonEmptyList[TypeName], repeated: Boolean) +case class Children(targetTypes: NonEmptyList[TypeName], repeated: Boolean) + +case class Subtype(name: TypeName) + +object IR { + + def from(nt: NodeType): Type = + if nt.subtypes.nonEmpty then fromUnion(nt) + else + fromProduct(nt) + + private def fromUnion(nt: NodeType): Type.Union = Type.Union( + name = nt.tpe, + subtypes = NonEmptyList.fromListUnsafe(nt.subtypes.map(subtype => Subtype(name = subtype.tpe))), + ) + + private def fromProduct(nt: NodeType): Type.Product = Type.Product( + name = nt.tpe, + fields = + nt.fields + .map { (fieldName, fieldInfo) => + Field( + name = fieldName, + targetTypes = NonEmptyList.fromListUnsafe(fieldInfo.types.map(_.tpe)), + repeated = fieldInfo.multiple, + ) + } + .toList, + children = nt.children.map { children => + Children( + targetTypes = NonEmptyList.fromListUnsafe(children.types.map(_.tpe)), + repeated = children.multiple, + ) + }, + ) + +} diff --git a/modules/parser-gen/src/main/scala/playground/parsergen/ParserGen.scala b/modules/parser-gen/src/main/scala/playground/parsergen/ParserGen.scala new file mode 100644 index 000000000..239de3751 --- /dev/null +++ b/modules/parser-gen/src/main/scala/playground/parsergen/ParserGen.scala @@ -0,0 +1,377 @@ +package playground.parsergen + +import cats.data.NonEmptyList +import cats.syntax.all.* +import monocle.syntax.all.* +import org.polyvariant.treesitter4s.Node +import smithy4s.Blob +import smithy4s.json.Json +import treesittersmithy.FieldName +import treesittersmithy.NodeType +import treesittersmithy.NodeTypes +import treesittersmithy.TypeName +import util.chaining.* + +import scala.annotation.targetName +import scala.meta.Dialect + +extension (tn: TypeName) { + @targetName("renderTypeName") + def render: String = tn.value.dropWhile(_ == '_').fromSnakeCase.ident + def renderProjection: String = show"as${tn.prettyName}".ident + def renderVisitorMethod: String = show"on${tn.prettyName}".ident + private def prettyName = tn.value.dropWhile(_ == '_').fromSnakeCase + def asChildName: FieldName = FieldName(tn.value) +} + +extension (fn: FieldName) { + @targetName("renderFieldName") + def render: String = fn.value.ident +} + +extension (tpe: NodeType) { + + def render: String = + IR.from(tpe) match { + case union: Type.Union => renderUnion(union) + case product: Type.Product => renderProduct(product) + } + +} + +private def renderUnion(u: Type.Union): String = { + val name = u.name.render + val underlyingType = u.subtypes.map(_.name.render).mkString_(" | ") + + val projections = u.subtypes.map { sub => + // format: off + show"""def ${sub.name.renderProjection}: Option[${sub.name.render}] = ${sub.name.render}.unapply(node)""" + // format: on + } + + val instanceMethods = + show"""extension (node: $name) { + |${projections.mkString_("\n").indentTrim(2)} + | def visit[A](visitor: Visitor[A]): A = visitor.visit(node) + |}""".stripMargin + + val applyMethod = { + val cases = u + .subtypes + .map(nodeType => show"""case ${nodeType.name.render}(node) => Right(node)""") + + show"""def apply(node: Node): Either[String, $name] = node match { + |${cases.mkString_("\n").indentTrim(2)} + | case _ => Left(s"Expected $name, got $${node.tpe}") + |}""".stripMargin + } + + val typedApplyMethod = show"""def apply(node: $underlyingType): $name = node""".stripMargin + + val visitor = + show""" + |trait Visitor[A] { + |${u + .subtypes + .map(sub => show"def ${sub.name.renderVisitorMethod}(node: ${sub.name.render}): A") + .mkString_("\n") + .indentTrim(2)} + | + | def visit(node: $name): A = (node: @nowarn("msg=match may not be exhaustive")) match { + |${u + .subtypes + .map(sub => show"case ${sub.name.render}(node) => ${sub.name.renderVisitorMethod}(node)") + .mkString_("\n") + .indentTrim(4)} + | } + |} + | + |object Visitor { + | abstract class Default[A] extends Visitor[A] { + | def default: A + | + |${u + .subtypes + .map(sub => + show"def ${sub.name.renderVisitorMethod}(node: ${sub.name.render}): A = default" + ) + .mkString_("\n") + .indentTrim(4)} + | } + |} + |""".stripMargin + + val selectorMethods = u + .subtypes + .map { subtype => + // format: off + show"""def ${subtype.name.asChildName.render} : ${subtype.name.render}.Selector = ${subtype.name.render}.Selector(path.flatMap(_.${subtype.name.renderProjection}))""" + // format: on + } + .mkString_("\n") + + show"""// Generated code! Do not modify by hand. + |package playground.generated.nodes + | + |import ${classOf[Node].getName()} + |import playground.treesitter4s.std.Selection + |import annotation.nowarn + | + |opaque type $name <: Node = $underlyingType + | + |object $name { + | + |${instanceMethods.indentTrim(2)} + | + |${applyMethod.indentTrim(2)} + | + |${typedApplyMethod.indentTrim(2)} + | + | def unsafeApply(node: Node): $name = apply(node).fold(sys.error, identity) + | + | def unapply(node: Node): Option[$name] = apply(node).toOption + | + |${visitor.indentTrim(2)} + | + | final case class Selector(path: List[$name]) extends Selection[$name] { + |${selectorMethods.indentTrim(4)} + | + | type Self = Selector + | protected val remake = Selector.apply + | } + |} + |""".stripMargin +} + +private def renderProduct(p: Type.Product): String = { + val name = p.name.render + + def renderTypeUnion(types: NonEmptyList[TypeName]) = types + .map(_.render) + .reduceLeft(_ + " | " + _) + + def renderFieldType(field: Field): String = renderTypeUnion(field.targetTypes).pipe { + case s if field.repeated => show"List[$s]" + case s => show"Option[$s]" + } + + def renderChildrenType(children: Children): String = renderTypeUnion(children.targetTypes).pipe { + case s if children.repeated => show"List[$s]" + case s => show"Option[$s]" + } + + def renderChildType(tpe: TypeName, repeated: Boolean): String = tpe.render.pipe { + case s if repeated => show"List[$s]" + case s => show"Option[$s]" + } + + val fieldGetters = p + .fields + .map { field => + val allFields = show"""node.fields.getOrElse(${field.name.value.literal}, Nil)""" + + val cases = field.targetTypes.map { tpe => + show"""case ${tpe.render}(node) => node""" + } + + val fieldValue = + if field.repeated then show"""$allFields.toList.collect { + |${cases.mkString_("\n").indentTrim(2)} + |}""".stripMargin + else + show"""$allFields.headOption.map { + |${cases.mkString_("\n").indentTrim(2)} + |}""".stripMargin + + show"""def ${field.name.render}: ${renderFieldType(field)} = $fieldValue""" + } + + val typedChildren = p.children.map { children => + val fieldTypeAnnotation = renderChildrenType(children) + + val allChildren = show"""node.children""" + + val cases = children.targetTypes.map { tpe => + show"""case ${tpe.render}(node) => node""" + } + + val fieldValue = + if children.repeated then show"""$allChildren.toList.collect { + |${cases.mkString_("\n").indentTrim(2)} + |}""".stripMargin + else + show"""$allChildren.collectFirst { + |${cases.mkString_("\n").indentTrim(2)} + |}""".stripMargin + + show"""def typedChildren: ${fieldTypeAnnotation} = $fieldValue""" + } + + val typedChildrenPrecise = p + .children + .toList + .flatMap { fieldInfo => + fieldInfo.targetTypes.map((fieldInfo.repeated, _)).toList + } + .map { (repeated, fieldType) => + val fieldTypeAnnotation = renderChildType(fieldType, repeated) + val childValue = + if repeated then show"""node.children.toList.collect { + | case ${fieldType.render}(node) => node + |}""".stripMargin + else + show"""node.children.collectFirst { + | case ${fieldType.render}(node) => node + |}""".stripMargin + + show"""def ${fieldType.asChildName.render}: $fieldTypeAnnotation = $childValue""".stripMargin + } + + val instanceMethods = + if (fieldGetters.nonEmpty || typedChildren.nonEmpty || typedChildrenPrecise.nonEmpty) { + show"""extension (node: $name) { + | def select[A](f: $name.Selector => Selection[A]): List[A] = f($name.Selector(List(node))).path + | // fields + |${fieldGetters.mkString_("\n\n").indentTrim(2)} + | // typed children + |${typedChildren.foldMap(_.indentTrim(2)): String} + | // precise typed children + |${typedChildrenPrecise.mkString_("\n\n").indentTrim(2)} + |}""".stripMargin + } else + "" + + val selectorMethods = p + .fields + .flatMap { + case field if field.targetTypes.size == 1 => + // format: off + show"""def ${field.name.render}: ${field.targetTypes.head.render}.Selector = ${field.targetTypes.head.render}.Selector(path.flatMap(_.${field.name.render}))""".stripMargin.some + // format: on + + case f => + System + .err + .println( + s"Skipping selector for field ${f.name} in product $name as it has multiple target types" + ) + none + } + .concat( + p.children.toList.flatMap(_.targetTypes.toList).map { tpe => + // format: off + show"""def ${tpe.asChildName.render}: ${tpe.render}.Selector = ${tpe.render}.Selector(path.flatMap(_.${tpe.asChildName.render}))""".stripMargin + // format: on + } + ) + .mkString_("\n") + + show"""// Generated code! Do not modify by hand. + |package playground.generated.nodes + | + |import ${classOf[Node].getName()} + |import playground.treesitter4s.std.Selection + | + |opaque type $name <: Node = Node + | + |object $name { + |${instanceMethods.indentTrim(2)} + | + | def apply(node: Node): Either[String, $name] = + | if node.tpe == ${p.name.value.literal} + | then Right(node) + | else Left(s"Expected ${p.name.render}, got $${node.tpe}") + | + | def unsafeApply(node: Node): $name = apply(node).fold(sys.error, identity) + | + | def unapply(node: Node): Option[$name] = apply(node).toOption + | + | final case class Selector(path: List[$name]) extends Selection[$name] { + |${selectorMethods.indentTrim(4)} + | + | type Self = Selector + | protected val remake = Selector.apply + | } + |} + |""".stripMargin + +} + +@main def parserGen = { + val types = + Json + .read[NodeTypes]( + Blob(os.read(os.pwd / "tree-sitter-smithyql" / "src" / "node-types.json")) + ) + .toTry + .get + .value + + val base = os.Path(sys.env("CODEGEN_TARGET")) + // os.pwd / "modules" / "treesitter" / "src" / "main" / "scala" / "playground" / "generated" / "nodes" + + val rendered = types + .filter(_.named) + .map( + // only render field types that are named + _.focus(_.fields.each.types) + .modify(_.filter(_.named)) + // don't render the field if it has no types + .focus(_.fields) + .modify(_.filter((_, v) => v.types.nonEmpty)) + ) + .fproduct( + _.render + ) + + os.remove.all(base) + + val files = rendered.map((tpe, code) => (base / s"${tpe.tpe.render}.scala", code)) + + files + .foreach { (path, code) => + os.write( + path, + code, + createFolders = true, + ) + println(path) + } + +} + +extension (s: String) { + + def indentTrim(n: Int): String = s + .linesIterator + .map { + case line if line.nonEmpty => " " * n + line + case line => line + } + .mkString("\n") + + def trimLines: String = s.linesIterator.map(_.stripTrailing()).mkString("\n") + + def literal: String = scala.meta.Lit.String(s).printSyntaxFor(scala.meta.dialects.Scala3) + + def ident: String = { + // etc. + val reserved = Set("List", "String", "Boolean", "Null") + if reserved(s) then s + "_" + else + scala.meta.Name(s).printSyntaxFor(scala.meta.dialects.Scala3) + } + + def fromSnakeCase: String = s.split('_').map(_.capitalize).mkString + +} + +extension [A](l: List[A]) { + + def requireOnly: A = + l match { + case a :: Nil => a + case _ => throw new IllegalArgumentException(s"Expected exactly one element, got $l") + } + +} diff --git a/modules/parser-gen/src/main/smithy/treesitter.smithy b/modules/parser-gen/src/main/smithy/treesitter.smithy new file mode 100644 index 000000000..3c7c687e6 --- /dev/null +++ b/modules/parser-gen/src/main/smithy/treesitter.smithy @@ -0,0 +1,58 @@ +$version: "2" + +namespace treesittersmithy + +list NodeTypes { + member: NodeType +} + +structure NodeType { + @required + @jsonName("type") + tpe: TypeName + + @required + named: Boolean + + @required + fields: NodeFields = {} + + children: FieldInfo + + @required + subtypes: NodeTypes = [] +} + +string TypeName + +map NodeFields { + key: FieldName + value: FieldInfo +} + +string FieldName + +structure FieldInfo { + @required + multiple: Boolean + + @required + required: Boolean + + @required + types: TypeList +} + +list TypeList { + member: TypeInfo +} + +// https://github.com/disneystreaming/smithy4s/issues/1618 +structure TypeInfo { + @required + @jsonName("type") + tpe: TypeName + + @required + named: Boolean +} diff --git a/modules/parser/src/test/scala/playground/smithyql/parser/ParserSuite.scala b/modules/parser/src/test/scala/playground/smithyql/parser/ParserSuite.scala index 290128e7d..5659402d8 100644 --- a/modules/parser/src/test/scala/playground/smithyql/parser/ParserSuite.scala +++ b/modules/parser/src/test/scala/playground/smithyql/parser/ParserSuite.scala @@ -8,6 +8,8 @@ import fs2.io.file.Path import io.circe.Codec import io.circe.Decoder import io.circe.syntax.* +import org.polyvariant.treesitter4s.Node +import org.polyvariant.treesitter4s.TreeSitterAPI import playground.Assertions.* import playground.smithyql.* import playground.smithyql.parser.v2.scanner.Scanner @@ -20,6 +22,8 @@ import java.nio.file.Paths trait ParserSuite extends SimpleIOSuite { + def treeSitterWrap(fileSource: String): String = fileSource + def loadParserTests[Alg[_[_]]: SourceParser]( prefix: String, // this isn't on by default because whitespace in full files (currently, 1-1 mapping to queries) is significant and should not be trimmed before parsing. @@ -56,6 +60,7 @@ trait ParserSuite extends SimpleIOSuite { } validTokensTest(testCase, trimWhitespace) + treeSitterTest(testCase, trimWhitespace) } private def validTokensTest( @@ -73,6 +78,27 @@ trait ParserSuite extends SimpleIOSuite { } } + private def treeSitterTest( + testCase: TestCase, + trimWhitespace: Boolean, + ) = + test(testCase.name + " (tree-sitter no errors)") { + testCase.readInput(trimWhitespace).map { input => + val src = treeSitterWrap(input) + val scanned = TreeSitterAPI.make("smithyql").parse(src).rootNode.get + + val errors = scanned + .fold[List[Node]](_ :: _.flatten.toList) + .filter(_.isError) + .map { node => + s"${node.source} (${node.startByte} to ${node.endByte}, ${node.selfAndParents.map(_.tpe).mkString(" -> ")})" + } + .mkString("\n") + + assert(errors.isEmpty) || failure("error in file: " + testCase.base) || failure(src) + } + } + // invalidTokens: a flag that tells the suite whether the file should contain invalid tokens. def loadNegativeParserTests[Alg[_[_]]: SourceParser]( prefix: String, @@ -91,8 +117,23 @@ trait ParserSuite extends SimpleIOSuite { if (!invalidTokens) validTokensTest(testCase, trimWhitespace) + treeSitterNegativeTest(testCase, trimWhitespace) } + private def treeSitterNegativeTest( + testCase: TestCase, + trimWhitespace: Boolean, + ) = + test(testCase.name + " (tree-sitter require errors)") { + testCase.readInput(trimWhitespace).map { input => + val scanned = TreeSitterAPI.make("smithyql").parse(input).rootNode.get + + val errors = scanned.fold[List[Node]](_ :: _.flatten.toList).find(_.isError) + + assert(errors.nonEmpty) + } + } + private def readText( path: Path ) = @@ -144,11 +185,12 @@ trait ParserSuite extends SimpleIOSuite { outputExtension: String, ) { + private val inputPath = base / "input.smithyql-test" private val outputPath = base / s"output$outputExtension" def readInput( trimWhitespace: Boolean - ): IO[String] = readText(base / "input.smithyql-test").map( + ): IO[String] = readText(inputPath).map( if (trimWhitespace) _.strip else diff --git a/modules/parser/src/test/scala/playground/smithyql/parser/generative/ListParserTests.scala b/modules/parser/src/test/scala/playground/smithyql/parser/generative/ListParserTests.scala index 8458527b9..f70cf43fc 100644 --- a/modules/parser/src/test/scala/playground/smithyql/parser/generative/ListParserTests.scala +++ b/modules/parser/src/test/scala/playground/smithyql/parser/generative/ListParserTests.scala @@ -7,5 +7,11 @@ import playground.smithyql.parser.ParserSuite import playground.smithyql.parser.SourceParser object ListParserTests extends ParserSuite { + + override def treeSitterWrap(fileSource: String): String = + s"""FakeCall { + | fakeField = $fileSource + |}""".stripMargin + loadParserTests[Listed]("listed", trimWhitespace = true) } diff --git a/modules/parser/src/test/scala/playground/smithyql/parser/generative/StructParserTests.scala b/modules/parser/src/test/scala/playground/smithyql/parser/generative/StructParserTests.scala index 1eebed2fd..9aba27743 100644 --- a/modules/parser/src/test/scala/playground/smithyql/parser/generative/StructParserTests.scala +++ b/modules/parser/src/test/scala/playground/smithyql/parser/generative/StructParserTests.scala @@ -6,5 +6,7 @@ import playground.smithyql.parser.Codecs.given import playground.smithyql.parser.ParserSuite object StructParserTests extends ParserSuite { + + override def treeSitterWrap(fileSource: String): String = s"FakeCall $fileSource" loadParserTests[Struct]("struct", trimWhitespace = true) } diff --git a/modules/treesitter/src/main/resources/darwin-aarch64/libtree-sitter-smithyql.dylib b/modules/treesitter/src/main/resources/darwin-aarch64/libtree-sitter-smithyql.dylib new file mode 100755 index 000000000..0c4dce8f3 Binary files /dev/null and b/modules/treesitter/src/main/resources/darwin-aarch64/libtree-sitter-smithyql.dylib differ diff --git a/modules/treesitter/src/main/resources/darwin-x86-64/libtree-sitter-smithyql.dylib b/modules/treesitter/src/main/resources/darwin-x86-64/libtree-sitter-smithyql.dylib new file mode 100755 index 000000000..a57fb005e Binary files /dev/null and b/modules/treesitter/src/main/resources/darwin-x86-64/libtree-sitter-smithyql.dylib differ diff --git a/modules/treesitter/src/main/resources/linux-aarch64/libtree-sitter-smithyql.so b/modules/treesitter/src/main/resources/linux-aarch64/libtree-sitter-smithyql.so new file mode 100755 index 000000000..9c44704bf Binary files /dev/null and b/modules/treesitter/src/main/resources/linux-aarch64/libtree-sitter-smithyql.so differ diff --git a/modules/treesitter/src/main/resources/linux-x86-64/libtree-sitter-smithyql.so b/modules/treesitter/src/main/resources/linux-x86-64/libtree-sitter-smithyql.so new file mode 100755 index 000000000..7a4b23899 Binary files /dev/null and b/modules/treesitter/src/main/resources/linux-x86-64/libtree-sitter-smithyql.so differ diff --git a/modules/treesitter/src/main/scala/playground/treesitter4s/std/Selection.scala b/modules/treesitter/src/main/scala/playground/treesitter4s/std/Selection.scala new file mode 100644 index 000000000..fbf2016aa --- /dev/null +++ b/modules/treesitter/src/main/scala/playground/treesitter4s/std/Selection.scala @@ -0,0 +1,10 @@ +package playground.treesitter4s.std + +trait Selection[A] { + type Self <: Selection[A] + def path: List[A] + protected def remake: List[A] => Self + + def transform(f: List[A] => List[A]): Self = remake(f(path)) + def find(f: A => Boolean): Self = transform(_.find(f).toList) +} diff --git a/modules/treesitter/src/test/scala/playground/smithyql/parser/v3/TreeSitterParserTests.scala b/modules/treesitter/src/test/scala/playground/smithyql/parser/v3/TreeSitterParserTests.scala new file mode 100644 index 000000000..10900b126 --- /dev/null +++ b/modules/treesitter/src/test/scala/playground/smithyql/parser/v3/TreeSitterParserTests.scala @@ -0,0 +1,96 @@ +package playground.smithyql.parser.v3 + +import org.polyvariant.treesitter4s.Node +import org.polyvariant.treesitter4s.TreeSitterAPI +import playground.generated.nodes.SourceFile +import weaver.* + +object TreeSitterParserTests extends FunSuite { + + private def parse(s: String): SourceFile = { + val p = TreeSitterAPI.make("smithyql") + SourceFile.unsafeApply(p.parse(s).rootNode.get) + } + + test("SourceFile fields") { + val in = parse("""use service foo.bar.baz.bax#Baz + |GetBaz{}""".stripMargin) + + assert.eql(in.prelude.map(_.use_clause.size), Some(1)) && + assert(in.statements.nonEmpty) + } + + test("All parents of deep child") { + val allNodes = parse("""use service foo.bar.baz.bax#Baz + |GetBaz { a = { x = 42 }}""".stripMargin) + .fold[List[Node]](_ :: _.flatten.toList) + + val parentTypesAndSources = allNodes + .find(_.source == "x") + .get + .selfAndParents + .map(n => n.tpe -> n.source) + .mkString("\n") + + val expected = List( + "identifier" -> "x", + "binding" -> "x = 42", + "struct" -> "{ x = 42 }", + "binding" -> "a = { x = 42 }", + "struct" -> "{ a = { x = 42 }}", + "run_query" -> "GetBaz { a = { x = 42 }}", + "top_level_statement" -> "GetBaz { a = { x = 42 }}", + "source_file" -> "use service foo.bar.baz.bax#Baz\nGetBaz { a = { x = 42 }}", + ).mkString("\n") + + assert.same(expected, parentTypesAndSources) + } + + test("Deep insight into field") { + val in = parse("""use service foo.bar.baz.bax#Baz + |GetBaz { a = { x = 42 } }""".stripMargin) + + val valueOfX = + in.select( + _.statements + .run_query + .input + .bindings + .find(_.key.get.source == "a") + .value + .struct + .bindings + .find(_.key.get.source == "x") + .value + .number + ).head + .source + .toInt + + assert.eql(42, valueOfX) + } + + test("Deep insight into field, but the file isn't valid") { + val in = parse("""use service fo o.b ar.b/az.bax/#//B//,,{}az + |GetBa z { a = { x = 42, 50 }, z, 42 }""".stripMargin) + + val valueOfX = + in.select( + _.statements + .run_query + .input + .bindings + .find(_.key.get.source == "a") + .value + .struct + .bindings + .find(_.key.get.source == "x") + .value + .number + ).head + .source + .toInt + + assert.eql(42, valueOfX) + } +} diff --git a/project/plugins.sbt b/project/plugins.sbt index f30b32c35..eafa35c74 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,5 +1,3 @@ -ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always - addSbtPlugin("org.typelevel" % "sbt-typelevel" % "0.8.0") addSbtPlugin("org.typelevel" % "sbt-typelevel-mergify" % "0.8.0") diff --git a/smithy-build.json b/smithy-build.json index b680bfa85..c411d8a30 100644 --- a/smithy-build.json +++ b/smithy-build.json @@ -1,18 +1,22 @@ { - "version": "1.0", - "sources": [ - "modules/examples/src/main/smithy", - "modules/examples/target/scala-3.6.4/src_managed/main/smithy", - "modules/protocol4s/src/main/smithy", - "modules/protocol4s/target/scala-3.6.4/src_managed/main/smithy", - "modules/examples/target/scala-3.7.1/src_managed/main/smithy", - "modules/protocol4s/target/scala-3.7.1/src_managed/main/smithy" - ], - "maven": { - "dependencies": [ - "com.disneystreaming.alloy:alloy-core:0.3.31", - "com.disneystreaming.smithy4s:smithy4s-protocol:0.18.37" + "version" : "1.0", + "sources" : [ + "modules/parser-gen/src/main/smithy", + "modules/parser-gen/target/scala-3.7.2/src_managed/main/smithy", + "modules/examples/src/main/smithy", + "modules/examples/target/scala-3.7.2/src_managed/main/smithy", + "modules/protocol4s/src/main/smithy", + "modules/protocol4s/target/scala-3.7.2/src_managed/main/smithy" ], - "repositories": [] - } -} + "maven" : { + "dependencies" : [ + "com.disneystreaming.alloy:alloy-core:0.3.29", + "com.disneystreaming.smithy4s:smithy4s-protocol:0.18.41" + ], + "repositories" : [ + { + "url" : "https://s01.oss.sonatype.org/content/repositories/snapshots" + } + ] + } +} \ No newline at end of file diff --git a/tree-sitter-smithyql/.gitattributes b/tree-sitter-smithyql/.gitattributes new file mode 100644 index 000000000..4cb10583b --- /dev/null +++ b/tree-sitter-smithyql/.gitattributes @@ -0,0 +1,11 @@ +* text=auto eol=lf + +src/*.json linguist-generated +src/parser.c linguist-generated +src/tree_sitter/* linguist-generated + +bindings/** linguist-generated +binding.gyp linguist-generated +setup.py linguist-generated +Makefile linguist-generated +Package.swift linguist-generated diff --git a/tree-sitter-smithyql/.gitignore b/tree-sitter-smithyql/.gitignore new file mode 100644 index 000000000..5d11846d9 --- /dev/null +++ b/tree-sitter-smithyql/.gitignore @@ -0,0 +1,54 @@ +# Rust artifacts +Cargo.lock +target/ + +# Node artifacts +build/ +prebuilds/ +node_modules/ +*.tgz + +# Swift artifacts +.build/ +Package.resolved + +# Go artifacts +go.sum +_obj/ + +# Python artifacts +.venv/ +dist/ +*.egg-info +*.whl + +# C artifacts +*.a +*.so +*.so.* +*.dylib +*.dll +*.pc + +# Example dirs +/examples/*/ + +# Grammar volatiles +*.wasm +*.obj +*.o + +src/ + +.editorconfig +Cargo.toml +Makefile +Package.swift +binding.gyp +bindings/ +go.mod +package.json +pyproject.toml +setup.py + +a.out.js diff --git a/tree-sitter-smithyql/example.smithyql b/tree-sitter-smithyql/example.smithyql new file mode 100644 index 000000000..8124bf33b --- /dev/null +++ b/tree-sitter-smithyql/example.smithyql @@ -0,0 +1,25 @@ +use service a.b#C + +hello { + a = 42, + b = 50, + c = { + d = "foo", + e = false, + f = true, + list = [ + 50, + 100, + 100, + [ + null, + false, + true, + null, + "a", + 40, + ], + ], + }, + nul = null, +} diff --git a/tree-sitter-smithyql/grammar.js b/tree-sitter-smithyql/grammar.js new file mode 100644 index 000000000..01ee9c700 --- /dev/null +++ b/tree-sitter-smithyql/grammar.js @@ -0,0 +1,92 @@ +// Comma-separated sequence of field, with an optional trailing comma. +function comma_separated_trailing(field_grammar) { + return prec.left( + 1, + seq(field_grammar, repeat(seq(",", field_grammar)), optional(",")) + ); +} + +module.exports = grammar({ + name: "smithyql", + + extras: ($) => [$.whitespace, $.comment], + rules: { + source_file: ($) => + seq( + field("prelude", optional($.prelude)), + field("statements", repeat($.top_level_statement)) + ), + + prelude: ($) => repeat1($.use_clause), + + // todo: use token.immediate to prevent comments? + // or just allow comments everywhere? + use_clause: ($) => + seq( + "use", + $.whitespace, + "service", + $.whitespace, + field("identifier", $.qualified_identifier) + ), + + top_level_statement: ($) => choice($.run_query), + + run_query: ($) => + seq( + field("operation_name", $.query_operation_name), + field("input", $.struct) + ), + + _namespace: ($) => seq($.identifier, repeat(seq(".", $.identifier))), + + qualified_identifier: ($) => + seq( + field("namespace", $._namespace), + "#", + field("selection", $.identifier) + ), + + query_operation_name: ($) => + choice( + seq( + field("service_identifier", $.qualified_identifier), + ".", + field("name", $.operation_name) + ), + field("name", $.operation_name) + ), + + operation_name: ($) => $.identifier, + + _input_node: ($) => + choice($.struct, $.list, $.number, $.string, $.boolean, $.null), + + struct: ($) => seq("{", field("bindings", optional($._bindings)), "}"), + list: ($) => seq("[", field("list_fields", optional($._list_fields)), "]"), + + _bindings: ($) => comma_separated_trailing($.binding), + + binding: ($) => + seq( + field("key", $.identifier), + choice("=", ":"), + field("value", $._input_node) + ), + + _list_fields: ($) => comma_separated_trailing($._input_node), + + identifier: ($) => /[a-zA-Z_][a-zA-Z0-9_]*/, + + boolean: ($) => choice("true", "false"), + number: ($) => /-?(0|[1-9]\d*)(\.\d+)?([eE][+-]?\d+)?/, + string: ($) => /"([^"\\]|\\.)*"/, + + null: ($) => "null", + + comment: ($) => token(seq("//", /.*/)), + whitespace: ($) => /\s+/, + }, + supertypes: ($) => [$._input_node], +}); +// diff --git a/tree-sitter-smithyql/test/corpus/simple-complete.txt b/tree-sitter-smithyql/test/corpus/simple-complete.txt new file mode 100644 index 000000000..24a054456 --- /dev/null +++ b/tree-sitter-smithyql/test/corpus/simple-complete.txt @@ -0,0 +1,44 @@ +================================================================================ +Simple complete valid example +================================================================================ +use service foo.bar#baz +Hello { a = 42, b = "false", c = true } +-------------------------------------------------------------------------------- +(source_file + (use_clause + (whitespace) + (whitespace) + (qualified_identifier + (identifier) + (identifier) + (identifier))) + (whitespace) + (top_level_statement + (operation_call + (operation_name + (identifier)) + (whitespace) + (struct + (whitespace) + (bindings + (binding + (identifier) + (whitespace) + (whitespace) + (input_node + (number))) + (whitespace) + (binding + (identifier) + (whitespace) + (whitespace) + (input_node + (string))) + (whitespace) + (binding + (identifier) + (whitespace) + (whitespace) + (input_node + (boolean)))) + (whitespace))))) diff --git a/update-libs.sh b/update-libs.sh new file mode 100755 index 000000000..1fe5c0844 --- /dev/null +++ b/update-libs.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +LIBS_PATH=$(nix build .#tree-sitter-smithyql-all --no-link --print-out-paths --print-build-logs) +mkdir -p modules/treesitter/src/main/resources +cp -R "$LIBS_PATH"/* modules/treesitter/src/main/resources +chmod -R 755 modules/treesitter/src/main/resources