Refactor project structure, implement conditional support for kv and fix tokenization bugs

Author: StefanH-AT

package.json

@@ -42,7 +42,7 @@
         "lint": "eslint ./src",
         "deploy": "vsce publish"
     },
-    "main": "./out/main.js",
+    "main": "./out/extension/main.js",
     "activationEvents": [
         "onLanguage:keyvalue3",
         "onLanguage:vmt",

src/extension/Kv.ts

@@ -0,0 +1,27 @@
+import { Range } from "vscode";
+
+export class KvPair {
+
+    public key: KvPiece;
+    public values: KvPiece[];
+
+    constructor(key: KvPiece, values: KvPiece[]) {
+        this.key = key;
+        this.values = values;
+    }
+
+    public get value(): KvPiece {
+        return this.values[0];
+    }
+
+}
+
+export class KvPiece {
+    public content: string;
+    public range: Range;
+
+    constructor(content: string, range: Range) {
+        this.content = content;
+        this.range = range;
+    }
+}

src/compiler/captions-compile.ts renamed to src/extension/compiler/captions-compile.ts

@@ -8,9 +8,13 @@ import { compileSomething } from "./compiler-base";
 
 let ccChannel: OutputChannel;
 
-export function init(context: ExtensionContext): void {
-    const ccCommand = commands.registerTextEditorCommand("captions.compile", compileCaptions);
-    context.subscriptions.push(ccCommand);
+export async function init(context: ExtensionContext): Promise<void> {
+    const commandList = await commands.getCommands(true);
+    const exists = commandList.some(c => c === "captions.compile");
+    if(!exists) {
+        const ccCommand = commands.registerTextEditorCommand("captions.compile", compileCaptions);
+        context.subscriptions.push(ccCommand);
+    }
 }
 
 async function compileCaptions(editor: TextEditor): Promise<void> {

src/compiler/compiler-base.ts renamed to src/extension/compiler/compiler-base.ts

@@ -1,40 +1,40 @@
-import { FileType, OutputChannel, ProgressLocation, TextEditor, Uri, window, workspace } from "vscode";
-import { isWhitespace } from "../kv-core/kv-string-util";
-import { config } from "../main";
+import vscode from "vscode";
+import { isWhitespace } from "../../kv-core/kv-string-util";
+import * as main from "../main";
 import * as fs from "fs";
 import { execFile } from "child_process";
 
 export interface CompileSettings {
-    editor: TextEditor;
-    channel: OutputChannel | undefined;
+    editor: vscode.TextEditor;
+    channel: vscode.OutputChannel | undefined;
     compilerName: string;
     configPrefix: string;
 }
 
 export async function compileSomething(settings: CompileSettings): Promise<void> {
 
-    const channel = settings.channel ?? window.createOutputChannel(settings.compilerName);
+    const channel = settings.channel ?? vscode.window.createOutputChannel(settings.compilerName);
 
-    const exePath: string = config.get(`${settings.configPrefix}.executablePath`) ?? "";
-    const shouldOpenOutputWindow: boolean = config.get(`${settings.configPrefix}.openOutputWindow`) ?? false;
-    const params: string[] = config.get(`${settings.configPrefix}.additionalParameters`) ?? [];
+    const exePath: string = main.config.get(`${settings.configPrefix}.executablePath`) ?? "";
+    const shouldOpenOutputWindow: boolean = main.config.get(`${settings.configPrefix}.openOutputWindow`) ?? false;
+    const params: string[] = main.config.get(`${settings.configPrefix}.additionalParameters`) ?? [];
 
     const fileUri = settings.editor.document.uri;
-    const fileStat = await workspace.fs.stat(fileUri);
-    if(fileStat.type !== FileType.File) {
-        window.showErrorMessage("The current document is not a file. Cannot compile. Honestly, I have no idea how this is possible, so if you can into this, please message me.");
+    const fileStat = await vscode.workspace.fs.stat(fileUri);
+    if(fileStat.type !== vscode.FileType.File) {
+        vscode.window.showErrorMessage("The current document is not a file. Cannot compile. Honestly, I have no idea how this is possible, so if you can into this, please message me.");
         return;
     }
     const filePath = fileUri.fsPath;
-    const workDir: string = config.get(`${settings.configPrefix}.workingDirectory`) ?? Uri.joinPath(fileUri, "..").fsPath;
+    const workDir: string = main.config.get(`${settings.configPrefix}.workingDirectory`) ?? vscode.Uri.joinPath(fileUri, "..").fsPath;
 
     if( exePath == null || isWhitespace(exePath) ) {
-        window.showErrorMessage(`${settings.compilerName} path is empty. Please configure!`);
+        vscode.window.showErrorMessage(`${settings.compilerName} path is empty. Please configure!`);
         return;
     }
 
     if( !fs.existsSync(exePath) ) {
-        window.showErrorMessage(`${settings.compilerName} executeable at '${exePath}' does not exist.`);
+        vscode.window.showErrorMessage(`${settings.compilerName} executeable at '${exePath}' does not exist.`);
         return;
     }
 
@@ -59,8 +59,8 @@ export async function compileSomething(settings: CompileSettings): Promise<void>
         if(stderr) channel.appendLine("! " + stderr);
     });
 
-    window.withProgress({
-        location: ProgressLocation.Notification,
+    vscode.window.withProgress({
+        location: vscode.ProgressLocation.Notification,
         title: settings.compilerName,
         cancellable: true
     }, (progress, token) => {

src/compiler/model-compile.ts renamed to src/extension/compiler/model-compile.ts

@@ -3,20 +3,19 @@
 // Adds commands to compile .qc (model) files.
 // ==========================================================================
 
-import { commands, ExtensionContext, OutputChannel, TextEditor, window } from "vscode";
-import { config } from "../main";
+import vscode from "vscode";
 import { compileSomething } from "./compiler-base";
 
-let qcChannel: OutputChannel;
+let qcChannel: vscode.OutputChannel;
 
-export function init(context: ExtensionContext): void {
-    const ccCommand = commands.registerTextEditorCommand("mdl.compile", compileModel);
+export function init(context: vscode.ExtensionContext): void {
+    const ccCommand = vscode.commands.registerTextEditorCommand("mdl.compile", compileModel);
     context.subscriptions.push(ccCommand);
 }
 
-async function compileModel(editor: TextEditor): Promise<void> {
+async function compileModel(editor: vscode.TextEditor): Promise<void> {
     if(qcChannel == null) {
-        qcChannel = window.createOutputChannel("Model Compiler");
+        qcChannel = vscode.window.createOutputChannel("Model Compiler");
     }
 
     await compileSomething({

src/extension/language/CaptionsColorsProvider.ts

@@ -0,0 +1,40 @@
+import vscode from "vscode";
+import KvDocument from "./KvDocument";
+import { populateColorTagMatches } from "../../kv-core/kv-caption-tag-matches";
+
+
+export class CaptionsColorsProvider implements vscode.DocumentColorProvider {
+
+    provideDocumentColors(document: vscode.TextDocument, cancellationToken: vscode.CancellationToken): vscode.ColorInformation[] {
+        const lines = document.lineCount;
+
+        const kvDoc = KvDocument.from(document);
+        if (kvDoc == null)
+            return [];
+
+        const colorInfos: vscode.ColorInformation[] = [];
+
+        for (let i = 0; i < lines; i++) {
+            if (cancellationToken.isCancellationRequested)
+                break;
+
+            // Get a line that isn't empty
+            const kv = kvDoc.getKeyValueAt(i);
+            if (kv == null)
+                continue;
+
+            const clrInfo = populateColorTagMatches(kv.value.content);
+            clrInfo.forEach((clr) => {
+                const colorInfo = new vscode.ColorInformation(kv.value.range.with(kv.value.range.start.translate(0, clr.start), kv.value.range.start.translate(0, clr.end)),
+                    new vscode.Color(clr.color.r / 255, clr.color.g / 255, clr.color.b / 255, 1.0));
+                colorInfos.push(colorInfo);
+            });
+        }
+
+        return colorInfos;
+    }
+
+    provideColorPresentations(color: vscode.Color, context: { document: vscode.TextDocument; range: vscode.Range; }, cancellationToken: vscode.CancellationToken): vscode.ColorPresentation[] {
+        return [];
+    }
+}

src/extension/language/CaptionsSemanticTokenProvider.ts

@@ -0,0 +1,34 @@
+import vscode from "vscode";
+import KvDocument from "./KvDocument";
+import { KvTokensProviderBase } from "./KvTokensProviderBase";
+import { KvSemanticProcessor, KvSemanticProcessorParams } from "./KvSemanticProcessor";
+
+
+export class CaptionsSemanticTokenProvider extends KvTokensProviderBase {
+
+    protected keyProcessors: KvSemanticProcessor[] = [
+        { regex: /.*/, processor: this.processKey }
+    ];
+    protected valueProcessors: KvSemanticProcessor[] = [
+        { regex: /.*/, processor: this.processValue }
+    ];
+
+    constructor() {
+        super(KvDocument.tokenLegend, vscode.languages.createDiagnosticCollection("captions"));
+    }
+
+    processKey(params: KvSemanticProcessorParams): boolean {
+        params.tokensBuilder.push(params.wholeRange, "parameter");
+        return true;
+    }
+
+    processValue(params: KvSemanticProcessorParams): boolean {
+
+        if (params.scope === ".lang.tokens") {
+            return true; // Don't add a semantic token to  lang values and let tmLanguage handle it.
+        }
+        params.tokensBuilder.push(params.wholeRange, "string");
+        return true;
+    }
+
+}

src/extension/language/KvDocument.ts

@@ -0,0 +1,103 @@
+import * as vscode from "vscode";
+import { KvPair, KvPiece } from "../Kv";
+import { Token, Tokenizer, TokenType } from "../../kv-core/kv-tokenizer";
+import { KvTokensProviderBase } from "./KvTokensProviderBase";
+
+export default class KvDocument {
+
+    protected _document: vscode.TextDocument;
+    protected _tokens: Token[];
+
+    public get document(): vscode.TextDocument {
+        return this._document;
+    }
+
+    public get tokens(): Token[] {
+        return this._tokens;
+    }
+
+    private static tokenizer: Tokenizer = new Tokenizer();
+
+    public static tokenize(document: vscode.TextDocument): Token[] {
+        const text = document.getText();
+        this.tokenizer.tokenizeFile(text);
+        const tokens = this.tokenizer.tokens;
+    
+        return tokens;
+    }
+
+    public static from(document: vscode.TextDocument): KvDocument {
+        return new KvDocument(document, this.tokenize(document));
+    }
+
+    public static tokenLegend = new vscode.SemanticTokensLegend([
+        "struct",
+        "comment",
+        "variable",
+        "string",
+        "number",
+        "operator",
+        "macro",
+        "boolean",
+        "keyword",
+        "parameter"
+    ], [
+        "declaration",
+        "readonly"
+    ]);
+
+    private constructor(document: vscode.TextDocument, tks: Token[]) {
+        this._document = document;
+        this._tokens = tks;
+    }
+
+    public getKeyValueAt(lineNumber: number): KvPair | null {
+
+        const line = this._document.lineAt(lineNumber);
+        if (line.isEmptyOrWhitespace)
+            return null;
+        const tokens = this.findTokensOnLine(lineNumber);
+
+        // Normal old keyvalue
+        if (tokens.length == 0)
+            return null;
+
+        let keyPiece: KvPiece | null = null;
+        const valuePieces: KvPiece[] = [];
+        for (const token of tokens) {
+            switch (token.type) {
+            case TokenType.Key:
+                keyPiece = this.getUnquotedToken(token); break;
+            case TokenType.Value:
+                valuePieces.push(this.getUnquotedToken(token)); break;
+            }
+        }
+
+        if (keyPiece == null)
+            return null;
+
+        return new KvPair(keyPiece, valuePieces);
+    }
+
+    public getAllValueTokens(): Token[] {
+        return this._tokens.filter(t => t.type === TokenType.Value);
+    }
+
+    public getAllTokens(): Token[] {
+        return this._tokens;
+    }
+
+    public findTokensOnLine(line: number): Token[] {
+        return this._tokens.filter(t => t.line == line);
+    }
+
+    private getTokenRange(token: Token): vscode.Range {
+        return new vscode.Range(this.document.positionAt(token.start), this.document.positionAt(token.end));
+    }
+
+    private getUnquotedToken(token: Token): KvPiece {
+        const range = this.getTokenRange(token);
+        return KvTokensProviderBase.unquoteToken(token, range);
+    }
+
+}

src/extension/language/KvFormatter.ts

@@ -0,0 +1,35 @@
+import vscode from "vscode";
+import { formatTokens } from "../../kv-core/kv-formatter";
+import KvDocument from "./KvDocument";
+
+
+
+export class KvDocumentFormatter implements vscode.DocumentFormattingEditProvider {
+
+
+
+    protected doPutBracesOnNewline(): boolean {
+        const config = vscode.workspace.getConfiguration("sourceEngine");
+        return config.get("keyvalueBracesOnNewline") ?? false;
+    }
+
+    provideDocumentFormattingEdits(document: vscode.TextDocument, options: vscode.FormattingOptions, token: vscode.CancellationToken): vscode.TextEdit[] {
+
+        const tokens = KvDocument.tokenize(document);
+        if (tokens == null)
+            return [];
+
+        const startPos = document.positionAt(0);
+
+        // We just delete and reconstruct the entire file. Much easier.
+        const edits: vscode.TextEdit[] = [
+            new vscode.TextEdit(new vscode.Range(startPos, document.positionAt(document.getText().length)), "")
+        ];
+
+        edits.push(new vscode.TextEdit(new vscode.Range(startPos, startPos), formatTokens(tokens, this.doPutBracesOnNewline())));
+
+        return edits;
+
+    }
+
+}

src/extension/language/KvSemanticProcessor.ts

@@ -0,0 +1,24 @@
+import { KvPiece } from "../Kv";
+import { Range, SemanticTokensBuilder } from "vscode";
+import KvDocument from "./KvDocument";
+
+export type KvSemanticProcessorFunction = (options: KvSemanticProcessorParams) => boolean;
+
+export interface KvSemanticProcessorParams {
+    kvPiece: KvPiece;
+    wholeRange: Range;
+    tokensBuilder: SemanticTokensBuilder;
+    captures: RegExpMatchArray;
+    kvDocument: KvDocument;
+    scope: string;
+}
+
+export class KvSemanticProcessor {
+    public processor: KvSemanticProcessorFunction;
+    public regex = /.+/;
+
+    constructor(processor: KvSemanticProcessorFunction, regex: RegExp) {
+        this.processor = processor;
+        this.regex = regex;
+    }
+}