Table Viewer optimization in ITC

client/src/connection/itc/ItcLibraryAdapter.ts

@@ -11,8 +11,9 @@ import {
   TableData,
   TableRow,
 } from "../../components/LibraryNavigator/types";
-import { Column, ColumnCollection } from "../rest/api/compute";
-import { runCode } from "./CodeRunner";
+import { ColumnCollection } from "../rest/api/compute";
+import { getColumnIconType } from "../util";
+import { executeRawCode, runCode } from "./CodeRunner";
 import { Config } from "./types";
 
 class ItcLibraryAdapter implements LibraryAdapter {
@@ -45,31 +46,14 @@ class ItcLibraryAdapter implements LibraryAdapter {
   }
 
   public async getColumns(item: LibraryItem): Promise<ColumnCollection> {
-    const sql = `
-      %let OUTPUT;
-      proc sql;
-        select catx(',', name, type, varnum) as column into: OUTPUT separated by '~'
-        from sashelp.vcolumn
-        where libname='${item.library}' and memname='${item.name}'
-        order by varnum;
-      quit;
-      %put <COLOUTPUT> &OUTPUT; %put </COLOUTPUT>;
+    const code = `
+      $runner.GetColumns("${item.library}", "${item.name}")
     `;
-
-    const columnLines = processQueryRows(
-      await this.runCode(sql, "<COLOUTPUT>", "</COLOUTPUT>"),
-    );
-
-    const columns = columnLines.map((lineText): Column => {
-      const [name, type, index] = lineText.split(",");
-
-      return {
-        name,
-        type,
-        index: parseInt(index, 10),
-      };
-    });
-
+    const output = await executeRawCode(code);
+    const columns = JSON.parse(output).map((column) => ({
+      ...column,
+      type: getColumnIconType(column),
+    }));
     return {
       items: columns,
       count: -1,
@@ -213,48 +197,16 @@ class ItcLibraryAdapter implements LibraryAdapter {
     start: number,
     limit: number,
   ): Promise<{ rows: Array<string[]>; count: number }> {
-    const maxTableNameLength = 32;
-    const tempTable = `${item.name}${hms()}${start}`.substring(
-      0,
-      maxTableNameLength,
-    );
+    const fullTableName = `${item.library}.${item.name}`;
     const code = `
-      options nonotes nosource nodate nonumber;
-      %let COUNT;
-      proc sql;
-        SELECT COUNT(1) into: COUNT FROM  ${item.library}.${item.name};
-      quit;
-      data work.${tempTable};
-        set ${item.library}.${item.name};
-        if ${start + 1} <= _N_ <= ${start + limit} then output;
-      run;
-
-      filename out temp;
-      proc json nokeys out=out pretty; export work.${tempTable}; run;
-
-      %put <TABLEDATA>;
-      %put <Count>&COUNT</Count>;
-      data _null_; infile out; input; put _infile_; run;
-      %put </TABLEDATA>;
-      proc datasets library=work nolist nodetails; delete ${tempTable}; run;
-      options notes source date number;
+      $runner.GetDatasetRecords("${fullTableName}", ${start}, ${limit})
     `;
-
-    let output = await this.runCode(code, "<TABLEDATA>", "</TABLEDATA>");
-
-    // Extract result count
-    const countRegex = /<Count>(.*)<\/Count>/;
-    const countMatches = output.match(countRegex);
-    const count = parseInt(countMatches[1].replace(/\s|\n/gm, ""), 10);
-    output = output.replace(countRegex, "");
-
-    const rows = output.replace(/\n|\t/gm, "").slice(output.indexOf("{"));
+    const output = await executeRawCode(code);
     try {
-      const tableData = JSON.parse(rows);
-      return { rows: tableData[`SASTableData+${tempTable}`], count };
+      return JSON.parse(output);
     } catch (e) {
       console.warn("Failed to load table data with error", e);
-      console.warn("Raw output", rows);
+      console.warn("Raw output", output);
       throw new Error(
         l10n.t(
           "An error was encountered when loading table data. This usually happens when a table is too large or the data couldn't be processed. See console for more details.",
@@ -263,18 +215,28 @@ class ItcLibraryAdapter implements LibraryAdapter {
     }
   }
 
-  protected async runCode(
-    code: string,
-    startTag: string = "",
-    endTag: string = "",
+  protected async executionHandler(
+    callback: () => Promise<string>,
   ): Promise<string> {
     try {
-      return await runCode(code, startTag, endTag);
+      return await callback();
     } catch (e) {
       onRunError(e);
       return "";
     }
   }
+
+  protected async runCode(
+    code: string,
+    startTag: string = "",
+    endTag: string = "",
+  ): Promise<string> {
+    return this.executionHandler(() => runCode(code, startTag, endTag));
+  }
+
+  protected async executeRawCode(code: string): Promise<string> {
+    return this.executionHandler(() => executeRawCode(code));
+  }
 }
 
 const processQueryRows = (response: string): string[] => {
@@ -288,9 +250,4 @@ const processQueryRows = (response: string): string[] => {
     .filter((value, index, array) => array.indexOf(value) === index);
 };
 
-const hms = () => {
-  const date = new Date();
-  return `${date.getHours()}${date.getMinutes()}${date.getSeconds()}`;
-};
-
 export default ItcLibraryAdapter;

client/src/connection/itc/script.ts

@@ -15,6 +15,7 @@ type ScriptProperties = {
 export const getScript = ({
   interopLibraryFolderPath = "",
 }: ScriptProperties) => `
+using namespace System.Collections.Generic
 function GetInteropDirectory {
   # try to load from user specified path first
   if ("${interopLibraryFolderPath}") {
@@ -51,6 +52,8 @@ try {
 
 class SASRunner{
   [System.__ComObject] $objSAS
+  [System.__ComObject] $objKeeper
+  [System.__ComObject] $dataConnection
 
   [void]ResolveSystemVars(){
     try {
@@ -89,6 +92,16 @@ class SASRunner{
             $password
         )
 
+        $this.objKeeper = New-Object -ComObject SASObjectManager.ObjectKeeper
+        $this.objKeeper.AddObject(1, "WorkspaceObject", $this.objSAS)
+
+        $this.dataConnection = New-Object -comobject ADODB.Connection
+        $this.dataConnection.Provider = "sas.IOMProvider"
+        $this.dataConnection.Properties("Data Source") = (
+          "iom-id://" + $this.objSAS.UniqueIdentifier
+        )
+        $this.dataConnection.Open()
+
         Write-Host "${LineCodes.SessionCreatedCode}"
     } catch {
       Write-Error "${ERROR_START_TAG}Setup error: $_${ERROR_END_TAG}"
@@ -151,6 +164,8 @@ class SASRunner{
 
   [void]Close(){
   try{
+        $this.dataConnection.Close()
+        $this.objKeeper.RemoveObject($this.objSAS)
         $this.objSAS.Close()
     }catch{
       Write-Error "${ERROR_START_TAG}Close error: $_${ERROR_END_TAG}"
@@ -228,6 +243,87 @@ class SASRunner{
 
     Write-Host "${LineCodes.ResultsFetchedCode}"
   }
+
+  [void]GetDatasetRecords([string]$tableName, [int]$start = 0, [int]$limit = 100) {
+    $objRecordSet = New-Object -comobject ADODB.Recordset
+    $objRecordSet.ActiveConnection = $this.dataConnection # This is needed to set the properties for sas formats.
+    $objRecordSet.Properties.Item("SAS Formats").Value = "_ALL_"
+
+    $objRecordSet.Open(
+      $tableName, 
+      [System.Reflection.Missing]::Value, # Use the active connection
+      2,  # adOpenDynamic
+      1,  # adLockReadOnly
+      512 # adCmdTableDirect
+    )
+
+    $records = [List[List[object]]]::new()
+    $fields = $objRecordSet.Fields.Count
+
+    if ($objRecordSet.EOF) {
+      Write-Host '{"rows": [], "count": 0}'
+      return
+    }
+
+    $objRecordSet.AbsolutePosition = $start + 1
+
+    for ($j = 0; $j -lt $limit -and $objRecordSet.EOF -eq $False; $j++) {
+      $cell = [List[object]]::new()
+      for ($i = 0; $i -lt $fields; $i++) {
+        $cell.Add($objRecordSet.Fields.Item($i).Value)
+      }
+      $records.Add($cell)
+      $objRecordSet.MoveNext()
+    }
+    $objRecordSet.Close()
+
+    $objRecordSet.Open(
+      "SELECT COUNT(1) FROM $tableName", 
+      $this.dataConnection, 3, 1, 1
+    ) # adOpenStatic, adLockReadOnly, adCmdText
+    $count = $objRecordSet.Fields.Item(0).Value
+    $objRecordSet.Close()
+
+    $result = New-Object psobject
+    $result | Add-Member -MemberType NoteProperty -Name "rows" -Value $records
+    $result | Add-Member -MemberType NoteProperty -Name "count" -Value $count
+
+    Write-Host $(ConvertTo-Json -Depth 10 -InputObject $result -Compress)
+  }
+
+  [void]GetColumns([string]$libname, [string]$memname) {
+    $objRecordSet = New-Object -comobject ADODB.Recordset
+    $objRecordSet.ActiveConnection = $this.dataConnection
+    $query = @"
+      select name, type, format 
+      from sashelp.vcolumn 
+      where libname='$libname' and memname='$memname';
+"@
+    $objRecordSet.Open(
+      $query,
+      [System.Reflection.Missing]::Value, # Use the active connection
+      2, # adOpenDynamic
+      1, # adLockReadOnly
+      1  # adCmdText
+    )
+
+    $rows = $objRecordSet.GetRows()
+
+    $objRecordSet.Close()
+
+    $parsedRows = @()
+    for ($i = 0; $i -lt $rows.GetLength(1); $i++) {
+      $parsedRow = [PSCustomObject]@{
+        index = $i + 1
+        name  = $rows[0, $i]
+        type  = $rows[1, $i]
+        format = $rows[2, $i]
+      }
+      $parsedRows += $parsedRow
+    }
+
+    Write-Host $(ConvertTo-Json -Depth 10 -InputObject $parsedRows -Compress)
+  }
 
   [void]DeleteItemAtPath([string]$filePath,[bool]$recursive) {
     if ($recursive) {

client/src/connection/util.ts

@@ -24,3 +24,76 @@ export const extractTextBetweenTags = (
         .replace(/\n$/, "")
     : text;
 };
+
+export const getColumnIconType = ({
+  type,
+  format,
+}: {
+  index: number;
+  type: string;
+  name: string;
+  format: string;
+}) => {
+  format = format.toUpperCase();
+
+  const isDateFormat = () =>
+    [
+      "DAT",
+      "MM",
+      "DD",
+      "YY",
+      "EURDF",
+      "JUL",
+      "YEAR",
+      "DAY",
+      "MONTH",
+      "MON",
+      "DOWNAME",
+    ].some((f) => format.includes(f)) &&
+    ![
+      "TIME",
+      "HH",
+      "SS",
+      "COMM",
+      "DATEAMPM",
+      "DATETIME",
+      "NLDATMTM",
+      "NLDATM",
+      "NLDATMAP",
+      "NLDATMW",
+    ].some((f) => format.includes(f));
+
+  const isTimeFormat = () =>
+    ["TIME", "TIMAP", "HOUR", "HH", "MM", "SS", "NLDATMTM"].some((f) =>
+      format.includes(f),
+    ) && !["DATEAMPM", "DATETIME", "COMMA"].some((f) => format.includes(f));
+
+  const isDateTimeFormat = () =>
+    ["DATEAMPM", "DATETIME", "NLDATM", "NLDATMAP", "NLDATMW"].some((f) =>
+      format.includes(f),
+    );
+
+  const isCurrencyFormat = () =>
+    ["NLMNI", "NLMNL", "NLMNY", "YEN", "DOLLAR", "EURO"].some((f) =>
+      format.includes(f),
+    );
+
+  if (type !== "num") {
+    return type;
+  }
+
+  if (isDateFormat()) {
+    return "date";
+  }
+  if (isTimeFormat()) {
+    return "time";
+  }
+  if (isDateTimeFormat()) {
+    return "datetime";
+  }
+  if (isCurrencyFormat()) {
+    return "currency";
+  }
+
+  return type;
+};

client/test/connection/itc/Coderunner.test.ts

@@ -6,7 +6,7 @@ import { runCode } from "../../../src/connection/itc/CodeRunner";
 import { Session } from "../../../src/connection/session";
 
 export class MockSession extends Session {
-  private _logFn;
+  protected _logFn;
   private _runMap: Record<string, string> | undefined;
   public sasSystemLine = "The Sas System";