Feature/add list write agent tool

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "sharepoint-online",
-    "version": "1.1.4",
+    "version": "1.1.5",
     "meta": {
         "label": "SharePoint Online",
         "description": "Read and write data from/to your SharePoint Online account",

python-agent-tools/sharepoint-online-write-list-tool/tool.json

@@ -0,0 +1,91 @@
+{
+    "id": "write-sharepoint-list",
+    "meta": {
+        "icon": "icon-cloud",
+        "label": "Write to a SharePoint Online list"
+    },
+
+    "params" : [
+        {
+            "name": "auth_type",
+            "label": "Type of authentication",
+            "type": "SELECT",
+            "selectChoices": [
+                {
+                    "value": "login",
+                    "label": "User name / password"
+                },
+                {
+                    "value": "oauth",
+                    "label": "Azure Single Sign On"
+                },
+                {
+                    "value": "site-app-permissions",
+                    "label": "Site App Permissions"
+                },
+                {
+                    "value": "app-certificate",
+                    "label": "Certificates"
+                }
+            ]
+        },
+        {
+            "name": "sharepoint_oauth",
+            "label": "Azure preset",
+            "type": "PRESET",
+            "parameterSetId": "oauth-login",
+            "visibilityCondition": "model.auth_type == 'oauth'"
+        },
+        {
+            "name": "sharepoint_sharepy",
+            "label": "SharePoint preset",
+            "type": "PRESET",
+            "parameterSetId": "sharepoint-login",
+            "visibilityCondition": "model.auth_type == 'login'"
+        },
+        {
+            "name": "site_app_permissions",
+            "label": "Site App preset",
+            "type": "PRESET",
+            "parameterSetId": "site-app-permissions",
+            "visibilityCondition": "model.auth_type == 'site-app-permissions'"
+        },
+        {
+            "name": "app_certificate",
+            "label": "Certificates",
+            "type": "PRESET",
+            "parameterSetId": "app-certificate",
+            "visibilityCondition": "model.auth_type == 'app-certificate'"
+        },
+        {
+            "name": "sharepoint_list_title",
+            "label": "List title",
+            "defaultValue":  "DSS_${projectKey}_",
+            "description": "",
+            "type": "STRING",
+            "mandatory": true
+        },
+        {
+            "name": "advanced_parameters",
+            "label": "Show advanced parameters",
+            "description": "",
+            "type": "BOOLEAN",
+            "defaultValue": false
+        },
+        {
+            "name": "sharepoint_site_overwrite",
+            "label": "Site path preset overwrite",
+            "type": "STRING",
+            "description": "sites/site_name/subsite...",
+            "visibilityCondition": "model.advanced_parameters == true"
+        },
+        {
+            "name": "sharepoint_list_view_title",
+            "label": "View name",
+            "description": "Read data from a specific view",
+            "type": "STRING",
+            "defaultValue": "",
+            "visibilityCondition": "model.advanced_parameters == true"
+        }
+    ]
+}

python-agent-tools/sharepoint-online-write-list-tool/tool.py

@@ -0,0 +1,81 @@
+from dataiku.llm.agent_tools import BaseAgentTool
+from sharepoint_client import SharePointClient
+from safe_logger import SafeLogger
+from dss_constants import DSSConstants
+
+logger = SafeLogger("sharepoint-online plugin", DSSConstants.SECRET_PARAMETERS_KEYS)
+
+
+class WriteToSharePointListTool(BaseAgentTool):
+
+    def set_config(self, config, plugin_config):
+        logger.info('SharePoint Online plugin list write tool v{}'.format(DSSConstants.PLUGIN_VERSION))
+        self.sharepoint_list_title = config.get("sharepoint_list_title")
+        self.auth_type = config.get('auth_type')
+        logger.info('init:sharepoint_list_title={}, auth_type={}'.format(self.sharepoint_list_title, self.auth_type))
+        self.expand_lookup = config.get("expand_lookup", False)
+        self.metadata_to_retrieve = config.get("metadata_to_retrieve", [])
+        advanced_parameters = config.get("advanced_parameters", False)
+        self.write_mode = "create"
+        if not advanced_parameters:
+            self.max_workers = 1  # no multithread per default
+            self.batch_size = 100
+            self.sharepoint_list_view_title = ""
+        else:
+            self.max_workers = config.get("max_workers", 1)
+            self.batch_size = config.get("batch_size", 100)
+            self.sharepoint_list_view_title = config.get("sharepoint_list_view_title", "")
+        logger.info("init:advanced_parameters={}, max_workers={}, batch_size={}".format(advanced_parameters, self.max_workers, self.batch_size))
+        self.metadata_to_retrieve.append("Title")
+        self.display_metadata = len(self.metadata_to_retrieve) > 0
+        self.client = SharePointClient(config)
+        self.sharepoint_list_view_id = None
+        if self.sharepoint_list_view_title:
+            self.sharepoint_list_view_id = self.client.get_view_id(self.sharepoint_list_title, self.sharepoint_list_view_title)
+        self.sharepoint_column_of_interest = config.get("sharepoint_column_of_interest")
+        self.output_schema = None
+
+    def get_descriptor(self, tool):
+        schema = self.client.get_read_schema(display_metadata=self.display_metadata, metadata_to_retrieve=self.metadata_to_retrieve, add_description=True)
+        columns = schema.get("columns", [])
+        properties = {}
+        required = []
+        output_columns = []
+        for column in columns:
+            column_description = column.get("description")
+            if column_description:
+                properties[column.get("name")] = {
+                    "type": column.get("type"),
+                    "name": column.get("name")
+                }
+                required.append(column.get("name"))  # For now...
+                output_columns.append({
+                    "type": column.get("type"),
+                    "name": column.get("name")
+                })
+        self.output_schema = {
+            "columns": output_columns
+        }
+        return {
+            "description": "This tool can be used to access lists on SharePoint Online. The input to this tool is a dictionary containing the new issue summary and description, e.g. '{'summary':'new issue summary', 'description':'new issue description'}'",
+            "inputSchema" : {
+                "$id": "https://dataiku.com/agents/tools/search/input",
+                "title": "Add an item to a SharePoint Online list tool",
+                "type": "object",
+                "properties" : properties
+            }
+        }
+
+    def invoke(self, input, trace):
+        sharepoint_writer = self.client.get_writer(
+            self.output_schema,
+            None, None, 1, 1,
+            "append"
+        )
+        row = input.get("input", {})
+        sharepoint_writer.write_row_dict(row)
+        sharepoint_writer.close()
+
+        return { 
+            "output" : 'The record was added'
+        }

python-lib/dss_constants.py

@@ -37,7 +37,7 @@ class DSSConstants(object):
         "sharepoint_oauth": "The access token is missing"
     }
     PATH = 'path'
-    PLUGIN_VERSION = "1.1.4"
+    PLUGIN_VERSION = "1.1.5-beta.1"
     SECRET_PARAMETERS_KEYS = ["Authorization", "sharepoint_username", "sharepoint_password", "client_secret", "client_certificate", "passphrase"]
     SITE_APP_DETAILS = {
         "sharepoint_tenant": "The tenant name is missing",
@@ -58,5 +58,6 @@ class DSSConstants(object):
         "bigint": "Integer",
         "smallint": "Integer",
         "tinyint": "Integer",
-        "date": "DateTime"
+        "date": "DateTime",
+        "datetimenotz": "DateTime"
     }

python-lib/sharepoint_client.py

@@ -7,6 +7,8 @@
 import time
 import json
 import re
+import dataiku
+import dataikuapi
 
 from xml.etree.ElementTree import Element, tostring
 from xml.dom import minidom
@@ -135,6 +137,28 @@ def __init__(self, config):
                 max_retries=SharePointConstants.MAX_RETRIES,
                 base_retry_timer_sec=SharePointConstants.WAIT_TIME_BEFORE_RETRY_SEC
             )
+        elif config.get('auth_type') == "dss-connection":
+            connection_name = config.get("dss_connection")
+            print("ALX:connection_name={}".format(connection_name))
+            client = dataiku.api_client()
+            print("ALX:client={}".format(client))
+            connection = client.get_connection(connection_name)
+            print("ALX:connection={}".format(connection))
+            connection_info = connection.get_info()
+            print("ALX:connection_info={}".format(connection_info))
+            credentials = connection_info.get_oauth2_credential()
+            print("ALX:credentials={}".format(credentials))
+            self.sharepoint_access_token = credentials.get("accessToken")
+            self.session.update_settings(session=SharePointSession(
+                    None,
+                    None,
+                    self.sharepoint_url,
+                    self.sharepoint_site,
+                    sharepoint_access_token=self.sharepoint_access_token
+                ),
+                max_retries=SharePointConstants.MAX_RETRIES,
+                base_retry_timer_sec=SharePointConstants.WAIT_TIME_BEFORE_RETRY_SEC
+            )
         else:
             raise SharePointClientError("The type of authentication is not selected")
         self.sharepoint_list_title = config.get("sharepoint_list_title")
@@ -349,6 +373,16 @@ def get_list_fields(self, list_title):
             return None
         return self.extract_results(json_response)
 
+    def search_list(self, list_title, column_to_query, query):
+        list_search_url = self.get_list_search_url(list_title, column_to_query, query)
+        response = self.session.get(
+            list_search_url
+        )
+        json_response = response.json()
+        if self.is_response_empty(json_response):
+            return None
+        return self.extract_results(json_response)
+
     @staticmethod
     def is_response_empty(response):
         return SharePointConstants.RESULTS_CONTAINER_V2 not in response or SharePointConstants.RESULTS not in response[SharePointConstants.RESULTS_CONTAINER_V2]
@@ -691,6 +725,13 @@ def get_list_add_item_using_path_url(self, list_title):
 
     def get_list_fields_url(self, list_title):
         return self.get_lists_by_title_url(list_title) + "/fields"
+
+    def get_list_search_url(self, list_title, column_to_query, query):
+        return self.get_lists_by_title_url(list_title) + "/Items?expand=File&$filter=substringof('{}',{})".format(
+            query,
+            column_to_query
+        )
+    # /Items?$expand=File&$filter=substringof('T15', Title)
 
     def get_lists_add_field_url(self, list_title):
         return self.get_base_url() + "/GetList(@a1)/Fields/CreateFieldAsXml?@a1='/{}/Lists/{}'".format(
@@ -930,7 +971,7 @@ def get_writer(self, dataset_schema, dataset_partitioning,
             write_mode=write_mode
         )
 
-    def get_read_schema(self, display_metadata=False, metadata_to_retrieve=[]):
+    def get_read_schema(self, display_metadata=False, metadata_to_retrieve=[], add_description=False):
         logger.info('get_read_schema')
         sharepoint_columns = self.get_list_fields(self.sharepoint_list_title)
         dss_columns = []
@@ -951,10 +992,18 @@ def get_read_schema(self, display_metadata=False, metadata_to_retrieve=[]):
                 sharepoint_type = get_dss_type(column[SharePointConstants.TYPE_AS_STRING])
                 self.column_sharepoint_type[column[SharePointConstants.STATIC_NAME]] = column[SharePointConstants.TYPE_AS_STRING]
                 if sharepoint_type is not None:
-                    dss_columns.append({
-                        SharePointConstants.NAME_COLUMN: column[SharePointConstants.TITLE_COLUMN],
-                        SharePointConstants.TYPE_COLUMN: sharepoint_type
-                    })
+                    if add_description:
+                        column_record = {
+                            SharePointConstants.NAME_COLUMN: column["InternalName"],
+                            SharePointConstants.TYPE_COLUMN: sharepoint_type,
+                            "description": column.get("Description")
+                        }
+                    else:
+                        column_record = {
+                            SharePointConstants.NAME_COLUMN: column[SharePointConstants.TITLE_COLUMN],
+                            SharePointConstants.TYPE_COLUMN: sharepoint_type
+                        }
+                    dss_columns.append(column_record)
                     self.column_ids[column[SharePointConstants.STATIC_NAME]] = sharepoint_type
                     self.column_names[column[SharePointConstants.STATIC_NAME]] = column[SharePointConstants.TITLE_COLUMN]
                     self.column_entity_property_name[column[SharePointConstants.STATIC_NAME]] = column[SharePointConstants.ENTITY_PROPERTY_NAME]