Start ingestion of tickets

Author: SamyPesse

bun.lock

@@ -4,7 +4,8 @@
     "private": true,
     "dependencies": {
         "@gitbook/runtime": "*",
-        "@gitbook/api": "*"
+        "@gitbook/api": "*",
+        "p-map": "^7.0.3"
     },
     "devDependencies": {
         "@gitbook/cli": "workspace:*",

integrations/zendesk-conversations/package.json

@@ -11,7 +11,7 @@ export function getZendeskOAuthConfig(context: ZendeskRuntimeContext) {
         redirectURL: `${context.environment.integration.urls.publicEndpoint}/oauth`,
         clientId: context.environment.secrets.CLIENT_ID,
         clientSecret: context.environment.secrets.CLIENT_SECRET,
-        scopes: ['tickets:read', 'webhooks:write', 'webhooks:read'],
+        scopes: ['read', 'webhooks:write'],
         authorizeURL: (installation) => {
             const subdomain = assertInstallationSubdomain(installation);
             return `https://${subdomain}.zendesk.com/oauth/authorizations/new`

integrations/zendesk-conversations/src/client.ts

@@ -0,0 +1,96 @@
+import pMap from "p-map";
+import { ZendeskClient, ZendeskTicket } from "./zendesk";
+import { ConversationInput, ConversationPartMessage } from '@gitbook/api';
+
+/**
+ * Ingest the last closed tickets from Zendesk.
+ */
+export async function ingestTickets(
+    client: ZendeskClient,
+    onConversations: (conversation: ConversationInput[]) => Promise<void>,
+    options: {
+        /** From when to ingest tickets. */
+        startTime: Date;
+        /** Maximum number of tickets to ingest. */
+        maxTickets: number;
+    }
+) {
+    let cursor: string | null = null;
+    let ticketsIngested = 0;
+
+    do {
+        const { tickets, after_cursor } = await client.listTicketsIncremental({
+            startTime: options.startTime,
+            cursor,
+        });
+        cursor = after_cursor;
+
+        const closedTickets = tickets.filter(isTicketClosed);
+        ticketsIngested += closedTickets.length;
+
+        const conversations = await pMap(closedTickets, async (ticket) => {
+            return await parseTicketAsConversation(client, ticket);
+        }, {
+            concurrency: 3
+        });
+
+        await onConversations(conversations);
+    } while (ticketsIngested < options.maxTickets && !!cursor);
+}
+
+/**
+ * Fetch the conversation from Zendesk and parse it into a GitBook conversation.
+ */
+export async function parseTicketAsConversation(client: ZendeskClient, ticket: ZendeskTicket): Promise<ConversationInput> {
+    if (!isTicketClosed(ticket)) {
+        throw new Error(`Ticket ${ticket.id} is not closed or solved`);
+    }
+
+    const comments = await client.listTicketComments(ticket.id);
+    const conversation: ConversationInput = {
+        id: `${client.subdomain}/${ticket.id}`,
+        subject: ticket.subject,
+        metadata: {
+            attributes: {
+                id: ticket.id.toString(),
+                subdomain: client.subdomain,
+            },
+            createdAt: ticket.created_at,
+        },
+        parts: await pMap(comments.comments, async (comment) => {
+            return {
+                type: 'message',
+                role: await getUserRole(client, comment.author_id),
+                body: comment.plain_body,
+            }
+        }, {
+            concurrency: 3,
+        }),
+    };
+
+    return conversation;
+}
+
+const cachedRoles = new Map<string, ConversationPartMessage['role']>();
+
+/**
+ * Evaluate if a user is an end-user or a team member.
+ * We cache the result to avoid making too many requests to Zendesk.
+ */
+async function getUserRole(client: ZendeskClient, userId: number): Promise<ConversationPartMessage['role']> {
+    const cacheKey = `${client.subdomain}/${userId}`;
+    const cachedRole = cachedRoles.get(cacheKey);
+    if (cachedRole) return cachedRole;
+
+    const { user } = await client.getUser(userId);
+    const role = user.role === 'end-user' ? 'user' : 'team-member';
+    cachedRoles.set(cacheKey, role);
+    return role;
+}
+
+/**
+ * Check if a ticket is closed or solved.
+ */
+function isTicketClosed(ticket: ZendeskTicket) {
+    return ticket.status === 'closed' || ticket.status === 'solved';
+}

integrations/zendesk-conversations/src/conversations.ts

@@ -1,37 +1,62 @@
 import {
     createIntegration,
-    createOAuthHandler,
+    createOAuthHandler
 } from '@gitbook/runtime';
 import { ZendeskRuntimeContext } from './types';
 import { configComponent } from './config';
 import { getZendeskClient, getZendeskOAuthConfig } from './client';
+import { ingestTickets } from './conversations';
 
 export default createIntegration<ZendeskRuntimeContext>({
-    fetch: (request, context) => {
-        const oauthHandler = createOAuthHandler(getZendeskOAuthConfig(context), { replace: false });
-        return oauthHandler(request, context);
+    fetch: async (request, context) => {
+        const url = new URL(request.url);
+
+        if (url.pathname.endsWith('/webhook')) {
+            const payload = await request.json();
+            console.log(`Webhook received: ${JSON.stringify(payload)}`);
+            return new Response('OK', { status: 200 });
+        }
+
+        if (url.pathname.endsWith('/oauth')) {
+            const oauthHandler = createOAuthHandler(getZendeskOAuthConfig(context), { replace: false });
+            return oauthHandler(request, context);
+
+        }
+
+        return new Response('Not found', { status: 404 });
     },
     components: [configComponent],
     events: {
+        /**
+         * When the integration is installed, we:
+         *   - Setup a webhook to be notified when tickets are closed
+         *   - Fetch all recent tickets and ingest them
+         */
         installation_setup: async (event, context) => {
             const { installation } = context.environment;
             if (installation?.configuration.subdomain && installation?.configuration.oauth_credentials) {
-                // Properly configured
                 const client = await getZendeskClient(context);
 
                 // Recreate the webhook.
                 const name = `GitBook - ${installation.target.organization}`;
                 const { webhooks } = await client.listWebhooks({ name });
                 await Promise.all(webhooks.map(webhook => client.deleteWebhook(webhook.id)));
-                const webhook = await client.createWebhook({
+                const { webhook } = await client.createWebhook({
                     name: `GitBook - ${installation.target.organization}`,
                     endpoint: `${installation.urls.publicEndpoint}/webhook`,
                     subscriptions: [
                         'zen:event-type:ticket.status_changed'
                     ]
                 });
-                
-                console.log('Webhook created', webhook);
+                console.log(`Webhook created: ${webhook.id}`);
+
+                await ingestTickets(client, async (conversations) => {
+                    console.log(`Ingesting ${conversations.length} conversations`);
+                    await context.api.orgs.ingestConversation(installation.target.organization, conversations);
+                }, {
+                    startTime: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30),
+                    maxTickets: 300,
+                });
             }
         },