Skip to content

Commit 51e5328

Browse files
lukkalukka
authored
handle speculative requests with the cache (#13599)
1 parent 58819c9 commit 51e5328

File tree

2 files changed

+47
-23
lines changed

2 files changed

+47
-23
lines changed

‎Extension/src/LanguageServer/copilotCompletionContextProvider.ts

Lines changed: 42 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,8 @@ export class CopilotCompletionContextProvider implements ContextResolver<Support
7979
private static readonly defaultTimeBudgetMs: number = 7;
8080
// Assume the cache is stale when the distance to the current caret is greater than this value.
8181
private static readonly defaultMaxCaretDistance = 8192;
82-
private static readonly defaultMaxSnippetCount = 15;
83-
private static readonly defaultMaxSnippetLength = 10 * 1024; // 10KB
82+
private static readonly defaultMaxSnippetCount = 7;
83+
private static readonly defaultMaxSnippetLength = 3 * 1024;
8484
private static readonly defaultDoAggregateSnippets = true;
8585
private completionContextCancellation = new vscode.CancellationTokenSource();
8686
private contextProviderDisposable: vscode.Disposable | undefined;
@@ -152,25 +152,24 @@ export class CopilotCompletionContextProvider implements ContextResolver<Support
152152
telemetry.addGetClientForElapsed(getClientForDuration);
153153
if (!client) { throw WellKnownErrors.clientNotFound(); }
154154
const getCompletionContextStartTime = performance.now();
155-
156155
const copilotCompletionContext: CopilotCompletionContextResult =
157156
await client.getCompletionContext(docUri, caretOffset, snippetsFeatureFlag, maxSnippetCount, maxSnippetLength, doAggregateSnippets, internalToken);
158157
telemetry.addRequestId(copilotCompletionContext.requestId);
159-
logMessage += `(id:${copilotCompletionContext.requestId}) (getClientFor elapsed:${getClientForDuration}ms)`;
158+
logMessage += `(id: ${copilotCompletionContext.requestId})(getClientFor elapsed:${getClientForDuration}ms)`;
160159
if (!copilotCompletionContext.areSnippetsMissing) {
161160
const resultMismatch = copilotCompletionContext.sourceFileUri !== docUri.toString();
162-
if (resultMismatch) { logMessage += ` (mismatch TU vs result)`; }
161+
if (resultMismatch) { logMessage += `(mismatch TU vs result)`; }
163162
}
164163
const cacheEntryId = randomUUID().toString();
165164
this.completionContextCache.set(copilotCompletionContext.sourceFileUri, [cacheEntryId, copilotCompletionContext]);
166165
const duration = CopilotCompletionContextProvider.getRoundedDuration(startTime);
167166
telemetry.addCacheComputedData(duration, cacheEntryId);
168167
logMessage += ` cached in ${duration}ms ${copilotCompletionContext.traits.length} trait(s)`;
169-
if (copilotCompletionContext.areSnippetsMissing) { logMessage += ` (missing code snippets) `; }
168+
if (copilotCompletionContext.areSnippetsMissing) { logMessage += `(missing code snippets)`; }
170169
else {
171170
logMessage += ` and ${copilotCompletionContext.snippets.length} snippet(s)`;
172-
logMessage += `, response.featureFlag:${copilotCompletionContext.featureFlag}, \
173-
response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotCompletionContext.caretOffset} `;
171+
logMessage += `(response.featureFlag:${copilotCompletionContext.featureFlag})`;
172+
logMessage += `(response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotCompletionContext.caretOffset})`;
174173
}
175174

176175
telemetry.addResponseMetadata(copilotCompletionContext.areSnippetsMissing, copilotCompletionContext.snippets.length,
@@ -181,7 +180,7 @@ response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotC
181180
} catch (e: any) {
182181
if (e instanceof vscode.CancellationError || e.message === CancellationError.Canceled) {
183182
telemetry.addInternalCanceled(CopilotCompletionContextProvider.getRoundedDuration(startTime));
184-
logMessage += ` (internal cancellation) `;
183+
logMessage += `(internal cancellation)`;
185184
throw InternalCancellationError;
186185
}
187186

@@ -338,11 +337,18 @@ response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotC
338337
} else { return [defaultValue, defaultValue ? CopilotCompletionKind.GotFromCache : CopilotCompletionKind.MissingCacheMiss]; }
339338
}
340339

340+
private static isStaleCacheHit(caretOffset: number, cacheCaretOffset: number, maxCaretDistance: number): boolean {
341+
return Math.abs(caretOffset - caretOffset) > maxCaretDistance;
342+
}
343+
344+
private static createContextItems(copilotCompletionContext: CopilotCompletionContextResult | undefined): SupportedContextItem[] {
345+
return [...copilotCompletionContext?.snippets ?? [], ...copilotCompletionContext?.traits ?? []] as SupportedContextItem[];
346+
}
347+
341348
public async resolve(context: ResolveRequest, copilotCancel: vscode.CancellationToken): Promise<SupportedContextItem[]> {
342349
const proposedEdits = context.documentContext.proposedEdits;
343-
if (proposedEdits) { return []; } // Ignore the request if there are proposed edits.
344350
const resolveStartTime = performance.now();
345-
let logMessage = `Copilot: resolve(${context.documentContext.uri}: ${context.documentContext.offset}):`;
351+
let logMessage = `Copilot: resolve(${context.documentContext.uri}:${context.documentContext.offset}):`;
346352
const cppTimeBudgetMs = await this.fetchTimeBudgetMs(context);
347353
const maxCaretDistance = await this.fetchMaxDistanceToCaret(context);
348354
const maxSnippetCount = await this.fetchMaxSnippetCount(context);
@@ -363,37 +369,49 @@ response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotC
363369
});
364370
if (featureFlag === undefined) { return []; }
365371
const cacheEntry: CacheEntry | undefined = this.completionContextCache.get(docUri.toString());
366-
const defaultValue = cacheEntry?.[1];
367-
[copilotCompletionContext, copilotCompletionContextKind] = await this.resolveResultAndKind(context, featureFlag,
368-
telemetry.fork(), defaultValue, resolveStartTime, cppTimeBudgetMs, maxSnippetCount, maxSnippetLength, doAggregateSnippets, copilotCancel);
372+
if (proposedEdits) {
373+
const defaultValue = cacheEntry?.[1];
374+
const isStaleCache = defaultValue !== undefined ? CopilotCompletionContextProvider.isStaleCacheHit(docOffset, defaultValue.caretOffset, maxCaretDistance) : true;
375+
const contextItems = isStaleCache ? [] : CopilotCompletionContextProvider.createContextItems(defaultValue);
376+
copilotCompletionContext = isStaleCache ? undefined : defaultValue;
377+
copilotCompletionContextKind = isStaleCache ? CopilotCompletionKind.StaleCacheHit : CopilotCompletionKind.GotFromCache;
378+
telemetry.addSpeculativeRequestMetadata(proposedEdits.length);
379+
if (cacheEntry?.[0]) {
380+
telemetry.addCacheHitEntryGuid(cacheEntry[0]);
381+
}
382+
return contextItems;
383+
}
384+
const [resultContext, resultKind] = await this.resolveResultAndKind(context, featureFlag,
385+
telemetry.fork(), cacheEntry?.[1], resolveStartTime, cppTimeBudgetMs, maxSnippetCount, maxSnippetLength, doAggregateSnippets, copilotCancel);
386+
copilotCompletionContext = resultContext;
387+
copilotCompletionContextKind = resultKind;
388+
logMessage += `(id: ${copilotCompletionContext?.requestId})`;
369389
// Fix up copilotCompletionContextKind accounting for stale-cache-hits.
370390
if (copilotCompletionContextKind === CopilotCompletionKind.GotFromCache &&
371391
copilotCompletionContext && cacheEntry) {
372392
telemetry.addCacheHitEntryGuid(cacheEntry[0]);
373393
const cachedData = cacheEntry[1];
374-
if (Math.abs(cachedData.caretOffset - context.documentContext.offset) > maxCaretDistance) {
394+
if (CopilotCompletionContextProvider.isStaleCacheHit(docOffset, cachedData.caretOffset, maxCaretDistance)) {
375395
copilotCompletionContextKind = CopilotCompletionKind.StaleCacheHit;
376396
copilotCompletionContext.snippets = [];
377397
}
378398
}
379-
telemetry.addCompletionContextKind(copilotCompletionContextKind);
380399
// Handle cancellation.
381400
if (copilotCompletionContextKind === CopilotCompletionKind.Canceled) {
382401
const duration: number = CopilotCompletionContextProvider.getRoundedDuration(resolveStartTime);
383402
telemetry.addCopilotCanceled(duration);
384403
throw new CopilotCancellationError();
385404
}
386-
logMessage += ` (id: ${copilotCompletionContext?.requestId})`;
387-
return [...copilotCompletionContext?.snippets ?? [], ...copilotCompletionContext?.traits ?? []] as SupportedContextItem[];
405+
return CopilotCompletionContextProvider.createContextItems(copilotCompletionContext);
388406
} catch (e: any) {
389407
if (e instanceof CopilotCancellationError) {
390408
telemetry.addCopilotCanceled(CopilotCompletionContextProvider.getRoundedDuration(resolveStartTime));
391-
logMessage += ` (copilot cancellation)`;
409+
logMessage += `(copilot cancellation)`;
392410
throw e;
393411
}
394412
if (e instanceof InternalCancellationError) {
395413
telemetry.addInternalCanceled(CopilotCompletionContextProvider.getRoundedDuration(resolveStartTime));
396-
logMessage += ` (internal cancellation) `;
414+
logMessage += `(internal cancellation)`;
397415
throw e;
398416
}
399417
if (e instanceof CancellationError) { throw e; }
@@ -403,13 +421,15 @@ response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotC
403421
throw e;
404422
} finally {
405423
const duration: number = CopilotCompletionContextProvider.getRoundedDuration(resolveStartTime);
406-
logMessage += `featureFlag:${featureFlag?.toString()}, `;
424+
logMessage += `(featureFlag:${featureFlag?.toString()})`;
425+
if (proposedEdits) { logMessage += `(speculative request, proposedEdits:${proposedEdits.length})`; }
407426
if (copilotCompletionContext === undefined) {
408427
logMessage += `result is undefined and no code snippets provided(${copilotCompletionContextKind.toString()}), elapsed time:${duration} ms`;
409428
} else {
410429
logMessage += `for ${docUri}:${docOffset} provided ${copilotCompletionContext.snippets.length} code snippet(s)(${copilotCompletionContextKind.toString()}\
411-
${copilotCompletionContext?.areSnippetsMissing ? ", missing code snippets" : ""}) and ${copilotCompletionContext.traits.length} trait(s), elapsed time:${duration} ms`;
430+
${copilotCompletionContext?.areSnippetsMissing ? "(missing code snippets)" : ""}) and ${copilotCompletionContext.traits.length} trait(s), elapsed time:${duration} ms`;
412431
}
432+
telemetry.addCompletionContextKind(copilotCompletionContextKind);
413433
telemetry.addResponseMetadata(copilotCompletionContext?.areSnippetsMissing ?? true,
414434
copilotCompletionContext?.snippets.length, copilotCompletionContext?.traits.length,
415435
copilotCompletionContext?.caretOffset, copilotCompletionContext?.featureFlag);
@@ -447,4 +467,3 @@ response.uri:${copilotCompletionContext.sourceFileUri || "<not-set>"}:${copilotC
447467
}
448468
}
449469
}
450-

‎Extension/src/LanguageServer/copilotCompletionContextTelemetry.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,11 @@ export class CopilotCompletionContextTelemetry {
7676
this.addMetric('getClientForElapsedMs', duration);
7777
}
7878

79+
public addSpeculativeRequestMetadata(proposedEditsCount: number): void {
80+
this.addProperty('request.isSpeculativeRequest', 'true');
81+
this.addMetric('request.proposedEditsCount', proposedEditsCount);
82+
}
83+
7984
public addResponseMetadata(areSnippetsMissing: boolean, codeSnippetsCount?: number, traitsCount?: number, caretOffset?: number,
8085
featureFlag?: CopilotCompletionContextFeatures): void {
8186
this.addProperty('response.areCodeSnippetsMissing', areSnippetsMissing.toString());

0 commit comments

Comments
 (0)