Skip to content

Commit 732282a

Browse files
author
Nicolas Dorseuil
committed
tag cache refactored to use tagKey
1 parent 84d3634 commit 732282a

File tree

11 files changed

+93
-76
lines changed

11 files changed

+93
-76
lines changed

packages/open-next/src/adapters/cache.ts

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import type {
66
import type { CacheKey } from "types/overrides";
77
import {
88
createCacheKey,
9+
createTagKey,
910
getTagsFromValue,
1011
hasBeenRevalidated,
1112
writeTags,
@@ -71,7 +72,7 @@ export default class Cache {
7172
const _tags = [...(tags ?? []), ...(softTags ?? [])];
7273
const _lastModified = cachedEntry.lastModified ?? Date.now();
7374
const _hasBeenRevalidated = await hasBeenRevalidated(
74-
baseKey,
75+
key,
7576
_tags,
7677
cachedEntry,
7778
);
@@ -90,7 +91,7 @@ export default class Cache {
9091
);
9192
if (path) {
9293
const hasPathBeenUpdated = await hasBeenRevalidated(
93-
path.replace("_N_T_/", ""),
94+
createCacheKey({key: path.replace("_N_T_/", ""), type: "cache"}),
9495
[],
9596
cachedEntry,
9697
);
@@ -132,7 +133,7 @@ export default class Cache {
132133
const tags = getTagsFromValue(cacheData);
133134
const _lastModified = cachedEntry.lastModified ?? Date.now();
134135
const _hasBeenRevalidated = await hasBeenRevalidated(
135-
baseKey,
136+
key,
136137
tags,
137138
cachedEntry,
138139
);
@@ -323,7 +324,7 @@ export default class Cache {
323324

324325
try {
325326
if (globalThis.tagCache.mode === "nextMode") {
326-
const paths = (await globalThis.tagCache.getPathsByTags?.(_tags)) ?? [];
327+
const paths = (await globalThis.tagCache.getPathsByTags?.(_tags.map(createTagKey))) ?? [];
327328

328329
await writeTags(_tags);
329330
if (paths.length > 0) {
@@ -349,7 +350,7 @@ export default class Cache {
349350
for (const tag of _tags) {
350351
debug("revalidateTag", tag);
351352
// Find all keys with the given tag
352-
const paths = await globalThis.tagCache.getByTag(tag);
353+
const paths = await globalThis.tagCache.getByTag(createTagKey(tag));
353354
debug("Items", paths);
354355
const toInsert = paths.map((path) => ({
355356
path,
@@ -360,11 +361,11 @@ export default class Cache {
360361
if (tag.startsWith("_N_T_/")) {
361362
for (const path of paths) {
362363
// We need to find all hard tags for a given path
363-
const _tags = await globalThis.tagCache.getByPath(path);
364+
const _tags = await globalThis.tagCache.getByPath(createTagKey(path));
364365
const hardTags = _tags.filter((t) => !t.startsWith("_N_T_/"));
365366
// For every hard tag, we need to find all paths and revalidate them
366367
for (const hardTag of hardTags) {
367-
const _paths = await globalThis.tagCache.getByTag(hardTag);
368+
const _paths = await globalThis.tagCache.getByTag(createTagKey(hardTag));
368369
debug({ hardTag, _paths });
369370
toInsert.push(
370371
..._paths.map((path) => ({
@@ -377,7 +378,10 @@ export default class Cache {
377378
}
378379

379380
// Update all keys with the given tag with revalidatedAt set to now
380-
await writeTags(toInsert);
381+
await writeTags(toInsert.map((t) => ({
382+
path: createTagKey(t.path),
383+
tag: createTagKey(t.tag),
384+
})));
381385

382386
// We can now invalidate all paths in the CDN
383387
// This only applies to `revalidateTag`, not to `res.revalidate()`
@@ -438,13 +442,13 @@ export default class Cache {
438442

439443
// Get all tags stored in dynamodb for the given key
440444
// If any of the derived tags are not stored in dynamodb for the given key, write them
441-
const storedTags = await globalThis.tagCache.getByPath(key);
445+
const storedTags = await globalThis.tagCache.getByPath(createTagKey(key));
442446
const tagsToWrite = derivedTags.filter((tag) => !storedTags.includes(tag));
443447
if (tagsToWrite.length > 0) {
444448
await writeTags(
445449
tagsToWrite.map((tag) => ({
446-
path: key,
447-
tag: tag,
450+
path: createTagKey(key),
451+
tag: createTagKey(tag),
448452
// In case the tags are not there we just need to create them
449453
// but we don't want them to return from `getLastModified` as they are not stale
450454
revalidatedAt: 1,

packages/open-next/src/adapters/composable-cache.ts

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { ComposableCacheEntry, ComposableCacheHandler } from "types/cache";
2-
import type { CacheKey } from "types/overrides";
3-
import { createCacheKey, writeTags } from "utils/cache";
2+
import type { CacheKey, TagKey } from "types/overrides";
3+
import { createCacheKey, createTagKey, writeTags } from "utils/cache";
44
import { fromReadableStream, toReadableStream } from "utils/stream";
55
import { debug, warn } from "./logger";
66

@@ -28,7 +28,7 @@ export default {
2828
result.value.tags.length > 0
2929
) {
3030
const hasBeenRevalidated = await globalThis.tagCache.hasBeenRevalidated(
31-
result.value.tags,
31+
result.value.tags.map(createTagKey),
3232
result.lastModified,
3333
);
3434
if (hasBeenRevalidated) return undefined;
@@ -38,7 +38,7 @@ export default {
3838
) {
3939
const hasBeenRevalidated =
4040
(await globalThis.tagCache.getLastModified(
41-
cacheKey.baseKey,
41+
cacheKey,
4242
result.lastModified,
4343
)) === -1;
4444
if (hasBeenRevalidated) return undefined;
@@ -66,11 +66,11 @@ export default {
6666
value: valueToStore,
6767
});
6868
if (globalThis.tagCache.mode === "original") {
69-
const storedTags = await globalThis.tagCache.getByPath(cacheKey.baseKey);
69+
const storedTags = await globalThis.tagCache.getByPath(cacheKey);
7070
const tagsToWrite = entry.tags.filter((tag) => !storedTags.includes(tag));
7171
if (tagsToWrite.length > 0) {
7272
await writeTags(
73-
tagsToWrite.map((tag) => ({ tag, path: cacheKey.baseKey })),
73+
tagsToWrite.map((tag) => ({ tag: createTagKey(tag), path: createTagKey(cacheKey.baseKey) })),
7474
);
7575
}
7676
}
@@ -82,7 +82,7 @@ export default {
8282
},
8383
async getExpiration(...tags: string[]) {
8484
if (globalThis.tagCache.mode === "nextMode") {
85-
return globalThis.tagCache.getLastRevalidated(tags);
85+
return globalThis.tagCache.getLastRevalidated(tags.map(createTagKey));
8686
}
8787
// We always return 0 here, original tag cache are handled directly in the get part
8888
// TODO: We need to test this more, i'm not entirely sure that this is working as expected
@@ -98,16 +98,16 @@ export default {
9898
// We need to find all paths linked to to these tags
9999
const pathsToUpdate = await Promise.all(
100100
tags.map(async (tag) => {
101-
const paths = await tagCache.getByTag(tag);
101+
const paths = await tagCache.getByTag(createTagKey(tag));
102102
return paths.map((path) => ({
103-
path,
104-
tag,
103+
path: createTagKey(path),
104+
tag: createTagKey(tag),
105105
revalidatedAt,
106106
}));
107107
}),
108108
);
109109
// We need to deduplicate paths, we use a set for that
110-
const setToWrite = new Set<{ path: string; tag: string }>();
110+
const setToWrite = new Set<{ path: TagKey; tag: TagKey; revalidatedAt: number }>();
111111
for (const entry of pathsToUpdate.flat()) {
112112
setToWrite.add(entry);
113113
}

packages/open-next/src/adapters/dynamo-provider.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
22

33
import { createGenericHandler } from "../core/createGenericHandler.js";
44
import { resolveTagCache } from "../core/resolve.js";
5+
import { createTagKey } from "utils/cache.js";
56

67
const PHYSICAL_RESOURCE_ID = "dynamodb-cache" as const;
78

@@ -60,8 +61,8 @@ async function insert(
6061
const data: DataType[] = JSON.parse(file);
6162

6263
const parsedData = data.map((item) => ({
63-
tag: item.tag.S,
64-
path: item.path.S,
64+
tag: createTagKey(item.tag.S),
65+
path: createTagKey(item.path.S),
6566
revalidatedAt: Number.parseInt(item.revalidatedAt.N),
6667
}));
6768

packages/open-next/src/core/routing/cacheInterceptor.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,7 @@ export async function cacheInterceptor(
226226
if (cachedData.value?.type === "app") {
227227
const tags = getTagsFromValue(cachedData.value);
228228
const _hasBeenRevalidated = await hasBeenRevalidated(
229-
localizedPath,
229+
createCacheKey({ key: localizedPath, type: "cache" }),
230230
tags,
231231
cachedData,
232232
);

packages/open-next/src/overrides/tagCache/dummy.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { TagCache } from "types/overrides";
1+
import type { TagCache, TagKey } from "types/overrides";
22

33
// We don't want to throw error on this one because we might use it when we don't need tag cache
44
const dummyTagCache: TagCache = {
@@ -10,7 +10,7 @@ const dummyTagCache: TagCache = {
1010
getByTag: async () => {
1111
return [];
1212
},
13-
getLastModified: async (_: string, lastModified) => {
13+
getLastModified: async (_: TagKey, lastModified) => {
1414
return lastModified ?? Date.now();
1515
},
1616
writeTags: async () => {

packages/open-next/src/overrides/tagCache/dynamodb-lite.ts

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import path from "node:path";
33

44
import { AwsClient } from "aws4fetch";
5-
import type { TagCache } from "types/overrides";
5+
import type { TagCache, TagKey } from "types/overrides";
66
import { RecoverableError } from "utils/error";
77
import { customFetchClient } from "utils/fetch";
88

@@ -50,17 +50,14 @@ const awsFetch = (
5050
);
5151
};
5252

53-
function buildDynamoKey(key: string) {
54-
const { NEXT_BUILD_ID } = process.env;
55-
// FIXME: We should probably use something else than path.join here
56-
// this could transform some fetch cache key into a valid path
57-
return path.posix.join(NEXT_BUILD_ID ?? "", key);
53+
function buildDynamoKey(key: TagKey) {
54+
return `${key.buildId ?? ""}_${key.baseKey}`;
5855
}
5956

60-
function buildDynamoObject(path: string, tags: string, revalidatedAt?: number) {
57+
function buildDynamoObject(path: TagKey, tag: TagKey, revalidatedAt?: number) {
6158
return {
6259
path: { S: buildDynamoKey(path) },
63-
tag: { S: buildDynamoKey(tags) },
60+
tag: { S: buildDynamoKey(tag) },
6461
revalidatedAt: { N: `${revalidatedAt ?? Date.now()}` },
6562
};
6663
}

packages/open-next/src/overrides/tagCache/dynamodb-nextMode.ts

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { NextModeTagCache } from "types/overrides";
1+
import type { NextModeTagCache, TagKey } from "types/overrides";
22

33
import { AwsClient } from "aws4fetch";
44
import { RecoverableError } from "utils/error";
@@ -49,17 +49,14 @@ const awsFetch = (
4949
);
5050
};
5151

52-
function buildDynamoKey(key: string) {
53-
const { NEXT_BUILD_ID } = process.env;
54-
// FIXME: We should probably use something else than path.join here
55-
// this could transform some fetch cache key into a valid path
56-
return path.posix.join(NEXT_BUILD_ID ?? "", "_tag", key);
52+
function buildDynamoKey(key: TagKey) {
53+
return `${key.buildId ?? ""}_${key.baseKey}`;
5754
}
5855

5956
// We use the same key for both path and tag
6057
// That's mostly for compatibility reason so that it's easier to use this with existing infra
6158
// FIXME: Allow a simpler object without an unnecessary path key
62-
function buildDynamoObject(tag: string, revalidatedAt?: number) {
59+
function buildDynamoObject(tag: TagKey, revalidatedAt?: number) {
6360
return {
6461
path: { S: buildDynamoKey(tag) },
6562
tag: { S: buildDynamoKey(tag) },
@@ -71,11 +68,11 @@ function buildDynamoObject(tag: string, revalidatedAt?: number) {
7168
export default {
7269
name: "ddb-nextMode",
7370
mode: "nextMode",
74-
getLastRevalidated: async (tags: string[]) => {
71+
getLastRevalidated: async (tags: TagKey[]) => {
7572
// Not supported for now
7673
return 0;
7774
},
78-
hasBeenRevalidated: async (tags: string[], lastModified?: number) => {
75+
hasBeenRevalidated: async (tags: TagKey[], lastModified?: number) => {
7976
if (globalThis.openNextConfig.dangerous?.disableTagCache) {
8077
return false;
8178
}
@@ -117,7 +114,7 @@ export default {
117114
debug("retrieved tags", revalidatedTags);
118115
return revalidatedTags.length > 0;
119116
},
120-
writeTags: async (tags: string[]) => {
117+
writeTags: async (tags: TagKey[]) => {
121118
try {
122119
const { CACHE_DYNAMO_TABLE } = process.env;
123120
if (globalThis.openNextConfig.dangerous?.disableTagCache) {

packages/open-next/src/overrides/tagCache/dynamodb.ts

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
DynamoDBClient,
77
QueryCommand,
88
} from "@aws-sdk/client-dynamodb";
9-
import type { TagCache } from "types/overrides";
9+
import type { TagCache, TagKey } from "types/overrides";
1010

1111
import { awsLogger, debug, error } from "../../adapters/logger";
1212
import { chunk, parseNumberFromEnv } from "../../adapters/util";
@@ -27,16 +27,14 @@ function parseDynamoClientConfigFromEnv(): DynamoDBClientConfig {
2727

2828
const dynamoClient = new DynamoDBClient(parseDynamoClientConfigFromEnv());
2929

30-
function buildDynamoKey(key: string) {
31-
// FIXME: We should probably use something else than path.join here
32-
// this could transform some fetch cache key into a valid path
33-
return path.posix.join(NEXT_BUILD_ID ?? "", key);
30+
function buildDynamoKey(key: TagKey) {
31+
return `${key.buildId ?? ""}_${key.baseKey}`;
3432
}
3533

36-
function buildDynamoObject(path: string, tags: string, revalidatedAt?: number) {
34+
function buildDynamoObject(path: TagKey, tag: TagKey, revalidatedAt?: number) {
3735
return {
3836
path: { S: buildDynamoKey(path) },
39-
tag: { S: buildDynamoKey(tags) },
37+
tag: { S: buildDynamoKey(tag) },
4038
revalidatedAt: { N: `${revalidatedAt ?? Date.now()}` },
4139
};
4240
}

packages/open-next/src/overrides/tagCache/fs-dev.ts

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import fs from "node:fs";
22
import path from "node:path";
33

4-
import type { TagCache } from "types/overrides";
4+
import type { TagCache, TagKey } from "types/overrides";
55
import { getMonorepoRelativePath } from "utils/normalize-path";
66

77
const tagFile = path.join(
@@ -19,20 +19,20 @@ let tags = JSON.parse(tagContent) as {
1919
const tagCache: TagCache = {
2020
name: "fs-dev",
2121
mode: "original",
22-
getByPath: async (path: string) => {
22+
getByPath: async (path: TagKey) => {
2323
return tags
24-
.filter((tagPathMapping) => tagPathMapping.path.S === path)
24+
.filter((tagPathMapping) => tagPathMapping.path.S === path.baseKey)
2525
.map((tag) => tag.tag.S);
2626
},
27-
getByTag: async (tag: string) => {
27+
getByTag: async (tag: TagKey) => {
2828
return tags
29-
.filter((tagPathMapping) => tagPathMapping.tag.S === tag)
29+
.filter((tagPathMapping) => tagPathMapping.tag.S === tag.baseKey)
3030
.map((tag) => tag.path.S);
3131
},
32-
getLastModified: async (path: string, lastModified?: number) => {
32+
getLastModified: async (path: TagKey, lastModified?: number) => {
3333
const revalidatedTags = tags.filter(
3434
(tagPathMapping) =>
35-
tagPathMapping.path.S === path &&
35+
tagPathMapping.path.S === path.baseKey &&
3636
Number.parseInt(tagPathMapping.revalidatedAt.N) > (lastModified ?? 0),
3737
);
3838
return revalidatedTags.length > 0 ? -1 : (lastModified ?? Date.now());
@@ -46,8 +46,8 @@ const tagCache: TagCache = {
4646
);
4747
tags = unchangedTags.concat(
4848
newTags.map((tag) => ({
49-
tag: { S: tag.tag },
50-
path: { S: tag.path },
49+
tag: { S: tag.tag.baseKey },
50+
path: { S: tag.path.baseKey },
5151
revalidatedAt: { N: String(tag.revalidatedAt ?? 1) },
5252
})),
5353
);

0 commit comments

Comments
 (0)