From fbf174335107340b4b4e737212ecdb1b051be74f Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 10:03:21 +0800 Subject: [PATCH 1/7] Revert "Revert "chore: static assets cdn"" --- apps/web/next.config.mjs | 18 +++++++ apps/web/package.json | 2 +- scripts/upload-static-assets.sh | 94 +++++++++++++++++++++++++++++++++ turbo.json | 7 ++- 4 files changed, 119 insertions(+), 2 deletions(-) create mode 100755 scripts/upload-static-assets.sh diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index ac818ad..8190010 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -5,6 +5,7 @@ import webpack from 'webpack' /** @type {import('next').NextConfig} */ const nextConfig = { + assetPrefix: getAssetPrefix(), reactStrictMode: false, env: { NEXT_PUBLIC_PGLITE_VERSION: await getPackageVersion('@electric-sql/pglite'), @@ -85,3 +86,20 @@ async function getPackageVersion(module) { const packageJson = await getPackageJson(module) return packageJson.version } + +function getAssetPrefix() { + // If not force enabled, but not production env, disable CDN + if (process.env.FORCE_ASSET_CDN !== '1' && process.env.VERCEL_ENV !== 'production') { + return undefined + } + + // Force disable CDN + if (process.env.FORCE_ASSET_CDN === '-1') { + return undefined + } + + // @ts-ignore + return `https://frontend-assets.supabase.com/${ + process.env.SITE_NAME + }/${process.env.VERCEL_GIT_COMMIT_SHA.substring(0, 12)}` +} diff --git a/apps/web/package.json b/apps/web/package.json index fe67c25..20e2206 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,7 +4,7 @@ "private": true, "scripts": { "dev": "npm run build:sw && next dev", - "build": "npm run build:sw && next build", + "build": "npm run build:sw && next build && ./../../scripts/upload-static-assets.sh", "build:sw": "vite build", "start": "next start", "lint": "next lint", diff --git a/scripts/upload-static-assets.sh b/scripts/upload-static-assets.sh new file mode 100755 index 0000000..f3bfc48 --- /dev/null +++ b/scripts/upload-static-assets.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +####### + +# This script is used to upload static build assets (JS, CSS, ...) and public static files (public folder) to a CDN. +# We're using Cloudflare R2 as CDN. +# By using a CDN, we can serve static assets extremely fast while saving big time on egress costs. +# An alternative is proxying via CF, but that comes with Orange-To-Orange issues (Cloudflare having issues with Cloudflare) and increased latency as there is a double TLS termination. +# The script is only supposed to run on production deployments and is not run on any previews. + +# By using a dynamic path including the env, app and commit hash, we can ensure that there are no conflicts. +# Static assets from previous deployments stick around for a while to ensure there are no "downtimes". + +# Advantages of the CDN approach we're using: + +# Get rid of egress costs for static assets across our apps on Vercel +# Disable CF proxying and get around these odd timeouts issues +# Save ~20ms or so for asset requests, as there is no additional CF proxying and we avoid terminating SSL twice +# Always hits the CDN, gonna be super quick +# Does not run on local or preview environments, only on staging/prod deployments +# There are no other disadvantages - you don't have to consider it when developing locally, previews still work, everything on Vercel works as we're used to + +####### + +# If asset CDN is specifically disabled (i.e. studio self-hosted), we skip +if [[ "$FORCE_ASSET_CDN" == "-1" ]]; then + echo "Skipping asset upload. Set FORCE_ASSET_CDN=1 or VERCEL_ENV=production to execute." + exit 0 +fi + +# Check for force env var or production environment +if [[ "$FORCE_ASSET_CDN" != "1" ]] && [[ "$VERCEL_ENV" != "production" ]]; then + echo "Skipping asset upload. Set FORCE_ASSET_CDN=1 or VERCEL_ENV=production to execute." + exit 0 +fi + +# Set the cdnBucket variable based on NEXT_PUBLIC_ENVIRONMENT +if [[ "$NEXT_PUBLIC_ENVIRONMENT" == "staging" ]]; then + BUCKET_NAME="frontend-assets-staging" +else + BUCKET_NAME="frontend-assets-prod" +fi + +STATIC_DIR=".next/static" +PUBLIC_DIR="public" + +# Colors for output +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# Install AWS CLI if not present +if ! command -v aws &> /dev/null; then + echo -e "${YELLOW}Setting up AWS CLI...${NC}" + curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.22.35.zip" -o "awscliv2.zip" + unzip -q awscliv2.zip + export PATH=$PWD/aws/dist:$PATH + rm awscliv2.zip +fi + +# Check if directory exists +if [ ! -d "$STATIC_DIR" ]; then + echo -e "${YELLOW}Directory $STATIC_DIR not found!${NC}" + echo "Make sure you're running this script from your Next.js project root." + exit 1 +fi + +# Upload files with cache configuration and custom endpoint +echo -e "${YELLOW}Uploading static files to R2...${NC}" +aws s3 sync "$STATIC_DIR" "s3://$BUCKET_NAME/$SITE_NAME/${VERCEL_GIT_COMMIT_SHA:0:12}/_next/static" \ + --endpoint-url "$ASSET_CDN_S3_ENDPOINT" \ + --cache-control "public,max-age=604800,immutable" \ + --region auto \ + --only-show-errors + +# Some public files may be referenced through CSS (relative path) and therefore they would be requested via the CDN url +# To ensure we don't run into some nasty debugging issues, we upload the public files to the CDN as well +echo -e "${YELLOW}Uploading public files to R2...${NC}" +aws s3 sync "$PUBLIC_DIR" "s3://$BUCKET_NAME/$SITE_NAME/${VERCEL_GIT_COMMIT_SHA:0:12}" \ + --endpoint-url "$ASSET_CDN_S3_ENDPOINT" \ + --cache-control "public,max-age=604800,immutable" \ + --region auto \ + --only-show-errors + +if [ $? -eq 0 ]; then + echo -e "${GREEN}Upload completed successfully!${NC}" + + # Clean up local static files so we prevent a double upload + echo -e "${YELLOW}Cleaning up local static files...${NC}" + rm -rf "$STATIC_DIR"/* + echo -e "${GREEN}Local static files cleaned up${NC}" + + # We still keep the public dir, as Next.js does not officially support serving the public files via CDN +fi \ No newline at end of file diff --git a/turbo.json b/turbo.json index 9a73227..d95984f 100644 --- a/turbo.json +++ b/turbo.json @@ -39,7 +39,12 @@ "KV_*", "SUPABASE_*", "LOGFLARE_*", - "REDIRECT_LEGACY_DOMAIN" + "REDIRECT_LEGACY_DOMAIN", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "FORCE_ASSET_CDN", + "ASSET_CDN_S3_ENDPOINT", + "SITE_NAME" ], "outputs": [".next/**", "!.next/cache/**"], "cache": true From 6b781c1d47d933150b789c7738738279068da6ad Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 14:24:48 +0800 Subject: [PATCH 2/7] trigger vercel From 12acc409bafc1a571f923c16c8ed1b75f58fd575 Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 14:39:06 +0800 Subject: [PATCH 3/7] add worker src csp --- apps/web/next.config.mjs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 8190010..2d4d5ea 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -67,6 +67,19 @@ const nextConfig = { return redirects }, + headers() { + return [ + { + source: '/:path*', + headers: [ + { + key: 'Content-Security-Policy', + value: "worker-src 'self' https://frontend-assets.supabase.com", + }, + ], + }, + ] + }, } export default nextConfig From b289a16b8f7caf1e10f70a8213d2c4caee014c6f Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 14:54:55 +0800 Subject: [PATCH 4/7] try blob approach --- apps/web/lib/db/index.ts | 4 ++-- apps/web/lib/embed/index.ts | 5 ++++- apps/web/lib/utils.ts | 10 ++++++++-- apps/web/next.config.mjs | 13 ------------- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/apps/web/lib/db/index.ts b/apps/web/lib/db/index.ts index dbeeb7d..a9918c1 100644 --- a/apps/web/lib/db/index.ts +++ b/apps/web/lib/db/index.ts @@ -4,7 +4,7 @@ import { PGliteWorker } from '@electric-sql/pglite/worker' import { Message as AiMessage, ToolInvocation } from 'ai' import { codeBlock } from 'common-tags' import { nanoid } from 'nanoid' -import { downloadFileFromUrl } from '../util' +import { getWorkerURL } from '../utils' export type Database = { id: string @@ -57,7 +57,7 @@ export class DbManager { // Note the below syntax is required by webpack in order to // identify the worker properly during static analysis // see: https://webpack.js.org/guides/web-workers/ - new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' }), + new Worker(getWorkerURL(new URL('./worker.ts', import.meta.url)), { type: 'module' }), { // Opt out of PGlite worker leader election / shared DBs id: options?.id ?? nanoid(), diff --git a/apps/web/lib/embed/index.ts b/apps/web/lib/embed/index.ts index 20227ca..9f981f3 100644 --- a/apps/web/lib/embed/index.ts +++ b/apps/web/lib/embed/index.ts @@ -1,5 +1,6 @@ import { FeatureExtractionPipelineOptions } from '@xenova/transformers' import * as Comlink from 'comlink' +import { getWorkerURL } from '../utils' type EmbedFn = (typeof import('./worker.ts'))['embed'] @@ -15,7 +16,9 @@ function getEmbedFn() { throw new Error('Embed function only available in the browser') } - const worker = new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' }) + const worker = new Worker(getWorkerURL(new URL('./worker.ts', import.meta.url)), { + type: 'module', + }) embedFn = Comlink.wrap(worker) return embedFn } diff --git a/apps/web/lib/utils.ts b/apps/web/lib/utils.ts index d084cca..8080cd3 100644 --- a/apps/web/lib/utils.ts +++ b/apps/web/lib/utils.ts @@ -1,6 +1,12 @@ -import { type ClassValue, clsx } from "clsx" -import { twMerge } from "tailwind-merge" +import { type ClassValue, clsx } from 'clsx' +import { twMerge } from 'tailwind-merge' export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)) } + +export function getWorkerURL(url: URL | string) { + return URL.createObjectURL( + new Blob([`importScripts("${url.toString()}");`], { type: 'text/javascript' }) + ) +} diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 2d4d5ea..8190010 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -67,19 +67,6 @@ const nextConfig = { return redirects }, - headers() { - return [ - { - source: '/:path*', - headers: [ - { - key: 'Content-Security-Policy', - value: "worker-src 'self' https://frontend-assets.supabase.com", - }, - ], - }, - ] - }, } export default nextConfig From 8e2c94d3b372e6e6ddaab09f30e7a58ef259da42 Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 15:06:52 +0800 Subject: [PATCH 5/7] es module worker imports --- apps/web/lib/utils.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/web/lib/utils.ts b/apps/web/lib/utils.ts index 8080cd3..2aca864 100644 --- a/apps/web/lib/utils.ts +++ b/apps/web/lib/utils.ts @@ -7,6 +7,11 @@ export function cn(...inputs: ClassValue[]) { export function getWorkerURL(url: URL | string) { return URL.createObjectURL( - new Blob([`importScripts("${url.toString()}");`], { type: 'text/javascript' }) + new Blob( + [ + /* JS */ `import * as workerModule from "${url.toString()}";Object.assign(self, workerModule);`, + ], + { type: 'text/javascript' } + ) ) } From 93acb9fd80420d543e6dc9df249b13d9634e1663 Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 15:12:22 +0800 Subject: [PATCH 6/7] yolo change to .js? --- apps/web/lib/db/index.ts | 2 +- apps/web/lib/embed/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/lib/db/index.ts b/apps/web/lib/db/index.ts index a9918c1..671118a 100644 --- a/apps/web/lib/db/index.ts +++ b/apps/web/lib/db/index.ts @@ -57,7 +57,7 @@ export class DbManager { // Note the below syntax is required by webpack in order to // identify the worker properly during static analysis // see: https://webpack.js.org/guides/web-workers/ - new Worker(getWorkerURL(new URL('./worker.ts', import.meta.url)), { type: 'module' }), + new Worker(getWorkerURL(new URL('./worker.js', import.meta.url)), { type: 'module' }), { // Opt out of PGlite worker leader election / shared DBs id: options?.id ?? nanoid(), diff --git a/apps/web/lib/embed/index.ts b/apps/web/lib/embed/index.ts index 9f981f3..4277e94 100644 --- a/apps/web/lib/embed/index.ts +++ b/apps/web/lib/embed/index.ts @@ -16,7 +16,7 @@ function getEmbedFn() { throw new Error('Embed function only available in the browser') } - const worker = new Worker(getWorkerURL(new URL('./worker.ts', import.meta.url)), { + const worker = new Worker(getWorkerURL(new URL('./worker.js', import.meta.url)), { type: 'module', }) embedFn = Comlink.wrap(worker) From 4c496b3494c6d791739a46b5af98f3ef0c2f45a5 Mon Sep 17 00:00:00 2001 From: Alaister Young Date: Thu, 6 Feb 2025 15:18:41 +0800 Subject: [PATCH 7/7] try csp again --- apps/web/lib/db/index.ts | 4 ++-- apps/web/lib/embed/index.ts | 5 +---- apps/web/lib/utils.ts | 11 ----------- apps/web/next.config.mjs | 13 +++++++++++++ 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/apps/web/lib/db/index.ts b/apps/web/lib/db/index.ts index 671118a..dbeeb7d 100644 --- a/apps/web/lib/db/index.ts +++ b/apps/web/lib/db/index.ts @@ -4,7 +4,7 @@ import { PGliteWorker } from '@electric-sql/pglite/worker' import { Message as AiMessage, ToolInvocation } from 'ai' import { codeBlock } from 'common-tags' import { nanoid } from 'nanoid' -import { getWorkerURL } from '../utils' +import { downloadFileFromUrl } from '../util' export type Database = { id: string @@ -57,7 +57,7 @@ export class DbManager { // Note the below syntax is required by webpack in order to // identify the worker properly during static analysis // see: https://webpack.js.org/guides/web-workers/ - new Worker(getWorkerURL(new URL('./worker.js', import.meta.url)), { type: 'module' }), + new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' }), { // Opt out of PGlite worker leader election / shared DBs id: options?.id ?? nanoid(), diff --git a/apps/web/lib/embed/index.ts b/apps/web/lib/embed/index.ts index 4277e94..20227ca 100644 --- a/apps/web/lib/embed/index.ts +++ b/apps/web/lib/embed/index.ts @@ -1,6 +1,5 @@ import { FeatureExtractionPipelineOptions } from '@xenova/transformers' import * as Comlink from 'comlink' -import { getWorkerURL } from '../utils' type EmbedFn = (typeof import('./worker.ts'))['embed'] @@ -16,9 +15,7 @@ function getEmbedFn() { throw new Error('Embed function only available in the browser') } - const worker = new Worker(getWorkerURL(new URL('./worker.js', import.meta.url)), { - type: 'module', - }) + const worker = new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' }) embedFn = Comlink.wrap(worker) return embedFn } diff --git a/apps/web/lib/utils.ts b/apps/web/lib/utils.ts index 2aca864..d32b0fe 100644 --- a/apps/web/lib/utils.ts +++ b/apps/web/lib/utils.ts @@ -4,14 +4,3 @@ import { twMerge } from 'tailwind-merge' export function cn(...inputs: ClassValue[]) { return twMerge(clsx(inputs)) } - -export function getWorkerURL(url: URL | string) { - return URL.createObjectURL( - new Blob( - [ - /* JS */ `import * as workerModule from "${url.toString()}";Object.assign(self, workerModule);`, - ], - { type: 'text/javascript' } - ) - ) -} diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs index 8190010..2d4d5ea 100644 --- a/apps/web/next.config.mjs +++ b/apps/web/next.config.mjs @@ -67,6 +67,19 @@ const nextConfig = { return redirects }, + headers() { + return [ + { + source: '/:path*', + headers: [ + { + key: 'Content-Security-Policy', + value: "worker-src 'self' https://frontend-assets.supabase.com", + }, + ], + }, + ] + }, } export default nextConfig