Compress npm polyfills

Author: ije

scripts/update-nodelibs.sh renamed to scripts/update-node-libs.sh

@@ -3,13 +3,12 @@
 tag="2.0.1"
 dlUrl="https://codeload.github.com/jspm/jspm-core/tar.gz/refs/tags/${tag}"
 
+cd $(dirname $0)
 curl -o "jspm-core-${tag}.tar.gz" $dlUrl
 tar -xzf jspm-core-${tag}.tar.gz
-cd jspm-core-${tag}/nodelibs
-rm -rf node
-mv browser node
-tar -czf nodelibs.tar.gz node
-mv nodelibs.tar.gz ../../server/embed/nodelibs.tar.gz
-cd ../../
+mv  jspm-core-${tag}/nodelibs/browser node
+tar -czf node-libs.tar.gz node
+mv node-libs.tar.gz ../server/embed/node-libs.tar.gz
 rm -rf jspm-core-${tag}
-rm -rf jspm-core-${tag}.tar.gz
+rm -rf node
+rm -f jspm-core-${tag}.tar.gz

scripts/update-npm-polyfills.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+tag="0.0.1"
+dlUrl="https://codeload.github.com/esm-dev/esm-npm-polyfills/tar.gz/refs/tags/${tag}"
+
+cd $(dirname $0)
+curl -o "esm-npm-polyfills-${tag}.tar.gz" $dlUrl
+mv "esm-npm-polyfills-${tag}.tar.gz" ../server/embed/npm-polyfills.tar.gz

server/build.go

@@ -427,15 +427,12 @@ func (ctx *BuildContext) buildModule() (result BuildResult, err error) {
 						}
 					}
 
-					if specifier != "node-fetch" && args.Kind != api.ResolveJSRequireCall && args.Kind != api.ResolveJSRequireResolve {
-						data, err := embedFS.ReadFile(("server/embed/polyfills/npm_" + specifier + ".js"))
-						if err == nil {
-							return api.OnResolveResult{
-								Path:       args.Path,
-								PluginData: data,
-								Namespace:  "purge-polyfill",
-							}, nil
-						}
+					if data, ok := npmPolyfills[specifier]; ok && args.Kind == api.ResolveJSImportStatement {
+						return api.OnResolveResult{
+							Path:       args.Path,
+							PluginData: data,
+							Namespace:  "purge-polyfill",
+						}, nil
 					}
 
 					// resolve specifier with package `browser` field

server/build_resolver.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"bytes"
 	"crypto/sha1"
 	"encoding/base64"
 	"encoding/hex"
@@ -776,20 +775,13 @@ func (ctx *BuildContext) resolveExternalModule(specifier string, kind api.Resolv
 	}
 
 	// replace some npm polyfills with native APIs
-
-	if specifier == "node-fetch" {
-		if ctx.target != "node" {
-			resolvedPath = "/npm_node-fetch.js"
-			return
-		}
-	} else {
-		data, err := embedFS.ReadFile(("server/embed/polyfills/npm_" + specifier + ".js"))
-		if err == nil {
-			data = bytes.ReplaceAll(data, []byte{';', '\n'}, []byte{';'})
-			data = bytes.TrimSuffix(data, []byte{';'})
-			resolvedPath = fmt.Sprintf("data:text/javascript;base64,%s", base64.StdEncoding.EncodeToString(data))
-			return
-		}
+	if data, ok := npmPolyfills[specifier]; ok {
+		resolvedPath = fmt.Sprintf("data:text/javascript;base64,%s", base64.StdEncoding.EncodeToString(data))
+		return
+	}
+	if specifier == "node-fetch" && ctx.target != "node" {
+		resolvedPath = "/npm_node-fetch.js"
+		return
 	}
 
 	// common npm dependency
@@ -1112,7 +1104,7 @@ func (ctx *BuildContext) cjsLexer(specifier string) (cjs cjsLexerResult, err err
 }
 
 func (ctx *BuildContext) esmLexer(specifier string) (isESM bool, namedExports []string, err error) {
-	isESM, namedExports, err = validateJS(path.Join(ctx.wd, "node_modules", ctx.pkg.Name, specifier))
+	isESM, namedExports, err = validateJSFile(path.Join(ctx.wd, "node_modules", ctx.pkg.Name, specifier))
 	if err != nil {
 		err = fmt.Errorf("esmLexer: %v", err)
 	}
@@ -1189,7 +1181,7 @@ func normalizeBuildEntry(ctx *BuildContext, entry *BuildEntry) {
 		}
 		// check if the cjs entry is an ESM
 		if entry.cjs != "" && strings.HasSuffix(entry.cjs, ".js") {
-			isESM, _, _ := validateJS(path.Join(ctx.pkgDir, entry.cjs))
+			isESM, _, _ := validateJSFile(path.Join(ctx.pkgDir, entry.cjs))
 			if isESM {
 				if entry.esm == "" {
 					entry.esm = entry.cjs

server/config.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 )
 
+// Config represents the configuration of esm.sh server.
 type Config struct {
 	Port               uint16                 `json:"port"`
 	TlsPort            uint16                 `json:"tlsPort"`

server/consts.go

@@ -1,5 +1,6 @@
 package server
 
+// esm.sh version
 const VERSION = 136
 
 // asset extensions

server/dts_walker.go

@@ -27,6 +27,7 @@ var (
 	bytesStripleSlash = []byte{'/', '/', '/'}
 )
 
+// Walks through a .d.ts file and resolves import paths.
 func walkDts(r io.Reader, buf *bytes.Buffer, resolve func(specifier string, kind string, position int) (resovledPath string, err error)) (err error) {
 	var multiLineComment bool
 	var importExportExpr bool

server/embed.go

@@ -8,11 +8,13 @@ import (
 	"io"
 	"os"
 	"path"
+	"strings"
 )
 
 var (
-	embedFS  EmbedFS
-	nodeLibs map[string]string
+	embedFS      EmbedFS
+	nodeLibs     map[string]string
+	npmPolyfills map[string][]byte
 )
 
 type EmbedFS interface {
@@ -32,7 +34,7 @@ func (fs MockEmbedFS) Lstat(name string) (os.FileInfo, error) {
 }
 
 func loadNodeLibs(fs EmbedFS) (err error) {
-	data, err := fs.ReadFile("server/embed/nodelibs.tar.gz")
+	data, err := fs.ReadFile("server/embed/node-libs.tar.gz")
 	if err != nil {
 		return
 	}
@@ -41,7 +43,6 @@ func loadNodeLibs(fs EmbedFS) (err error) {
 		return
 	}
 	tr := tar.NewReader(gr)
-	nodeLibs = make(map[string]string)
 	for {
 		h, err := tr.Next()
 		if err != nil {
@@ -70,7 +71,39 @@ func loadNodeLibs(fs EmbedFS) (err error) {
 	return nil
 }
 
+func loadNpmPolyfills(fs EmbedFS) (err error) {
+	data, err := fs.ReadFile("server/embed/npm-polyfills.tar.gz")
+	if err != nil {
+		return
+	}
+	gr, err := gzip.NewReader(bytes.NewReader(data))
+	if err != nil {
+		return
+	}
+	tr := tar.NewReader(gr)
+	for {
+		h, err := tr.Next()
+		if err != nil {
+			break
+		}
+		if h.Typeflag == tar.TypeReg {
+			data, err := io.ReadAll(tr)
+			if err != nil {
+				return err
+			}
+			if strings.HasSuffix(h.Name, ".mjs") {
+				name := strings.TrimSuffix(path.Base(h.Name), ".mjs")
+				data = bytes.ReplaceAll(data, []byte{';', '\n'}, []byte{';'})
+				data = bytes.TrimSuffix(data, []byte{';'})
+				npmPolyfills[name] = data
+			}
+		}
+	}
+	return nil
+}
+
 func init() {
 	embedFS = &embed.FS{}
 	nodeLibs = make(map[string]string)
+	npmPolyfills = make(map[string][]byte)
 }

server/embed/nodelibs.tar.gz renamed to server/embed/node-libs.tar.gz

@@ -12,9 +12,7 @@ import (
 	"github.com/ije/esbuild-internal/logger"
 )
 
-var (
-	jsExts = []string{".mjs", ".js", ".jsx", ".mts", ".ts", ".tsx", ".cjs"}
-)
+var jsExts = []string{".mjs", ".js", ".jsx", ".mts", ".ts", ".tsx", ".cjs"}
 
 // stripModuleExt strips the module extension from the given string.
 func stripModuleExt(s string, exts ...string) string {
@@ -29,7 +27,8 @@ func stripModuleExt(s string, exts ...string) string {
 	return s
 }
 
-func validateJS(filename string) (isESM bool, namedExports []string, err error) {
+// validateJSFile validates the given javascript file.
+func validateJSFile(filename string) (isESM bool, namedExports []string, err error) {
 	data, err := os.ReadFile(filename)
 	if err != nil {
 		return
@@ -64,6 +63,7 @@ func validateJS(filename string) (isESM bool, namedExports []string, err error)
 	return
 }
 
+// minify minifies the given javascript code.
 func minify(code string, target api.Target, loader api.Loader) ([]byte, error) {
 	ret := api.Transform(code, api.TransformOptions{
 		Target:            target,

server/embed/npm-polyfills.tar.gz

@@ -24,7 +24,7 @@ import (
 const npmRegistry = "https://registry.npmjs.org/"
 const jsrRegistry = "https://npm.jsr.io/"
 
-// ref https://github.com/npm/validate-npm-package-name
+// base https://github.com/npm/validate-npm-package-name
 var npmNaming = valid.Validator{valid.FromTo{'a', 'z'}, valid.FromTo{'A', 'Z'}, valid.FromTo{'0', '9'}, valid.Eq('.'), valid.Eq('-'), valid.Eq('_')}
 
 // NpmPackageVerions defines versions of a NPM package

server/embed/polyfills/npm_abort-controller.js

@@ -49,6 +49,15 @@ const (
 	ctCSS            = "text/css; charset=utf-8"
 )
 
+func auth(secret string) rex.Handle {
+	return func(ctx *rex.Context) interface{} {
+		if secret != "" && ctx.R.Header.Get("Authorization") != "Bearer "+secret {
+			return rex.Status(401, "Unauthorized")
+		}
+		return nil
+	}
+}
+
 func router() rex.Handle {
 	startTime := time.Now()
 	globalETag := fmt.Sprintf(`W/"v%d"`, VERSION)
@@ -1203,15 +1212,6 @@ func router() rex.Handle {
 	}
 }
 
-func appendVaryHeader(header http.Header, key string) {
-	vary := header.Get("Vary")
-	if vary == "" {
-		header.Set("Vary", key)
-	} else {
-		header.Set("Vary", vary+", "+key)
-	}
-}
-
 func throwErrorJS(ctx *rex.Context, message string, static bool) interface{} {
 	buf := bytes.NewBuffer(nil)
 	fmt.Fprintf(buf, "/* esm.sh - error */\n")

server/embed/polyfills/npm_array-every.js

@@ -23,7 +23,7 @@ var (
 	log        *logger.Logger
 )
 
-// Serve serves ESM server
+// Serve serves the esm.sh server
 func Serve(efs EmbedFS) {
 	var (
 		cfile string
@@ -89,6 +89,12 @@ func Serve(efs EmbedFS) {
 	}
 	log.Debugf("%d node libs loaded", len(nodeLibs))
 
+	err = loadNpmPolyfills(efs)
+	if err != nil {
+		log.Fatalf("load npm polyfills: %v", err)
+	}
+	log.Debugf("%d npm polyfills loaded", len(npmPolyfills))
+
 	var accessLogger *logger.Logger
 	if config.LogDir == "" {
 		accessLogger = &logger.Logger{}
@@ -160,17 +166,3 @@ func Serve(efs EmbedFS) {
 	log.FlushBuffer()
 	accessLogger.FlushBuffer()
 }
-
-func init() {
-	config = &Config{}
-	log = &logger.Logger{}
-}
-
-func auth(secret string) rex.Handle {
-	return func(ctx *rex.Context) interface{} {
-		if secret != "" && ctx.R.Header.Get("Authorization") != "Bearer "+secret {
-			return rex.Status(401, "Unauthorized")
-		}
-		return nil
-	}
-}