serve css/js assets under digest urls, cache-control (#929)

Changes the caching strategy for assets.

All assets {PATH} (e.g. css/main.css) are now served under two URLs:

1. /{PATH} (Cache-control header allows caching for 1 day)
2. /_/{DIGEST}/{PATH} (Cache-control header allows caching for a very long time and declares the file immutable)

Digests are looked up via assets.ml (produced by ocaml-crunch with mode plain). They are a hash of the file content.

In the templates, there is now a parameter digest_url : string -> string which renders the path containing the digest (only for assets from assets.ml, as we do not expect files from media.ml to change).

Note: Cache-control: immutable tells the browser that checking whether the file was modified is unnecessary.

Resolves #839 except for the playground.

Note that, since inter.css has a relative import for the font files, all the font files are served under inter.css's digest. So, if anyone ever needs to update the font files, they have to modify inter.css to change its digest.

Author: sabine

src/ocamlorg_frontend/dune

@@ -1,6 +1,6 @@
 (library
  (name ocamlorg_frontend)
- (libraries dream ood timedesc))
+ (libraries dream ood timedesc ocamlorg_static))
 
 (include_subdirs unqualified)
 

src/ocamlorg_frontend/layouts/layout.eml

@@ -34,17 +34,17 @@ inner =
     <link rel="icon" type="image/x-icon" href="/favicon.ico">
     <link rel="manifest" href="/manifest.json">
     <% (match styles with | Some styles -> styles |> List.iter (fun style -> %>
-    <link rel="stylesheet" href="<%s style %>">
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url style %>">
     <% ) | None -> %>
-    <link rel="stylesheet" href="/css/main.css">
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url "css/main.css" %>">
     <% ); %>
-    <link rel="stylesheet" href="/vendors/font-files/inter.css">
-    <script defer src="/vendors/alpine.min.js"></script>
-    <script defer src="/vendors/htmx.min.js"></script>
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url "vendors/font-files/inter.css" %>">
+    <script defer src="<%s Ocamlorg_static.asset_url "vendors/alpine.min.js" %>"></script>
+    <script defer src="<%s Ocamlorg_static.asset_url "vendors/htmx.min.js" %>"></script>
     <% if use_swiper then ( %>
-    <link rel="stylesheet" href="/vendors/swiper-bundle.min.css">
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url "vendors/swiper-bundle.min.css" %>">
     <link rel="alternate" type="application/rss+xml" title="OCaml RSS Feed" href="/feed.xml">
-    <script src="/vendors/swiper-bundle.min.js"></script>
+    <script src="<%s Ocamlorg_static.asset_url "vendors/swiper-bundle.min.js" %>"></script>
     <% ); %>
     <title><%s title %></title>
   </head>

src/ocamlorg_frontend/pages/package_documentation.eml

@@ -50,7 +50,7 @@ Package_layout.render
 ~package
 ~path
 ~canonical:(Url.package_doc package.name ~version:package.version ?page:path_page)
-~styles:["/css/main.css"; "/css/doc.css"] @@
+~styles:["css/main.css"; "css/doc.css"] @@
 <div x-data="{ open: false, sidebar: window.innerWidth > 1024 && true, showOnMobile: false}" @resize.window="sidebar = window.innerWidth > 1024" x-on:close-sidebar="sidebar=window.innerWidth > 1024 && true">
   <button :title='(sidebar? "close" : "open")+" sidebar"' class="flex items-center bg-primary-600 p-3 z-30 rounded-full text-white shadow-md bottom-20 fixed lg:hidden right-10"
     x-on:click="sidebar = ! sidebar">

src/ocamlorg_frontend/pages/playground.eml

@@ -17,10 +17,10 @@ let render () =
     <link rel="canonical" href="https://ocaml.org<%s Url.playground %>">
     <link rel="icon" type="image/x-icon" href="/favicon.ico">
     <link rel="manifest" href="/manifest.json">
-    <link rel="stylesheet" href="/css/main.css">
-    <link rel="stylesheet" href="/css/codemirror.css">
-    <link rel="stylesheet" href="/vendors/font-files/inter.css">
-    <script defer src="/vendors/alpine.min.js"></script>
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url "css/main.css" %>">
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url "css/codemirror.css" %>">
+    <link rel="stylesheet" href="<%s Ocamlorg_static.asset_url "vendors/font-files/inter.css" %>">
+    <script defer src="<%s Ocamlorg_static.asset_url "vendors/alpine.min.js" %>"></script>
     <title>OCaml Playground</title>
   </head>
 

src/ocamlorg_static/dune

@@ -0,0 +1,28 @@
+(library
+ (name ocamlorg_static)
+ (libraries fmt dream))
+
+(rule
+ (targets asset.ml)
+ (deps
+  %{workspace_root}/asset/css/main.css
+  (source_tree %{workspace_root}/asset))
+ (action
+  (with-stdout-to
+   %{null}
+   (run %{bin:ocaml-crunch} -m plain %{workspace_root}/asset -o asset.ml))))
+
+(rule
+ (targets media.ml)
+ (deps
+  (source_tree %{workspace_root}/data/media))
+ (action
+  (with-stdout-to
+   %{null}
+   (run
+    %{bin:ocaml-crunch}
+    -m
+    plain
+    %{workspace_root}/data/media
+    -o
+    media.ml))))

src/ocamlorg_static/file.ml

@@ -0,0 +1,13 @@
+let to_url_path ?digest filepath =
+  match digest with
+  | None -> filepath
+  | Some digest -> Fmt.str "/_/%s/%s" digest filepath
+
+type t = { filepath : string; digest : string option }
+
+let of_url_path path =
+  let xs = String.split_on_char '/' path in
+  match xs with
+  | "_" :: x :: xs -> Some { digest = Some x; filepath = String.concat "/" xs }
+  | "_" :: _ | [] -> None
+  | _ -> Some { digest = None; filepath = path }

src/ocamlorg_static/file.mli

@@ -0,0 +1,24 @@
+(* We serve static files (e.g. assets) under two relative URLs:
+
+   /{PATH}
+
+   and
+
+   /_/{DIGEST}/{PATH}
+
+   For the latter path, we allow browsers to cache the file received for a very
+   long time, treating it like an immutable resource with a unique URL based on
+   a file content digest.
+
+   This module implements references to static files (with or without digest)
+   and conversions between (1) a file path + optional digest, and (2) the
+   corresponding relative URL under which the static file is served via HTTP. *)
+
+(* renders a relative URL from the optional [digest] and [filepath] *)
+val to_url_path : ?digest:string -> string -> string
+
+type t = { filepath : string; digest : string option }
+
+(* tries to convert a relative URL to a static file reference by extracting the
+   digest (if applicable) and filepath *)
+val of_url_path : string -> t option

src/ocamlorg_static/ocamlorg_static.ml

@@ -0,0 +1,18 @@
+module Asset = Asset
+module Media = Media
+
+let of_url_path = File.of_url_path
+
+(* given the path of a file from `assets.ml`: 1. looks up the file's digest in
+   and 2. returns the corresponding digest URL for use in templates *)
+let asset_url filepath =
+  let digest =
+    Option.map (fun d -> Dream.to_base64url d) (Asset.hash filepath)
+  in
+  if digest = None then
+    raise
+      (Invalid_argument
+         (Fmt.str
+            "ERROR: '%s' is rendered via asset_url, but it is not an asset!"
+            filepath));
+  File.to_url_path ?digest filepath

src/ocamlorg_web/lib/dune

@@ -11,22 +11,3 @@
   ood
   timedesc
   mirage-kv-mem))
-
-(rule
- (targets asset.ml asset.mli)
- (deps
-  %{workspace_root}/asset/css/main.css
-  (source_tree %{workspace_root}/asset))
- (action
-  (with-stdout-to
-   %{null}
-   (run %{bin:ocaml-crunch} -m lwt %{workspace_root}/asset -o asset.ml))))
-
-(rule
- (targets media.ml media.mli)
- (deps
-  (source_tree %{workspace_root}/data/media))
- (action
-  (with-stdout-to
-   %{null}
-   (run %{bin:ocaml-crunch} -m lwt %{workspace_root}/data/media -o media.ml))))

src/ocamlorg_web/lib/router.ml

@@ -1,27 +1,17 @@
 module Url = Ocamlorg_frontend.Url
 
 let asset_loader =
-  let open Lwt.Syntax in
-  let store = Asset.connect () in
   Static.loader
-    ~read:(fun _root path ->
-      let* store = store in
-      Asset.get store (Mirage_kv.Key.v path))
-    ~last_modified:(fun _root path ->
-      let* store = store in
-      Asset.last_modified store (Mirage_kv.Key.v path))
-    ~not_cached:[ "css/main.css"; "/css/main.css"; "robots.txt"; "/robots.txt" ]
+    ~read:(fun _root path -> Ocamlorg_static.Asset.read path)
+    ~digest:(fun _root path ->
+      Option.map Dream.to_base64url (Ocamlorg_static.Asset.hash path))
+    ~not_cached:[ "robots.txt"; "/robots.txt" ]
 
 let media_loader =
-  let open Lwt.Syntax in
-  let store = Media.connect () in
   Static.loader
-    ~read:(fun _root path ->
-      let* store = store in
-      Media.get store (Mirage_kv.Key.v path))
-    ~last_modified:(fun _root path ->
-      let* store = store in
-      Media.last_modified store (Mirage_kv.Key.v path))
+    ~read:(fun _root path -> Ocamlorg_static.Media.read path)
+    ~digest:(fun _root path ->
+      Option.map Dream.to_base64url @@ Ocamlorg_static.Media.hash path)
 
 let page_routes =
   Dream.scope ""

src/ocamlorg_web/lib/static.ml

@@ -36,29 +36,39 @@ let not_modified ~last_modified request =
   | None -> false
   | Some date -> String.equal date last_modified
 
-let max_age = 60 * 60 * 24 (* one day *)
+let loader ~read ~digest ?(not_cached = []) local_root path request =
+  let not_cached = List.mem path not_cached in
+  let maybe_static_file = Ocamlorg_static.of_url_path path in
+  match maybe_static_file with
+  | None -> Dream.not_found request
+  | Some static_file -> (
+      let filepath = static_file.filepath in
+      let result = read local_root filepath in
+      match result with
+      | None -> Handler.not_found request
+      | Some asset when not_cached ->
+          Dream.respond
+            ~headers:
+              ([ ("Cache-Control", "no-store, max-age=0") ]
+              @ Dream.mime_lookup path)
+            asset
+      | Some asset ->
+          let digest = digest local_root filepath in
+          if
+            static_file.digest <> None
+            && not (Option.equal ( = ) digest static_file.digest)
+          then
+            Dream.log "asset %s exists but digest does not match: %s != %s"
+              filepath
+              (Option.value ~default:"" static_file.digest)
+              (Dream.to_base64url (Option.value ~default:"" digest));
 
-let loader ~read ~last_modified ?(not_cached = []) local_root path request =
-  let open Lwt.Syntax in
-  let* last_modified = last_modified local_root path in
-  match last_modified with
-  | Error _ -> Handler.not_found request
-  | Ok last_modified -> (
-      let last_modified = Last_modified.ptime_to_http_date last_modified in
-      if not_modified ~last_modified request then
-        Dream.respond ~status:`Not_Modified ""
-      else
-        let* result = read local_root path in
-        match result with
-        | Error _ -> Handler.not_found request
-        | Ok asset when List.mem path not_cached ->
-            Dream.respond ~headers:(Dream.mime_lookup path) asset
-        | Ok asset ->
-            Dream.respond
-              ~headers:
-                ([
-                   ("Cache-Control", Fmt.str "max-age=%d" max_age);
-                   ("Last-Modified", last_modified);
-                 ]
-                @ Dream.mime_lookup path)
-              asset)
+          let cache_control =
+            match static_file.digest with
+            | None -> "max-age=86400" (* one day *)
+            | Some _ -> "max-age=31536000, immutable"
+          in
+          Dream.respond
+            ~headers:
+              ([ ("Cache-Control", cache_control) ] @ Dream.mime_lookup path)
+            asset)