Move asset hashing into the CLI (DioxusLabs#3988)

* Fix deduping assets

* hash assets in the linker

* fix extracting assets

* Fix the hashed asset name

* deserialize the asset back from the link section

* prevent the read from being inlined

* remove some extra logs

* Fix liveview routing

* Only look for direct dependencies on dioxus features

* fix wasm split test

* On wasm, create a new object file with the asset information

* Fix multiple assets

* add a regression test for 3467

* fix image link

* fix linker out file

* fix linux linker args

* add windows linker command

* fix linux link args

* fix windows target triple

* Expose an option to build assets in the linker intercept

* Fix the standalone optimizer and document it in the dx help command

* include the cli opt version in the hash

* fix clippy

* fix ssg port

* add a safety note to the read_volatile call

* move found assets log

* fix some merge conflicts

* fix build

* make most env var for the linker optional

* guess the target triple when dx acts as the linker intercept

* automatically find the linker for normal builds

* use the right linker in wasm mode

* fix mac linker

* look inside archive files (rlibs)

* remove out.txt

* fix clippy

* fix typo

* fix clippy

* fix build

* fix dx link name in docs

* ignore libcompiler_builtins warnings

* fix playwright

* pass typocheck

* fix printing sysroot

* Modify final binary

* fix wasm assets

* remove old link section logic

* fix tests

* forward manganis export args

* fail to evaluate relocatable address

* retain all exports in the same pattern as other platforms

* Fix adding assets during a wasm hot patch

* less nesting and remove logs

* fix clippy

* log pdb files

* read from pdb

* print data section contents

* print with object

* normal windows builds working

* fix hot patch windows

* clean up logs

* fix clippy

* clean up assets

* use the most recent pdb

* fix doc test

* fix typo

* tiny nits

* make bundled a reference instead of a pointer

* add helpful comment to impl

* match on file format

* Fix asset hashing comment

---------

Co-authored-by: Jonathan Kelley <jkelleyrtp@gmail.com>

Author: 2 people

Cargo.lock

@@ -72,7 +72,7 @@ pub fn serve_asset_from_raw_path(path: &str) -> Result<Response<Vec<u8>>, AssetS
 /// - [ ] Linux (rpm)
 /// - [ ] Linux (deb)
 /// - [ ] Android
-#[allow(unused)]
+#[allow(unreachable_code)]
 fn get_asset_root() -> PathBuf {
     let cur_exe = std::env::current_exe().unwrap();
 

packages/asset-resolver/src/lib.rs

@@ -67,3 +67,6 @@ swc_parallel = { version = "=1.0.1", default-features = false }
 swc_timer = { version = "=1.0.0", default-features = false }
 swc_visit = { version = "=2.0.0", default-features = false }
 browserslist-rs = { version = "=0.16.0" }
+
+[build-dependencies]
+built = { version = "0.7.5", features = ["git2"] }

packages/cli-opt/Cargo.toml

@@ -0,0 +1,3 @@
+fn main() {
+    built::write_built_file().expect("Failed to acquire build-time information");
+}

packages/cli-opt/build.rs

@@ -0,0 +1,10 @@
+// The file has been placed there by the build script.
+include!(concat!(env!("OUT_DIR"), "/built.rs"));
+
+pub(crate) fn version() -> String {
+    format!(
+        "{} ({})",
+        PKG_VERSION,
+        GIT_COMMIT_HASH_SHORT.unwrap_or("was built without git repository")
+    )
+}

packages/cli-opt/src/build_info.rs

@@ -1,4 +1,4 @@
-use std::path::Path;
+use std::{hash::Hasher, path::Path};
 
 use anyhow::{anyhow, Context};
 use codemap::SpanLoc;
@@ -146,12 +146,11 @@ pub(crate) fn minify_css(css: &str) -> anyhow::Result<String> {
     Ok(res.code)
 }
 
-/// Process an scss/sass file into css.
-pub(crate) fn process_scss(
+/// Compile scss with grass
+pub(crate) fn compile_scss(
     scss_options: &CssAssetOptions,
     source: &Path,
-    output_path: &Path,
-) -> anyhow::Result<()> {
+) -> anyhow::Result<String> {
     let style = match scss_options.minified() {
         true => OutputStyle::Compressed,
         false => OutputStyle::Expanded,
@@ -162,7 +161,18 @@ pub(crate) fn process_scss(
         .quiet(false)
         .logger(&ScssLogger {});
 
-    let css = grass::from_path(source, &options)?;
+    let css = grass::from_path(source, &options)
+        .with_context(|| format!("Failed to compile scss file: {}", source.display()))?;
+    Ok(css)
+}
+
+/// Process an scss/sass file into css.
+pub(crate) fn process_scss(
+    scss_options: &CssAssetOptions,
+    source: &Path,
+    output_path: &Path,
+) -> anyhow::Result<()> {
+    let css = compile_scss(scss_options, source)?;
     let minified = minify_css(&css)?;
 
     std::fs::write(output_path, minified).with_context(|| {
@@ -199,3 +209,19 @@ impl grass::Logger for ScssLogger {
         );
     }
 }
+
+/// Hash the inputs to the scss file
+pub(crate) fn hash_scss(
+    scss_options: &CssAssetOptions,
+    source: &Path,
+    hasher: &mut impl Hasher,
+) -> anyhow::Result<()> {
+    // Grass doesn't expose the ast for us to traverse the imports in the file. Instead of parsing scss ourselves
+    // we just hash the expanded version of the file for now
+    let css = compile_scss(scss_options, source)?;
+
+    // Hash the compiled css
+    hasher.write(css.as_bytes());
+
+    Ok(())
+}

packages/cli-opt/src/css.rs

@@ -1,4 +1,5 @@
 use anyhow::Context;
+use manganis::{CssModuleAssetOptions, FolderAssetOptions};
 use manganis_core::{AssetOptions, CssAssetOptions, ImageAssetOptions, JsAssetOptions};
 use std::path::Path;
 
@@ -33,7 +34,7 @@ pub(crate) fn process_file_to_with_options(
     }
     if let Some(parent) = output_path.parent() {
         if !parent.exists() {
-            std::fs::create_dir_all(parent)?;
+            std::fs::create_dir_all(parent).context("Failed to create directory")?;
         }
     }
 
@@ -47,63 +48,94 @@ pub(crate) fn process_file_to_with_options(
             .unwrap_or_default()
             .to_string_lossy()
     ));
+    let resolved_options = resolve_asset_options(source, options);
 
-    match options {
-        AssetOptions::Unknown => match source.extension().map(|e| e.to_string_lossy()).as_deref() {
-            Some("css") => {
-                process_css(&CssAssetOptions::new(), source, &temp_path)?;
-            }
-            Some("scss" | "sass") => {
-                process_scss(&CssAssetOptions::new(), source, &temp_path)?;
-            }
-            Some("js") => {
-                process_js(&JsAssetOptions::new(), source, &temp_path, !in_folder)?;
-            }
-            Some("json") => {
-                process_json(source, &temp_path)?;
-            }
-            Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
-                process_image(&ImageAssetOptions::new(), source, &temp_path)?;
-            }
-            Some(_) | None => {
-                if source.is_dir() {
-                    process_folder(source, &temp_path)?;
-                } else {
-                    let source_file = std::fs::File::open(source)?;
-                    let mut reader = std::io::BufReader::new(source_file);
-                    let output_file = std::fs::File::create(&temp_path)?;
-                    let mut writer = std::io::BufWriter::new(output_file);
-                    std::io::copy(&mut reader, &mut writer).with_context(|| {
-                        format!(
-                            "Failed to write file to output location: {}",
-                            temp_path.display()
-                        )
-                    })?;
-                }
-            }
-        },
-        AssetOptions::Css(options) => {
+    match &resolved_options {
+        ResolvedAssetType::Css(options) => {
             process_css(options, source, &temp_path)?;
         }
-        AssetOptions::CssModule(options) => {
+        ResolvedAssetType::CssModule(options) => {
             process_css_module(options, source, output_path, &temp_path)?;
         }
-        AssetOptions::Js(options) => {
+        ResolvedAssetType::Scss(options) => {
+            process_scss(options, source, &temp_path)?;
+        }
+        ResolvedAssetType::Js(options) => {
             process_js(options, source, &temp_path, !in_folder)?;
         }
-        AssetOptions::Image(options) => {
+        ResolvedAssetType::Image(options) => {
             process_image(options, source, &temp_path)?;
         }
-        AssetOptions::Folder(_) => {
+        ResolvedAssetType::Json => {
+            process_json(source, &temp_path)?;
+        }
+        ResolvedAssetType::Folder(_) => {
             process_folder(source, &temp_path)?;
         }
-        _ => {
-            tracing::warn!("Unknown asset options: {:?}", options);
+        ResolvedAssetType::File => {
+            let source_file = std::fs::File::open(source)?;
+            let mut reader = std::io::BufReader::new(source_file);
+            let output_file = std::fs::File::create(&temp_path)?;
+            let mut writer = std::io::BufWriter::new(output_file);
+            std::io::copy(&mut reader, &mut writer).with_context(|| {
+                format!(
+                    "Failed to write file to output location: {}",
+                    temp_path.display()
+                )
+            })?;
         }
     }
 
     // If everything was successful, rename the temp file to the final output path
-    std::fs::rename(temp_path, output_path)?;
+    std::fs::rename(temp_path, output_path).context("Failed to rename output file")?;
 
     Ok(())
 }
+
+pub(crate) enum ResolvedAssetType {
+    /// An image asset
+    Image(ImageAssetOptions),
+    /// A css asset
+    Css(CssAssetOptions),
+    /// A css module asset
+    CssModule(CssModuleAssetOptions),
+    /// A SCSS asset
+    Scss(CssAssetOptions),
+    /// A javascript asset
+    Js(JsAssetOptions),
+    /// A json asset
+    Json,
+    /// A folder asset
+    Folder(FolderAssetOptions),
+    /// A generic file
+    File,
+}
+
+pub(crate) fn resolve_asset_options(source: &Path, options: &AssetOptions) -> ResolvedAssetType {
+    match options {
+        AssetOptions::Image(image) => ResolvedAssetType::Image(*image),
+        AssetOptions::Css(css) => ResolvedAssetType::Css(*css),
+        AssetOptions::CssModule(css) => ResolvedAssetType::CssModule(*css),
+        AssetOptions::Js(js) => ResolvedAssetType::Js(*js),
+        AssetOptions::Folder(folder) => ResolvedAssetType::Folder(*folder),
+        AssetOptions::Unknown => resolve_unknown_asset_options(source),
+        _ => {
+            tracing::warn!("Unknown asset options... you may need to update the Dioxus CLI. Defaulting to a generic file: {:?}", options);
+            resolve_unknown_asset_options(source)
+        }
+    }
+}
+
+fn resolve_unknown_asset_options(source: &Path) -> ResolvedAssetType {
+    match source.extension().map(|e| e.to_string_lossy()).as_deref() {
+        Some("scss" | "sass") => ResolvedAssetType::Scss(CssAssetOptions::new()),
+        Some("css") => ResolvedAssetType::Css(CssAssetOptions::new()),
+        Some("js") => ResolvedAssetType::Js(JsAssetOptions::new()),
+        Some("json") => ResolvedAssetType::Json,
+        Some("jpg" | "jpeg" | "png" | "webp" | "avif") => {
+            ResolvedAssetType::Image(ImageAssetOptions::new())
+        }
+        _ if source.is_dir() => ResolvedAssetType::Folder(FolderAssetOptions::new()),
+        _ => ResolvedAssetType::File,
+    }
+}