Fix a panic when opening a directory with truncate(true)

This is intended to address bytecodealliance/wasmtime#10577

Author: alexcrichton

cap-primitives/src/windows/fs/open_unchecked.rs

@@ -205,12 +205,14 @@ fn handle_open_result(
             }
 
             // Windows truncates symlinks into normal files, so truncation
-            // may be disabled above; do it manually if needed.
+            // may be disabled above; do it manually if needed. Note that this
+            // is expected to always succeed for normal files, but this will
+            // fail if a directory was opened as directories don't support
+            // truncation.
             if manually_trunc {
-                // Unwrap is ok because 0 never overflows, and we'll only
-                // have `manually_trunc` set when the file is opened for
-                // writing.
-                f.set_len(0).unwrap();
+                if let Err(e) = f.set_len(0) {
+                    return Err(OpenUncheckedError::Other(e));
+                }
             }
             Ok(f)
         }

tests/fs.rs

@@ -1645,3 +1645,19 @@ fn create_dir_long_paths() {
         std::io::ErrorKind::NotFound
     );
 }
+
+/// Ensure that opening a directory with the truncation flag set is an error,
+/// not a panic inside of cap-std.
+#[test]
+fn open_directory_with_truncate_is_error() {
+    use cap_fs_ext::OpenOptionsMaybeDirExt;
+    let tmpdir = tmpdir();
+    let mut options = OpenOptions::new();
+    options
+        .truncate(true)
+        .maybe_dir(true)
+        .read(true)
+        .write(true);
+    tmpdir.create_dir("test").unwrap();
+    assert!(tmpdir.open_with("test", &options).is_err());
+}