Virtual file support (#701)

* Add support for virtual files (eg, not backed by an OS file).

Virtual files are implemented through trait objects, with a default
implementation that tries to behave like on-disk files, but entirely
backed by in-memory structures.

Co-authored-by: Dan Gohman <sunfish@mozilla.com>

crates/test-programs/build.rs

@@ -11,11 +11,19 @@ fn main() {
 #[cfg(feature = "test_programs")]
 mod wasi_tests {
     use std::env;
-    use std::fs::{read_dir, DirEntry, File};
+    use std::fs::{read_dir, File};
     use std::io::{self, Write};
     use std::path::{Path, PathBuf};
     use std::process::{Command, Stdio};
 
+    #[derive(Clone, Copy, Debug)]
+    enum PreopenType {
+        /// Preopens should be satisfied with real OS files.
+        OS,
+        /// Preopens should be satisfied with virtual files.
+        Virtual,
+    }
+
     pub(super) fn build_and_generate_tests() {
         // Validate if any of test sources are present and if they changed
         // This should always work since there is no submodule to init anymore
@@ -103,8 +111,21 @@ mod wasi_tests {
                 .replace("-", "_")
         )?;
         writeln!(out, "    use super::{{runtime, utils, setup_log}};")?;
+        writeln!(out, "    use runtime::PreopenType;")?;
         for dir_entry in dir_entries {
-            write_testsuite_tests(out, dir_entry, testsuite)?;
+            let test_path = dir_entry.path();
+            let stemstr = test_path
+                .file_stem()
+                .expect("file_stem")
+                .to_str()
+                .expect("to_str");
+
+            if no_preopens(testsuite, stemstr) {
+                write_testsuite_tests(out, &test_path, testsuite, PreopenType::OS)?;
+            } else {
+                write_testsuite_tests(out, &test_path, testsuite, PreopenType::OS)?;
+                write_testsuite_tests(out, &test_path, testsuite, PreopenType::Virtual)?;
+            }
         }
         writeln!(out, "}}")?;
         Ok(())
@@ -112,10 +133,10 @@ mod wasi_tests {
 
     fn write_testsuite_tests(
         out: &mut File,
-        dir_entry: DirEntry,
+        path: &Path,
         testsuite: &str,
+        preopen_type: PreopenType,
     ) -> io::Result<()> {
-        let path = dir_entry.path();
         let stemstr = path
             .file_stem()
             .expect("file_stem")
@@ -123,14 +144,19 @@ mod wasi_tests {
             .expect("to_str");
 
         writeln!(out, "    #[test]")?;
-        if ignore(testsuite, stemstr) {
+        let test_fn_name = format!(
+            "{}{}",
+            &stemstr.replace("-", "_"),
+            if let PreopenType::Virtual = preopen_type {
+                "_virtualfs"
+            } else {
+                ""
+            }
+        );
+        if ignore(testsuite, &test_fn_name) {
             writeln!(out, "    #[ignore]")?;
         }
-        writeln!(
-            out,
-            "    fn r#{}() -> anyhow::Result<()> {{",
-            &stemstr.replace("-", "_")
-        )?;
+        writeln!(out, "    fn r#{}() -> anyhow::Result<()> {{", test_fn_name,)?;
         writeln!(out, "        setup_log();")?;
         writeln!(
             out,
@@ -145,16 +171,25 @@ mod wasi_tests {
         let workspace = if no_preopens(testsuite, stemstr) {
             "None"
         } else {
-            writeln!(
-                out,
-                "        let workspace = utils::prepare_workspace(&bin_name)?;"
-            )?;
-            "Some(workspace.path())"
+            match preopen_type {
+                PreopenType::OS => {
+                    writeln!(
+                        out,
+                        "        let workspace = utils::prepare_workspace(&bin_name)?;"
+                    )?;
+                    "Some(workspace.path())"
+                }
+                PreopenType::Virtual => "Some(std::path::Path::new(&bin_name))",
+            }
         };
         writeln!(
             out,
-            "        runtime::instantiate(&data, &bin_name, {})",
-            workspace
+            "        runtime::instantiate(&data, &bin_name, {}, {})",
+            workspace,
+            match preopen_type {
+                PreopenType::OS => "PreopenType::OS",
+                PreopenType::Virtual => "PreopenType::Virtual",
+            }
         )?;
         writeln!(out, "    }}")?;
         writeln!(out)?;
@@ -164,8 +199,30 @@ mod wasi_tests {
     cfg_if::cfg_if! {
         if #[cfg(not(windows))] {
             /// Ignore tests that aren't supported yet.
-            fn ignore(_testsuite: &str, _name: &str) -> bool {
-                false
+            fn ignore(testsuite: &str, name: &str) -> bool {
+                if testsuite == "wasi-tests" {
+                    match name {
+                        // TODO: virtfs files cannot be poll_oneoff'd yet
+                        "poll_oneoff_virtualfs" => true,
+                        // TODO: virtfs does not support filetimes yet.
+                        "path_filestat_virtualfs" |
+                        "fd_filestat_set_virtualfs" => true,
+                        // TODO: virtfs does not support symlinks yet.
+                        "nofollow_errors_virtualfs" |
+                        "path_link_virtualfs" |
+                        "readlink_virtualfs" |
+                        "readlink_no_buffer_virtualfs" |
+                        "dangling_symlink_virtualfs" |
+                        "symlink_loop_virtualfs" |
+                        "path_symlink_trailing_slashes_virtualfs" => true,
+                        // TODO: virtfs does not support rename yet.
+                        "path_rename_trailing_slashes_virtualfs" |
+                        "path_rename_virtualfs" => true,
+                        _ => false,
+                    }
+                } else {
+                    unreachable!()
+                }
             }
         } else {
             /// Ignore tests that aren't supported yet.
@@ -178,6 +235,22 @@ mod wasi_tests {
                         "truncation_rights" => true,
                         "path_link" => true,
                         "dangling_fd" => true,
+                        // TODO: virtfs files cannot be poll_oneoff'd yet
+                        "poll_oneoff_virtualfs" => true,
+                        // TODO: virtfs does not support filetimes yet.
+                        "path_filestat_virtualfs" |
+                        "fd_filestat_set_virtualfs" => true,
+                        // TODO: virtfs does not support symlinks yet.
+                        "nofollow_errors_virtualfs" |
+                        "path_link_virtualfs" |
+                        "readlink_virtualfs" |
+                        "readlink_no_buffer_virtualfs" |
+                        "dangling_symlink_virtualfs" |
+                        "symlink_loop_virtualfs" |
+                        "path_symlink_trailing_slashes_virtualfs" => true,
+                        // TODO: virtfs does not support rename yet.
+                        "path_rename_trailing_slashes_virtualfs" |
+                        "path_rename_virtualfs" => true,
                         _ => false,
                     }
                 } else {

crates/test-programs/tests/wasm_tests/runtime.rs

@@ -1,30 +1,44 @@
 use anyhow::{bail, Context};
 use std::fs::File;
 use std::path::Path;
+use wasi_common::VirtualDirEntry;
 use wasmtime::{Instance, Module, Store};
 
-pub fn instantiate(data: &[u8], bin_name: &str, workspace: Option<&Path>) -> anyhow::Result<()> {
+#[derive(Clone, Copy, Debug)]
+pub enum PreopenType {
+    /// Preopens should be satisfied with real OS files.
+    OS,
+    /// Preopens should be satisfied with virtual files.
+    Virtual,
+}
+
+pub fn instantiate(
+    data: &[u8],
+    bin_name: &str,
+    workspace: Option<&Path>,
+    preopen_type: PreopenType,
+) -> anyhow::Result<()> {
     let store = Store::default();
 
-    let get_preopens = |workspace: Option<&Path>| -> anyhow::Result<Vec<_>> {
-        if let Some(workspace) = workspace {
-            let preopen_dir = wasi_common::preopen_dir(workspace)
-                .context(format!("error while preopening {:?}", workspace))?;
-
-            Ok(vec![(".".to_owned(), preopen_dir)])
-        } else {
-            Ok(vec![])
-        }
-    };
-
     // Create our wasi context with pretty standard arguments/inheritance/etc.
-    // Additionally register andy preopened directories if we have them.
+    // Additionally register any preopened directories if we have them.
     let mut builder = wasi_common::WasiCtxBuilder::new();
 
     builder.arg(bin_name).arg(".").inherit_stdio();
 
-    for (dir, file) in get_preopens(workspace)? {
-        builder.preopened_dir(file, dir);
+    if let Some(workspace) = workspace {
+        match preopen_type {
+            PreopenType::OS => {
+                let preopen_dir = wasi_common::preopen_dir(workspace)
+                    .context(format!("error while preopening {:?}", workspace))?;
+                builder.preopened_dir(preopen_dir, ".");
+            }
+            PreopenType::Virtual => {
+                // we can ignore the workspace path for virtual preopens because virtual preopens
+                // don't exist in the filesystem anyway - no name conflict concerns.
+                builder.preopened_virt(VirtualDirEntry::empty_directory(), ".");
+            }
+        }
     }
 
     // The nonstandard thing we do with `WasiCtxBuilder` is to ensure that

crates/test-programs/wasi-tests/src/bin/fd_flags_set.rs

@@ -50,7 +50,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
         )
         .expect("reading file"),
         buffer.len(),
-        "shoudl read {} bytes",
+        "should read {} bytes",
         buffer.len()
     );
 
@@ -87,7 +87,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
         )
         .expect("reading file"),
         buffer.len(),
-        "shoudl read {} bytes",
+        "should read {} bytes",
         buffer.len()
     );
 
@@ -126,7 +126,7 @@ unsafe fn test_fd_fdstat_set_flags(dir_fd: wasi::Fd) {
         )
         .expect("reading file"),
         buffer.len(),
-        "shoudl read {} bytes",
+        "should read {} bytes",
         buffer.len()
     );
 

crates/test-programs/wasi-tests/src/bin/remove_directory_trailing_slashes.rs

@@ -11,14 +11,14 @@ unsafe fn test_remove_directory_trailing_slashes(dir_fd: wasi::Fd) {
 
     wasi::path_create_directory(dir_fd, "dir").expect("creating a directory");
 
-    // Test that removing it with a trailing flash succeeds.
+    // Test that removing it with a trailing slash succeeds.
     wasi::path_remove_directory(dir_fd, "dir/")
         .expect("remove_directory with a trailing slash on a directory should succeed");
 
     // Create a temporary file.
     create_file(dir_fd, "file");
 
-    // Test that removing it with no trailing flash fails.
+    // Test that removing it with no trailing slash fails.
     assert_eq!(
         wasi::path_remove_directory(dir_fd, "file")
             .expect_err("remove_directory without a trailing slash on a file should fail")
@@ -27,7 +27,7 @@ unsafe fn test_remove_directory_trailing_slashes(dir_fd: wasi::Fd) {
         "errno should be ERRNO_NOTDIR"
     );
 
-    // Test that removing it with a trailing flash fails.
+    // Test that removing it with a trailing slash fails.
     assert_eq!(
         wasi::path_remove_directory(dir_fd, "file/")
             .expect_err("remove_directory with a trailing slash on a file should fail")