Fixes CraneStation/wasmtime#396

This commit fixes an issue with incorrect handling of /dev/(u)random
on Linux. It turns out that `nix::unistd::isatty` call handled only
the POSIX spec case where `ENOTTY` is returned in case the passed
in file descriptor is OK but not a TTY, whereas on Linux this is not
always the case. On Linux, it can be the case that `EINVAL` is returned
instead and this case AFAIK is not handled by the `nix` crate. This
commit fixes this by using `libc::isatty` syscall directly and checking
the return values.

src/sys/unix/bsd/mod.rs

@@ -1,2 +1,19 @@
 pub(crate) mod hostcalls_impl;
 pub(crate) mod osfile;
+
+pub(crate) mod fdentry_impl {
+    use crate::{sys::host_impl, Result};
+    use std::os::unix::prelude::AsRawFd;
+
+    pub(crate) unsafe fn isatty(fd: &impl AsRawFd) -> Result<bool> {
+        let res = libc::isatty(fd.as_raw_fd());
+        if res == 0 {
+            Ok(true)
+        } else {
+            match nix::errno::Errno::last() {
+                nix::errno::Errno::ENOTTY => Ok(false),
+                x => Err(host_impl::errno_from_nix(x)),
+            }
+        }
+    }
+}

src/sys/unix/fdentry_impl.rs

@@ -6,6 +6,7 @@ use std::os::unix::prelude::{AsRawFd, FileTypeExt, FromRawFd, RawFd};
 cfg_if::cfg_if! {
     if #[cfg(target_os = "linux")] {
         pub(crate) use super::linux::osfile::*;
+        pub(crate) use super::linux::fdentry_impl::*;
     } else if #[cfg(any(
             target_os = "macos",
             target_os = "netbsd",
@@ -15,6 +16,7 @@ cfg_if::cfg_if! {
             target_os = "dragonfly"
     ))] {
         pub(crate) use super::bsd::osfile::*;
+        pub(crate) use super::bsd::fdentry_impl::*;
     }
 }
 
@@ -65,13 +67,15 @@ pub(crate) unsafe fn determine_type_rights<Fd: AsRawFd>(
         let file = std::mem::ManuallyDrop::new(std::fs::File::from_raw_fd(fd.as_raw_fd()));
         let ft = file.metadata()?.file_type();
         if ft.is_block_device() {
+            log::debug!("Fd {:?} is a block device", fd.as_raw_fd());
             (
                 host::__WASI_FILETYPE_BLOCK_DEVICE,
                 host::RIGHTS_BLOCK_DEVICE_BASE,
                 host::RIGHTS_BLOCK_DEVICE_INHERITING,
             )
         } else if ft.is_char_device() {
-            if nix::unistd::isatty(fd.as_raw_fd())? {
+            log::debug!("Fd {:?} is a char device", fd.as_raw_fd());
+            if isatty(fd)? {
                 (
                     host::__WASI_FILETYPE_CHARACTER_DEVICE,
                     host::RIGHTS_TTY_BASE,
@@ -85,18 +89,21 @@ pub(crate) unsafe fn determine_type_rights<Fd: AsRawFd>(
                 )
             }
         } else if ft.is_dir() {
+            log::debug!("Fd {:?} is a directory", fd.as_raw_fd());
             (
                 host::__WASI_FILETYPE_DIRECTORY,
                 host::RIGHTS_DIRECTORY_BASE,
                 host::RIGHTS_DIRECTORY_INHERITING,
             )
         } else if ft.is_file() {
+            log::debug!("Fd {:?} is a file", fd.as_raw_fd());
             (
                 host::__WASI_FILETYPE_REGULAR_FILE,
                 host::RIGHTS_REGULAR_FILE_BASE,
                 host::RIGHTS_REGULAR_FILE_INHERITING,
             )
         } else if ft.is_socket() {
+            log::debug!("Fd {:?} is a socket", fd.as_raw_fd());
             use nix::sys::socket;
             match socket::getsockopt(fd.as_raw_fd(), socket::sockopt::SockType)? {
                 socket::SockType::Datagram => (
@@ -112,12 +119,14 @@ pub(crate) unsafe fn determine_type_rights<Fd: AsRawFd>(
                 _ => return Err(Error::EINVAL),
             }
         } else if ft.is_fifo() {
+            log::debug!("Fd {:?} is a fifo", fd.as_raw_fd());
             (
                 host::__WASI_FILETYPE_UNKNOWN,
                 host::RIGHTS_REGULAR_FILE_BASE,
                 host::RIGHTS_REGULAR_FILE_INHERITING,
             )
         } else {
+            log::debug!("Fd {:?} is unknown", fd.as_raw_fd());
             return Err(Error::EINVAL);
         }
     };

src/sys/unix/hostcalls_impl/fs.rs

@@ -120,6 +120,10 @@ pub(crate) fn path_open(
     // Call openat. Use mode 0o666 so that we follow whatever the user's
     // umask is, but don't set the executable flag, because it isn't yet
     // meaningful for WASI programs to create executable files.
+
+    log::debug!("path_open resolved = {:?}", resolved);
+    log::debug!("path_open oflags = {:?}", nix_all_oflags);
+
     let new_fd = match openat(
         resolved.dirfd().as_raw_fd(),
         resolved.path(),
@@ -172,6 +176,8 @@ pub(crate) fn path_open(
         }
     };
 
+    log::debug!("path_open new_fd = {:?}", new_fd);
+
     // Determine the type of the new file descriptor and which rights contradict with this type
     Ok(unsafe { File::from_raw_fd(new_fd) })
 }

src/sys/unix/linux/mod.rs

@@ -1,2 +1,26 @@
 pub(crate) mod hostcalls_impl;
 pub(crate) mod osfile;
+
+pub(crate) mod fdentry_impl {
+    use crate::{sys::host_impl, Result};
+    use std::os::unix::prelude::AsRawFd;
+
+    pub(crate) unsafe fn isatty(fd: &impl AsRawFd) -> Result<bool> {
+        use nix::errno::Errno;
+
+        let res = libc::isatty(fd.as_raw_fd());
+        if res == 0 {
+            Ok(true)
+        } else {
+            match Errno::last() {
+                // While POSIX specifies ENOTTY if the passed
+                // fd is *not* a tty, on Linux, some implementations
+                // may return EINVAL instead.
+                //
+                // https://linux.die.net/man/3/isatty
+                Errno::ENOTTY | Errno::EINVAL => Ok(false),
+                x => Err(host_impl::errno_from_nix(x)),
+            }
+        }
+    }
+}