wasmtime/crates/jit-icache-coherence/src/libc.rs

#![allow(unused)]

use libc::{syscall, EINVAL, EPERM};
use std::ffi::c_void;
use std::io::{Error, Result};

const MEMBARRIER_CMD_GLOBAL: libc::c_int = 1;
const MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 32;
const MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE: libc::c_int = 64;

/// See docs on [crate::pipeline_flush_mt] for a description of what this function is trying to do.
#[inline]
pub(crate) fn pipeline_flush_mt() -> Result<()> {
    // Ensure that no processor has fetched a stale instruction stream.
    //
    // On AArch64 we try to do this by executing a "broadcast" `ISB` which is not something that the
    // architecture provides us but we can emulate it using the membarrier kernel interface.
    //
    // This behaviour was documented in a patch, however it seems that it hasn't been upstreamed yet
    // Nevertheless it clearly explains the guarantees that the Linux kernel provides us regarding the
    // membarrier interface, and how to use it for JIT contexts.
    // https://lkml.kernel.org/lkml/07a8b963002cb955b7516e61bad19514a3acaa82.1623813516.git.luto@kernel.org/
    //
    // I couldn't find the follow up for that patch but there doesn't seem to be disagreement about
    // that specific part in the replies.
    // TODO: Check if the kernel has updated the membarrier documentation
    //
    // See the following issues for more info:
    //  * https://github.com/bytecodealliance/wasmtime/pull/3426
    //  * https://github.com/bytecodealliance/wasmtime/pull/4997
    //
    // TODO: x86 and s390x have coherent caches so they don't need this, but RISCV does not
    // guarantee that, so we may need to do something similar for it. However as noted in the above
    // kernel patch the SYNC_CORE membarrier has different guarantees on each architecture
    // so we need follow up and check what it provides us.
    // See: https://github.com/bytecodealliance/wasmtime/issues/5033
    #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
    match membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE) {
        Ok(_) => {}

        // EPERM happens if the calling process hasn't yet called the register membarrier.
        // We can call the register membarrier now, and then retry the actual membarrier,
        //
        // This does have some overhead since on the first time we call this function we
        // actually execute three membarriers, but this only happens once per process and only
        // one slow membarrier is actually executed (The last one, which actually generates an IPI).
        Err(e) if e.raw_os_error().unwrap() == EPERM => {
            membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE)?;
            membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE)?;
        }

        // On kernels older than 4.16 the above syscall does not exist, so we can
        // fallback to MEMBARRIER_CMD_GLOBAL which is an alias for MEMBARRIER_CMD_SHARED
        // that has existed since 4.3. GLOBAL is a lot slower, but allows us to have
        // compatibility with older kernels.
        Err(e) if e.raw_os_error().unwrap() == EINVAL => {
            membarrier(MEMBARRIER_CMD_GLOBAL)?;
        }

        // In any other case we got an actual error, so lets propagate that up
        e => e?,
    }

    Ok(())
}

#[cfg(target_os = "linux")]
fn membarrier(barrier: libc::c_int) -> Result<()> {
    let flags: libc::c_int = 0;
    let res = unsafe { syscall(libc::SYS_membarrier, barrier, flags) };
    if res == 0 {
        Ok(())
    } else {
        Err(Error::last_os_error())
    }
}

/// See docs on [crate::clear_cache] for a description of what this function is trying to do.
#[inline]
pub(crate) fn clear_cache(_ptr: *const c_void, _len: usize) -> Result<()> {
    // TODO: On AArch64 we currently rely on the `mprotect` call that switches the memory from W+R to R+X
    // to do this for us, however that is an implementation detail and should not be relied upon
    // We should call some implementation of `clear_cache` here
    //
    // See: https://github.com/bytecodealliance/wasmtime/issues/3310

    Ok(())
}