Call membarrier() after making JIT mappings executable on AArch64 Linux

The membarrier() system call ensures that no processor has fetched
a stale instruction stream.

Copyright (c) 2021, Arm Limited.

cranelift/jit/src/backend.rs

@@ -428,6 +428,14 @@ impl JITModule {
         self.memory.readonly.set_readonly();
         self.memory.code.set_readable_and_executable();
 
+        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
+        {
+            let cmd: libc::c_int = 32; // MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE
+
+            // Ensure that no processor has fetched a stale instruction stream.
+            unsafe { libc::syscall(libc::SYS_membarrier, cmd) };
+        }
+
         for update in self.pending_got_updates.drain(..) {
             unsafe { update.entry.as_ref() }.store(update.ptr as *mut _, Ordering::SeqCst);
         }
@@ -489,6 +497,15 @@ impl JITModule {
             module.libcall_plt_entries.insert(libcall, plt_entry);
         }
 
+        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
+        {
+            let cmd: libc::c_int = 64; // MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE
+
+            // This is a requirement of the membarrier() call executed by
+            // the finalize_definitions() method.
+            unsafe { libc::syscall(libc::SYS_membarrier, cmd) };
+        }
+
         module
     }
 

crates/jit/src/code_memory.rs

@@ -54,6 +54,15 @@ impl CodeMemory {
     /// The returned `CodeMemory` manages the internal `MmapVec` and the
     /// `publish` method is used to actually make the memory executable.
     pub fn new(mmap: MmapVec) -> Self {
+        #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
+        {
+            // This is a requirement of the `membarrier` call executed by the `publish` method.
+            rsix::process::membarrier(
+                rsix::process::MembarrierCommand::RegisterPrivateExpeditedSyncCore,
+            )
+            .unwrap();
+        }
+
         Self {
             mmap: ManuallyDrop::new(mmap),
             unwind_registration: ManuallyDrop::new(None),
@@ -159,6 +168,15 @@ impl CodeMemory {
                 .make_executable(text_range.clone())
                 .expect("unable to make memory executable");
 
+            #[cfg(all(target_arch = "aarch64", target_os = "linux"))]
+            {
+                // Ensure that no processor has fetched a stale instruction stream.
+                rsix::process::membarrier(
+                    rsix::process::MembarrierCommand::PrivateExpeditedSyncCore,
+                )
+                .unwrap();
+            }
+
             // With all our memory set up use the platform-specific
             // `UnwindRegistration` implementation to inform the general
             // runtime that there's unwinding information available for all