Ability to disable cache after it's been configured on Config (#5542)

* Wasmtime: Add `Config::disable_cache`

* bench-api: Always disable the cache

* bench-api: Always get a `Config` from CLI flags

This commit fixes an issue that I ran into just now where benchmarking
one `*.so` with `--engine-flags` was giving wildly unexpected results
comparing to something without `--engine-flags`. The root cause here
appears to that when specifying `--engine-flags` the CLI parsing code is
used to create a `Config` and when omitted a `Config::new` instance is
created. The main difference between these is that for the CLI caching
is enabled by default and for `Config::new` it is not. Coupled with the
fact that caching doesn't really work for the `main` branch this ended
up giving wild results.

The fix here is to first always use the CLI parsing code to create a
`Config` to ensure that a config is consistently created. Next the
`--disable-cache` flag is unconditionally passed to the CLI parsing to
ensure that compilation actually happens.

Once applied this enables comparing an engine without flags and an
engine with flags which provides consistent results.

* Fix compile error

Co-authored-by: Alex Crichton <alex@alexcrichton.com>

crates/cli-flags/src/lib.rs

@@ -16,7 +16,7 @@
     )
 )]
 
-use anyhow::{bail, Context, Result};
+use anyhow::{bail, Result};
 use clap::Parser;
 use std::collections::HashMap;
 use std::path::PathBuf;
@@ -234,17 +234,6 @@ pub struct CommonOptions {
 }
 
 impl CommonOptions {
-    pub fn parse_from_str(s: &str) -> Result<Self> {
-        let parts = s.split(" ").filter(|s| !s.is_empty());
-        // The first argument is the name of the executable, which we don't use
-        // here, but have to provide because `clap` skips over it, and otherwise
-        // our first CLI flag will be ignored.
-        let parts = Some("wasmtime").into_iter().chain(parts);
-        let options =
-            Self::try_parse_from(parts).context("unable to parse options from passed flags")?;
-        Ok(options)
-    }
-
     pub fn init_logging(&self) {
         if self.disable_logging {
             return;
@@ -723,24 +712,4 @@ mod test {
             }
         );
     }
-
-    #[test]
-    fn test_parse_from_str() {
-        fn use_func(flags: &str) -> CommonOptions {
-            CommonOptions::parse_from_str(flags).unwrap()
-        }
-        fn use_clap_parser(flags: &[&str]) -> CommonOptions {
-            CommonOptions::try_parse_from(flags).unwrap()
-        }
-
-        assert_eq!(use_func(""), use_clap_parser(&[]));
-        assert_eq!(
-            use_func("--wasm-features=threads"),
-            use_clap_parser(&["foo", "--wasm-features=threads"])
-        );
-        assert_eq!(
-            use_func("--cranelift-set enable_simd=true"),
-            use_clap_parser(&["foo", "--cranelift-set", "enable_simd=true"])
-        );
-    }
 }