diff --git a/ci/build-tarballs.sh b/ci/build-tarballs.sh
index 1ad907f978..80f210897f 100755
--- a/ci/build-tarballs.sh
+++ b/ci/build-tarballs.sh
@@ -32,7 +32,7 @@ mktarball() {
 # Create the main tarball of binaries
 bin_pkgname=wasmtime-$TAG-$platform
 mkdir tmp/$bin_pkgname
-cp LICENSE README.md CACHE_CONFIGURATION.md tmp/$bin_pkgname
+cp LICENSE README.md docs/CACHE_CONFIGURATION.md tmp/$bin_pkgname
 mv bins-$src/{wasmtime,wasm2obj}$exe tmp/$bin_pkgname
 chmod +x tmp/$bin_pkgname/{wasmtime,wasm2obj}$exe
 mktarball $bin_pkgname
diff --git a/crates/environ/src/cache/config.rs b/crates/environ/src/cache/config.rs
index f993bf6586..605f8a3409 100644
--- a/crates/environ/src/cache/config.rs
+++ b/crates/environ/src/cache/config.rs
@@ -171,7 +171,7 @@ pub fn create_new_config<P: AsRef<Path> + Debug>(
     let content = "\
 # Comment out certain settings to use default values.
 # For more settings, please refer to the documentation:
-# https://github.com/CraneStation/wasmtime/blob/master/CACHE_CONFIGURATION.md
+# https://github.com/CraneStation/wasmtime/blob/master/docs/CACHE_CONFIGURATION.md
 
 [cache]
 enabled = true
diff --git a/CACHE_CONFIGURATION.md b/docs/CACHE_CONFIGURATION.md
similarity index 100%
rename from CACHE_CONFIGURATION.md
rename to docs/CACHE_CONFIGURATION.md
diff --git a/installer/msi/wasmtime.wxs b/installer/msi/wasmtime.wxs
index 32fd74ed96..a9590809c2 100644
--- a/installer/msi/wasmtime.wxs
+++ b/installer/msi/wasmtime.wxs
@@ -59,7 +59,7 @@
                 <File Id="README.md" Source="README.md" KeyPath="yes" Checksum="yes"/>
             </Component>
             <Component Id="CACHE_CONFIGURATION" Guid="*">
-                <File Id="CACHE_CONFIGURATION.md" Source="CACHE_CONFIGURATION.md" KeyPath="yes" Checksum="yes"/>
+                <File Id="CACHE_CONFIGURATION.md" Source="docs\CACHE_CONFIGURATION.md" KeyPath="yes" Checksum="yes"/>
             </Component>
         </DirectoryRef>