diff --git a/setup.py b/setup.py
index b4212cb9d..f3e9690f2 100644
--- a/setup.py
+++ b/setup.py
@@ -165,13 +165,12 @@ def _build_wheel(
         for whlfile in glob.glob(os.path.join(self.dist_dir, "*.whl")):
             os.makedirs("wheelhouse", exist_ok=True)
             if InWheel:
-                with InWheel(
-                    in_wheel=whlfile,
-                    out_wheel=os.path.join("wheelhouse", os.path.basename(whlfile)),
-                ):
+                wheelhouse_whl = os.path.join("wheelhouse", os.path.basename(whlfile))
+                shutil.move(whlfile, wheelhouse_whl)
+                with InWheel(in_wheel=wheelhouse_whl, out_wheel=whlfile):
                     print(f"Updating RECORD file of {whlfile}")
         print("Copying new wheels")
-        shutil.move("wheelhouse", self.dist_dir)
+        shutil.rmtree("wheelhouse")
 
     def _download_and_extract_local_driver(
         self,