diff --git a/libcxx/utils/ssh.py b/libcxx/utils/ssh.py
--- a/libcxx/utils/ssh.py
+++ b/libcxx/utils/ssh.py
@@ -60,17 +60,25 @@
 
         # Ensure the test dependencies exist, tar them up and copy the tarball
         # over to the remote host.
-        with tempfile.NamedTemporaryFile(suffix='.tar') as tmpTar:
-            with tarfile.open(fileobj=tmpTar, mode='w') as tarball:
-                for dep in args.dependencies:
-                    if not os.path.exists(dep):
-                        sys.stderr.write('Missing file or directory "{}" marked as a dependency of a test'.format(dep))
-                        return 1
-                    tarball.add(dep, arcname=os.path.basename(dep))
-
-            remoteTarball = pathOnRemote(tmpTar.name)
-            tmpTar.flush()
-            subprocess.check_call(scp(tmpTar.name, remoteTarball))
+        try:
+            with tempfile.NamedTemporaryFile(suffix='.tar', delete=False) as tmpTar:
+                with tarfile.open(fileobj=tmpTar, mode='w') as tarball:
+                    for dep in args.dependencies:
+                        if not os.path.exists(dep):
+                            sys.stderr.write('Missing file or directory "{}" marked as a dependency of a test'.format(dep))
+                            return 1
+                        tarball.add(dep, arcname=os.path.basename(dep))
+
+                remoteTarball = pathOnRemote(tmpTar.name)
+                tmpTar.flush()
+                subprocess.check_call(scp(tmpTar.name, remoteTarball))
+        finally:
+            # We delete it manually because otherwise on Windows
+            # it gets deleted before we transfer it to the remote host.
+            # This is because on Windows, file handles with the O_TEMPORARY flag
+            # (which is set if we pass `delete=True`) are deleted as soon as
+            # they're closed.
+            os.remove(tmpTar.name)
 
         # Untar the dependencies in the temporary directory and remove the tarball.
         remoteCommands = [