diff --git a/internal/cache/backend.go b/internal/cache/backend.go
index 7048fd4d4..824a35753 100644
--- a/internal/cache/backend.go
+++ b/internal/cache/backend.go
@@ -107,16 +107,17 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 	}
 
 	// test again, maybe the file was cached in the meantime
-	if b.Cache.Has(h) {
-		return nil
-	}
+	if !b.Cache.Has(h) {
 
-	err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
-		return b.Cache.Save(h, rd)
-	})
-	if err != nil {
-		// try to remove from the cache, ignore errors
-		_ = b.Cache.Remove(h)
+		// nope, it's still not in the cache, pull it from the repo and save it
+
+		err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
+			return b.Cache.Save(h, rd)
+		})
+		if err != nil {
+			// try to remove from the cache, ignore errors
+			_ = b.Cache.Remove(h)
+		}
 	}
 
 	// signal other waiting goroutines that the file may now be cached