Cancel current command if cache becomes unusable

If the cache suddenly disappears, the current command will now fail.

changelog/unreleased/pull-4166

@@ -0,0 +1,7 @@
+Enhancement: Cancel current command if cache becomes unusable
+
+If the cache directory was removed or ran out of space while restic was
+running, this caused further caching attempts to fail and drastically slow down
+the command execution. Now, the currently running command is canceled instead.
+
+https://github.com/restic/restic/pull/4166

internal/cache/backend.go

@@ -83,7 +83,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
 	if err != nil {
 		debug.Log("unable to save %v to cache: %v", h, err)
 		_ = b.Cache.remove(h)
-		return nil
+		return err
 	}
 
 	return nil
@@ -106,11 +106,19 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 		return nil
 	}
 
+	defer func() {
+		// signal other waiting goroutines that the file may now be cached
+		close(finish)
+
+		// remove the finish channel from the map
+		b.inProgressMutex.Lock()
+		delete(b.inProgress, h)
+		b.inProgressMutex.Unlock()
+	}()
+
 	// test again, maybe the file was cached in the meantime
 	if !b.Cache.Has(h) {
-
 		// nope, it's still not in the cache, pull it from the repo and save it
-
 		err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
 			return b.Cache.Save(h, rd)
 		})
@@ -118,16 +126,9 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
 			// try to remove from the cache, ignore errors
 			_ = b.Cache.remove(h)
 		}
+		return err
 	}
 
-	// signal other waiting goroutines that the file may now be cached
-	close(finish)
-
-	// remove the finish channel from the map
-	b.inProgressMutex.Lock()
-	delete(b.inProgress, h)
-	b.inProgressMutex.Unlock()
-
 	return nil
 }
 
@@ -178,11 +179,13 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
 
 	debug.Log("auto-store %v in the cache", h)
 	err = b.cacheFile(ctx, h)
-	if err == nil {
-		inCache, err = b.loadFromCache(ctx, h, length, offset, consumer)
-		if inCache {
-			return err
-		}
+	if err != nil {
+		return err
+	}
+
+	inCache, err = b.loadFromCache(ctx, h, length, offset, consumer)
+	if inCache {
+		return err
 	}
 
 	debug.Log("error caching %v: %v, falling back to backend", h, err)

internal/cache/file_test.go

@@ -11,8 +11,10 @@ import (
 	"time"
 
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 
 	"golang.org/x/sync/errgroup"
 )
@@ -266,3 +268,19 @@ func TestFileSaveConcurrent(t *testing.T) {
 	saved := load(t, c, h)
 	test.Equals(t, data, saved)
 }
+
+func TestFileSaveAfterDamage(t *testing.T) {
+	c := TestNewCache(t)
+	rtest.OK(t, fs.RemoveAll(c.path))
+
+	// save a few bytes of data in the cache
+	data := test.Random(123456789, 42)
+	id := restic.Hash(data)
+	h := restic.Handle{
+		Type: restic.PackFile,
+		Name: id.String(),
+	}
+	if err := c.Save(h, bytes.NewReader(data)); err == nil {
+		t.Fatal("Missing error when saving to deleted cache directory")
+	}
+}