From e47e08a68803baf8b240a9abbeb540b492695b2d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 31 May 2024 14:12:06 +0200 Subject: [PATCH] restorer: separately track skipped files --- internal/restorer/restorer.go | 3 +-- internal/restorer/restorer_unix_test.go | 27 +++++++++++-------------- internal/ui/restore/json.go | 8 ++++++++ internal/ui/restore/json_test.go | 20 +++++++++++++++--- internal/ui/restore/progress.go | 14 +++++++++++++ internal/ui/restore/progress_test.go | 26 +++++++++++++++++------- internal/ui/restore/text.go | 6 ++++++ internal/ui/restore/text_test.go | 20 +++++++++++++++--- 8 files changed, 94 insertions(+), 30 deletions(-) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 267b2898c..ae622874b 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -373,8 +373,7 @@ func (res *Restorer) withOverwriteCheck(node *restic.Node, target, location stri if isHardlink { size = 0 } - res.progress.AddFile(size) - res.progress.AddProgress(location, size, size) + res.progress.AddSkippedFile(size) return nil } return cb() diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index 95a83cdf7..97d2dd07d 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -70,16 +70,13 @@ func getBlockCount(t *testing.T, filename string) int64 { } type printerMock struct { - filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 + s restoreui.State } -func (p *printerMock) Update(_, _, _, _ uint64, _ time.Duration) { +func (p *printerMock) Update(_ restoreui.State, _ time.Duration) { } -func (p *printerMock) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) { - p.filesFinished = filesFinished - p.filesTotal = filesTotal - p.allBytesWritten = allBytesWritten - p.allBytesTotal = allBytesTotal +func (p *printerMock) Finish(s restoreui.State, _ time.Duration) { + p.s = s } func TestRestorerProgressBar(t *testing.T) { @@ -112,12 +109,12 @@ func TestRestorerProgressBar(t *testing.T) { rtest.OK(t, err) progress.Finish() - const filesFinished = 4 - const filesTotal = filesFinished - const allBytesWritten = 10 - const allBytesTotal = allBytesWritten - rtest.Assert(t, mock.filesFinished == filesFinished, "filesFinished: expected %v, got %v", filesFinished, mock.filesFinished) - rtest.Assert(t, mock.filesTotal == filesTotal, "filesTotal: expected %v, got %v", filesTotal, mock.filesTotal) - rtest.Assert(t, mock.allBytesWritten == allBytesWritten, "allBytesWritten: expected %v, got %v", allBytesWritten, mock.allBytesWritten) - rtest.Assert(t, mock.allBytesTotal == allBytesTotal, "allBytesTotal: expected %v, got %v", allBytesTotal, mock.allBytesTotal) + rtest.Equals(t, restoreui.State{ + FilesFinished: 4, + FilesTotal: 4, + FilesSkipped: 0, + AllBytesWritten: 10, + AllBytesTotal: 10, + AllBytesSkipped: 0, + }, mock.s) } diff --git a/internal/ui/restore/json.go b/internal/ui/restore/json.go index 50d4fe0f7..512640a7a 100644 --- a/internal/ui/restore/json.go +++ b/internal/ui/restore/json.go @@ -26,8 +26,10 @@ func (t *jsonPrinter) Update(p State, duration time.Duration) { SecondsElapsed: uint64(duration / time.Second), TotalFiles: p.FilesTotal, FilesRestored: p.FilesFinished, + FilesSkipped: p.FilesSkipped, TotalBytes: p.AllBytesTotal, BytesRestored: p.AllBytesWritten, + BytesSkipped: p.AllBytesSkipped, } if p.AllBytesTotal > 0 { @@ -43,8 +45,10 @@ func (t *jsonPrinter) Finish(p State, duration time.Duration) { SecondsElapsed: uint64(duration / time.Second), TotalFiles: p.FilesTotal, FilesRestored: p.FilesFinished, + FilesSkipped: p.FilesSkipped, TotalBytes: p.AllBytesTotal, BytesRestored: p.AllBytesWritten, + BytesSkipped: p.AllBytesSkipped, } t.print(status) } @@ -55,8 +59,10 @@ type statusUpdate struct { PercentDone float64 `json:"percent_done"` TotalFiles uint64 `json:"total_files,omitempty"` FilesRestored uint64 `json:"files_restored,omitempty"` + FilesSkipped uint64 `json:"files_skipped,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"` BytesRestored uint64 `json:"bytes_restored,omitempty"` + BytesSkipped uint64 `json:"bytes_skipped,omitempty"` } type summaryOutput struct { @@ -64,6 +70,8 @@ type summaryOutput struct { SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` TotalFiles uint64 `json:"total_files,omitempty"` FilesRestored uint64 `json:"files_restored,omitempty"` + FilesSkipped uint64 `json:"files_skipped,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"` BytesRestored uint64 `json:"bytes_restored,omitempty"` + BytesSkipped uint64 `json:"bytes_skipped,omitempty"` } diff --git a/internal/ui/restore/json_test.go b/internal/ui/restore/json_test.go index 7ce7b58f3..37983f7d7 100644 --- a/internal/ui/restore/json_test.go +++ b/internal/ui/restore/json_test.go @@ -10,20 +10,34 @@ import ( func TestJSONPrintUpdate(t *testing.T) { term := &mockTerm{} printer := NewJSONProgress(term) - printer.Update(State{3, 11, 29, 47}, 5*time.Second) + printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) } +func TestJSONPrintUpdateWithSkipped(t *testing.T) { + term := &mockTerm{} + printer := NewJSONProgress(term) + printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.output) +} + func TestJSONPrintSummaryOnSuccess(t *testing.T) { term := &mockTerm{} printer := NewJSONProgress(term) - printer.Finish(State{11, 11, 47, 47}, 5*time.Second) + printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second) test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output) } func TestJSONPrintSummaryOnErrors(t *testing.T) { term := &mockTerm{} printer := NewJSONProgress(term) - printer.Finish(State{3, 11, 29, 47}, 5*time.Second) + printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) } + +func TestJSONPrintSummaryOnSuccessWithSkipped(t *testing.T) { + term := &mockTerm{} + printer := NewJSONProgress(term) + printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.output) +} diff --git a/internal/ui/restore/progress.go b/internal/ui/restore/progress.go index 5e501c4b3..7e8bcfd25 100644 --- a/internal/ui/restore/progress.go +++ b/internal/ui/restore/progress.go @@ -10,8 +10,10 @@ import ( type State struct { FilesFinished uint64 FilesTotal uint64 + FilesSkipped uint64 AllBytesWritten uint64 AllBytesTotal uint64 + AllBytesSkipped uint64 } type Progress struct { @@ -97,6 +99,18 @@ func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTot } } +func (p *Progress) AddSkippedFile(size uint64) { + if p == nil { + return + } + + p.m.Lock() + defer p.m.Unlock() + + p.s.FilesSkipped++ + p.s.AllBytesSkipped += size +} + func (p *Progress) Finish() { p.updater.Done() } diff --git a/internal/ui/restore/progress_test.go b/internal/ui/restore/progress_test.go index 728b74350..56f5f62ce 100644 --- a/internal/ui/restore/progress_test.go +++ b/internal/ui/restore/progress_test.go @@ -45,7 +45,7 @@ func TestNew(t *testing.T) { return false }) test.Equals(t, printerTrace{ - printerTraceEntry{State{0, 0, 0, 0}, 0, false}, + printerTraceEntry{State{0, 0, 0, 0, 0, 0}, 0, false}, }, result) } @@ -57,7 +57,7 @@ func TestAddFile(t *testing.T) { return false }) test.Equals(t, printerTrace{ - printerTraceEntry{State{0, 1, 0, fileSize}, 0, false}, + printerTraceEntry{State{0, 1, 0, 0, fileSize, 0}, 0, false}, }, result) } @@ -71,7 +71,7 @@ func TestFirstProgressOnAFile(t *testing.T) { return false }) test.Equals(t, printerTrace{ - printerTraceEntry{State{0, 1, expectedBytesWritten, expectedBytesTotal}, 0, false}, + printerTraceEntry{State{0, 1, 0, expectedBytesWritten, expectedBytesTotal, 0}, 0, false}, }, result) } @@ -86,7 +86,7 @@ func TestLastProgressOnAFile(t *testing.T) { return false }) test.Equals(t, printerTrace{ - printerTraceEntry{State{1, 1, fileSize, fileSize}, 0, false}, + printerTraceEntry{State{1, 1, 0, fileSize, fileSize, 0}, 0, false}, }, result) } @@ -102,7 +102,7 @@ func TestLastProgressOnLastFile(t *testing.T) { return false }) test.Equals(t, printerTrace{ - printerTraceEntry{State{2, 2, 50 + fileSize, 50 + fileSize}, 0, false}, + printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, 0, false}, }, result) } @@ -117,7 +117,7 @@ func TestSummaryOnSuccess(t *testing.T) { return true }) test.Equals(t, printerTrace{ - printerTraceEntry{State{2, 2, 50 + fileSize, 50 + fileSize}, mockFinishDuration, true}, + printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, mockFinishDuration, true}, }, result) } @@ -132,6 +132,18 @@ func TestSummaryOnErrors(t *testing.T) { return true }) test.Equals(t, printerTrace{ - printerTraceEntry{State{1, 2, 50 + fileSize/2, 50 + fileSize}, mockFinishDuration, true}, + printerTraceEntry{State{1, 2, 0, 50 + fileSize/2, 50 + fileSize, 0}, mockFinishDuration, true}, + }, result) +} + +func TestSkipFile(t *testing.T) { + fileSize := uint64(100) + + result := testProgress(func(progress *Progress) bool { + progress.AddSkippedFile(fileSize) + return true + }) + test.Equals(t, printerTrace{ + printerTraceEntry{State{0, 0, 1, 0, 0, fileSize}, mockFinishDuration, true}, }, result) } diff --git a/internal/ui/restore/text.go b/internal/ui/restore/text.go index 9da388e51..28a6eb965 100644 --- a/internal/ui/restore/text.go +++ b/internal/ui/restore/text.go @@ -24,6 +24,9 @@ func (t *textPrinter) Update(p State, duration time.Duration) { allPercent := ui.FormatPercent(p.AllBytesWritten, p.AllBytesTotal) progress := fmt.Sprintf("[%s] %s %v files/dirs %s, total %v files/dirs %v", timeLeft, allPercent, p.FilesFinished, formattedAllBytesWritten, p.FilesTotal, formattedAllBytesTotal) + if p.FilesSkipped > 0 { + progress += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped)) + } t.terminal.SetStatus([]string{progress}) } @@ -42,6 +45,9 @@ func (t *textPrinter) Finish(p State, duration time.Duration) { summary = fmt.Sprintf("Summary: Restored %d / %d files/dirs (%s / %s) in %s", p.FilesFinished, p.FilesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) } + if p.FilesSkipped > 0 { + summary += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped)) + } t.terminal.Print(summary) } diff --git a/internal/ui/restore/text_test.go b/internal/ui/restore/text_test.go index 2a1723943..3b776a7df 100644 --- a/internal/ui/restore/text_test.go +++ b/internal/ui/restore/text_test.go @@ -22,20 +22,34 @@ func (m *mockTerm) SetStatus(lines []string) { func TestPrintUpdate(t *testing.T) { term := &mockTerm{} printer := NewTextProgress(term) - printer.Update(State{3, 11, 29, 47}, 5*time.Second) + printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output) } +func TestPrintUpdateWithSkipped(t *testing.T) { + term := &mockTerm{} + printer := NewTextProgress(term) + printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second) + test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.output) +} + func TestPrintSummaryOnSuccess(t *testing.T) { term := &mockTerm{} printer := NewTextProgress(term) - printer.Finish(State{11, 11, 47, 47}, 5*time.Second) + printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second) test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output) } func TestPrintSummaryOnErrors(t *testing.T) { term := &mockTerm{} printer := NewTextProgress(term) - printer.Finish(State{3, 11, 29, 47}, 5*time.Second) + printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output) } + +func TestPrintSummaryOnSuccessWithSkipped(t *testing.T) { + term := &mockTerm{} + printer := NewTextProgress(term) + printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs 59 B"}, term.output) +}