From 41d0271579a54a6e6eeff59023526aa876f8c81d Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Thu, 5 Sep 2024 10:48:13 -0400 Subject: [PATCH 1/2] Update restic to v0.17.0 Signed-off-by: Tesshu Flower --- controllers/mover/restic/mover.go | 17 +- mover-restic/SOURCE_VERSIONS | 2 +- .../restic/.github/workflows/docker.yml | 8 +- .../restic/.github/workflows/tests.yml | 31 +- mover-restic/restic/.gitignore | 1 + mover-restic/restic/.golangci.yml | 16 + mover-restic/restic/.readthedocs.yaml | 4 + mover-restic/restic/CHANGELOG.md | 4163 ++++++++++------- mover-restic/restic/CONTRIBUTING.md | 5 +- mover-restic/restic/README.md | 18 +- mover-restic/restic/VERSION | 2 +- .../changelog/0.10.0_2020-09-19/pull-2195 | 2 +- .../changelog/0.10.0_2020-09-19/pull-2668 | 2 +- .../changelog/0.11.0_2020-11-05/issue-1756 | 2 +- .../changelog/0.11.0_2020-11-05/issue-340 | 2 +- .../changelog/0.12.0_2021-02-14/issue-3232 | 4 +- .../changelog/0.12.0_2021-02-14/pull-3106 | 2 +- .../changelog/0.16.0_2023-07-31/issue-3941 | 2 +- .../changelog/0.17.0_2024-07-26/issue-1786 | 20 + .../changelog/0.17.0_2024-07-26/issue-2348 | 12 + .../changelog/0.17.0_2024-07-26/issue-3600 | 11 + .../changelog/0.17.0_2024-07-26/issue-3806 | 12 + .../changelog/0.17.0_2024-07-26/issue-4048 | 6 + .../changelog/0.17.0_2024-07-26/issue-4209 | 7 + .../changelog/0.17.0_2024-07-26/issue-4251 | 16 + .../changelog/0.17.0_2024-07-26/issue-4287 | 12 + .../changelog/0.17.0_2024-07-26/issue-4437 | 11 + .../changelog/0.17.0_2024-07-26/issue-4472 | 18 + .../changelog/0.17.0_2024-07-26/issue-4540 | 7 + .../changelog/0.17.0_2024-07-26/issue-4547 | 7 + .../changelog/0.17.0_2024-07-26/issue-4549 | 11 + .../changelog/0.17.0_2024-07-26/issue-4568 | 16 + .../changelog/0.17.0_2024-07-26/issue-4583 | 13 + .../changelog/0.17.0_2024-07-26/issue-4601 | 9 + .../changelog/0.17.0_2024-07-26/issue-4602 | 22 + .../changelog/0.17.0_2024-07-26/issue-4627 | 33 + .../changelog/0.17.0_2024-07-26/issue-4656 | 7 + .../changelog/0.17.0_2024-07-26/issue-4676 | 8 + .../changelog/0.17.0_2024-07-26/issue-4678 | 8 + .../changelog/0.17.0_2024-07-26/issue-4707 | 14 + .../changelog/0.17.0_2024-07-26/issue-4733 | 12 + .../changelog/0.17.0_2024-07-26/issue-4744 | 9 + .../changelog/0.17.0_2024-07-26/issue-4760 | 8 + .../changelog/0.17.0_2024-07-26/issue-4768 | 8 + .../changelog/0.17.0_2024-07-26/issue-4781 | 8 + .../changelog/0.17.0_2024-07-26/issue-4817 | 26 + .../changelog/0.17.0_2024-07-26/issue-4850 | 8 + .../changelog/0.17.0_2024-07-26/issue-4902 | 8 + .../changelog/0.17.0_2024-07-26/issue-662 | 11 + .../changelog/0.17.0_2024-07-26/issue-693 | 13 + .../changelog/0.17.0_2024-07-26/issue-828 | 11 + .../changelog/0.17.0_2024-07-26/pull-3067 | 25 + .../changelog/0.17.0_2024-07-26/pull-4006 | 15 + .../changelog/0.17.0_2024-07-26/pull-4354 | 7 + .../changelog/0.17.0_2024-07-26/pull-4503 | 8 + .../changelog/0.17.0_2024-07-26/pull-4526 | 12 + .../changelog/0.17.0_2024-07-26/pull-4573 | 6 + .../changelog/0.17.0_2024-07-26/pull-4590 | 6 + .../changelog/0.17.0_2024-07-26/pull-4611 | 9 + .../changelog/0.17.0_2024-07-26/pull-4615 | 6 + .../changelog/0.17.0_2024-07-26/pull-4664 | 10 + .../changelog/0.17.0_2024-07-26/pull-4703 | 11 + .../changelog/0.17.0_2024-07-26/pull-4708 | 13 + .../changelog/0.17.0_2024-07-26/pull-4709 | 10 + .../changelog/0.17.0_2024-07-26/pull-4737 | 6 + .../changelog/0.17.0_2024-07-26/pull-4764 | 10 + .../changelog/0.17.0_2024-07-26/pull-4796 | 8 + .../changelog/0.17.0_2024-07-26/pull-4807 | 6 + .../changelog/0.17.0_2024-07-26/pull-4839 | 7 + .../changelog/0.17.0_2024-07-26/pull-4884 | 11 + .../changelog/0.8.0_2017-11-26/pull-1040 | 2 +- .../changelog/0.8.0_2017-11-26/pull-1319 | 2 +- .../changelog/0.8.2_2018-02-17/issue-1506 | 2 +- .../changelog/0.8.2_2018-02-17/pull-1595 | 2 +- .../changelog/0.8.3_2018-02-26/pull-1623 | 2 +- .../changelog/0.9.0_2018-05-21/pull-1735 | 2 +- .../changelog/0.9.3_2018-10-13/pull-1876 | 2 +- .../changelog/0.9.6_2019-11-22/issue-2179 | 2 +- mover-restic/restic/cmd/restic/cleanup.go | 84 +- mover-restic/restic/cmd/restic/cmd_backup.go | 145 +- .../cmd/restic/cmd_backup_integration_test.go | 117 +- mover-restic/restic/cmd/restic/cmd_cache.go | 7 +- mover-restic/restic/cmd/restic/cmd_cat.go | 31 +- mover-restic/restic/cmd/restic/cmd_check.go | 202 +- .../cmd/restic/cmd_check_integration_test.go | 8 +- .../restic/cmd/restic/cmd_check_test.go | 87 +- mover-restic/restic/cmd/restic/cmd_copy.go | 52 +- .../cmd/restic/cmd_copy_integration_test.go | 25 +- mover-restic/restic/cmd/restic/cmd_debug.go | 145 +- mover-restic/restic/cmd/restic/cmd_diff.go | 48 +- mover-restic/restic/cmd/restic/cmd_dump.go | 51 +- .../restic/cmd/restic/cmd_features.go | 59 + mover-restic/restic/cmd/restic/cmd_find.go | 102 +- mover-restic/restic/cmd/restic/cmd_forget.go | 203 +- .../cmd/restic/cmd_forget_integration_test.go | 59 +- .../restic/cmd/restic/cmd_generate.go | 31 +- mover-restic/restic/cmd/restic/cmd_init.go | 7 +- mover-restic/restic/cmd/restic/cmd_key.go | 255 +- mover-restic/restic/cmd/restic/cmd_key_add.go | 137 + .../cmd/restic/cmd_key_integration_test.go | 105 +- .../restic/cmd/restic/cmd_key_list.go | 109 + .../restic/cmd/restic/cmd_key_passwd.go | 83 + .../restic/cmd/restic/cmd_key_remove.go | 69 + mover-restic/restic/cmd/restic/cmd_list.go | 32 +- .../cmd/restic/cmd_list_integration_test.go | 2 +- mover-restic/restic/cmd/restic/cmd_ls.go | 258 +- .../cmd/restic/cmd_ls_integration_test.go | 40 +- mover-restic/restic/cmd/restic/cmd_ls_test.go | 198 +- mover-restic/restic/cmd/restic/cmd_migrate.go | 51 +- mover-restic/restic/cmd/restic/cmd_mount.go | 80 +- .../cmd/restic/cmd_mount_integration_test.go | 36 +- mover-restic/restic/cmd/restic/cmd_options.go | 5 +- mover-restic/restic/cmd/restic/cmd_prune.go | 740 +-- .../cmd/restic/cmd_prune_integration_test.go | 45 +- mover-restic/restic/cmd/restic/cmd_recover.go | 31 +- .../restic/cmd/restic/cmd_repair_index.go | 119 +- .../cmd_repair_index_integration_test.go | 31 +- .../restic/cmd/restic/cmd_repair_packs.go | 116 +- .../restic/cmd/restic/cmd_repair_snapshots.go | 34 +- .../cmd_repair_snapshots_integration_test.go | 4 +- mover-restic/restic/cmd/restic/cmd_restore.go | 189 +- .../restic/cmd_restore_integration_test.go | 173 +- mover-restic/restic/cmd/restic/cmd_rewrite.go | 166 +- .../restic/cmd_rewrite_integration_test.go | 81 +- .../restic/cmd/restic/cmd_self_update.go | 5 +- .../restic/cmd/restic/cmd_snapshots.go | 51 +- mover-restic/restic/cmd/restic/cmd_stats.go | 80 +- mover-restic/restic/cmd/restic/cmd_tag.go | 29 +- .../cmd/restic/cmd_tag_integration_test.go | 1 + mover-restic/restic/cmd/restic/cmd_unlock.go | 5 +- mover-restic/restic/cmd/restic/cmd_version.go | 33 +- mover-restic/restic/cmd/restic/delete.go | 68 - mover-restic/restic/cmd/restic/exclude.go | 20 +- mover-restic/restic/cmd/restic/find.go | 22 +- mover-restic/restic/cmd/restic/find_test.go | 61 + mover-restic/restic/cmd/restic/global.go | 258 +- .../restic/cmd/restic/global_debug.go | 82 +- .../restic/cmd/restic/global_release.go | 3 + mover-restic/restic/cmd/restic/global_test.go | 24 + mover-restic/restic/cmd/restic/include.go | 100 + .../restic/cmd/restic/include_test.go | 59 + .../restic/integration_filter_pattern_test.go | 42 +- .../cmd/restic/integration_helpers_test.go | 42 +- .../restic/cmd/restic/integration_test.go | 38 +- mover-restic/restic/cmd/restic/lock.go | 318 +- mover-restic/restic/cmd/restic/main.go | 43 +- mover-restic/restic/cmd/restic/progress.go | 39 +- .../restic/cmd/restic/secondary_repo.go | 22 +- .../restic/cmd/restic/secondary_repo_test.go | 5 +- mover-restic/restic/cmd/restic/termstatus.go | 41 + .../repo-restore-permissions-test.tar.gz | Bin 4174 -> 4256 bytes mover-restic/restic/doc/020_installation.rst | 18 +- .../restic/doc/030_preparing_a_new_repo.rst | 53 +- mover-restic/restic/doc/040_backup.rst | 266 +- .../restic/doc/045_working_with_repos.rst | 184 +- .../doc/047_tuning_backup_parameters.rst | 31 +- mover-restic/restic/doc/050_restore.rst | 92 +- mover-restic/restic/doc/060_forget.rst | 23 +- mover-restic/restic/doc/075_scripting.rst | 257 +- .../restic/doc/077_troubleshooting.rst | 17 +- mover-restic/restic/doc/REST_backend.rst | 16 +- mover-restic/restic/doc/bash-completion.sh | 513 +- mover-restic/restic/doc/design.rst | 8 +- .../restic/doc/developer_information.rst | 42 +- mover-restic/restic/doc/faq.rst | 2 +- mover-restic/restic/doc/fish-completion.fish | 2 +- mover-restic/restic/doc/man/restic-backup.1 | 24 +- mover-restic/restic/doc/man/restic-cache.1 | 11 +- mover-restic/restic/doc/man/restic-cat.1 | 13 +- mover-restic/restic/doc/man/restic-check.1 | 13 +- mover-restic/restic/doc/man/restic-copy.1 | 24 +- mover-restic/restic/doc/man/restic-diff.1 | 21 +- mover-restic/restic/doc/man/restic-dump.1 | 23 +- mover-restic/restic/doc/man/restic-find.1 | 18 +- mover-restic/restic/doc/man/restic-forget.1 | 26 +- mover-restic/restic/doc/man/restic-generate.1 | 11 +- mover-restic/restic/doc/man/restic-init.1 | 15 +- mover-restic/restic/doc/man/restic-key-add.1 | 149 + mover-restic/restic/doc/man/restic-key-list.1 | 135 + .../restic/doc/man/restic-key-passwd.1 | 150 + .../restic/doc/man/restic-key-remove.1 | 134 + mover-restic/restic/doc/man/restic-key.1 | 32 +- mover-restic/restic/doc/man/restic-list.1 | 13 +- mover-restic/restic/doc/man/restic-ls.1 | 21 +- mover-restic/restic/doc/man/restic-migrate.1 | 13 +- mover-restic/restic/doc/man/restic-mount.1 | 19 +- mover-restic/restic/doc/man/restic-prune.1 | 13 +- mover-restic/restic/doc/man/restic-recover.1 | 13 +- .../restic/doc/man/restic-repair-index.1 | 13 +- .../restic/doc/man/restic-repair-packs.1 | 16 +- .../restic/doc/man/restic-repair-snapshots.1 | 17 +- mover-restic/restic/doc/man/restic-repair.1 | 8 + mover-restic/restic/doc/man/restic-restore.1 | 53 +- mover-restic/restic/doc/man/restic-rewrite.1 | 25 +- .../restic/doc/man/restic-self-update.1 | 13 +- .../restic/doc/man/restic-snapshots.1 | 17 +- mover-restic/restic/doc/man/restic-stats.1 | 19 +- mover-restic/restic/doc/man/restic-tag.1 | 17 +- mover-restic/restic/doc/man/restic-unlock.1 | 11 +- mover-restic/restic/doc/man/restic-version.1 | 11 +- mover-restic/restic/doc/man/restic.1 | 8 + mover-restic/restic/doc/manual_rest.rst | 44 +- .../restic/doc/powershell-completion.ps1 | 6 +- mover-restic/restic/docker/Dockerfile | 2 +- mover-restic/restic/go.mod | 51 +- mover-restic/restic/go.sum | 158 +- .../helpers/build-release-binaries/main.go | 3 +- .../restic/helpers/prepare-release/main.go | 7 +- .../restic/internal/archiver/archiver.go | 203 +- .../restic/internal/archiver/archiver_test.go | 332 +- .../internal/archiver/archiver_unix_test.go | 48 + .../restic/internal/archiver/blob_saver.go | 8 +- .../internal/archiver/blob_saver_test.go | 22 +- .../restic/internal/archiver/file_saver.go | 11 +- .../internal/archiver/file_saver_test.go | 6 +- .../restic/internal/archiver/scanner.go | 10 +- .../restic/internal/archiver/scanner_test.go | 14 +- .../restic/internal/archiver/testing.go | 37 +- .../restic/internal/archiver/testing_test.go | 14 +- mover-restic/restic/internal/archiver/tree.go | 2 +- .../restic/internal/archiver/tree_saver.go | 10 +- .../internal/archiver/tree_saver_test.go | 2 +- .../restic/internal/archiver/tree_test.go | 6 +- .../restic/internal/backend/azure/azure.go | 56 +- .../internal/backend/azure/azure_test.go | 4 +- .../restic/internal/backend/azure/config.go | 4 +- mover-restic/restic/internal/backend/b2/b2.go | 69 +- .../restic/internal/backend/b2/config.go | 4 +- .../internal/{restic => backend}/backend.go | 29 +- .../restic/internal/backend/backend_test.go | 38 + .../internal/{ => backend}/cache/backend.go | 73 +- .../internal/backend/cache/backend_test.go | 240 + .../internal/{ => backend}/cache/cache.go | 6 +- .../{ => backend}/cache/cache_test.go | 0 .../internal/{ => backend}/cache/dir.go | 0 .../internal/{ => backend}/cache/dir_test.go | 0 .../internal/{ => backend}/cache/file.go | 72 +- .../internal/{ => backend}/cache/file_test.go | 47 +- .../internal/{ => backend}/cache/testing.go | 0 .../internal/backend/dryrun/dry_backend.go | 29 +- .../backend/dryrun/dry_backend_test.go | 24 +- .../internal/{restic => backend}/file.go | 8 +- .../internal/{restic => backend}/file_test.go | 2 +- .../restic/internal/backend/gs/config.go | 4 +- mover-restic/restic/internal/backend/gs/gs.go | 70 +- .../restic/internal/backend/http_transport.go | 34 +- .../backend/httpuseragent_roundtripper.go | 25 + .../httpuseragent_roundtripper_test.go | 50 + .../restic/internal/backend/layout/layout.go | 22 +- .../internal/backend/layout/layout_default.go | 28 +- .../internal/backend/layout/layout_rest.go | 14 +- .../backend/layout/layout_s3legacy.go | 26 +- .../internal/backend/layout/layout_test.go | 83 +- .../internal/backend/limiter/limiter.go | 4 +- .../backend/limiter/limiter_backend.go | 22 +- .../backend/limiter/limiter_backend_test.go | 12 +- .../backend/limiter/static_limiter_test.go | 1 + .../restic/internal/backend/local/config.go | 2 +- .../internal/backend/local/layout_test.go | 6 +- .../restic/internal/backend/local/local.go | 66 +- .../backend/local/local_internal_test.go | 4 +- .../backend/location/display_location_test.go | 4 +- .../backend/location/location_test.go | 4 +- .../internal/backend/location/registry.go | 16 +- .../restic/internal/backend/logger/log.go | 22 +- .../internal/backend/mem/mem_backend.go | 61 +- .../restic/internal/backend/mock/backend.go | 47 +- .../restic/internal/backend/rclone/backend.go | 8 +- .../internal/backend/rclone/internal_test.go | 6 +- .../restic/internal/backend/readerat.go | 9 +- .../restic/internal/backend/rest/config.go | 4 +- .../internal/backend/rest/config_test.go | 7 + .../restic/internal/backend/rest/rest.go | 183 +- .../internal/backend/rest/rest_int_test.go | 14 +- .../restic/internal/backend/rest/rest_test.go | 146 +- .../internal/backend/rest/rest_unix_test.go | 30 + .../internal/backend/retry/backend_retry.go | 154 +- .../backend/retry/backend_retry_test.go | 257 +- .../{restic => backend}/rewind_reader.go | 2 +- .../{restic => backend}/rewind_reader_test.go | 2 +- .../restic/internal/backend/s3/config.go | 17 +- mover-restic/restic/internal/backend/s3/s3.go | 219 +- .../restic/internal/backend/s3/s3_test.go | 4 +- .../restic/internal/backend/sema/backend.go | 28 +- .../internal/backend/sema/backend_test.go | 64 +- .../restic/internal/backend/sftp/config.go | 2 +- .../internal/backend/sftp/layout_test.go | 6 +- .../restic/internal/backend/sftp/sftp.go | 82 +- .../restic/internal/backend/shell_split.go | 2 +- .../restic/internal/backend/swift/config.go | 4 +- .../restic/internal/backend/swift/swift.go | 64 +- .../internal/backend/swift/swift_test.go | 4 +- .../internal/backend/test/benchmarks.go | 13 +- .../restic/internal/backend/test/doc.go | 2 +- .../restic/internal/backend/test/suite.go | 13 +- .../restic/internal/backend/test/tests.go | 155 +- .../restic/internal/backend/util/defaults.go | 50 + .../internal/backend/util/defaults_test.go | 64 + .../internal/backend/{ => util}/errdot_119.go | 2 +- .../internal/backend/{ => util}/errdot_old.go | 2 +- .../internal/backend/{ => util}/foreground.go | 2 +- .../backend/{ => util}/foreground_sysv.go | 4 +- .../backend/{ => util}/foreground_test.go | 6 +- .../backend/{ => util}/foreground_unix.go | 2 +- .../backend/{ => util}/foreground_windows.go | 2 +- .../internal/backend/util/limited_reader.go | 15 + .../internal/backend/{ => util}/paths.go | 2 +- mover-restic/restic/internal/backend/utils.go | 142 - .../restic/internal/backend/utils_test.go | 246 - .../internal/backend/watchdog_roundtriper.go | 119 + .../backend/watchdog_roundtriper_test.go | 204 + mover-restic/restic/internal/bloblru/cache.go | 57 +- .../restic/internal/bloblru/cache_test.go | 67 + .../restic/internal/cache/backend_test.go | 202 - .../restic/internal/checker/checker.go | 276 +- .../restic/internal/checker/checker_test.go | 199 +- .../restic/internal/checker/testing.go | 31 +- mover-restic/restic/internal/crypto/crypto.go | 43 +- mover-restic/restic/internal/debug/debug.go | 4 +- mover-restic/restic/internal/dump/common.go | 103 +- .../restic/internal/dump/common_test.go | 2 +- mover-restic/restic/internal/errors/errors.go | 37 + .../restic/internal/feature/features.go | 140 + .../restic/internal/feature/features_test.go | 151 + .../restic/internal/feature/registry.go | 25 + .../restic/internal/feature/testing.go | 33 + .../restic/internal/feature/testing_test.go | 19 + mover-restic/restic/internal/fs/ea_windows.go | 285 ++ .../restic/internal/fs/ea_windows_test.go | 247 + mover-restic/restic/internal/fs/file.go | 36 + mover-restic/restic/internal/fs/file_unix.go | 2 +- .../restic/internal/fs/file_windows.go | 47 + .../restic/internal/fs/fs_local_vss.go | 154 +- .../restic/internal/fs/fs_local_vss_test.go | 285 ++ .../restic/internal/fs/fs_reader_command.go | 97 + .../internal/fs/fs_reader_command_test.go | 48 + mover-restic/restic/internal/fs/fs_track.go | 2 +- mover-restic/restic/internal/fs/sd_windows.go | 439 ++ .../restic/internal/fs/sd_windows_test.go | 60 + .../internal/fs/sd_windows_test_helpers.go | 126 + mover-restic/restic/internal/fs/stat_test.go | 4 +- mover-restic/restic/internal/fs/vss.go | 12 +- .../restic/internal/fs/vss_windows.go | 324 +- mover-restic/restic/internal/fuse/dir.go | 4 +- mover-restic/restic/internal/fuse/file.go | 16 +- .../restic/internal/fuse/fuse_test.go | 8 +- .../internal/fuse/snapshots_dirstruct.go | 2 +- .../restic/internal/migrations/s3_layout.go | 10 +- .../internal/migrations/upgrade_repo_v2.go | 93 +- .../migrations/upgrade_repo_v2_test.go | 78 +- .../restic/internal/repository/check.go | 209 + .../restic/internal/repository/fuzz_test.go | 2 +- .../{ => repository}/hashing/reader.go | 0 .../{ => repository}/hashing/reader_test.go | 0 .../{ => repository}/hashing/writer.go | 0 .../{ => repository}/hashing/writer_test.go | 0 .../repository/index/associated_data.go | 156 + .../repository/index/associated_data_test.go | 154 + .../internal/{ => repository}/index/index.go | 147 +- .../{ => repository}/index/index_parallel.go | 4 +- .../index/index_parallel_test.go | 12 +- .../{ => repository}/index/index_test.go | 22 +- .../{ => repository}/index/indexmap.go | 35 +- .../{ => repository}/index/indexmap_test.go | 42 + .../{ => repository}/index/master_index.go | 368 +- .../index/master_index_test.go | 158 +- .../{ => repository}/index/testing.go | 9 +- .../restic/internal/repository/key.go | 38 +- .../restic/internal/repository/lock.go | 274 ++ .../repository}/lock_test.go | 206 +- .../internal/{ => repository}/pack/doc.go | 0 .../internal/{ => repository}/pack/pack.go | 14 +- .../pack/pack_internal_test.go | 0 .../{ => repository}/pack/pack_test.go | 10 +- .../internal/repository/packer_manager.go | 34 +- .../repository/packer_manager_test.go | 6 +- .../internal/repository/packer_uploader.go | 12 +- .../restic/internal/repository/prune.go | 640 +++ .../restic/internal/repository/prune_test.go | 108 + .../restic/internal/repository/raw.go | 56 + .../restic/internal/repository/raw_test.go | 108 + .../restic/internal/repository/repack.go | 15 +- .../restic/internal/repository/repack_test.go | 124 +- .../internal/repository/repair_index.go | 139 + .../internal/repository/repair_index_test.go | 75 + .../restic/internal/repository/repair_pack.go | 72 + .../internal/repository/repair_pack_test.go | 130 + .../restic/internal/repository/repository.go | 584 ++- .../repository/repository_internal_test.go | 373 +- .../internal/repository/repository_test.go | 472 +- .../restic/internal/repository/s3_backend.go | 12 + .../restic/internal/repository/testing.go | 56 +- .../internal/repository/upgrade_repo.go | 103 + .../internal/repository/upgrade_repo_test.go | 82 + .../restic/internal/restic/backend_find.go | 14 +- .../internal/restic/backend_find_test.go | 52 +- .../restic/internal/restic/backend_test.go | 38 - mover-restic/restic/internal/restic/blob.go | 9 + mover-restic/restic/internal/restic/config.go | 22 +- .../internal/restic/counted_blob_set.go | 68 - .../internal/restic/counted_blob_set_test.go | 45 - mover-restic/restic/internal/restic/find.go | 6 +- .../restic/internal/restic/find_test.go | 2 +- mover-restic/restic/internal/restic/idset.go | 6 + .../restic/internal/restic/idset_test.go | 3 + mover-restic/restic/internal/restic/lister.go | 52 + .../restic/internal/restic/lister_test.go | 68 + mover-restic/restic/internal/restic/lock.go | 89 +- .../restic/internal/restic/lock_test.go | 61 +- mover-restic/restic/internal/restic/node.go | 298 +- .../restic/internal/restic/node_aix.go | 35 +- .../restic/internal/restic/node_netbsd.go | 36 +- .../restic/internal/restic/node_openbsd.go | 36 +- .../restic/internal/restic/node_test.go | 52 +- .../restic/internal/restic/node_unix_test.go | 2 +- .../restic/internal/restic/node_windows.go | 343 +- .../internal/restic/node_windows_test.go | 331 ++ .../restic/internal/restic/node_xattr.go | 93 +- .../internal/restic/node_xattr_all_test.go | 56 + .../restic/internal/restic/node_xattr_test.go | 28 + .../restic/internal/restic/parallel.go | 51 +- .../restic/internal/restic/repository.go | 107 +- .../restic/internal/restic/snapshot.go | 26 +- .../restic/internal/restic/snapshot_find.go | 4 +- .../internal/restic/snapshot_find_test.go | 8 +- .../restic/internal/restic/snapshot_group.go | 14 + .../internal/restic/snapshot_group_test.go | 2 +- .../restic/internal/restic/snapshot_policy.go | 16 +- .../restic/internal/restic/snapshot_test.go | 2 +- .../restic/testdata/policy_keep_snapshots_0 | 1781 +------ .../restic/testdata/policy_keep_snapshots_36 | 412 +- .../restic/testdata/policy_keep_snapshots_37 | 515 +- .../restic/testdata/policy_keep_snapshots_38 | 362 +- .../restic/testdata/policy_keep_snapshots_39 | 67 +- .../restic/internal/restic/testing.go | 19 + .../restic/internal/restic/testing_test.go | 23 +- .../restic/internal/restic/tree_stream.go | 2 +- .../restic/internal/restic/tree_test.go | 8 +- .../restic/internal/restorer/filerestorer.go | 178 +- .../internal/restorer/filerestorer_test.go | 139 +- .../restic/internal/restorer/fileswriter.go | 165 +- .../restorer/fileswriter_other_test.go | 10 + .../internal/restorer/fileswriter_test.go | 136 +- .../restorer/fileswriter_windows_test.go | 7 + .../internal/restorer/hardlinks_index.go | 24 +- .../internal/restorer/hardlinks_index_test.go | 6 +- .../restic/internal/restorer/restorer.go | 568 ++- .../restic/internal/restorer/restorer_test.go | 809 +++- .../restic/internal/restorer/restorer_unix.go | 10 + .../internal/restorer/restorer_unix_test.go | 81 +- .../internal/restorer/restorer_windows.go | 13 + .../restorer/restorer_windows_test.go | 540 +++ .../restic/internal/restorer/sparsewrite.go | 3 - mover-restic/restic/internal/test/helpers.go | 22 +- .../restic/internal/ui/backup/json.go | 13 +- .../restic/internal/ui/backup/progress.go | 51 +- .../internal/ui/backup/progress_test.go | 5 +- .../restic/internal/ui/backup/text.go | 12 +- mover-restic/restic/internal/ui/format.go | 23 + .../restic/internal/ui/format_test.go | 18 + mover-restic/restic/internal/ui/message.go | 14 +- .../restic/internal/ui/progress/printer.go | 65 + .../restic/internal/ui/restore/json.go | 79 +- .../restic/internal/ui/restore/json_test.go | 51 +- .../restic/internal/ui/restore/progress.go | 80 +- .../internal/ui/restore/progress_test.go | 119 +- .../restic/internal/ui/restore/text.go | 68 +- .../restic/internal/ui/restore/text_test.go | 52 +- .../restic/internal/ui/stdio_wrapper.go | 72 - .../restic/internal/ui/table/table.go | 16 +- .../restic/internal/ui/table/table_test.go | 19 +- .../restic/internal/ui/termstatus/status.go | 45 +- .../internal/ui/termstatus/status_test.go | 14 +- .../internal/ui/termstatus/stdio_wrapper.go | 47 + .../ui/{ => termstatus}/stdio_wrapper_test.go | 2 +- .../ui/termstatus/terminal_windows.go | 4 +- .../restic/internal/walker/rewriter.go | 33 +- .../restic/internal/walker/rewriter_test.go | 56 +- mover-restic/restic/internal/walker/walker.go | 84 +- .../restic/internal/walker/walker_test.go | 230 +- 480 files changed, 24413 insertions(+), 12027 deletions(-) create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-1786 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-2348 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3600 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3806 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4048 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4209 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4251 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4287 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4437 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4472 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4540 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4547 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4549 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4568 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4583 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4601 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4602 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4627 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4656 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4676 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4678 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4707 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4733 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4744 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4760 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4768 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4781 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4817 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4850 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4902 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-662 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-693 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/issue-828 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-3067 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4006 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4354 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4503 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4526 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4573 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4590 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4611 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4615 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4664 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4703 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4708 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4709 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4737 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4764 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4796 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4807 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4839 create mode 100644 mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4884 create mode 100644 mover-restic/restic/cmd/restic/cmd_features.go create mode 100644 mover-restic/restic/cmd/restic/cmd_key_add.go create mode 100644 mover-restic/restic/cmd/restic/cmd_key_list.go create mode 100644 mover-restic/restic/cmd/restic/cmd_key_passwd.go create mode 100644 mover-restic/restic/cmd/restic/cmd_key_remove.go delete mode 100644 mover-restic/restic/cmd/restic/delete.go create mode 100644 mover-restic/restic/cmd/restic/find_test.go create mode 100644 mover-restic/restic/cmd/restic/include.go create mode 100644 mover-restic/restic/cmd/restic/include_test.go create mode 100644 mover-restic/restic/cmd/restic/termstatus.go create mode 100644 mover-restic/restic/doc/man/restic-key-add.1 create mode 100644 mover-restic/restic/doc/man/restic-key-list.1 create mode 100644 mover-restic/restic/doc/man/restic-key-passwd.1 create mode 100644 mover-restic/restic/doc/man/restic-key-remove.1 rename mover-restic/restic/internal/{restic => backend}/backend.go (77%) create mode 100644 mover-restic/restic/internal/backend/backend_test.go rename mover-restic/restic/internal/{ => backend}/cache/backend.go (70%) create mode 100644 mover-restic/restic/internal/backend/cache/backend_test.go rename mover-restic/restic/internal/{ => backend}/cache/cache.go (97%) rename mover-restic/restic/internal/{ => backend}/cache/cache_test.go (100%) rename mover-restic/restic/internal/{ => backend}/cache/dir.go (100%) rename mover-restic/restic/internal/{ => backend}/cache/dir_test.go (100%) rename mover-restic/restic/internal/{ => backend}/cache/file.go (65%) rename mover-restic/restic/internal/{ => backend}/cache/file_test.go (82%) rename mover-restic/restic/internal/{ => backend}/cache/testing.go (100%) rename mover-restic/restic/internal/{restic => backend}/file.go (92%) rename mover-restic/restic/internal/{restic => backend}/file_test.go (98%) create mode 100644 mover-restic/restic/internal/backend/httpuseragent_roundtripper.go create mode 100644 mover-restic/restic/internal/backend/httpuseragent_roundtripper_test.go create mode 100644 mover-restic/restic/internal/backend/rest/rest_unix_test.go rename mover-restic/restic/internal/{restic => backend}/rewind_reader.go (99%) rename mover-restic/restic/internal/{restic => backend}/rewind_reader_test.go (99%) create mode 100644 mover-restic/restic/internal/backend/util/defaults.go create mode 100644 mover-restic/restic/internal/backend/util/defaults_test.go rename mover-restic/restic/internal/backend/{ => util}/errdot_119.go (97%) rename mover-restic/restic/internal/backend/{ => util}/errdot_old.go (95%) rename mover-restic/restic/internal/backend/{ => util}/foreground.go (97%) rename mover-restic/restic/internal/backend/{ => util}/foreground_sysv.go (85%) rename mover-restic/restic/internal/backend/{ => util}/foreground_test.go (85%) rename mover-restic/restic/internal/backend/{ => util}/foreground_unix.go (98%) rename mover-restic/restic/internal/backend/{ => util}/foreground_windows.go (96%) create mode 100644 mover-restic/restic/internal/backend/util/limited_reader.go rename mover-restic/restic/internal/backend/{ => util}/paths.go (97%) delete mode 100644 mover-restic/restic/internal/backend/utils.go delete mode 100644 mover-restic/restic/internal/backend/utils_test.go create mode 100644 mover-restic/restic/internal/backend/watchdog_roundtriper.go create mode 100644 mover-restic/restic/internal/backend/watchdog_roundtriper_test.go delete mode 100644 mover-restic/restic/internal/cache/backend_test.go create mode 100644 mover-restic/restic/internal/feature/features.go create mode 100644 mover-restic/restic/internal/feature/features_test.go create mode 100644 mover-restic/restic/internal/feature/registry.go create mode 100644 mover-restic/restic/internal/feature/testing.go create mode 100644 mover-restic/restic/internal/feature/testing_test.go create mode 100644 mover-restic/restic/internal/fs/ea_windows.go create mode 100644 mover-restic/restic/internal/fs/ea_windows_test.go create mode 100644 mover-restic/restic/internal/fs/fs_local_vss_test.go create mode 100644 mover-restic/restic/internal/fs/fs_reader_command.go create mode 100644 mover-restic/restic/internal/fs/fs_reader_command_test.go create mode 100644 mover-restic/restic/internal/fs/sd_windows.go create mode 100644 mover-restic/restic/internal/fs/sd_windows_test.go create mode 100644 mover-restic/restic/internal/fs/sd_windows_test_helpers.go create mode 100644 mover-restic/restic/internal/repository/check.go rename mover-restic/restic/internal/{ => repository}/hashing/reader.go (100%) rename mover-restic/restic/internal/{ => repository}/hashing/reader_test.go (100%) rename mover-restic/restic/internal/{ => repository}/hashing/writer.go (100%) rename mover-restic/restic/internal/{ => repository}/hashing/writer_test.go (100%) create mode 100644 mover-restic/restic/internal/repository/index/associated_data.go create mode 100644 mover-restic/restic/internal/repository/index/associated_data_test.go rename mover-restic/restic/internal/{ => repository}/index/index.go (86%) rename mover-restic/restic/internal/{ => repository}/index/index_parallel.go (92%) rename mover-restic/restic/internal/{ => repository}/index/index_parallel_test.go (64%) rename mover-restic/restic/internal/{ => repository}/index/index_test.go (96%) rename mover-restic/restic/internal/{ => repository}/index/indexmap.go (88%) rename mover-restic/restic/internal/{ => repository}/index/indexmap_test.go (77%) rename mover-restic/restic/internal/{ => repository}/index/master_index.go (52%) rename mover-restic/restic/internal/{ => repository}/index/master_index_test.go (68%) rename mover-restic/restic/internal/{ => repository}/index/testing.go (66%) create mode 100644 mover-restic/restic/internal/repository/lock.go rename mover-restic/restic/{cmd/restic => internal/repository}/lock_test.go (53%) rename mover-restic/restic/internal/{ => repository}/pack/doc.go (100%) rename mover-restic/restic/internal/{ => repository}/pack/pack.go (96%) rename mover-restic/restic/internal/{ => repository}/pack/pack_internal_test.go (100%) rename mover-restic/restic/internal/{ => repository}/pack/pack_test.go (89%) create mode 100644 mover-restic/restic/internal/repository/prune.go create mode 100644 mover-restic/restic/internal/repository/prune_test.go create mode 100644 mover-restic/restic/internal/repository/raw.go create mode 100644 mover-restic/restic/internal/repository/raw_test.go create mode 100644 mover-restic/restic/internal/repository/repair_index.go create mode 100644 mover-restic/restic/internal/repository/repair_index_test.go create mode 100644 mover-restic/restic/internal/repository/repair_pack.go create mode 100644 mover-restic/restic/internal/repository/repair_pack_test.go create mode 100644 mover-restic/restic/internal/repository/s3_backend.go create mode 100644 mover-restic/restic/internal/repository/upgrade_repo.go create mode 100644 mover-restic/restic/internal/repository/upgrade_repo_test.go delete mode 100644 mover-restic/restic/internal/restic/backend_test.go delete mode 100644 mover-restic/restic/internal/restic/counted_blob_set.go delete mode 100644 mover-restic/restic/internal/restic/counted_blob_set_test.go create mode 100644 mover-restic/restic/internal/restic/lister.go create mode 100644 mover-restic/restic/internal/restic/lister_test.go create mode 100644 mover-restic/restic/internal/restic/node_windows_test.go create mode 100644 mover-restic/restic/internal/restic/node_xattr_all_test.go create mode 100644 mover-restic/restic/internal/restic/node_xattr_test.go create mode 100644 mover-restic/restic/internal/restorer/fileswriter_other_test.go create mode 100644 mover-restic/restic/internal/restorer/fileswriter_windows_test.go create mode 100644 mover-restic/restic/internal/restorer/restorer_unix.go create mode 100644 mover-restic/restic/internal/restorer/restorer_windows.go create mode 100644 mover-restic/restic/internal/ui/progress/printer.go delete mode 100644 mover-restic/restic/internal/ui/stdio_wrapper.go create mode 100644 mover-restic/restic/internal/ui/termstatus/stdio_wrapper.go rename mover-restic/restic/internal/ui/{ => termstatus}/stdio_wrapper_test.go (98%) diff --git a/controllers/mover/restic/mover.go b/controllers/mover/restic/mover.go index c984d617b..de7f9a1f1 100644 --- a/controllers/mover/restic/mover.go +++ b/controllers/mover/restic/mover.go @@ -368,14 +368,14 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume utils.EnvFromSecret(repo.Name, "AWS_SECRET_ACCESS_KEY", true), utils.EnvFromSecret(repo.Name, "AWS_SESSION_TOKEN", true), // New in v0.14.0 utils.EnvFromSecret(repo.Name, "AWS_DEFAULT_REGION", true), - /* These vars are in restic main but not in an official release yet (as of v0.16.3) - utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_ARN", true), - utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_SESSION_NAME", true), - utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID", true), - utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_POLICY", true), - utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_REGION", true), - utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT", true), - */ + utils.EnvFromSecret(repo.Name, "AWS_PROFILE", true), + // AWS_SHARED_CREDENTIALS_FILE <- not implementing + utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_ARN", true), // New in v0.17.0 + utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_SESSION_NAME", true), // New in v0.17.0 + utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID", true), // New in v0.17.0 + utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_POLICY", true), // New in v0.17.0 + utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_REGION", true), // New in v0.17.0 + utils.EnvFromSecret(repo.Name, "RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT", true), // New in v0.17.0 utils.EnvFromSecret(repo.Name, "ST_AUTH", true), utils.EnvFromSecret(repo.Name, "ST_USER", true), utils.EnvFromSecret(repo.Name, "ST_KEY", true), @@ -403,6 +403,7 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume utils.EnvFromSecret(repo.Name, "AZURE_ACCOUNT_KEY", true), utils.EnvFromSecret(repo.Name, "AZURE_ACCOUNT_SAS", true), // New in v0.14.0 utils.EnvFromSecret(repo.Name, "AZURE_ENDPOINT_SUFFIX", true), // New in v0.16.0 + // AZURE_FORCE_CLI_CREDENTIAL <- not implementing, requires azure cli or local credentials stored from cli? utils.EnvFromSecret(repo.Name, "GOOGLE_PROJECT_ID", true), utils.EnvFromSecret(repo.Name, "RESTIC_REST_USERNAME", true), // New in v0.16.1 utils.EnvFromSecret(repo.Name, "RESTIC_REST_PASSWORD", true), // New in v0.16.1 diff --git a/mover-restic/SOURCE_VERSIONS b/mover-restic/SOURCE_VERSIONS index d4746ab0c..b86a53d0a 100644 --- a/mover-restic/SOURCE_VERSIONS +++ b/mover-restic/SOURCE_VERSIONS @@ -1,2 +1,2 @@ -https://github.com/restic/restic.git v0.16.5 fe9f142b5249f7db1a7f2bad1bedf9321c885e51 +https://github.com/restic/restic.git v0.17.0 277c8f5029a12bd882c2c1d2088f435caec67bb8 https://github.com/minio/minio-go.git v7.0.66 5415e6c72a71610108fe05ee747ac760dd40094f diff --git a/mover-restic/restic/.github/workflows/docker.yml b/mover-restic/restic/.github/workflows/docker.yml index a19767849..a943d1b15 100644 --- a/mover-restic/restic/.github/workflows/docker.yml +++ b/mover-restic/restic/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -33,7 +33,7 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} tags: | @@ -45,7 +45,7 @@ jobs: uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 + uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20 - name: Ensure consistent binaries run: | @@ -55,7 +55,7 @@ jobs: if: github.ref != 'refs/heads/master' - name: Build and push Docker image - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + uses: docker/build-push-action@15560696de535e4014efeff63c48f16952e52dd1 with: push: true context: . diff --git a/mover-restic/restic/.github/workflows/tests.yml b/mover-restic/restic/.github/workflows/tests.yml index 45681c6c5..3ca7a9edb 100644 --- a/mover-restic/restic/.github/workflows/tests.yml +++ b/mover-restic/restic/.github/workflows/tests.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - latest_go: "1.21.x" + latest_go: "1.22.x" GO111MODULE: on jobs: @@ -23,27 +23,32 @@ jobs: # list of jobs to run: include: - job_name: Windows - go: 1.21.x + go: 1.22.x os: windows-latest - job_name: macOS - go: 1.21.x + go: 1.22.x os: macOS-latest test_fuse: false - job_name: Linux - go: 1.21.x + go: 1.22.x os: ubuntu-latest test_cloud_backends: true test_fuse: true check_changelog: true - job_name: Linux (race) - go: 1.21.x + go: 1.22.x os: ubuntu-latest test_fuse: true test_opts: "-race" + - job_name: Linux + go: 1.21.x + os: ubuntu-latest + test_fuse: true + - job_name: Linux go: 1.20.x os: ubuntu-latest @@ -69,7 +74,7 @@ jobs: - name: Get programs (Linux/macOS) run: | echo "build Go tools" - go install github.com/restic/rest-server/cmd/rest-server@latest + go install github.com/restic/rest-server/cmd/rest-server@master echo "install minio server" mkdir $HOME/bin @@ -101,7 +106,7 @@ jobs: $ProgressPreference = 'SilentlyContinue' echo "build Go tools" - go install github.com/restic/rest-server/... + go install github.com/restic/rest-server/cmd/rest-server@master echo "install minio server" mkdir $Env:USERPROFILE/bin @@ -242,6 +247,10 @@ jobs: lint: name: lint runs-on: ubuntu-latest + permissions: + contents: read + # allow annotating code in the PR + checks: write steps: - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v5 @@ -252,10 +261,10 @@ jobs: uses: actions/checkout@v4 - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + uses: golangci/golangci-lint-action@v6 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.55.2 + version: v1.57.1 args: --verbose --timeout 5m # only run golangci-lint for pull requests, otherwise ALL hints get @@ -293,7 +302,7 @@ jobs: - name: Docker meta id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: # list of Docker images to use as base name for tags images: | @@ -316,7 +325,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: push: false context: . diff --git a/mover-restic/restic/.gitignore b/mover-restic/restic/.gitignore index b7201c26b..c8c3aa69a 100644 --- a/mover-restic/restic/.gitignore +++ b/mover-restic/restic/.gitignore @@ -1,3 +1,4 @@ +/.idea /restic /restic.exe /.vagrant diff --git a/mover-restic/restic/.golangci.yml b/mover-restic/restic/.golangci.yml index 98b5f9e03..e632965bb 100644 --- a/mover-restic/restic/.golangci.yml +++ b/mover-restic/restic/.golangci.yml @@ -35,6 +35,11 @@ linters: # parse and typecheck code - typecheck + # ensure that http response bodies are closed + - bodyclose + + - importas + issues: # don't use the default exclude rules, this hides (among others) ignored # errors from Close() calls @@ -51,3 +56,14 @@ issues: # staticcheck: there's no easy way to replace these packages - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated" - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated" + + exclude-rules: + # revive: ignore unused parameters in tests + - path: (_test\.go|testing\.go|backend/.*/tests\.go) + text: "unused-parameter:" + +linters-settings: + importas: + alias: + - pkg: github.com/restic/restic/internal/test + alias: rtest diff --git a/mover-restic/restic/.readthedocs.yaml b/mover-restic/restic/.readthedocs.yaml index 2a7769e9c..acec8519b 100644 --- a/mover-restic/restic/.readthedocs.yaml +++ b/mover-restic/restic/.readthedocs.yaml @@ -8,6 +8,10 @@ build: tools: python: "3.11" +# Build HTMLZip +formats: + - htmlzip + # Build documentation in the docs/ directory with Sphinx sphinx: configuration: doc/conf.py diff --git a/mover-restic/restic/CHANGELOG.md b/mover-restic/restic/CHANGELOG.md index b7ab57158..2a6926755 100644 --- a/mover-restic/restic/CHANGELOG.md +++ b/mover-restic/restic/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.17.0](#changelog-for-restic-0170-2024-07-26) * [Changelog for 0.16.5](#changelog-for-restic-0165-2024-07-01) * [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) * [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14) @@ -34,6 +35,712 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.17.0 (2024-07-26) +The following sections list the changes in restic 0.17.0 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #3600: Handle unreadable xattrs in folders above `backup` source + * Fix #4209: Fix slow SFTP upload performance + * Fix #4503: Correct hardlink handling in `stats` command + * Fix #4568: Prevent `forget --keep-tags ` from deleting all snapshots + * Fix #4615: Make `find` not sometimes ignore directories + * Fix #4656: Properly report ID of newly added keys + * Fix #4703: Shutdown cleanly when receiving SIGTERM + * Fix #4709: Correct `--no-lock` handling of `ls` and `tag` commands + * Fix #4760: Fix possible error on concurrent cache cleanup + * Fix #4850: Handle UTF-16 password files in `key` command correctly + * Fix #4902: Update snapshot summary on `rewrite` + * Chg #956: Return exit code 10 and 11 for non-existing and locked repository + * Chg #4540: Require at least ARMv6 for ARM binaries + * Chg #4602: Deprecate legacy index format and `s3legacy` repository layout + * Chg #4627: Redesign backend error handling to improve reliability + * Chg #4707: Disable S3 anonymous authentication by default + * Chg #4744: Include full key ID in JSON output of `key list` + * Enh #662: Optionally skip snapshot creation if nothing changed + * Enh #693: Include snapshot size in `snapshots` output + * Enh #805: Add bitrot detection to `diff` command + * Enh #828: Improve features of the `repair packs` command + * Enh #1786: Support repositories with empty password + * Enh #2348: Add `--delete` option to `restore` command + * Enh #3067: Add extended options to configure Windows Shadow Copy Service + * Enh #3406: Improve `dump` performance for large files + * Enh #3806: Optimize and make `prune` command resumable + * Enh #4006: (alpha) Store deviceID only for hardlinks + * Enh #4048: Add support for FUSE-T with `mount` on macOS + * Enh #4251: Support reading backup from a command's standard output + * Enh #4287: Support connection to rest-server using unix socket + * Enh #4354: Significantly reduce `prune` memory usage + * Enh #4437: Make `check` command create non-existent cache directory + * Enh #4472: Support AWS Assume Role for S3 backend + * Enh #4547: Add `--json` option to `version` command + * Enh #4549: Add `--ncdu` option to `ls` command + * Enh #4573: Support rewriting host and time metadata in snapshots + * Enh #4583: Ignore `s3.storage-class` archive tiers for metadata + * Enh #4590: Speed up `mount` command's error detection + * Enh #4601: Add support for feature flags + * Enh #4611: Back up more file metadata on Windows + * Enh #4664: Make `ls` use `message_type` field in JSON output + * Enh #4676: Make `key` command's actions separate sub-commands + * Enh #4678: Add `--target` option to the `dump` command + * Enh #4708: Back up and restore SecurityDescriptors on Windows + * Enh #4733: Allow specifying `--host` via environment variable + * Enh #4737: Include snapshot ID in `reason` field of `forget` JSON output + * Enh #4764: Support forgetting all snapshots + * Enh #4768: Allow specifying custom User-Agent for outgoing requests + * Enh #4781: Add `restore` options to read include/exclude patterns from files + * Enh #4807: Support Extended Attributes on Windows NTFS + * Enh #4817: Make overwrite behavior of `restore` customizable + * Enh #4839: Add dry-run support to `restore` command + +## Details + + * Bugfix #3600: Handle unreadable xattrs in folders above `backup` source + + When backup sources are specified using absolute paths, `backup` also includes + information about the parent folders of the backup sources in the snapshot. + + If the extended attributes for some of these folders could not be read due to + missing permissions, this caused the backup to fail. This has now been fixed. + + https://github.com/restic/restic/issues/3600 + https://github.com/restic/restic/pull/4668 + https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216 + + * Bugfix #4209: Fix slow SFTP upload performance + + Since restic 0.12.1, the upload speed of the sftp backend to a remote server has + regressed significantly. This has now been fixed. + + https://github.com/restic/restic/issues/4209 + https://github.com/restic/restic/pull/4782 + + * Bugfix #4503: Correct hardlink handling in `stats` command + + If files on different devices had the same inode ID, the `stats` command did not + correctly calculate the snapshot size. This has now been fixed. + + https://github.com/restic/restic/pull/4503 + https://github.com/restic/restic/pull/4006 + https://forum.restic.net/t/possible-bug-in-stats/6461/8 + + * Bugfix #4568: Prevent `forget --keep-tags ` from deleting all snapshots + + Running `forget --keep-tags `, where `` is a tag that does not + exist in the repository, would remove all snapshots. This is especially + problematic if the tag name contains a typo. + + The `forget` command now fails with an error if all snapshots in a snapshot + group would be deleted. This prevents the above example from deleting all + snapshots. + + It is possible to temporarily disable the new check by setting the environment + variable `RESTIC_FEATURES=safe-forget-keep-tags=false`. Note that this feature + flag will be removed in the next minor restic version. + + https://github.com/restic/restic/pull/4568 + https://github.com/restic/restic/pull/4764 + + * Bugfix #4615: Make `find` not sometimes ignore directories + + In some cases, the `find` command ignored empty or moved directories. This has + now been fixed. + + https://github.com/restic/restic/pull/4615 + + * Bugfix #4656: Properly report ID of newly added keys + + `restic key add` now reports the ID of the newly added key. This simplifies + selecting a specific key using the `--key-hint key` option. + + https://github.com/restic/restic/issues/4656 + https://github.com/restic/restic/pull/4657 + + * Bugfix #4703: Shutdown cleanly when receiving SIGTERM + + Previously, when restic received the SIGTERM signal it would terminate + immediately, skipping cleanup and potentially causing issues like stale locks + being left behind. This primarily effected containerized restic invocations that + use SIGTERM, but could also be triggered via a simple `killall restic`. + + This has now been fixed, such that restic shuts down cleanly when receiving the + SIGTERM signal. + + https://github.com/restic/restic/pull/4703 + + * Bugfix #4709: Correct `--no-lock` handling of `ls` and `tag` commands + + The `ls` command never locked the repository. This has now been fixed, with the + old behavior still being supported using `ls --no-lock`. The latter invocation + also works with older restic versions. + + The `tag` command erroneously accepted the `--no-lock` command. This command now + always requires an exclusive lock. + + https://github.com/restic/restic/pull/4709 + + * Bugfix #4760: Fix possible error on concurrent cache cleanup + + If multiple restic processes concurrently cleaned up no longer existing files + from the cache, this could cause some of the processes to fail with an `no such + file or directory` error. This has now been fixed. + + https://github.com/restic/restic/issues/4760 + https://github.com/restic/restic/pull/4761 + + * Bugfix #4850: Handle UTF-16 password files in `key` command correctly + + Previously, `key add` and `key passwd` did not properly decode UTF-16 encoded + passwords read from a password file. This has now been fixed to correctly match + the encoding when opening a repository. + + https://github.com/restic/restic/issues/4850 + https://github.com/restic/restic/pull/4851 + + * Bugfix #4902: Update snapshot summary on `rewrite` + + Restic previously did not recalculate the total number of files and bytes + processed when files were excluded from a snapshot by the `rewrite` command. + This has now been fixed. + + https://github.com/restic/restic/issues/4902 + https://github.com/restic/restic/pull/4905 + + * Change #956: Return exit code 10 and 11 for non-existing and locked repository + + If a repository does not exist or cannot be locked, restic previously always + returned exit code 1. This made it difficult to distinguish these cases from + other errors. + + Restic now returns exit code 10 if the repository does not exist, and exit code + 11 if the repository could be not locked due to a conflicting lock. + + https://github.com/restic/restic/issues/956 + https://github.com/restic/restic/pull/4884 + + * Change #4540: Require at least ARMv6 for ARM binaries + + The official release binaries of restic now require at least ARMv6 support for + ARM platforms. + + https://github.com/restic/restic/issues/4540 + https://github.com/restic/restic/pull/4542 + + * Change #4602: Deprecate legacy index format and `s3legacy` repository layout + + Support for the legacy index format used by restic before version 0.2.0 has been + deprecated and will be removed in the next minor restic version. You can use + `restic repair index` to update the index to the current format. + + It is possible to temporarily reenable support for the legacy index format by + setting the environment variable `RESTIC_FEATURES=deprecate-legacy-index=false`. + Note that this feature flag will be removed in the next minor restic version. + + Support for the `s3legacy` repository layout used for the S3 backend before + restic 0.7.0 has been deprecated and will be removed in the next minor restic + version. You can migrate your S3 repository to the current layout using + `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`. + + It is possible to temporarily reenable support for the `s3legacy` layout by + setting the environment variable + `RESTIC_FEATURES=deprecate-s3-legacy-layout=false`. Note that this feature flag + will be removed in the next minor restic version. + + https://github.com/restic/restic/issues/4602 + https://github.com/restic/restic/pull/4724 + https://github.com/restic/restic/pull/4743 + + * Change #4627: Redesign backend error handling to improve reliability + + Restic now downloads pack files in large chunks instead of using a streaming + download. This prevents failures due to interrupted streams. The `restore` + command now also retries downloading individual blobs that could not be + retrieved. + + HTTP requests that are stuck for more than two minutes while uploading or + downloading are now forcibly interrupted. This ensures that stuck requests are + retried after a short timeout. + + Attempts to access a missing or truncated file will no longer be retried. This + avoids unnecessary retries in those cases. All other backend requests are + retried for up to 15 minutes. This ensures that temporarily interrupted network + connections can be tolerated. + + If a download yields a corrupt file or blob, then the download will be retried + once. + + Most parts of the new backend error handling can temporarily be disabled by + setting the environment variable `RESTIC_FEATURES=backend-error-redesign=false`. + Note that this feature flag will be removed in the next minor restic version. + + https://github.com/restic/restic/issues/4627 + https://github.com/restic/restic/issues/4193 + https://github.com/restic/restic/issues/4515 + https://github.com/restic/restic/issues/1523 + https://github.com/restic/restic/pull/4605 + https://github.com/restic/restic/pull/4792 + https://github.com/restic/restic/pull/4520 + https://github.com/restic/restic/pull/4800 + https://github.com/restic/restic/pull/4784 + https://github.com/restic/restic/pull/4844 + + * Change #4707: Disable S3 anonymous authentication by default + + When using the S3 backend with anonymous authentication, it continuously tried + to retrieve new authentication credentials, causing bad performance. + + Now, to use anonymous authentication, it is necessary to pass the extended + option `-o s3.unsafe-anonymous-auth=true` to restic. + + It is possible to temporarily revert to the old behavior by setting the + environment variable `RESTIC_FEATURES=explicit-s3-anonymous-auth=false`. Note + that this feature flag will be removed in the next minor restic version. + + https://github.com/restic/restic/issues/4707 + https://github.com/restic/restic/pull/4908 + + * Change #4744: Include full key ID in JSON output of `key list` + + The JSON output of the `key list` command has changed to include the full key ID + instead of just a shortened version of the ID, as the latter can be ambiguous in + some rare cases. To derive the short ID, please truncate the full ID down to + eight characters. + + https://github.com/restic/restic/issues/4744 + https://github.com/restic/restic/pull/4745 + + * Enhancement #662: Optionally skip snapshot creation if nothing changed + + The `backup` command always created a snapshot even if nothing in the backup set + changed compared to the parent snapshot. + + Restic now supports the `--skip-if-unchanged` option for the `backup` command, + which omits creating a snapshot if the new snapshot's content would be identical + to that of the parent snapshot. + + https://github.com/restic/restic/issues/662 + https://github.com/restic/restic/pull/4816 + + * Enhancement #693: Include snapshot size in `snapshots` output + + The `snapshots` command now prints the size for snapshots created using this or + a future restic version. To achieve this, the `backup` command now stores the + backup summary statistics in the snapshot. + + The text output of the `snapshots` command only shows the snapshot size. The + other statistics are only included in the JSON output. To inspect these + statistics use `restic snapshots --json` or `restic cat snapshot `. + + https://github.com/restic/restic/issues/693 + https://github.com/restic/restic/pull/4705 + https://github.com/restic/restic/pull/4913 + + * Enhancement #805: Add bitrot detection to `diff` command + + The output of the `diff` command now includes the modifier `?` for files to + indicate bitrot in backed up files. The `?` will appear whenever there is a + difference in content while the metadata is exactly the same. + + Since files with unchanged metadata are normally not read again when creating a + backup, the detection is only effective when the right-hand side of the diff has + been created with `backup --force`. + + https://github.com/restic/restic/issues/805 + https://github.com/restic/restic/pull/4526 + + * Enhancement #828: Improve features of the `repair packs` command + + The `repair packs` command has been improved to also be able to process + truncated pack files. The `check` and `check --read-data` command will provide + instructions on using the command if necessary to repair a repository. See the + guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html for + further instructions. + + https://github.com/restic/restic/issues/828 + https://github.com/restic/restic/pull/4644 + https://github.com/restic/restic/pull/4882 + + * Enhancement #1786: Support repositories with empty password + + Restic previously required a password to create or operate on repositories. + Using the new option `--insecure-no-password` it is now possible to disable this + requirement. Restic will not prompt for a password when using this option. + + For security reasons, the option must always be specified when operating on + repositories with an empty password, and specifying `--insecure-no-password` + while also passing a password to restic via a CLI option or environment variable + results in an error. + + The `init` and `copy` commands add the related `--from-insecure-no-password` + option, which applies to the source repository. The `key add` and `key passwd` + commands add the `--new-insecure-no-password` option to add or set an empty + password. + + https://github.com/restic/restic/issues/1786 + https://github.com/restic/restic/issues/4326 + https://github.com/restic/restic/pull/4698 + https://github.com/restic/restic/pull/4808 + + * Enhancement #2348: Add `--delete` option to `restore` command + + The `restore` command now supports a `--delete` option that allows removing + files and directories from the target directory that do not exist in the + snapshot. This option also allows files in the snapshot to replace non-empty + directories having the same name. + + To check that only expected files are deleted, add the `--dry-run --verbose=2` + options. + + https://github.com/restic/restic/issues/2348 + https://github.com/restic/restic/pull/4881 + + * Enhancement #3067: Add extended options to configure Windows Shadow Copy Service + + Previous, restic always used a 120 seconds timeout and unconditionally created + VSS snapshots for all volume mount points on disk. This behavior can now be + fine-tuned by the following new extended options (available only on Windows): + + - `-o vss.timeout`: Time that VSS can spend creating snapshot before timing out + (default: 120s) - `-o vss.exclude-all-mount-points`: Exclude mountpoints from + snapshotting on all volumes (default: false) - `-o vss.exclude-volumes`: + Semicolon separated list of volumes to exclude from snapshotting - `-o + vss.provider`: VSS provider identifier which will be used for snapshotting + + For example, change VSS timeout to five minutes and disable snapshotting of + mount points on all volumes: + + Restic backup --use-fs-snapshot -o vss.timeout=5m -o + vss.exclude-all-mount-points=true + + Exclude drive `d:`, mount point `c:\mnt` and a specific volume from + snapshotting: + + Restic backup --use-fs-snapshot -o + vss.exclude-volumes="d:\;c:\mnt\;\\?\Volume{e2e0315d-9066-4f97-8343-eb5659b35762}" + + Uses 'Microsoft Software Shadow Copy provider 1.0' instead of the default + provider: + + Restic backup --use-fs-snapshot -o + vss.provider={b5946137-7b9f-4925-af80-51abd60b20d5} + + https://github.com/restic/restic/pull/3067 + + * Enhancement #3406: Improve `dump` performance for large files + + The `dump` command now retrieves the data chunks for a file in parallel. This + improves the download performance by up to as many times as the configured + number of parallel backend connections. + + https://github.com/restic/restic/issues/3406 + https://github.com/restic/restic/pull/4796 + + * Enhancement #3806: Optimize and make `prune` command resumable + + Previously, if the `prune` command was interrupted, a later `prune` run would + start repacking pack files from the start, as `prune` did not update the index + while repacking. + + The `prune` command now supports resuming interrupted prune runs. The update of + the repository index has also been optimized to use less memory and only rewrite + parts of the index that have changed. + + https://github.com/restic/restic/issues/3806 + https://github.com/restic/restic/pull/4812 + + * Enhancement #4006: (alpha) Store deviceID only for hardlinks + + Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature. The + feature flag will be removed after repository format version 3 becomes available + or be replaced with a different solution. + + When creating backups from a filesystem snapshot, for example created using + BTRFS subvolumes, the deviceID of the filesystem changes compared to previous + snapshots. This prevented restic from deduplicating the directory metadata of a + snapshot. + + When this alpha feature is enabled, the deviceID is only stored for hardlinks, + which significantly reduces the metadata duplication for most backups. + + https://github.com/restic/restic/pull/4006 + + * Enhancement #4048: Add support for FUSE-T with `mount` on macOS + + The restic `mount` command now supports creating FUSE mounts using FUSE-T on + macOS. + + https://github.com/restic/restic/issues/4048 + https://github.com/restic/restic/pull/4825 + + * Enhancement #4251: Support reading backup from a command's standard output + + The `backup` command now supports the `--stdin-from-command` option. When using + this option, the arguments to `backup` are interpreted as a command instead of + paths to back up. `backup` then executes the given command and stores the + standard output from it in the backup, similar to the what the `--stdin` option + does. This also enables restic to verify that the command completes with exit + code zero. A non-zero exit code causes the backup to fail. + + Note that the `--stdin` option does not have to be specified at the same time, + and that the `--stdin-filename` option also applies to `--stdin-from-command`. + + Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump + [...]` + + https://github.com/restic/restic/issues/4251 + https://github.com/restic/restic/pull/4410 + + * Enhancement #4287: Support connection to rest-server using unix socket + + Restic now supports using a unix socket to connect to a rest-server version + 0.13.0 or later. This allows running restic as follows: + + ``` + rest-server --listen unix:/tmp/rest.socket --data /path/to/data & + restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...] + ``` + + https://github.com/restic/restic/issues/4287 + https://github.com/restic/restic/pull/4655 + + * Enhancement #4354: Significantly reduce `prune` memory usage + + The `prune` command has been optimized to use up to 60% less memory. The memory + usage should now be roughly similar to creating a backup. + + https://github.com/restic/restic/pull/4354 + https://github.com/restic/restic/pull/4812 + + * Enhancement #4437: Make `check` command create non-existent cache directory + + Previously, if a custom cache directory was specified for the `check` command, + but the directory did not exist, `check` continued with the cache disabled. + + The `check` command now attempts to create the cache directory before + initializing the cache. + + https://github.com/restic/restic/issues/4437 + https://github.com/restic/restic/pull/4805 + https://github.com/restic/restic/pull/4883 + + * Enhancement #4472: Support AWS Assume Role for S3 backend + + Previously only credentials discovered via the Minio discovery methods were used + to authenticate. + + However, there are many circumstances where the discovered credentials have + lower permissions and need to assume a specific role. This is now possible using + the following new environment variables: + + - RESTIC_AWS_ASSUME_ROLE_ARN - RESTIC_AWS_ASSUME_ROLE_SESSION_NAME - + RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID - RESTIC_AWS_ASSUME_ROLE_REGION (defaults to + us-east-1) - RESTIC_AWS_ASSUME_ROLE_POLICY - RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT + + https://github.com/restic/restic/issues/4472 + https://github.com/restic/restic/pull/4474 + + * Enhancement #4547: Add `--json` option to `version` command + + Restic now supports outputting restic version along with the Go version, OS and + architecture used to build restic in JSON format using `version --json`. + + https://github.com/restic/restic/issues/4547 + https://github.com/restic/restic/pull/4553 + + * Enhancement #4549: Add `--ncdu` option to `ls` command + + NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. It has + an option to save a directory tree and analyse it later. + + The `ls` command now supports outputting snapshot information in the NCDU format + using the `--ncdu` option. Example usage: `restic ls latest --ncdu | ncdu -f -` + + https://github.com/restic/restic/issues/4549 + https://github.com/restic/restic/pull/4550 + https://github.com/restic/restic/pull/4911 + + * Enhancement #4573: Support rewriting host and time metadata in snapshots + + The `rewrite` command now supports rewriting the host and/or time metadata of a + snapshot using the new `--new-host` and `--new-time` options. + + https://github.com/restic/restic/pull/4573 + + * Enhancement #4583: Ignore `s3.storage-class` archive tiers for metadata + + Restic used to store all files on S3 using the specified `s3.storage-class`. + + Now, restic will only use non-archive storage tiers for metadata, to avoid + problems when accessing a repository. To restore any data, it is still necessary + to manually warm up the required data beforehand. + + NOTE: There is no official cold storage support in restic, use this option at + your own risk. + + https://github.com/restic/restic/issues/4583 + https://github.com/restic/restic/pull/4584 + + * Enhancement #4590: Speed up `mount` command's error detection + + The `mount` command now checks for the existence of the mountpoint before + opening the repository, leading to quicker error detection. + + https://github.com/restic/restic/pull/4590 + + * Enhancement #4601: Add support for feature flags + + Restic now supports feature flags that can be used to enable and disable + experimental features. The flags can be set using the environment variable + `RESTIC_FEATURES`. To get a list of currently supported feature flags, use the + `features` command. + + https://github.com/restic/restic/issues/4601 + https://github.com/restic/restic/pull/4666 + + * Enhancement #4611: Back up more file metadata on Windows + + Previously, restic did not back up all common Windows-specific metadata. + + Restic now stores file creation time and file attributes like the hidden, + read-only and encrypted flags when backing up files and folders on Windows. + + https://github.com/restic/restic/pull/4611 + + * Enhancement #4664: Make `ls` use `message_type` field in JSON output + + The `ls` command was the only restic command that used the `struct_type` field + in its JSON output format to specify the message type. + + The JSON output of the `ls` command now also includes the `message_type` field, + which is consistent with other commands. The `struct_type` field is still + included, but now deprecated. + + https://github.com/restic/restic/pull/4664 + + * Enhancement #4676: Make `key` command's actions separate sub-commands + + Each of the `add`, `list`, `remove` and `passwd` actions provided by the `key` + command is now a separate sub-command and have its own documentation which can + be invoked using `restic key --help`. + + https://github.com/restic/restic/issues/4676 + https://github.com/restic/restic/pull/4685 + + * Enhancement #4678: Add `--target` option to the `dump` command + + Restic `dump` always printed to the standard output. It now supports specifying + a `--target` file to write its output to. + + https://github.com/restic/restic/issues/4678 + https://github.com/restic/restic/pull/4682 + https://github.com/restic/restic/pull/4692 + + * Enhancement #4708: Back up and restore SecurityDescriptors on Windows + + Restic now backs up and restores SecurityDescriptors for files and folders on + Windows which includes owner, group, discretionary access control list (DACL) + and system access control list (SACL). + + This requires the user to be a member of backup operators or the application + must be run as admin. If that is not the case, only the current user's owner, + group and DACL will be backed up, and during restore only the DACL of the backed + up file will be restored, with the current user's owner and group being set on + the restored file. + + https://github.com/restic/restic/pull/4708 + + * Enhancement #4733: Allow specifying `--host` via environment variable + + Restic commands that operate on snapshots, such as `restic backup` and `restic + snapshots`, support the `--host` option to specify the hostname for grouping + snapshots. + + Such commands now also support specifying the hostname via the environment + variable `RESTIC_HOST`. Note that `--host` still takes precedence over the + environment variable. + + https://github.com/restic/restic/issues/4733 + https://github.com/restic/restic/pull/4734 + + * Enhancement #4737: Include snapshot ID in `reason` field of `forget` JSON output + + The JSON output of the `forget` command now includes `id` and `short_id` of + snapshots in the `reason` field. + + https://github.com/restic/restic/pull/4737 + + * Enhancement #4764: Support forgetting all snapshots + + The `forget` command now supports the `--unsafe-allow-remove-all` option, which + removes all snapshots in the repository. + + This option must always be combined with a snapshot filter (by host, path or + tag). For example, the command `forget --tag example --unsafe-allow-remove-all` + removes all snapshots with the tag "example". + + https://github.com/restic/restic/pull/4764 + + * Enhancement #4768: Allow specifying custom User-Agent for outgoing requests + + Restic now supports setting a custom `User-Agent` for outgoing HTTP requests + using the global option `--http-user-agent` or the `RESTIC_HTTP_USER_AGENT` + environment variable. + + https://github.com/restic/restic/issues/4768 + https://github.com/restic/restic/pull/4810 + + * Enhancement #4781: Add `restore` options to read include/exclude patterns from files + + Restic now supports reading include and exclude patterns from files using the + `--include-file`, `--exclude-file`, `--iinclude-file` and `--iexclude-file` + options of the `restore` command. + + https://github.com/restic/restic/issues/4781 + https://github.com/restic/restic/pull/4811 + + * Enhancement #4807: Support Extended Attributes on Windows NTFS + + Restic now backs up and restores Extended Attributes for files and folders on + Windows NTFS. + + https://github.com/restic/restic/pull/4807 + + * Enhancement #4817: Make overwrite behavior of `restore` customizable + + The `restore` command now supports an `--overwrite` option to configure whether + already existing files are overwritten. The overwrite behavior can be configured + using the following option values: + + - `--overwrite always` (default): Always overwrites already existing files. The + `restore` command will verify the existing file content and only restore + mismatching parts to minimize downloads. Updates the metadata of all files. - + `--overwrite if-changed`: Like `always`, but speeds up the file content check by + assuming that files with matching size and modification time (mtime) are already + up to date. In case of a mismatch, the full file content is verified like with + `always`. Updates the metadata of all files. - `--overwrite if-newer`: Like + `always`, but only overwrites existing files when the file in the snapshot has a + newer modification time (mtime) than the existing file. - `--overwrite never`: + Never overwrites existing files. + + https://github.com/restic/restic/issues/4817 + https://github.com/restic/restic/issues/200 + https://github.com/restic/restic/issues/407 + https://github.com/restic/restic/issues/2662 + https://github.com/restic/restic/pull/4837 + https://github.com/restic/restic/pull/4838 + https://github.com/restic/restic/pull/4864 + https://github.com/restic/restic/pull/4921 + + * Enhancement #4839: Add dry-run support to `restore` command + + The `restore` command now supports the `--dry-run` option to perform a dry run. + Pass the `--verbose=2` option to see which files would remain unchanged, and + which would be updated or freshly restored. + + https://github.com/restic/restic/pull/4839 + + # Changelog for restic 0.16.5 (2024-07-01) The following sections list the changes in restic 0.16.5 relevant to restic users. The changes are ordered by importance. @@ -47,8 +754,8 @@ restic users. The changes are ordered by importance. * Enhancement #4799: Add option to force use of Azure CLI credential - A new environment variable `AZURE_FORCE_CLI_CREDENTIAL=true` allows forcing the use of - Azure CLI credential, ignoring other credentials like managed identity. + A new environment variable `AZURE_FORCE_CLI_CREDENTIAL=true` allows forcing the + use of Azure CLI credential, ignoring other credentials like managed identity. https://github.com/restic/restic/pull/4799 @@ -73,35 +780,39 @@ restic users. The changes are ordered by importance. * Bugfix #4677: Downgrade zstd library to fix rare data corruption at max. compression - In restic 0.16.3, backups where the compression level was set to `max` (using `--compression - max`) could in rare and very specific circumstances result in data corruption due to a bug in the - library used for compressing data. Restic 0.16.1 and 0.16.2 were not affected. + In restic 0.16.3, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. Restic + 0.16.1 and 0.16.2 were not affected. - Restic now uses the previous version of the library used to compress data, the same version used - by restic 0.16.2. Please note that the `auto` compression level (which restic uses by default) - was never affected, and even if you used `max` compression, chances of being affected by this - issue are small. + Restic now uses the previous version of the library used to compress data, the + same version used by restic 0.16.2. Please note that the `auto` compression + level (which restic uses by default) was never affected, and even if you used + `max` compression, chances of being affected by this issue are small. - To check a repository for any corruption, run `restic check --read-data`. This will download - and verify the whole repository and can be used at any time to completely verify the integrity of - a repository. If the `check` command detects anomalies, follow the suggested steps. + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. https://github.com/restic/restic/issues/4677 https://github.com/restic/restic/pull/4679 * Enhancement #4529: Add extra verification of data integrity before upload - Hardware issues, or a bug in restic or its dependencies, could previously cause corruption in - the files restic created and stored in the repository. Detecting such corruption previously - required explicitly running the `check --read-data` or `check --read-data-subset` - commands. + Hardware issues, or a bug in restic or its dependencies, could previously cause + corruption in the files restic created and stored in the repository. Detecting + such corruption previously required explicitly running the `check --read-data` + or `check --read-data-subset` commands. - To further ensure data integrity, even in the case of hardware issues or software bugs, restic - now performs additional verification of the files about to be uploaded to the repository. + To further ensure data integrity, even in the case of hardware issues or + software bugs, restic now performs additional verification of the files about to + be uploaded to the repository. - These extra checks will increase CPU usage during backups. They can therefore, if absolutely - necessary, be disabled using the `--no-extra-verify` global option. Please note that this - should be combined with more active checking using the previously mentioned check commands. + These extra checks will increase CPU usage during backups. They can therefore, + if absolutely necessary, be disabled using the `--no-extra-verify` global + option. Please note that this should be combined with more active checking using + the previously mentioned check commands. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 @@ -123,13 +834,14 @@ restic users. The changes are ordered by importance. * Bugfix #4560: Improve errors for irregular files on Windows - Since Go 1.21, most filesystem reparse points on Windows are considered to be irregular files. - This caused restic to show an `error: invalid node type ""` error message for those files. + Since Go 1.21, most filesystem reparse points on Windows are considered to be + irregular files. This caused restic to show an `error: invalid node type ""` + error message for those files. - This error message has now been improved and includes the relevant file path: `error: - nodeFromFileInfo path/to/file: unsupported file type "irregular"`. As irregular files are - not required to behave like regular files, it is not possible to provide a generic way to back up - those files. + This error message has now been improved and includes the relevant file path: + `error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. As + irregular files are not required to behave like regular files, it is not + possible to provide a generic way to back up those files. https://github.com/restic/restic/issues/4560 https://github.com/restic/restic/pull/4620 @@ -137,9 +849,10 @@ restic users. The changes are ordered by importance. * Bugfix #4574: Support backup of deduplicated files on Windows again - With the official release builds of restic 0.16.1 and 0.16.2, it was not possible to back up - files that were deduplicated by the corresponding Windows Server feature. This also applied - to restic versions built using Go 1.21.0-1.21.4. + With the official release builds of restic 0.16.1 and 0.16.2, it was not + possible to back up files that were deduplicated by the corresponding Windows + Server feature. This also applied to restic versions built using Go + 1.21.0-1.21.4. The Go version used to build restic has now been updated to fix this. @@ -148,10 +861,10 @@ restic users. The changes are ordered by importance. * Bugfix #4612: Improve error handling for `rclone` backend - Since restic 0.16.0, if rclone encountered an error while listing files, this could in rare - circumstances cause restic to assume that there are no files. Although unlikely, this - situation could result in data loss if it were to happen right when the `prune` command is - listing existing snapshots. + Since restic 0.16.0, if rclone encountered an error while listing files, this + could in rare circumstances cause restic to assume that there are no files. + Although unlikely, this situation could result in data loss if it were to happen + right when the `prune` command is listing existing snapshots. Error handling has now been improved to detect and work around this case. @@ -160,9 +873,10 @@ restic users. The changes are ordered by importance. * Bugfix #4624: Correct `restore` progress information if an error occurs - If an error occurred while restoring a snapshot, this could cause the `restore` progress bar to - show incorrect information. In addition, if a data file could not be loaded completely, then - errors would also be reported for some already restored files. + If an error occurred while restoring a snapshot, this could cause the `restore` + progress bar to show incorrect information. In addition, if a data file could + not be loaded completely, then errors would also be reported for some already + restored files. Error reporting of the `restore` command has now been made more accurate. @@ -171,11 +885,12 @@ restic users. The changes are ordered by importance. * Bugfix #4626: Improve reliability of restoring large files - In some cases restic failed to restore large files that frequently contain the same file chunk. - In combination with certain backends, this could result in network connection timeouts that - caused incomplete restores. + In some cases restic failed to restore large files that frequently contain the + same file chunk. In combination with certain backends, this could result in + network connection timeouts that caused incomplete restores. - Restic now includes special handling for such file chunks to ensure reliable restores. + Restic now includes special handling for such file chunks to ensure reliable + restores. https://github.com/restic/restic/pull/4626 https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 @@ -194,16 +909,18 @@ restic users. The changes are ordered by importance. * Bugfix #4540: Restore ARMv5 support for ARM binaries - The official release binaries for restic 0.16.1 were accidentally built to require ARMv7. The - build process is now updated to restore support for ARMv5. + The official release binaries for restic 0.16.1 were accidentally built to + require ARMv7. The build process is now updated to restore support for ARMv5. - Please note that restic 0.17.0 will drop support for ARMv5 and require at least ARMv6. + Please note that restic 0.17.0 will drop support for ARMv5 and require at least + ARMv6. https://github.com/restic/restic/issues/4540 * Bugfix #4545: Repair documentation build on Read the Docs - For restic 0.16.1, no documentation was available at https://restic.readthedocs.io/ . + For restic 0.16.1, no documentation was available at + https://restic.readthedocs.io/ . The documentation build process is now updated to work again. @@ -230,65 +947,67 @@ restic users. The changes are ordered by importance. * Bugfix #4513: Make `key list` command honor `--no-lock` - The `key list` command now supports the `--no-lock` options. This allows determining which - keys a repo can be accessed by without the need for having write access (e.g., read-only sftp - access, filesystem snapshot). + The `key list` command now supports the `--no-lock` options. This allows + determining which keys a repo can be accessed by without the need for having + write access (e.g., read-only sftp access, filesystem snapshot). https://github.com/restic/restic/issues/4513 https://github.com/restic/restic/pull/4514 * Bugfix #4516: Do not try to load password on command line autocomplete - The command line autocompletion previously tried to load the repository password. This could - cause the autocompletion not to work. Now, this step gets skipped. + The command line autocompletion previously tried to load the repository + password. This could cause the autocompletion not to work. Now, this step gets + skipped. https://github.com/restic/restic/issues/4516 https://github.com/restic/restic/pull/4526 * Bugfix #4523: Update zstd library to fix possible data corruption at max. compression - In restic 0.16.0, backups where the compression level was set to `max` (using `--compression - max`) could in rare and very specific circumstances result in data corruption due to a bug in the - library used for compressing data. + In restic 0.16.0, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. - Restic now uses the latest version of the library used to compress data, which includes a fix for - this issue. Please note that the `auto` compression level (which restic uses by default) was - never affected, and even if you used `max` compression, chances of being affected by this issue - were very small. + Restic now uses the latest version of the library used to compress data, which + includes a fix for this issue. Please note that the `auto` compression level + (which restic uses by default) was never affected, and even if you used `max` + compression, chances of being affected by this issue were very small. - To check a repository for any corruption, run `restic check --read-data`. This will download - and verify the whole repository and can be used at any time to completely verify the integrity of - a repository. If the `check` command detects anomalies, follow the suggested steps. + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. - To simplify any needed repository repair and minimize data loss, there is also a new and - experimental `repair packs` command that salvages all valid data from the affected pack files - (see `restic help repair packs` for more information). + To simplify any needed repository repair and minimize data loss, there is also a + new and experimental `repair packs` command that salvages all valid data from + the affected pack files (see `restic help repair packs` for more information). https://github.com/restic/restic/issues/4523 https://github.com/restic/restic/pull/4530 * Change #4532: Update dependencies and require Go 1.19 or newer - We have updated all dependencies. Since some libraries require newer Go standard library - features, support for Go 1.18 has been dropped, which means that restic now requires at least Go - 1.19 to build. + We have updated all dependencies. Since some libraries require newer Go standard + library features, support for Go 1.18 has been dropped, which means that restic + now requires at least Go 1.19 to build. https://github.com/restic/restic/pull/4532 https://github.com/restic/restic/pull/4533 * Enhancement #229: Show progress bar while loading the index - Restic did not provide any feedback while loading index files. Now, there is a progress bar that - shows the index loading progress. + Restic did not provide any feedback while loading index files. Now, there is a + progress bar that shows the index loading progress. https://github.com/restic/restic/issues/229 https://github.com/restic/restic/pull/4419 * Enhancement #4128: Automatically set `GOMAXPROCS` in resource-constrained containers - When running restic in a Linux container with CPU-usage limits, restic now automatically - adjusts `GOMAXPROCS`. This helps to reduce the memory consumption on hosts with many CPU - cores. + When running restic in a Linux container with CPU-usage limits, restic now + automatically adjusts `GOMAXPROCS`. This helps to reduce the memory consumption + on hosts with many CPU cores. https://github.com/restic/restic/issues/4128 https://github.com/restic/restic/pull/4485 @@ -296,32 +1015,33 @@ restic users. The changes are ordered by importance. * Enhancement #4480: Allow setting REST password and username via environment variables - Previously, it was only possible to specify the REST-server username and password in the - repository URL, or by using the `--repository-file` option. This meant it was not possible to - use authentication in contexts where the repository URL is stored in publicly accessible way. + Previously, it was only possible to specify the REST-server username and + password in the repository URL, or by using the `--repository-file` option. This + meant it was not possible to use authentication in contexts where the repository + URL is stored in publicly accessible way. - Restic now allows setting the username and password using the `RESTIC_REST_USERNAME` and - `RESTIC_REST_PASSWORD` variables. + Restic now allows setting the username and password using the + `RESTIC_REST_USERNAME` and `RESTIC_REST_PASSWORD` variables. https://github.com/restic/restic/pull/4480 * Enhancement #4511: Include inode numbers in JSON output for `find` and `ls` commands - Restic used to omit the inode numbers in the JSON messages emitted for nodes by the `ls` command - as well as for matches by the `find` command. It now includes those values whenever they are - available. + Restic used to omit the inode numbers in the JSON messages emitted for nodes by + the `ls` command as well as for matches by the `find` command. It now includes + those values whenever they are available. https://github.com/restic/restic/pull/4511 * Enhancement #4519: Add config option to set SFTP command arguments - When using the `sftp` backend, scenarios where a custom identity file was needed for the SSH - connection, required the full command to be specified: `-o sftp.command='ssh - user@host:port -i /ssh/my_private_key -s sftp'` + When using the `sftp` backend, scenarios where a custom identity file was needed + for the SSH connection, required the full command to be specified: `-o + sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'` - Now, the `-o sftp.args=...` option can be passed to restic to specify custom arguments for the - SSH command executed by the SFTP backend. This simplifies the above example to `-o - sftp.args='-i /ssh/my_private_key'`. + Now, the `-o sftp.args=...` option can be passed to restic to specify custom + arguments for the SSH command executed by the SFTP backend. This simplifies the + above example to `-o sftp.args='-i /ssh/my_private_key'`. https://github.com/restic/restic/issues/4241 https://github.com/restic/restic/pull/4519 @@ -367,31 +1087,32 @@ restic users. The changes are ordered by importance. * Bugfix #2565: Support "unlimited" in `forget --keep-*` options - Restic would previously forget snapshots that should have been kept when a negative value was - passed to the `--keep-*` options. Negative values are now forbidden. To keep all snapshots, - the special value `unlimited` is now supported. For example, `--keep-monthly unlimited` - will keep all monthly snapshots. + Restic would previously forget snapshots that should have been kept when a + negative value was passed to the `--keep-*` options. Negative values are now + forbidden. To keep all snapshots, the special value `unlimited` is now + supported. For example, `--keep-monthly unlimited` will keep all monthly + snapshots. https://github.com/restic/restic/issues/2565 https://github.com/restic/restic/pull/4234 * Bugfix #3311: Support non-UTF8 paths as symlink target - Earlier restic versions did not correctly `backup` and `restore` symlinks that contain a - non-UTF8 target. Note that this only affected systems that still use a non-Unicode encoding - for filesystem paths. + Earlier restic versions did not correctly `backup` and `restore` symlinks that + contain a non-UTF8 target. Note that this only affected systems that still use a + non-Unicode encoding for filesystem paths. - The repository format is now extended to add support for such symlinks. Please note that - snapshots must have been created with at least restic version 0.16.0 for `restore` to - correctly handle non-UTF8 symlink targets when restoring them. + The repository format is now extended to add support for such symlinks. Please + note that snapshots must have been created with at least restic version 0.16.0 + for `restore` to correctly handle non-UTF8 symlink targets when restoring them. https://github.com/restic/restic/issues/3311 https://github.com/restic/restic/pull/3802 * Bugfix #4199: Avoid lock refresh issues on slow network connections - On network connections with a low upload speed, backups and other operations could fail with - the error message `Fatal: failed to refresh lock in time`. + On network connections with a low upload speed, backups and other operations + could fail with the error message `Fatal: failed to refresh lock in time`. This has now been fixed by reworking the lock refresh handling. @@ -400,21 +1121,21 @@ restic users. The changes are ordered by importance. * Bugfix #4274: Improve lock refresh handling after standby - If the restic process was stopped or the host running restic entered standby during a long - running operation such as a backup, this previously resulted in the operation failing with - `Fatal: failed to refresh lock in time`. + If the restic process was stopped or the host running restic entered standby + during a long running operation such as a backup, this previously resulted in + the operation failing with `Fatal: failed to refresh lock in time`. - This has now been fixed such that restic first checks whether it is safe to continue the current - operation and only throws an error if not. + This has now been fixed such that restic first checks whether it is safe to + continue the current operation and only throws an error if not. https://github.com/restic/restic/issues/4274 https://github.com/restic/restic/pull/4374 * Bugfix #4319: Correctly clean up status bar output of the `backup` command - Due to a regression in restic 0.15.2, the status bar of the `backup` command could leave some - output behind. This happened if filenames were printed that are wider than the current - terminal width. This has now been fixed. + Due to a regression in restic 0.15.2, the status bar of the `backup` command + could leave some output behind. This happened if filenames were printed that are + wider than the current terminal width. This has now been fixed. https://github.com/restic/restic/issues/4319 https://github.com/restic/restic/pull/4318 @@ -425,25 +1146,26 @@ restic users. The changes are ordered by importance. * Bugfix #4400: Ignore missing folders in `rest` backend - If a repository accessed via the REST backend was missing folders, then restic would fail with - an error while trying to list the data in the repository. This has been now fixed. + If a repository accessed via the REST backend was missing folders, then restic + would fail with an error while trying to list the data in the repository. This + has been now fixed. https://github.com/restic/rest-server/issues/235 https://github.com/restic/restic/pull/4400 * Change #4176: Fix JSON message type of `scan_finished` for the `backup` command - Restic incorrectly set the `message_type` of the `scan_finished` message to `status` - instead of `verbose_status`. This has now been corrected so that the messages report the - correct type. + Restic incorrectly set the `message_type` of the `scan_finished` message to + `status` instead of `verbose_status`. This has now been corrected so that the + messages report the correct type. https://github.com/restic/restic/pull/4176 * Change #4201: Require Go 1.20 for Solaris builds - Building restic on Solaris now requires Go 1.20, as the library used to access Azure uses the - mmap syscall, which is only available on Solaris starting from Go 1.20. All other platforms - however continue to build with Go 1.18. + Building restic on Solaris now requires Go 1.20, as the library used to access + Azure uses the mmap syscall, which is only available on Solaris starting from Go + 1.20. All other platforms however continue to build with Go 1.18. https://github.com/restic/restic/pull/4201 @@ -464,8 +1186,8 @@ restic users. The changes are ordered by importance. * Enhancement #719: Add `--retry-lock` option - This option allows specifying a duration for which restic will wait if the repository is - already locked. + This option allows specifying a duration for which restic will wait if the + repository is already locked. https://github.com/restic/restic/issues/719 https://github.com/restic/restic/pull/2214 @@ -473,24 +1195,25 @@ restic users. The changes are ordered by importance. * Enhancement #1495: Sort snapshots by timestamp in `restic find` - The `find` command used to print snapshots in an arbitrary order. Restic now prints snapshots - sorted by timestamp. + The `find` command used to print snapshots in an arbitrary order. Restic now + prints snapshots sorted by timestamp. https://github.com/restic/restic/issues/1495 https://github.com/restic/restic/pull/4409 * Enhancement #1759: Add `repair index` and `repair snapshots` commands - The `rebuild-index` command has been renamed to `repair index`. The old name will still work, - but is deprecated. + The `rebuild-index` command has been renamed to `repair index`. The old name + will still work, but is deprecated. - When a snapshot was damaged, the only option up to now was to completely forget the snapshot, - even if only some unimportant files in it were damaged and other files were still fine. + When a snapshot was damaged, the only option up to now was to completely forget + the snapshot, even if only some unimportant files in it were damaged and other + files were still fine. - Restic now has a `repair snapshots` command, which can salvage any non-damaged files and parts - of files in the snapshots by removing damaged directories and missing file contents. Please - note that the damaged data may still be lost and see the "Troubleshooting" section in the - documentation for more details. + Restic now has a `repair snapshots` command, which can salvage any non-damaged + files and parts of files in the snapshots by removing damaged directories and + missing file contents. Please note that the damaged data may still be lost and + see the "Troubleshooting" section in the documentation for more details. https://github.com/restic/restic/issues/1759 https://github.com/restic/restic/issues/1714 @@ -502,19 +1225,20 @@ restic users. The changes are ordered by importance. * Enhancement #1926: Allow certificate paths to be passed through environment variables - Restic will now read paths to certificates from the environment variables `RESTIC_CACERT` or - `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` are not specified. + Restic will now read paths to certificates from the environment variables + `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` + are not specified. https://github.com/restic/restic/issues/1926 https://github.com/restic/restic/pull/4384 * Enhancement #2359: Provide multi-platform Docker images - The official Docker images are now built for the architectures linux/386, linux/amd64, - linux/arm and linux/arm64. + The official Docker images are now built for the architectures linux/386, + linux/amd64, linux/arm and linux/arm64. - As an alternative to the Docker Hub, the Docker images are also available on ghcr.io, the GitHub - Container Registry. + As an alternative to the Docker Hub, the Docker images are also available on + ghcr.io, the GitHub Container Registry. https://github.com/restic/restic/issues/2359 https://github.com/restic/restic/issues/4269 @@ -524,25 +1248,26 @@ restic users. The changes are ordered by importance. The `azure` backend previously only supported storages using the global domain `core.windows.net`. This meant that backups to other domains such as Azure China - (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not supported. - Restic now allows overriding the global domain using the environment variable - `AZURE_ENDPOINT_SUFFIX`. + (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not + supported. Restic now allows overriding the global domain using the environment + variable `AZURE_ENDPOINT_SUFFIX`. https://github.com/restic/restic/issues/2468 https://github.com/restic/restic/pull/4387 * Enhancement #2679: Reduce file fragmentation for local backend - Before this change, local backend files could become fragmented. Now restic will try to - preallocate space for pack files to avoid their fragmentation. + Before this change, local backend files could become fragmented. Now restic will + try to preallocate space for pack files to avoid their fragmentation. https://github.com/restic/restic/issues/2679 https://github.com/restic/restic/pull/3261 * Enhancement #3328: Reduce memory usage by up to 25% - The in-memory index has been optimized to be more garbage collection friendly. Restic now - defaults to `GOGC=50` to run the Go garbage collector more frequently. + The in-memory index has been optimized to be more garbage collection friendly. + Restic now defaults to `GOGC=50` to run the Go garbage collector more + frequently. https://github.com/restic/restic/issues/3328 https://github.com/restic/restic/pull/4352 @@ -550,21 +1275,21 @@ restic users. The changes are ordered by importance. * Enhancement #3397: Improve accuracy of ETA displayed during backup - Restic's `backup` command displayed an ETA that did not adapt when the rate of progress made - during the backup changed during the course of the backup. + Restic's `backup` command displayed an ETA that did not adapt when the rate of + progress made during the backup changed during the course of the backup. - Restic now uses recent progress when computing the ETA. It is important to realize that the - estimate may still be wrong, because restic cannot predict the future, but the hope is that the - ETA will be more accurate in most cases. + Restic now uses recent progress when computing the ETA. It is important to + realize that the estimate may still be wrong, because restic cannot predict the + future, but the hope is that the ETA will be more accurate in most cases. https://github.com/restic/restic/issues/3397 https://github.com/restic/restic/pull/3563 * Enhancement #3624: Keep oldest snapshot when there are not enough snapshots - The `forget` command now additionally preserves the oldest snapshot if fewer snapshots than - allowed by the `--keep-*` parameters would otherwise be kept. This maximizes the amount of - history kept within the specified limits. + The `forget` command now additionally preserves the oldest snapshot if fewer + snapshots than allowed by the `--keep-*` parameters would otherwise be kept. + This maximizes the amount of history kept within the specified limits. https://github.com/restic/restic/issues/3624 https://github.com/restic/restic/pull/4366 @@ -572,99 +1297,106 @@ restic users. The changes are ordered by importance. * Enhancement #3698: Add support for Managed / Workload Identity to `azure` backend - Restic now additionally supports authenticating to Azure using Workload Identity or Managed - Identity credentials, which are automatically injected in several environments such as a - managed Kubernetes cluster. + Restic now additionally supports authenticating to Azure using Workload Identity + or Managed Identity credentials, which are automatically injected in several + environments such as a managed Kubernetes cluster. https://github.com/restic/restic/issues/3698 https://github.com/restic/restic/pull/4029 * Enhancement #3871: Support `:` syntax to select subfolders - Commands like `diff` or `restore` always worked with the full snapshot. This did not allow - comparing only a specific subfolder or only restoring that folder (`restore --include - subfolder` filters the restored files, but still creates the directories included in - `subfolder`). + Commands like `diff` or `restore` always worked with the full snapshot. This did + not allow comparing only a specific subfolder or only restoring that folder + (`restore --include subfolder` filters the restored files, but still creates the + directories included in `subfolder`). - The commands `diff`, `dump`, `ls` and `restore` now support the `:` - syntax, where `snapshot` is the ID of a snapshot (or the string `latest`) and `subfolder` is a - path within the snapshot. The commands will then only work with the specified path of the - snapshot. The `subfolder` must be a path to a folder as returned by `ls`. Two examples: + The commands `diff`, `dump`, `ls` and `restore` now support the + `:` syntax, where `snapshot` is the ID of a snapshot (or + the string `latest`) and `subfolder` is a path within the snapshot. The commands + will then only work with the specified path of the snapshot. The `subfolder` + must be a path to a folder as returned by `ls`. Two examples: `restic restore -t target latest:/some/path` `restic diff 12345678:/some/path 90abcef:/some/path` - For debugging purposes, the `cat` command now supports `cat tree :` to - return the directory metadata for the given subfolder. + For debugging purposes, the `cat` command now supports `cat tree + :` to return the directory metadata for the given + subfolder. https://github.com/restic/restic/issues/3871 https://github.com/restic/restic/pull/4334 * Enhancement #3941: Support `--group-by` for backup parent selection - Previously, the `backup` command by default selected the parent snapshot based on the - hostname and the backup targets. When the backup path list changed, the `backup` command was - unable to determine a suitable parent snapshot and had to read all files again. + Previously, the `backup` command by default selected the parent snapshot based + on the hostname and the backup paths. When the backup path list changed, the + `backup` command was unable to determine a suitable parent snapshot and had to + read all files again. - The new `--group-by` option for the `backup` command allows filtering snapshots for the - parent selection by `host`, `paths` and `tags`. It defaults to `host,paths` which selects the - latest snapshot with hostname and paths matching those of the backup run. This matches the - behavior of prior restic versions. + The new `--group-by` option for the `backup` command allows filtering snapshots + for the parent selection by `host`, `paths` and `tags`. It defaults to + `host,paths` which selects the latest snapshot with hostname and paths matching + those of the backup run. This matches the behavior of prior restic versions. - The new `--group-by` option should be set to the same value as passed to `forget --group-by`. + The new `--group-by` option should be set to the same value as passed to `forget + --group-by`. https://github.com/restic/restic/issues/3941 https://github.com/restic/restic/pull/4081 * Enhancement #4130: Cancel current command if cache becomes unusable - If the cache directory was removed or ran out of space while restic was running, this would - previously cause further caching attempts to fail and thereby drastically slow down the - command execution. Now, the currently running command is instead canceled. + If the cache directory was removed or ran out of space while restic was running, + this would previously cause further caching attempts to fail and thereby + drastically slow down the command execution. Now, the currently running command + is instead canceled. https://github.com/restic/restic/issues/4130 https://github.com/restic/restic/pull/4166 * Enhancement #4159: Add `--human-readable` option to `ls` and `find` commands - Previously, when using the `-l` option with the `ls` and `find` commands, the displayed size - was always in bytes, without an option for a more human readable format such as MiB or GiB. + Previously, when using the `-l` option with the `ls` and `find` commands, the + displayed size was always in bytes, without an option for a more human readable + format such as MiB or GiB. - The new `--human-readable` option will convert longer size values into more human friendly - values with an appropriate suffix depending on the output size. For example, a size of - `14680064` will be shown as `14.000 MiB`. + The new `--human-readable` option will convert longer size values into more + human friendly values with an appropriate suffix depending on the output size. + For example, a size of `14680064` will be shown as `14.000 MiB`. https://github.com/restic/restic/issues/4159 https://github.com/restic/restic/pull/4351 * Enhancement #4188: Include restic version in snapshot metadata - The restic version used to backup a snapshot is now included in its metadata and shown when - inspecting a snapshot using `restic cat snapshot ` or `restic snapshots - --json`. + The restic version used to backup a snapshot is now included in its metadata and + shown when inspecting a snapshot using `restic cat snapshot ` or + `restic snapshots --json`. https://github.com/restic/restic/issues/4188 https://github.com/restic/restic/pull/4378 * Enhancement #4220: Add `jq` binary to Docker image - The Docker image now contains `jq`, which can be useful to process JSON data output by restic. + The Docker image now contains `jq`, which can be useful to process JSON data + output by restic. https://github.com/restic/restic/pull/4220 * Enhancement #4226: Allow specifying region of new buckets in the `gs` backend - Previously, buckets used by the Google Cloud Storage backend would always get created in the - "us" region. It is now possible to specify the region where a bucket should be created by using - the `-o gs.region=us` option. + Previously, buckets used by the Google Cloud Storage backend would always get + created in the "us" region. It is now possible to specify the region where a + bucket should be created by using the `-o gs.region=us` option. https://github.com/restic/restic/pull/4226 * Enhancement #4375: Add support for extended attributes on symlinks - Restic now supports extended attributes on symlinks when backing up, restoring, or - FUSE-mounting snapshots. This includes, for example, the `security.selinux` xattr on Linux - distributions that use SELinux. + Restic now supports extended attributes on symlinks when backing up, restoring, + or FUSE-mounting snapshots. This includes, for example, the `security.selinux` + xattr on Linux distributions that use SELinux. https://github.com/restic/restic/issues/4375 https://github.com/restic/restic/pull/4379 @@ -693,12 +1425,12 @@ restic users. The changes are ordered by importance. * Bugfix #2260: Sanitize filenames printed by `backup` during processing - The `backup` command would previously not sanitize the filenames it printed during - processing, potentially causing newlines or terminal control characters to mangle the - status output or even change the state of a terminal. + The `backup` command would previously not sanitize the filenames it printed + during processing, potentially causing newlines or terminal control characters + to mangle the status output or even change the state of a terminal. - Filenames are now checked and quoted if they contain non-printable or non-Unicode - characters. + Filenames are now checked and quoted if they contain non-printable or + non-Unicode characters. https://github.com/restic/restic/issues/2260 https://github.com/restic/restic/issues/4191 @@ -707,44 +1439,47 @@ restic users. The changes are ordered by importance. * Bugfix #4211: Make `dump` interpret `--host` and `--path` correctly A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and - `--path=` options: it looked for snapshots with paths called `` from hosts - called ``. It now treats the options as intended. + `--path=` options: it looked for snapshots with paths called `` from + hosts called ``. It now treats the options as intended. https://github.com/restic/restic/issues/4211 https://github.com/restic/restic/pull/4212 * Bugfix #4239: Correct number of blocks reported in mount point - Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for - files and links due to a rounding bug. In particular, empty files were reported as taking one - block instead of zero. + Restic mount points reported an incorrect number of 512-byte (POSIX standard) + blocks for files and links due to a rounding bug. In particular, empty files + were reported as taking one block instead of zero. - The rounding is now fixed: the number of blocks reported is the file size (or link target size) - divided by 512 and rounded up to a whole number. + The rounding is now fixed: the number of blocks reported is the file size (or + link target size) divided by 512 and rounded up to a whole number. https://github.com/restic/restic/issues/4239 https://github.com/restic/restic/pull/4240 * Bugfix #4253: Minimize risk of spurious filesystem loops with `mount` - When a backup contains a directory that has the same name as its parent, say `a/b/b`, and the GNU - `find` command was run on this backup in a restic mount, `find` would refuse to traverse the - lowest `b` directory, instead printing `File system loop detected`. This was due to the way the - restic mount command generates inode numbers for directories in the mount point. + When a backup contains a directory that has the same name as its parent, say + `a/b/b`, and the GNU `find` command was run on this backup in a restic mount, + `find` would refuse to traverse the lowest `b` directory, instead printing `File + system loop detected`. This was due to the way the restic mount command + generates inode numbers for directories in the mount point. - The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again - to avoid this issue. A perfect rule does not exist, but the probability of this behavior - occurring is now extremely small. + The rule for generating these inode numbers was changed in 0.15.0. It has now + been changed again to avoid this issue. A perfect rule does not exist, but the + probability of this behavior occurring is now extremely small. - When it does occur, the mount point is not broken, and scripts that traverse the mount point - should work as long as they don't rely on inode numbers for detecting filesystem loops. + When it does occur, the mount point is not broken, and scripts that traverse the + mount point should work as long as they don't rely on inode numbers for + detecting filesystem loops. https://github.com/restic/restic/issues/4253 https://github.com/restic/restic/pull/4255 * Enhancement #4180: Add release binaries for riscv64 architecture on Linux - Builds for the `riscv64` architecture on Linux are now included in the release binaries. + Builds for the `riscv64` architecture on Linux are now included in the release + binaries. https://github.com/restic/restic/pull/4180 @@ -771,8 +1506,8 @@ restic users. The changes are ordered by importance. * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend - In some cases the B2 backend could print `b2_download_file_by_name: 404: : b2.b2err` - warnings. These are only debug messages and can be safely ignored. + In some cases the B2 backend could print `b2_download_file_by_name: 404: : + b2.b2err` warnings. These are only debug messages and can be safely ignored. Restic now uses an updated library for accessing B2, which removes the warning. @@ -782,19 +1517,19 @@ restic users. The changes are ordered by importance. * Bugfix #4147: Make `prune --quiet` not print progress bar - A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar while deciding how - to process each pack files. This has now been fixed. + A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar + while deciding how to process each pack files. This has now been fixed. https://github.com/restic/restic/issues/4147 https://github.com/restic/restic/pull/4153 * Bugfix #4163: Make `self-update --output` work with new filename on Windows - Since restic 0.14.0 the `self-update` command did not work when a custom output filename was - specified via the `--output` option. This has now been fixed. + Since restic 0.14.0 the `self-update` command did not work when a custom output + filename was specified via the `--output` option. This has now been fixed. - As a workaround, either use an older restic version to run the self-update or create an empty - file with the output filename before updating e.g. using CMD: + As a workaround, either use an older restic version to run the self-update or + create an empty file with the output filename before updating e.g. using CMD: `type nul > new-file.exe` `restic self-update --output new-file.exe` @@ -803,24 +1538,27 @@ restic users. The changes are ordered by importance. * Bugfix #4167: Add missing ETA in `backup` progress bar - A regression in restic 0.15.0 caused the ETA to be missing from the progress bar displayed by the - `backup` command. This has now been fixed. + A regression in restic 0.15.0 caused the ETA to be missing from the progress bar + displayed by the `backup` command. This has now been fixed. https://github.com/restic/restic/pull/4167 * Enhancement #4143: Ignore empty lock files - With restic 0.15.0 the checks for stale locks became much stricter than before. In particular, - empty or unreadable locks were no longer silently ignored. This made restic to complain with - `Load(, 0, 0) returned error, retrying after 552.330144ms: - load(): invalid data returned` and fail in the end. + With restic 0.15.0 the checks for stale locks became much stricter than before. + In particular, empty or unreadable locks were no longer silently ignored. This + made restic to complain with `Load(, 0, 0) returned error, + retrying after 552.330144ms: load(): invalid data returned` and + fail in the end. - The error message is now clarified and the implementation changed to ignore empty lock files - which are sometimes created as the result of a failed uploads on some backends. + The error message is now clarified and the implementation changed to ignore + empty lock files which are sometimes created as the result of a failed uploads + on some backends. - Please note that unreadable lock files still have to cleaned up manually. To do so, you can run - `restic unlock --remove-all` which removes all existing lock files. But first make sure that - no other restic process is currently using the repository. + Please note that unreadable lock files still have to cleaned up manually. To do + so, you can run `restic unlock --remove-all` which removes all existing lock + files. But first make sure that no other restic process is currently using the + repository. https://github.com/restic/restic/issues/4143 https://github.com/restic/restic/pull/4152 @@ -876,63 +1614,65 @@ restic users. The changes are ordered by importance. * Bugfix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT - To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal to restic. This - used to cause restic to exit with a non-zero exit code. + To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal + to restic. This used to cause restic to exit with a non-zero exit code. - The exit code has now been changed to zero as the above is the expected way to stop the `mount` - command and should therefore be considered successful. + The exit code has now been changed to zero as the above is the expected way to + stop the `mount` command and should therefore be considered successful. https://github.com/restic/restic/issues/2015 https://github.com/restic/restic/pull/3894 * Bugfix #2578: Make `restore` replace existing symlinks - When restoring a symlink, restic used to report an error if the target path already existed. - This has now been fixed such that the potentially existing target path is first removed before - the symlink is restored. + When restoring a symlink, restic used to report an error if the target path + already existed. This has now been fixed such that the potentially existing + target path is first removed before the symlink is restored. https://github.com/restic/restic/issues/2578 https://github.com/restic/restic/pull/3780 * Bugfix #2591: Don't read password from stdin for `backup --stdin` - The `backup` command when used with `--stdin` previously tried to read first the password, - then the data to be backed up from standard input. This meant it would often confuse part of the - data for the password. + The `backup` command when used with `--stdin` previously tried to read first the + password, then the data to be backed up from standard input. This meant it would + often confuse part of the data for the password. - From now on, it will instead exit with the message `Fatal: cannot read both password and data - from stdin` unless the password is passed in some other way (such as - `--restic-password-file`, `RESTIC_PASSWORD`, etc). + From now on, it will instead exit with the message `Fatal: cannot read both + password and data from stdin` unless the password is passed in some other way + (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc). - To enter the password interactively a password command has to be used. For example on Linux, - `mysqldump somedatabase | restic backup --stdin --password-command='sh -c - "systemd-ask-password < /dev/tty"'` securely reads the password from the terminal. + To enter the password interactively a password command has to be used. For + example on Linux, `mysqldump somedatabase | restic backup --stdin + --password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads the + password from the terminal. https://github.com/restic/restic/issues/2591 https://github.com/restic/restic/pull/4011 * Bugfix #3161: Delete files on Backblaze B2 more reliably - Restic used to only delete the latest version of files stored in B2. In most cases this worked - well as there was only a single version of the file. However, due to retries while uploading it is - possible for multiple file versions to be stored at B2. This could lead to various problems for - files that should have been deleted but still existed. + Restic used to only delete the latest version of files stored in B2. In most + cases this worked well as there was only a single version of the file. However, + due to retries while uploading it is possible for multiple file versions to be + stored at B2. This could lead to various problems for files that should have + been deleted but still existed. - The implementation has now been changed to delete all versions of files, which doubles the - amount of Class B transactions necessary to delete files, but assures that no file versions are - left behind. + The implementation has now been changed to delete all versions of files, which + doubles the amount of Class B transactions necessary to delete files, but + assures that no file versions are left behind. https://github.com/restic/restic/issues/3161 https://github.com/restic/restic/pull/3885 * Bugfix #3336: Make SFTP backend report no space left on device - Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when the remote disk - was full. Restic now reports "sftp: no space left on device" and exits immediately when it - detects this condition. + Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when + the remote disk was full. Restic now reports "sftp: no space left on device" and + exits immediately when it detects this condition. - A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix itself - contained a bug that prevented it from taking effect. + A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix + itself contained a bug that prevented it from taking effect. https://github.com/restic/restic/issues/3336 https://github.com/restic/restic/pull/3345 @@ -940,9 +1680,10 @@ restic users. The changes are ordered by importance. * Bugfix #3567: Improve handling of interrupted syscalls in `mount` command - Accessing restic's FUSE mount could result in "input/output" errors when using programs in - which syscalls can be interrupted. This is for example the case for Go programs. This has now - been fixed by improved error handling of interrupted syscalls. + Accessing restic's FUSE mount could result in "input/output" errors when using + programs in which syscalls can be interrupted. This is for example the case for + Go programs. This has now been fixed by improved error handling of interrupted + syscalls. https://github.com/restic/restic/issues/3567 https://github.com/restic/restic/issues/3694 @@ -950,50 +1691,53 @@ restic users. The changes are ordered by importance. * Bugfix #3897: Fix stuck `copy` command when `-o .connections=1` - When running the `copy` command with `-o .connections=1` the command would be - infinitely stuck. This has now been fixed. + When running the `copy` command with `-o .connections=1` the command + would be infinitely stuck. This has now been fixed. https://github.com/restic/restic/issues/3897 https://github.com/restic/restic/pull/3898 * Bugfix #3918: Correct prune statistics for partially compressed repositories - In a partially compressed repository, one data blob can exist both in an uncompressed and a - compressed version. This caused the `prune` statistics to become inaccurate and e.g. report a - too high value for the unused size, such as "unused size after prune: 16777215.991 TiB". This - has now been fixed. + In a partially compressed repository, one data blob can exist both in an + uncompressed and a compressed version. This caused the `prune` statistics to + become inaccurate and e.g. report a too high value for the unused size, such as + "unused size after prune: 16777215.991 TiB". This has now been fixed. https://github.com/restic/restic/issues/3918 https://github.com/restic/restic/pull/3980 * Bugfix #3951: Make `ls` return exit code 1 if snapshot cannot be loaded - The `ls` command used to show a warning and return exit code 0 when failing to load a snapshot. - This has now been fixed such that it instead returns exit code 1 (still showing a warning). + The `ls` command used to show a warning and return exit code 0 when failing to + load a snapshot. This has now been fixed such that it instead returns exit code + 1 (still showing a warning). https://github.com/restic/restic/pull/3951 * Bugfix #4003: Make `backup` no longer hang on Solaris when seeing a FIFO file - The `backup` command used to hang on Solaris whenever it encountered a FIFO file (named pipe), - due to a bug in the handling of extended attributes. This bug has now been fixed. + The `backup` command used to hang on Solaris whenever it encountered a FIFO file + (named pipe), due to a bug in the handling of extended attributes. This bug has + now been fixed. https://github.com/restic/restic/issues/4003 https://github.com/restic/restic/pull/4053 * Bugfix #4016: Support ExFAT-formatted local backends on macOS Ventura - ExFAT-formatted disks could not be used as local backends starting from macOS Ventura. Restic - commands would fail with an "inappropriate ioctl for device" error. This has now been fixed. + ExFAT-formatted disks could not be used as local backends starting from macOS + Ventura. Restic commands would fail with an "inappropriate ioctl for device" + error. This has now been fixed. https://github.com/restic/restic/issues/4016 https://github.com/restic/restic/pull/4021 * Bugfix #4085: Make `init` ignore "Access Denied" errors when creating S3 buckets - In restic 0.9.0 through 0.13.0, the `init` command ignored some permission errors from S3 - backends when trying to check for bucket existence, so that manually created buckets with - custom permissions could be used for backups. + In restic 0.9.0 through 0.13.0, the `init` command ignored some permission + errors from S3 backends when trying to check for bucket existence, so that + manually created buckets with custom permissions could be used for backups. This feature became broken in 0.14.0, but has now been restored again. @@ -1002,20 +1746,21 @@ restic users. The changes are ordered by importance. * Bugfix #4100: Make `self-update` enabled by default only in release builds - The `self-update` command was previously included by default in all builds of restic as - opposed to only in official release builds, even if the `selfupdate` tag was not explicitly - enabled when building. + The `self-update` command was previously included by default in all builds of + restic as opposed to only in official release builds, even if the `selfupdate` + tag was not explicitly enabled when building. - This has now been corrected, and the `self-update` command is only available if restic was - built with `-tags selfupdate` (as done for official release builds by `build.go`). + This has now been corrected, and the `self-update` command is only available if + restic was built with `-tags selfupdate` (as done for official release builds by + `build.go`). https://github.com/restic/restic/pull/4100 * Bugfix #4103: Don't generate negative UIDs and GIDs in tar files from `dump` - When using a 32-bit build of restic, the `dump` command could in some cases create tar files - containing negative UIDs and GIDs, which cannot be read by GNU tar. This corner case especially - applies to backups from stdin on Windows. + When using a 32-bit build of restic, the `dump` command could in some cases + create tar files containing negative UIDs and GIDs, which cannot be read by GNU + tar. This corner case especially applies to backups from stdin on Windows. This is now fixed such that `dump` creates valid tar files in these cases too. @@ -1024,48 +1769,50 @@ restic users. The changes are ordered by importance. * Change #2724: Include full snapshot ID in JSON output of `backup` - We have changed the JSON output of the backup command to include the full snapshot ID instead of - just a shortened version, as the latter can be ambiguous in some rare cases. To derive the short - ID, please truncate the full ID down to eight characters. + We have changed the JSON output of the backup command to include the full + snapshot ID instead of just a shortened version, as the latter can be ambiguous + in some rare cases. To derive the short ID, please truncate the full ID down to + eight characters. https://github.com/restic/restic/issues/2724 https://github.com/restic/restic/pull/3993 * Change #3929: Make `unlock` display message only when locks were actually removed - The `unlock` command used to print the "successfully removed locks" message whenever it was - run, regardless of lock files having being removed or not. + The `unlock` command used to print the "successfully removed locks" message + whenever it was run, regardless of lock files having being removed or not. - This has now been changed such that it only prints the message if any lock files were actually - removed. In addition, it also reports the number of removed lock files. + This has now been changed such that it only prints the message if any lock files + were actually removed. In addition, it also reports the number of removed lock + files. https://github.com/restic/restic/issues/3929 https://github.com/restic/restic/pull/3935 * Change #4033: Don't print skipped snapshots by default in `copy` command - The `copy` command used to print each snapshot that was skipped because it already existed in - the target repository. The amount of this output could practically bury the list of snapshots - that were actually copied. + The `copy` command used to print each snapshot that was skipped because it + already existed in the target repository. The amount of this output could + practically bury the list of snapshots that were actually copied. - From now on, the skipped snapshots are by default not printed at all, but this can be re-enabled - by increasing the verbosity level of the command. + From now on, the skipped snapshots are by default not printed at all, but this + can be re-enabled by increasing the verbosity level of the command. https://github.com/restic/restic/issues/4033 https://github.com/restic/restic/pull/4066 * Change #4041: Update dependencies and require Go 1.18 or newer - Most dependencies have been updated. Since some libraries require newer language features, - support for Go 1.15-1.17 has been dropped, which means that restic now requires at least Go 1.18 - to build. + Most dependencies have been updated. Since some libraries require newer language + features, support for Go 1.15-1.17 has been dropped, which means that restic now + requires at least Go 1.18 to build. https://github.com/restic/restic/pull/4041 * Enhancement #14: Implement `rewrite` command - Restic now has a `rewrite` command which allows to rewrite existing snapshots to remove - unwanted files. + Restic now has a `rewrite` command which allows to rewrite existing snapshots to + remove unwanted files. https://github.com/restic/restic/issues/14 https://github.com/restic/restic/pull/2731 @@ -1073,15 +1820,15 @@ restic users. The changes are ordered by importance. * Enhancement #79: Restore files with long runs of zeros as sparse files - When using `restore --sparse`, the restorer may now write files containing long runs of zeros - as sparse files (also called files with holes), where the zeros are not actually written to - disk. + When using `restore --sparse`, the restorer may now write files containing long + runs of zeros as sparse files (also called files with holes), where the zeros + are not actually written to disk. - How much space is saved by writing sparse files depends on the operating system, file system and - the distribution of zeros in the file. + How much space is saved by writing sparse files depends on the operating system, + file system and the distribution of zeros in the file. - During backup restic still reads the whole file including sparse regions, but with optimized - processing speed of sparse regions. + During backup restic still reads the whole file including sparse regions, but + with optimized processing speed of sparse regions. https://github.com/restic/restic/issues/79 https://github.com/restic/restic/issues/3903 @@ -1091,9 +1838,9 @@ restic users. The changes are ordered by importance. * Enhancement #1078: Support restoring symbolic links on Windows - The `restore` command now supports restoring symbolic links on Windows. Because of Windows - specific restrictions this is only possible when running restic with the - `SeCreateSymbolicLinkPrivilege` privilege or as an administrator. + The `restore` command now supports restoring symbolic links on Windows. Because + of Windows specific restrictions this is only possible when running restic with + the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator. https://github.com/restic/restic/issues/1078 https://github.com/restic/restic/issues/2699 @@ -1101,14 +1848,14 @@ restic users. The changes are ordered by importance. * Enhancement #1734: Inform about successful retries after errors - When a recoverable error is encountered, restic shows a warning message saying that it's - retrying, e.g.: + When a recoverable error is encountered, restic shows a warning message saying + that it's retrying, e.g.: `Save() returned error, retrying after 357.131936ms: ...` - This message can be confusing in that it never clearly states whether the retry is successful or - not. This has now been fixed such that restic follows up with a message confirming a successful - retry, e.g.: + This message can be confusing in that it never clearly states whether the retry + is successful or not. This has now been fixed such that restic follows up with a + message confirming a successful retry, e.g.: `Save() operation successful after 1 retries` @@ -1117,12 +1864,12 @@ restic users. The changes are ordered by importance. * Enhancement #1866: Improve handling of directories with duplicate entries - If for some reason a directory contains a duplicate entry, the `backup` command would - previously fail with a `node "path/to/file" already present` or `nodes are not ordered got - "path/to/file", last "path/to/file"` error. + If for some reason a directory contains a duplicate entry, the `backup` command + would previously fail with a `node "path/to/file" already present` or `nodes are + not ordered got "path/to/file", last "path/to/file"` error. - The error handling has been improved to only report a warning in this case. Make sure to check - that the filesystem in question is not damaged if you see this! + The error handling has been improved to only report a warning in this case. Make + sure to check that the filesystem in question is not damaged if you see this! https://github.com/restic/restic/issues/1866 https://github.com/restic/restic/issues/3937 @@ -1130,29 +1877,31 @@ restic users. The changes are ordered by importance. * Enhancement #2134: Support B2 API keys restricted to hiding but not deleting files - When the B2 backend does not have the necessary permissions to permanently delete files, it now - automatically falls back to hiding files. This allows using restic with an application key - which is not allowed to delete files. This can prevent an attacker from deleting backups with - such an API key. + When the B2 backend does not have the necessary permissions to permanently + delete files, it now automatically falls back to hiding files. This allows using + restic with an application key which is not allowed to delete files. This can + prevent an attacker from deleting backups with such an API key. - To use this feature create an application key without the `deleteFiles` capability. It is - recommended to restrict the key to just one bucket. For example using the `b2` command line - tool: + To use this feature create an application key without the `deleteFiles` + capability. It is recommended to restrict the key to just one bucket. For + example using the `b2` command line tool: `b2 create-key --bucket listBuckets,readFiles,writeFiles,listFiles` - Alternatively, you can use the S3 backend to access B2, as described in the documentation. In - this mode, files are also only hidden instead of being deleted permanently. + Alternatively, you can use the S3 backend to access B2, as described in the + documentation. In this mode, files are also only hidden instead of being deleted + permanently. https://github.com/restic/restic/issues/2134 https://github.com/restic/restic/pull/2398 * Enhancement #2152: Make `init` open only one connection for the SFTP backend - The `init` command using the SFTP backend used to connect twice to the repository. This could be - inconvenient if the user must enter a password, or cause `init` to fail if the server does not - correctly close the first SFTP connection. + The `init` command using the SFTP backend used to connect twice to the + repository. This could be inconvenient if the user must enter a password, or + cause `init` to fail if the server does not correctly close the first SFTP + connection. This has now been fixed by reusing the first/initial SFTP connection opened. @@ -1161,40 +1910,44 @@ restic users. The changes are ordered by importance. * Enhancement #2533: Handle cache corruption on disk and in downloads - In rare situations, like for example after a system crash, the data stored in the cache might be - corrupted. This could cause restic to fail and required manually deleting the cache. + In rare situations, like for example after a system crash, the data stored in + the cache might be corrupted. This could cause restic to fail and required + manually deleting the cache. - Restic now automatically removes broken data from the cache, allowing it to recover from such a - situation without user intervention. In addition, restic retries downloads which return - corrupt data in order to also handle temporary download problems. + Restic now automatically removes broken data from the cache, allowing it to + recover from such a situation without user intervention. In addition, restic + retries downloads which return corrupt data in order to also handle temporary + download problems. https://github.com/restic/restic/issues/2533 https://github.com/restic/restic/pull/3521 * Enhancement #2715: Stricter repository lock handling - Previously, restic commands kept running even if they failed to refresh their locks in time. - This could be a problem e.g. in case the client system running a backup entered the standby power - mode while the backup was still in progress (which would prevent the client from refreshing its - lock), and after a short delay another host successfully runs `unlock` and `prune` on the - repository, which would remove all data added by the in-progress backup. If the backup client - later continues its backup, even though its lock had expired in the meantime, this would lead to - an incomplete snapshot. - - To address this, lock handling is now much stricter. Commands requiring a lock are canceled if - the lock is not refreshed successfully in time. In addition, if a lock file is not readable - restic will not allow starting a command. It may be necessary to remove invalid lock files - manually or use `unlock --remove-all`. Please make sure that no other restic processes are - running concurrently before doing this, however. + Previously, restic commands kept running even if they failed to refresh their + locks in time. This could be a problem e.g. in case the client system running a + backup entered the standby power mode while the backup was still in progress + (which would prevent the client from refreshing its lock), and after a short + delay another host successfully runs `unlock` and `prune` on the repository, + which would remove all data added by the in-progress backup. If the backup + client later continues its backup, even though its lock had expired in the + meantime, this would lead to an incomplete snapshot. + + To address this, lock handling is now much stricter. Commands requiring a lock + are canceled if the lock is not refreshed successfully in time. In addition, if + a lock file is not readable restic will not allow starting a command. It may be + necessary to remove invalid lock files manually or use `unlock --remove-all`. + Please make sure that no other restic processes are running concurrently before + doing this, however. https://github.com/restic/restic/issues/2715 https://github.com/restic/restic/pull/3569 * Enhancement #2750: Make backup file read concurrency configurable - The `backup` command now supports a `--read-concurrency` option which allows tuning restic - for very fast storage like NVMe disks by controlling the number of concurrent file reads during - the backup process. + The `backup` command now supports a `--read-concurrency` option which allows + tuning restic for very fast storage like NVMe disks by controlling the number of + concurrent file reads during the backup process. https://github.com/restic/restic/pull/2750 @@ -1209,75 +1962,78 @@ restic users. The changes are ordered by importance. * Enhancement #3096: Make `mount` command support macOS using macFUSE 4.x - Restic now uses a different FUSE library for mounting snapshots and making them available as a - FUSE filesystem using the `mount` command. This adds support for macFUSE 4.x which can be used - to make this work on recent macOS versions. + Restic now uses a different FUSE library for mounting snapshots and making them + available as a FUSE filesystem using the `mount` command. This adds support for + macFUSE 4.x which can be used to make this work on recent macOS versions. https://github.com/restic/restic/issues/3096 https://github.com/restic/restic/pull/4024 * Enhancement #3124: Support JSON output for the `init` command - The `init` command used to ignore the `--json` option, but now outputs a JSON message if the - repository was created successfully. + The `init` command used to ignore the `--json` option, but now outputs a JSON + message if the repository was created successfully. https://github.com/restic/restic/issues/3124 https://github.com/restic/restic/pull/3132 * Enhancement #3899: Optimize prune memory usage - The `prune` command needs large amounts of memory in order to determine what to keep and what to - remove. This is now optimized to use up to 30% less memory. + The `prune` command needs large amounts of memory in order to determine what to + keep and what to remove. This is now optimized to use up to 30% less memory. https://github.com/restic/restic/pull/3899 * Enhancement #3905: Improve speed of parent snapshot detection in `backup` command - Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw` - options could require a long time to find the parent snapshot. This has been improved. + Backing up a large number of files using `--files-from-verbatim` or + `--files-from-raw` options could require a long time to find the parent + snapshot. This has been improved. https://github.com/restic/restic/pull/3905 * Enhancement #3915: Add compression statistics to the `stats` command - When executed with `--mode raw-data` on a repository that supports compression, the `stats` - command now calculates and displays, for the selected repository or snapshots: the - uncompressed size of the data; the compression progress (percentage of data that has been - compressed); the compression ratio of the compressed data; the total space saving. + When executed with `--mode raw-data` on a repository that supports compression, + the `stats` command now calculates and displays, for the selected repository or + snapshots: the uncompressed size of the data; the compression progress + (percentage of data that has been compressed); the compression ratio of the + compressed data; the total space saving. - It also takes into account both the compressed and uncompressed data if the repository is only - partially compressed. + It also takes into account both the compressed and uncompressed data if the + repository is only partially compressed. https://github.com/restic/restic/pull/3915 * Enhancement #3925: Provide command completion for PowerShell - Restic already provided generation of completion files for bash, fish and zsh. Now powershell - is supported, too. + Restic already provided generation of completion files for bash, fish and zsh. + Now powershell is supported, too. https://github.com/restic/restic/pull/3925/files * Enhancement #3931: Allow `backup` file tree scanner to be disabled - The `backup` command walks the file tree in a separate scanner process to find the total size and - file/directory count, and uses this to provide an ETA. This can slow down backups, especially - of network filesystems. + The `backup` command walks the file tree in a separate scanner process to find + the total size and file/directory count, and uses this to provide an ETA. This + can slow down backups, especially of network filesystems. - The command now has a new option `--no-scan` which can be used to disable this scanning in order - to speed up backups when needed. + The command now has a new option `--no-scan` which can be used to disable this + scanning in order to speed up backups when needed. https://github.com/restic/restic/pull/3931 * Enhancement #3932: Improve handling of ErrDot errors in rclone and sftp backends - Since Go 1.19, restic can no longer implicitly run relative executables which are found in the - current directory (e.g. `rclone` if found in `.`). This is a security feature of Go to prevent - against running unintended and possibly harmful executables. + Since Go 1.19, restic can no longer implicitly run relative executables which + are found in the current directory (e.g. `rclone` if found in `.`). This is a + security feature of Go to prevent against running unintended and possibly + harmful executables. - The error message for this was just "cannot run executable found relative to current - directory". This has now been improved to yield a more specific error message, informing the - user how to explicitly allow running the executable using the `-o rclone.program` and `-o - sftp.command` extended options with `./`. + The error message for this was just "cannot run executable found relative to + current directory". This has now been improved to yield a more specific error + message, informing the user how to explicitly allow running the executable using + the `-o rclone.program` and `-o sftp.command` extended options with `./`. https://github.com/restic/restic/issues/3932 https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory @@ -1285,20 +2041,21 @@ restic users. The changes are ordered by importance. * Enhancement #3943: Ignore additional/unknown files in repository - If a restic repository had additional files in it (not created by restic), commands like `find` - and `restore` could become confused and fail with an `multiple IDs with prefix "12345678" - found` error. These commands now ignore such additional files. + If a restic repository had additional files in it (not created by restic), + commands like `find` and `restore` could become confused and fail with an + `multiple IDs with prefix "12345678" found` error. These commands now ignore + such additional files. https://github.com/restic/restic/pull/3943 https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17 * Enhancement #3955: Improve `backup` performance for small files - When backing up small files restic was slower than it could be. In particular this affected - backups using maximum compression. + When backing up small files restic was slower than it could be. In particular + this affected backups using maximum compression. - This has been fixed by reworking the internal parallelism of the backup command, making it back - up small files around two times faster. + This has been fixed by reworking the internal parallelism of the backup command, + making it back up small files around two times faster. https://github.com/restic/restic/pull/3955 @@ -1347,22 +2104,23 @@ restic users. The changes are ordered by importance. * Bugfix #2248: Support `self-update` on Windows - Restic `self-update` would fail in situations where the operating system locks running - binaries, including Windows. The new behavior works around this by renaming the running file - and swapping the updated file in place. + Restic `self-update` would fail in situations where the operating system locks + running binaries, including Windows. The new behavior works around this by + renaming the running file and swapping the updated file in place. https://github.com/restic/restic/issues/2248 https://github.com/restic/restic/pull/3675 * Bugfix #3428: List snapshots in backend at most once to resolve snapshot IDs - Many commands support specifying a list of snapshot IDs which are then used to determine the - snapshots to be processed by the command. To resolve snapshot IDs or `latest`, and check that - these exist, restic previously listed all snapshots stored in the repository. Depending on - the backend this could be a slow and/or expensive operation. + Many commands support specifying a list of snapshot IDs which are then used to + determine the snapshots to be processed by the command. To resolve snapshot IDs + or `latest`, and check that these exist, restic previously listed all snapshots + stored in the repository. Depending on the backend this could be a slow and/or + expensive operation. - Restic now lists the snapshots only once and remembers the result in order to resolve all - further snapshot IDs swiftly. + Restic now lists the snapshots only once and remembers the result in order to + resolve all further snapshot IDs swiftly. https://github.com/restic/restic/issues/3428 https://github.com/restic/restic/pull/3570 @@ -1370,27 +2128,28 @@ restic users. The changes are ordered by importance. * Bugfix #3432: Fix rare 'not found in repository' error for `copy` command - In rare cases `copy` (and other commands) would report that `LoadTree(...)` returned an `id - [...] not found in repository` error. This could be caused by a backup or copy command running - concurrently. The error was only temporary; running the failed restic command a second time as - a workaround did resolve the error. + In rare cases `copy` (and other commands) would report that `LoadTree(...)` + returned an `id [...] not found in repository` error. This could be caused by a + backup or copy command running concurrently. The error was only temporary; + running the failed restic command a second time as a workaround did resolve the + error. - This issue has now been fixed by correcting the order in which restic reads data from the - repository. It is now guaranteed that restic only loads snapshots for which all necessary data - is already available. + This issue has now been fixed by correcting the order in which restic reads data + from the repository. It is now guaranteed that restic only loads snapshots for + which all necessary data is already available. https://github.com/restic/restic/issues/3432 https://github.com/restic/restic/pull/3570 * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows - In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when - Ctrl+C is pressed on Windows. The solution was to create the subprocess with its console - detached from the restic console. + In #3602 a fix was introduced to address the problem of `rclone` prematurely + exiting when Ctrl+C is pressed on Windows. The solution was to create the + subprocess with its console detached from the restic console. - However, this solution failed when using `rclone` installed by Scoop or using `sftp` with a - passphrase-protected private key. We've now fixed this by using a different approach to - prevent Ctrl-C from passing down too early. + However, this solution failed when using `rclone` installed by Scoop or using + `sftp` with a passphrase-protected private key. We've now fixed this by using a + different approach to prevent Ctrl-C from passing down too early. https://github.com/restic/restic/issues/3681 https://github.com/restic/restic/issues/3692 @@ -1398,28 +2157,28 @@ restic users. The changes are ordered by importance. * Bugfix #3685: The `diff` command incorrectly listed some files as added - There was a bug in the `diff` command, causing it to always show files in a removed directory as - added. This has now been fixed. + There was a bug in the `diff` command, causing it to always show files in a + removed directory as added. This has now been fixed. https://github.com/restic/restic/issues/3685 https://github.com/restic/restic/pull/3686 * Bugfix #3716: Print "wrong password" to stderr instead of stdout - If an invalid password was entered, the error message was printed on stdout and not on stderr as - intended. This has now been fixed. + If an invalid password was entered, the error message was printed on stdout and + not on stderr as intended. This has now been fixed. https://github.com/restic/restic/pull/3716 https://forum.restic.net/t/4965 * Bugfix #3720: Directory sync errors for repositories accessed via SMB - On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to - save the lock file, yielding the following errors: + On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in + restic failing to save the lock file, yielding the following errors: - Save() returned error, retrying after 552.330144ms: sync /repo/locks: - no such file or directory Save() returned error, retrying after - 552.330144ms: sync /repo/locks: invalid argument + Save() returned error, retrying after 552.330144ms: sync + /repo/locks: no such file or directory Save() returned error, + retrying after 552.330144ms: sync /repo/locks: invalid argument This has now been fixed by ignoring the relevant error codes. @@ -1429,22 +2188,23 @@ restic users. The changes are ordered by importance. * Bugfix #3736: The `stats` command miscalculated restore size for multiple snapshots - Since restic 0.10.0 the restore size calculated by the `stats` command for multiple snapshots - was too low. The hardlink detection was accidentally applied across multiple snapshots and - thus ignored many files. This has now been fixed. + Since restic 0.10.0 the restore size calculated by the `stats` command for + multiple snapshots was too low. The hardlink detection was accidentally applied + across multiple snapshots and thus ignored many files. This has now been fixed. https://github.com/restic/restic/issues/3736 https://github.com/restic/restic/pull/3740 * Bugfix #3772: Correctly rebuild index for legacy repositories - After running `rebuild-index` on a legacy repository containing mixed pack files (that is, - pack files which store both metadata and file data), `check` printed warnings like `pack - 12345678 contained in several indexes: ...`. This warning was not critical, but has now - nonetheless been fixed by properly handling mixed pack files while rebuilding the index. + After running `rebuild-index` on a legacy repository containing mixed pack files + (that is, pack files which store both metadata and file data), `check` printed + warnings like `pack 12345678 contained in several indexes: ...`. This warning + was not critical, but has now nonetheless been fixed by properly handling mixed + pack files while rebuilding the index. - Running `prune` for such legacy repositories will also fix the warning by reorganizing the - pack files which caused it. + Running `prune` for such legacy repositories will also fix the warning by + reorganizing the pack files which caused it. https://github.com/restic/restic/pull/3772 https://github.com/restic/restic/pull/3884 @@ -1452,18 +2212,20 @@ restic users. The changes are ordered by importance. * Bugfix #3776: Limit number of key files tested while opening a repository - Previously, restic tested the password against every key in the repository when opening a - repository. The more keys there were in the repository, the slower this operation became. + Previously, restic tested the password against every key in the repository when + opening a repository. The more keys there were in the repository, the slower + this operation became. - Restic now tests the password against up to 20 key files in the repository. Alternatively, you - can use the `--key-hint=` option to specify a specific key file to use instead. + Restic now tests the password against up to 20 key files in the repository. + Alternatively, you can use the `--key-hint=` option to specify a + specific key file to use instead. https://github.com/restic/restic/pull/3776 * Bugfix #3861: Yield error on invalid policy to `forget` - The `forget` command previously silently ignored invalid/unsupported units in the duration - options, such as e.g. `--keep-within-daily 2w`. + The `forget` command previously silently ignored invalid/unsupported units in + the duration options, such as e.g. `--keep-within-daily 2w`. Specifying an invalid/unsupported duration unit now results in an error. @@ -1472,71 +2234,78 @@ restic users. The changes are ordered by importance. * Change #1842: Support debug log creation in release builds - Creating a debug log was only possible in debug builds which required users to manually build - restic. We changed the release builds to allow creating debug logs by simply setting the - environment variable `DEBUG_LOG=logname.log`. + Creating a debug log was only possible in debug builds which required users to + manually build restic. We changed the release builds to allow creating debug + logs by simply setting the environment variable `DEBUG_LOG=logname.log`. https://github.com/restic/restic/issues/1842 https://github.com/restic/restic/pull/3826 * Change #3295: Deprecate `check --check-unused` and add further checks - Since restic 0.12.0, it is expected to still have unused blobs after running `prune`. This made - the `--check-unused` option of the `check` command rather useless and tended to confuse - users. This option has been deprecated and is now ignored. + Since restic 0.12.0, it is expected to still have unused blobs after running + `prune`. This made the `--check-unused` option of the `check` command rather + useless and tended to confuse users. This option has been deprecated and is now + ignored. - The `check` command now also warns if a repository is using either the legacy S3 layout or mixed - pack files with both tree and data blobs. The latter is known to cause performance problems. + The `check` command now also warns if a repository is using either the legacy S3 + layout or mixed pack files with both tree and data blobs. The latter is known to + cause performance problems. https://github.com/restic/restic/issues/3295 https://github.com/restic/restic/pull/3730 * Change #3680: Update dependencies and require Go 1.15 or newer - We've updated most dependencies. Since some libraries require newer language features we're - dropping support for Go 1.14, which means that restic now requires at least Go 1.15 to build. + We've updated most dependencies. Since some libraries require newer language + features we're dropping support for Go 1.14, which means that restic now + requires at least Go 1.15 to build. https://github.com/restic/restic/issues/3680 https://github.com/restic/restic/issues/3883 * Change #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo` - The `init` and `copy` commands can read data from another repository. However, confusingly - `--repo2` referred to the repository *from* which the `init` command copies parameters, but - for the `copy` command `--repo2` referred to the copy *destination*. + The `init` and `copy` commands can read data from another repository. However, + confusingly `--repo2` referred to the repository *from* which the `init` command + copies parameters, but for the `copy` command `--repo2` referred to the copy + *destination*. - We've introduced a new option, `--from-repo`, which always refers to the source repository - for both commands. The old parameter names have been deprecated but still work. To create a new - repository and copy all snapshots to it, the commands are now as follows: + We've introduced a new option, `--from-repo`, which always refers to the source + repository for both commands. The old parameter names have been deprecated but + still work. To create a new repository and copy all snapshots to it, the + commands are now as follows: - ``` restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo - --copy-chunker-params restic -r /srv/restic-repo-copy copy --from-repo - /srv/restic-repo ``` + ``` + restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params + restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo + ``` https://github.com/restic/restic/pull/3742 https://forum.restic.net/t/5017 * Enhancement #21: Add compression support - We've added compression support to the restic repository format. To create a repository using - the new format run `init --repository-version 2`. Please note that the repository cannot be - read by restic versions prior to 0.14.0. + We've added compression support to the restic repository format. To create a + repository using the new format run `init --repository-version 2`. Please note + that the repository cannot be read by restic versions prior to 0.14.0. - You can configure whether data is compressed with the option `--compression`. It can be set to - `auto` (the default, which will compress very fast), `max` (which will trade backup speed and - CPU usage for better compression), or `off` (which disables compression). Each setting is - only applied for the current run of restic and does *not* apply to future runs. The option can - also be set via the environment variable `RESTIC_COMPRESSION`. + You can configure whether data is compressed with the option `--compression`. It + can be set to `auto` (the default, which will compress very fast), `max` (which + will trade backup speed and CPU usage for better compression), or `off` (which + disables compression). Each setting is only applied for the current run of + restic and does *not* apply to future runs. The option can also be set via the + environment variable `RESTIC_COMPRESSION`. - To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the documentation - for more details. The migration checks the repository integrity and upgrades the repository - format, but will not change any data. Afterwards, prune will rewrite the metadata to make use of - compression. + To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the + documentation for more details. The migration checks the repository integrity + and upgrades the repository format, but will not change any data. Afterwards, + prune will rewrite the metadata to make use of compression. - As an alternative you can use the `copy` command to migrate snapshots; First create a new - repository using `init --repository-version 2 --copy-chunker-params --repo2 - path/to/old/repo`, and then use the `copy` command to copy all snapshots to the new - repository. + As an alternative you can use the `copy` command to migrate snapshots; First + create a new repository using `init --repository-version 2 --copy-chunker-params + --repo2 path/to/old/repo`, and then use the `copy` command to copy all snapshots + to the new repository. https://github.com/restic/restic/issues/21 https://github.com/restic/restic/issues/3779 @@ -1546,25 +2315,28 @@ restic users. The changes are ordered by importance. * Enhancement #1153: Support pruning even when the disk is full - When running out of disk space it was no longer possible to add or remove data from a repository. - To help with recovering from such a deadlock, the prune command now supports an - `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to - read the documentation first! + When running out of disk space it was no longer possible to add or remove data + from a repository. To help with recovering from such a deadlock, the prune + command now supports an `--unsafe-recover-no-free-space` option to recover from + these situations. Make sure to read the documentation first! https://github.com/restic/restic/issues/1153 https://github.com/restic/restic/pull/3481 * Enhancement #2162: Adaptive IO concurrency based on backend connections - Many commands used hard-coded limits for the number of concurrent operations. This prevented - speed improvements by increasing the number of connections used by a backend. + Many commands used hard-coded limits for the number of concurrent operations. + This prevented speed improvements by increasing the number of connections used + by a backend. - These limits have now been replaced by using the configured number of backend connections - instead, which can be controlled using the `-o .connections=5` option. - Commands will then automatically scale their parallelism accordingly. + These limits have now been replaced by using the configured number of backend + connections instead, which can be controlled using the `-o + .connections=5` option. Commands will then automatically scale + their parallelism accordingly. - To limit the number of CPU cores used by restic, you can set the environment variable - `GOMAXPROCS` accordingly. For example to use a single CPU core, use `GOMAXPROCS=1`. + To limit the number of CPU cores used by restic, you can set the environment + variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use + `GOMAXPROCS=1`. https://github.com/restic/restic/issues/2162 https://github.com/restic/restic/issues/1467 @@ -1572,45 +2344,47 @@ restic users. The changes are ordered by importance. * Enhancement #2291: Allow pack size customization - Restic now uses a target pack size of 16 MiB by default. This can be customized using the - `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB. + Restic now uses a target pack size of 16 MiB by default. This can be customized + using the `--pack-size size` option. Supported pack sizes range between 4 and + 128 MiB. - It is possible to migrate an existing repository to _larger_ pack files using `prune - --repack-small`. This will rewrite every pack file which is significantly smaller than the - target size. + It is possible to migrate an existing repository to _larger_ pack files using + `prune --repack-small`. This will rewrite every pack file which is significantly + smaller than the target size. https://github.com/restic/restic/issues/2291 https://github.com/restic/restic/pull/3731 * Enhancement #2295: Allow use of SAS token to authenticate to Azure - Previously restic only supported AccountKeys to authenticate to Azure storage accounts, - which necessitates giving a significant amount of access. + Previously restic only supported AccountKeys to authenticate to Azure storage + accounts, which necessitates giving a significant amount of access. - We added support for Azure SAS tokens which are a more fine-grained and time-limited manner of - granting access. Set the `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_SAS` environment - variables to use a SAS token for authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it - will take precedence. + We added support for Azure SAS tokens which are a more fine-grained and + time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME` and + `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for authentication. + Note that if `AZURE_ACCOUNT_KEY` is set, it will take precedence. https://github.com/restic/restic/issues/2295 https://github.com/restic/restic/pull/3661 * Enhancement #2351: Use config file permissions to control file group access - Previously files in a local/SFTP repository would always end up with very restrictive access - permissions, allowing access only to the owner. This prevented a number of valid use-cases - involving groups and ACLs. + Previously files in a local/SFTP repository would always end up with very + restrictive access permissions, allowing access only to the owner. This + prevented a number of valid use-cases involving groups and ACLs. - We now use the permissions of the config file in the repository to decide whether group access - should be given to newly created repository files or not. We arrange for repository files to be - created group readable exactly when the repository config file is group readable. + We now use the permissions of the config file in the repository to decide + whether group access should be given to newly created repository files or not. + We arrange for repository files to be created group readable exactly when the + repository config file is group readable. - To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config - file can be used. For repositories that should be writable by group members a tad more setup is - required, see the docs. + To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent + on the config file can be used. For repositories that should be writable by + group members a tad more setup is required, see the docs. - Posix ACLs can also be used now that the group permissions being forced to zero no longer masks - the effect of ACL entries. + Posix ACLs can also be used now that the group permissions being forced to zero + no longer masks the effect of ACL entries. https://github.com/restic/restic/issues/2351 https://github.com/restic/restic/pull/3419 @@ -1618,27 +2392,29 @@ restic users. The changes are ordered by importance. * Enhancement #2696: Improve backup speed with many small files - We have restructured the backup pipeline to continue reading files while all upload - connections are busy. This allows the backup to already prepare the next data file such that the - upload can continue as soon as a connection becomes available. This can especially improve the - backup performance for high latency backends. + We have restructured the backup pipeline to continue reading files while all + upload connections are busy. This allows the backup to already prepare the next + data file such that the upload can continue as soon as a connection becomes + available. This can especially improve the backup performance for high latency + backends. - The upload concurrency is now controlled using the `-o .connections=5` - option. + The upload concurrency is now controlled using the `-o + .connections=5` option. https://github.com/restic/restic/issues/2696 https://github.com/restic/restic/pull/3489 * Enhancement #2907: Make snapshot directory structure of `mount` command customizable - We've added the possibility to customize the snapshot directory structure of the `mount` - command using templates passed to the `--snapshot-template` option. The formatting of - snapshots' timestamps is now controlled using `--time-template` and supports - subdirectories to for example group snapshots by year. Please see `restic help mount` for - further details. + We've added the possibility to customize the snapshot directory structure of the + `mount` command using templates passed to the `--snapshot-template` option. The + formatting of snapshots' timestamps is now controlled using `--time-template` + and supports subdirectories to for example group snapshots by year. Please see + `restic help mount` for further details. - Characters in tag names which are not allowed in a filename are replaced by underscores `_`. For - example a tag `foo/bar` will result in a directory name of `foo_bar`. + Characters in tag names which are not allowed in a filename are replaced by + underscores `_`. For example a tag `foo/bar` will result in a directory name of + `foo_bar`. https://github.com/restic/restic/issues/2907 https://github.com/restic/restic/pull/2913 @@ -1646,8 +2422,9 @@ restic users. The changes are ordered by importance. * Enhancement #2923: Improve speed of `copy` command - The `copy` command could require a long time to copy snapshots for non-local backends. This has - been improved to provide a throughput comparable to the `restore` command. + The `copy` command could require a long time to copy snapshots for non-local + backends. This has been improved to provide a throughput comparable to the + `restore` command. Additionally, `copy` now displays a progress bar. @@ -1656,21 +2433,23 @@ restic users. The changes are ordered by importance. * Enhancement #3114: Optimize handling of duplicate blobs in `prune` - Restic `prune` always used to repack all data files containing duplicate blobs. This - effectively removed all duplicates during prune. However, as a consequence all these data - files were repacked even if the unused repository space threshold could be reached with less - work. + Restic `prune` always used to repack all data files containing duplicate blobs. + This effectively removed all duplicates during prune. However, as a consequence + all these data files were repacked even if the unused repository space threshold + could be reached with less work. - This is now changed and `prune` works nice and fast even when there are lots of duplicate blobs. + This is now changed and `prune` works nice and fast even when there are lots of + duplicate blobs. https://github.com/restic/restic/issues/3114 https://github.com/restic/restic/pull/3290 * Enhancement #3465: Improve handling of temporary files on Windows - In some cases restic failed to delete temporary files, causing the current command to fail. - This has now been fixed by ensuring that Windows automatically deletes the file. In addition, - temporary files are only written to disk when necessary, reducing disk writes. + In some cases restic failed to delete temporary files, causing the current + command to fail. This has now been fixed by ensuring that Windows automatically + deletes the file. In addition, temporary files are only written to disk when + necessary, reducing disk writes. https://github.com/restic/restic/issues/3465 https://github.com/restic/restic/issues/1551 @@ -1678,22 +2457,23 @@ restic users. The changes are ordered by importance. * Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend - Restic did not support limiting the IO concurrency / number of connections for accessing - repositories stored using the local or SFTP backends. The number of connections is now limited - as for other backends, and can be configured via the `-o local.connections=2` and `-o - sftp.connections=5` options. This ensures that restic does not overwhelm the backend with - concurrent IO operations. + Restic did not support limiting the IO concurrency / number of connections for + accessing repositories stored using the local or SFTP backends. The number of + connections is now limited as for other backends, and can be configured via the + `-o local.connections=2` and `-o sftp.connections=5` options. This ensures that + restic does not overwhelm the backend with concurrent IO operations. https://github.com/restic/restic/pull/3475 * Enhancement #3484: Stream data in `check` and `prune` commands - The commands `check --read-data` and `prune` previously downloaded data files into - temporary files which could end up being written to disk. This could cause a large amount of data - being written to disk. + The commands `check --read-data` and `prune` previously downloaded data files + into temporary files which could end up being written to disk. This could cause + a large amount of data being written to disk. - The pack files are now instead streamed, which removes the need for temporary files. Please - note that *uploads* during `backup` and `prune` still require temporary files. + The pack files are now instead streamed, which removes the need for temporary + files. Please note that *uploads* during `backup` and `prune` still require + temporary files. https://github.com/restic/restic/issues/3710 https://github.com/restic/restic/pull/3484 @@ -1702,19 +2482,19 @@ restic users. The changes are ordered by importance. * Enhancement #3709: Validate exclude patterns before backing up Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or - `--iexclude-file` previously weren't validated. As a consequence, invalid patterns - resulted in files that were meant to be excluded being backed up. + `--iexclude-file` previously weren't validated. As a consequence, invalid + patterns resulted in files that were meant to be excluded being backed up. - Restic now validates all patterns before running the backup and aborts with a fatal error if an - invalid pattern is detected. + Restic now validates all patterns before running the backup and aborts with a + fatal error if an invalid pattern is detected. https://github.com/restic/restic/issues/3709 https://github.com/restic/restic/pull/3734 * Enhancement #3729: Display full IDs in `check` warnings - When running commands to inspect or repair a damaged repository, it is often necessary to - supply the full IDs of objects stored in the repository. + When running commands to inspect or repair a damaged repository, it is often + necessary to supply the full IDs of objects stored in the repository. The output of `check` now includes full IDs instead of their shortened variant. @@ -1722,28 +2502,29 @@ restic users. The changes are ordered by importance. * Enhancement #3773: Optimize memory usage for directories with many files - Backing up a directory with hundreds of thousands or more files caused restic to require large - amounts of memory. We've now optimized the `backup` command such that it requires up to 30% less - memory. + Backing up a directory with hundreds of thousands or more files caused restic to + require large amounts of memory. We've now optimized the `backup` command such + that it requires up to 30% less memory. https://github.com/restic/restic/pull/3773 * Enhancement #3819: Validate include/exclude patterns before restoring Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` and - `--iinclude` weren't validated before running the restore. Invalid patterns would result in - error messages being printed repeatedly, and possibly unwanted files being restored. + `--iinclude` weren't validated before running the restore. Invalid patterns + would result in error messages being printed repeatedly, and possibly unwanted + files being restored. - Restic now validates all patterns before running the restore, and aborts with a fatal error if - an invalid pattern is detected. + Restic now validates all patterns before running the restore, and aborts with a + fatal error if an invalid pattern is detected. https://github.com/restic/restic/pull/3819 * Enhancement #3837: Improve SFTP repository initialization over slow links - The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the - backend concurrently. This reduces the waiting times when creating a repository over a very - slow connection. + The `init` command, when used on an SFTP backend, now sends multiple `mkdir` + commands to the backend concurrently. This reduces the waiting times when + creating a repository over a very slow connection. https://github.com/restic/restic/issues/3837 https://github.com/restic/restic/pull/3840 @@ -1794,9 +2575,9 @@ restic users. The changes are ordered by importance. * Bugfix #1106: Never lock repository for `list locks` - The `list locks` command previously locked to the repository by default. This had the problem - that it wouldn't work for an exclusively locked repository and that the command would also - display its own lock file which can be confusing. + The `list locks` command previously locked to the repository by default. This + had the problem that it wouldn't work for an exclusively locked repository and + that the command would also display its own lock file which can be confusing. Now, the `list locks` command never locks the repository. @@ -1805,22 +2586,24 @@ restic users. The changes are ordered by importance. * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes - The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of - restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a - temporary location before being moved to their proper location. + The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state + in the event of restic (or the OS) crashing. This is now less likely to occur as + files are downloaded to a temporary location before being moved to their proper + location. - This also allows multiple concurrent restic processes to operate on a single repository - without conflicts. Previously, concurrent operations could cause segfaults because the - processes saw each other's partially downloaded files. + This also allows multiple concurrent restic processes to operate on a single + repository without conflicts. Previously, concurrent operations could cause + segfaults because the processes saw each other's partially downloaded files. https://github.com/restic/restic/issues/2345 https://github.com/restic/restic/pull/2838 * Bugfix #2452: Improve error handling of repository locking - Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly - created one. Instead it continued trying to delete the old (usually no longer existing) lock - file and thus over time lots of lock files accumulated. This has now been fixed. + Previously, when the lock refresh failed to delete the old lock file, it forgot + about the newly created one. Instead it continued trying to delete the old + (usually no longer existing) lock file and thus over time lots of lock files + accumulated. This has now been fixed. https://github.com/restic/restic/issues/2452 https://github.com/restic/restic/issues/2473 @@ -1829,43 +2612,45 @@ restic users. The changes are ordered by importance. * Bugfix #2738: Don't print progress for `backup --json --quiet` - Unlike the text output, the `--json` output format still printed progress information even in - `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode. + Unlike the text output, the `--json` output format still printed progress + information even in `--quiet` mode. This has now been fixed by always disabling + the progress output in quiet mode. https://github.com/restic/restic/issues/2738 https://github.com/restic/restic/pull/3264 * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable - Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable, - which caused problems in certain system/usage configurations. This has now been fixed. + Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment + variable, which caused problems in certain system/usage configurations. This has + now been fixed. https://github.com/restic/restic/issues/3382 https://github.com/restic/restic/pull/3474 * Bugfix #3488: `rebuild-index` failed if an index file was damaged - Previously, the `rebuild-index` command would fail with an error if an index file was damaged - or truncated. This has now been fixed. + Previously, the `rebuild-index` command would fail with an error if an index + file was damaged or truncated. This has now been fixed. - On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or - to manually delete the damaged index. + On older restic versions, a (slow) workaround is to use `rebuild-index + --read-all-packs` or to manually delete the damaged index. https://github.com/restic/restic/pull/3488 * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository - The `copy` command previously did not respect the `--no-lock` option for the source - repository, causing failures with read-only storage backends. This has now been fixed such - that the option is now respected. + The `copy` command previously did not respect the `--no-lock` option for the + source repository, causing failures with read-only storage backends. This has + now been fixed such that the option is now respected. https://github.com/restic/restic/issues/3518 https://github.com/restic/restic/pull/3589 * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error - Previously, if a request failed with an SSL unknown certificate authority error, the B2 - backend retried indefinitely and restic would appear to hang. + Previously, if a request failed with an SSL unknown certificate authority error, + the B2 backend retried indefinitely and restic would appear to hang. This has now been fixed and restic instead fails with an error message. @@ -1875,95 +2660,103 @@ restic users. The changes are ordered by importance. * Bugfix #3591: Fix handling of `prune --max-repack-size=0` - Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been - fixed. + Restic ignored the `--max-repack-size` option when passing a value of 0. This + has now been fixed. - As a workaround, `--max-repack-size=1` can be used with older versions of restic. + As a workaround, `--max-repack-size=1` can be used with older versions of + restic. https://github.com/restic/restic/pull/3591 * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows - Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the - backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time - as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone - stdio connection already closed" errors. + Previously, pressing Ctrl+C in a Windows console where restic was running with + rclone as the backend would cause rclone to exit prematurely due to getting a + `SIGINT` signal at the same time as restic. Restic would then wait for a long + time for time with "unexpected EOF" and "rclone stdio connection already closed" + errors. - This has now been fixed by restic starting the rclone process detached from the console restic - runs in (similar to starting processes in a new process group on Linux), which enables restic to - gracefully clean up rclone (which now never gets the `SIGINT`). + This has now been fixed by restic starting the rclone process detached from the + console restic runs in (similar to starting processes in a new process group on + Linux), which enables restic to gracefully clean up rclone (which now never gets + the `SIGINT`). https://github.com/restic/restic/issues/3601 https://github.com/restic/restic/pull/3602 * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot - The `backup` command, when a `--parent` was not provided, previously chose the most recent - matching snapshot as the parent snapshot. However, this didn't make sense when the user passed - `--time` to create a new snapshot older than the most recent snapshot. + The `backup` command, when a `--parent` was not provided, previously chose the + most recent matching snapshot as the parent snapshot. However, this didn't make + sense when the user passed `--time` to create a new snapshot older than the most + recent snapshot. - Instead, `backup` now chooses the most recent snapshot which is not newer than the - snapshot-being-created's timestamp, to avoid any time travel. + Instead, `backup` now chooses the most recent snapshot which is not newer than + the snapshot-being-created's timestamp, to avoid any time travel. https://github.com/restic/restic/pull/3619 * Bugfix #3667: The `mount` command now reports symlinks sizes - Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They - now have a size equal to the byte length of their target path, as required by POSIX. + Symlinks used to have size zero in restic mountpoints, confusing some + third-party tools. They now have a size equal to the byte length of their target + path, as required by POSIX. https://github.com/restic/restic/issues/3667 https://github.com/restic/restic/pull/3668 * Change #3519: Require Go 1.14 or newer - Restic now requires Go 1.14 to build. This allows it to use new standard library features - instead of an external dependency. + Restic now requires Go 1.14 to build. This allows it to use new standard library + features instead of an external dependency. https://github.com/restic/restic/issues/3519 * Change #3641: Ignore parent snapshot for `backup --stdin` - Restic uses a parent snapshot to speed up directory scanning when performing backups, but this - only wasted time and memory when the backup source is stdin (using the `--stdin` option of the - `backup` command), since no directory scanning is performed in this case. + Restic uses a parent snapshot to speed up directory scanning when performing + backups, but this only wasted time and memory when the backup source is stdin + (using the `--stdin` option of the `backup` command), since no directory + scanning is performed in this case. - Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to - skip some startup operations and saves a bit of resources. + Snapshots made with `backup --stdin` no longer have a parent snapshot, which + allows restic to skip some startup operations and saves a bit of resources. - The `--parent` option is still available for `backup --stdin`, but is now ignored. + The `--parent` option is still available for `backup --stdin`, but is now + ignored. https://github.com/restic/restic/issues/3641 https://github.com/restic/restic/pull/3645 * Enhancement #233: Support negative include/exclude patterns - If a pattern starts with an exclamation mark and it matches a file that was previously matched by - a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to - cancel the exclusion of some files. + If a pattern starts with an exclamation mark and it matches a file that was + previously matched by a regular pattern, the match is cancelled. Notably, this + can be used with `--exclude-file` to cancel the exclusion of some files. - It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it - is not possible to include files inside the directory. + It works similarly to `.gitignore`, with the same limitation; Once a directory + is excluded, it is not possible to include files inside the directory. Example of use as an exclude pattern for the `backup` command: $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...] - node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...] + node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git + # [...] https://github.com/restic/restic/issues/233 https://github.com/restic/restic/pull/2311 * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command - Testing exclude filters and other configuration options was error prone as wrong filters - could cause files to be uploaded unintentionally. It was also not possible to estimate - beforehand how much data would be uploaded. + Testing exclude filters and other configuration options was error prone as wrong + filters could cause files to be uploaded unintentionally. It was also not + possible to estimate beforehand how much data would be uploaded. - The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of - a backup without actually writing anything to the repository. + The `backup` command now has a `--dry-run`/`-n` option, which performs all the + normal steps of a backup without actually writing anything to the repository. - Passing -vv will log information about files that would be added, allowing for verification of - source and exclusion options before running the real backup. + Passing -vv will log information about files that would be added, allowing for + verification of source and exclusion options before running the real backup. https://github.com/restic/restic/issues/1542 https://github.com/restic/restic/pull/2308 @@ -1972,14 +2765,14 @@ restic users. The changes are ordered by importance. * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends - Previously only the B2 and partially the Swift backends verified the integrity of uploaded - (encrypted) files. The verification works by informing the backend about the expected hash of - the uploaded file. The backend then verifies the upload and thereby rules out any data - corruption during upload. + Previously only the B2 and partially the Swift backends verified the integrity + of uploaded (encrypted) files. The verification works by informing the backend + about the expected hash of the uploaded file. The backend then verifies the + upload and thereby rules out any data corruption during upload. - We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides - integrity checking for uploads also means that restic can now be used to store backups in S3 - buckets which have Object Lock enabled. + We have now added upload checksums for the Azure, GS, S3 and Swift backends, + which besides integrity checking for uploads also means that restic can now be + used to store backups in S3 buckets which have Object Lock enabled. https://github.com/restic/restic/issues/2202 https://github.com/restic/restic/issues/2700 @@ -1988,65 +2781,68 @@ restic users. The changes are ordered by importance. * Enhancement #2388: Add warning for S3 if partial credentials are provided - Previously restic did not notify about incomplete credentials when using the S3 backend, - instead just reporting access denied. + Previously restic did not notify about incomplete credentials when using the S3 + backend, instead just reporting access denied. - Restic now checks that both the AWS key ID and secret environment variables are set before - connecting to the remote server, and reports an error if not. + Restic now checks that both the AWS key ID and secret environment variables are + set before connecting to the remote server, and reports an error if not. https://github.com/restic/restic/issues/2388 https://github.com/restic/restic/pull/3532 * Enhancement #2508: Support JSON output and quiet mode for the `diff` command - The `diff` command now supports outputting machine-readable output in JSON format. To enable - this, pass the `--json` option to the command. To only print the summary and suppress detailed - output, pass the `--quiet` option. + The `diff` command now supports outputting machine-readable output in JSON + format. To enable this, pass the `--json` option to the command. To only print + the summary and suppress detailed output, pass the `--quiet` option. https://github.com/restic/restic/issues/2508 https://github.com/restic/restic/pull/3592 * Enhancement #2594: Speed up the `restore --verify` command - The `--verify` option lets the `restore` command verify the file content after it has restored - a snapshot. The performance of this operation has now been improved by up to a factor of two. + The `--verify` option lets the `restore` command verify the file content after + it has restored a snapshot. The performance of this operation has now been + improved by up to a factor of two. https://github.com/restic/restic/pull/2594 * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates - There is now an `--insecure-tls` global option in restic, which disables TLS verification for - self-signed certificates in order to support some development workflows. + There is now an `--insecure-tls` global option in restic, which disables TLS + verification for self-signed certificates in order to support some development + workflows. https://github.com/restic/restic/issues/2656 https://github.com/restic/restic/pull/2657 * Enhancement #2816: The `backup` command no longer updates file access times on Linux - When reading files during backup, restic used to cause the operating system to update the - files' access times. Note that this did not apply to filesystems with disabled file access - times. + When reading files during backup, restic used to cause the operating system to + update the files' access times. Note that this did not apply to filesystems with + disabled file access times. - Restic now instructs the operating system not to update the file access time, if the user - running restic is the file owner or has root permissions. + Restic now instructs the operating system not to update the file access time, if + the user running restic is the file owner or has root permissions. https://github.com/restic/restic/pull/2816 * Enhancement #2880: Make `recover` collect only unreferenced trees - Previously, the `recover` command used to generate a snapshot containing *all* root trees, - even those which were already referenced by a snapshot. + Previously, the `recover` command used to generate a snapshot containing *all* + root trees, even those which were already referenced by a snapshot. - This has been improved such that it now only processes trees not already referenced by any - snapshot. + This has been improved such that it now only processes trees not already + referenced by any snapshot. https://github.com/restic/restic/pull/2880 * Enhancement #3003: Atomic uploads for the SFTP backend - The SFTP backend did not upload files atomically. An interrupted upload could leave an - incomplete file behind which could prevent restic from accessing the repository. This has now - been fixed and uploads in the SFTP backend are done atomically. + The SFTP backend did not upload files atomically. An interrupted upload could + leave an incomplete file behind which could prevent restic from accessing the + repository. This has now been fixed and uploads in the SFTP backend are done + atomically. https://github.com/restic/restic/issues/3003 https://github.com/restic/restic/pull/3524 @@ -2060,25 +2856,27 @@ restic users. The changes are ordered by importance. * Enhancement #3429: Verify that new or modified keys are stored correctly - When adding a new key or changing the password of a key, restic used to just create the new key (and - remove the old one, when changing the password). There was no verification that the new key was - stored correctly and works properly. As the repository cannot be decrypted without a valid key - file, this could in rare cases cause the repository to become inaccessible. + When adding a new key or changing the password of a key, restic used to just + create the new key (and remove the old one, when changing the password). There + was no verification that the new key was stored correctly and works properly. As + the repository cannot be decrypted without a valid key file, this could in rare + cases cause the repository to become inaccessible. - Restic now checks that new key files actually work before continuing. This can protect against - some (rare) cases of hardware or storage problems. + Restic now checks that new key files actually work before continuing. This can + protect against some (rare) cases of hardware or storage problems. https://github.com/restic/restic/pull/3429 * Enhancement #3436: Improve local backend's resilience to (system) crashes - Restic now ensures that files stored using the `local` backend are created atomically (that - is, files are either stored completely or not at all). This ensures that no incomplete files are - left behind even if restic is terminated while writing a file. + Restic now ensures that files stored using the `local` backend are created + atomically (that is, files are either stored completely or not at all). This + ensures that no incomplete files are left behind even if restic is terminated + while writing a file. - In addition, restic now tries to ensure that the directory in the repository which contains a - newly uploaded file is also written to disk. This can prevent missing files if the system - crashes or the disk is not properly unmounted. + In addition, restic now tries to ensure that the directory in the repository + which contains a newly uploaded file is also written to disk. This can prevent + missing files if the system crashes or the disk is not properly unmounted. https://github.com/restic/restic/pull/3436 @@ -2086,54 +2884,56 @@ restic users. The changes are ordered by importance. Restic used to silently ignore the `--no-lock` option of the `forget` command. - It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If - `--no-lock` option is specified without `--dry-run`, restic prints a warning message to - stderr. + It now skips creation of lock file in case both `--dry-run` and `--no-lock` are + specified. If `--no-lock` option is specified without `--dry-run`, restic prints + a warning message to stderr. https://github.com/restic/restic/issues/3464 https://github.com/restic/restic/pull/3623 * Enhancement #3490: Support random subset by size in `check --read-data-subset` - The `--read-data-subset` option of the `check` command now supports a third way of specifying - the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or - t/T. + The `--read-data-subset` option of the `check` command now supports a third way + of specifying the subset to check, namely `nS` where `n` is a size in bytes with + suffix `S` as k/K, m/M, g/G or t/T. https://github.com/restic/restic/issues/3490 https://github.com/restic/restic/pull/3548 * Enhancement #3508: Cache blobs read by the `dump` command - When dumping a file using the `dump` command, restic did not cache blobs in any way, so even - consecutive runs of the same blob were loaded from the repository again and again, slowing down - the dump. + When dumping a file using the `dump` command, restic did not cache blobs in any + way, so even consecutive runs of the same blob were loaded from the repository + again and again, slowing down the dump. - Now, the caching mechanism already used by the `fuse` command is also used by the `dump` - command. This makes dumping much faster, especially for sparse files. + Now, the caching mechanism already used by the `fuse` command is also used by + the `dump` command. This makes dumping much faster, especially for sparse files. https://github.com/restic/restic/pull/3508 * Enhancement #3511: Support configurable timeout for the rclone backend - A slow rclone backend could cause restic to time out while waiting for the repository to open. - Restic now offers an `-o rclone.timeout` option to make this timeout configurable. + A slow rclone backend could cause restic to time out while waiting for the + repository to open. Restic now offers an `-o rclone.timeout` option to make this + timeout configurable. https://github.com/restic/restic/issues/3511 https://github.com/restic/restic/pull/3514 * Enhancement #3541: Improve handling of temporary B2 delete errors - Deleting files on B2 could sometimes fail temporarily, which required restic to retry the - delete operation. In some cases the file was deleted nevertheless, causing the retries and - ultimately the restic command to fail. This has now been fixed. + Deleting files on B2 could sometimes fail temporarily, which required restic to + retry the delete operation. In some cases the file was deleted nevertheless, + causing the retries and ultimately the restic command to fail. This has now been + fixed. https://github.com/restic/restic/issues/3541 https://github.com/restic/restic/pull/3544 * Enhancement #3542: Add file mode in symbolic notation to `ls --json` - The `ls --json` command now provides the file mode in symbolic notation (using the - `permissions` key), aligned with `find --json`. + The `ls --json` command now provides the file mode in symbolic notation (using + the `permissions` key), aligned with `find --json`. https://github.com/restic/restic/issues/3542 https://github.com/restic/restic/pull/3573 @@ -2141,11 +2941,12 @@ restic users. The changes are ordered by importance. * Enhancement #3593: Improve `copy` performance by parallelizing IO - Restic copy previously only used a single thread for copying blobs between repositories, - which resulted in limited performance when copying small blobs to/from a high latency backend - (i.e. any remote backend, especially b2). + Restic copy previously only used a single thread for copying blobs between + repositories, which resulted in limited performance when copying small blobs + to/from a high latency backend (i.e. any remote backend, especially b2). - Copying will now use 8 parallel threads to increase the throughput of the copy operation. + Copying will now use 8 parallel threads to increase the throughput of the copy + operation. https://github.com/restic/restic/pull/3593 @@ -2183,9 +2984,9 @@ restic users. The changes are ordered by importance. * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2 - When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect - when no data was returned at all. This could cause for example the `check` command to report the - following error: + When retrieving data from the rclone / REST backend while also using HTTP2 + restic did not detect when no data was returned at all. This could cause for + example the `check` command to report the following error: Pack ID does not match, want [...], got e3b0c442 @@ -2197,98 +2998,105 @@ restic users. The changes are ordered by importance. * Bugfix #3111: Fix terminal output redirection for PowerShell - When redirecting the output of restic using PowerShell on Windows, the output contained - terminal escape characters. This has been fixed by properly detecting the terminal type. + When redirecting the output of restic using PowerShell on Windows, the output + contained terminal escape characters. This has been fixed by properly detecting + the terminal type. - In addition, the mintty terminal now shows progress output for the backup command. + In addition, the mintty terminal now shows progress output for the backup + command. https://github.com/restic/restic/issues/3111 https://github.com/restic/restic/pull/3325 * Bugfix #3184: `backup --quiet` no longer prints status information - A regression in the latest restic version caused the output of `backup --quiet` to contain - large amounts of backup progress information when run using an interactive terminal. This is - fixed now. + A regression in the latest restic version caused the output of `backup --quiet` + to contain large amounts of backup progress information when run using an + interactive terminal. This is fixed now. - A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`. + A workaround for this bug is to run restic as follows: `restic backup --quiet + [..] | cat -`. https://github.com/restic/restic/issues/3184 https://github.com/restic/restic/pull/3186 * Bugfix #3214: Treat an empty password as a fatal error for repository init - When attempting to initialize a new repository, if an empty password was supplied, the - repository would be created but the init command would return an error with a stack trace. Now, - if an empty password is provided, it is treated as a fatal error, and no repository is created. + When attempting to initialize a new repository, if an empty password was + supplied, the repository would be created but the init command would return an + error with a stack trace. Now, if an empty password is provided, it is treated + as a fatal error, and no repository is created. https://github.com/restic/restic/issues/3214 https://github.com/restic/restic/pull/3283 * Bugfix #3267: `copy` failed to copy snapshots in rare cases - The `copy` command could in rare cases fail with the error message `SaveTree(...) returned - unexpected id ...`. This has been fixed. + The `copy` command could in rare cases fail with the error message + `SaveTree(...) returned unexpected id ...`. This has been fixed. - On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths. - Note that, due to limitations in the repository format, these are not stored properly and - should be avoided if possible. + On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 + target paths. Note that, due to limitations in the repository format, these are + not stored properly and should be avoided if possible. https://github.com/restic/restic/issues/3267 https://github.com/restic/restic/pull/3310 * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository - The command `restic check --read-data-subset=x%` crashed when run for an empty repository. - This has been fixed. + The command `restic check --read-data-subset=x%` crashed when run for an empty + repository. This has been fixed. https://github.com/restic/restic/issues/3296 https://github.com/restic/restic/pull/3309 * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend - The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend - when the `data` folder in the repository contained files. This has been fixed. + The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the + local backend when the `data` folder in the repository contained files. This has + been fixed. https://github.com/restic/restic/issues/3302 https://github.com/restic/restic/pull/3308 * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error - When using `--json` output it happened from time to time that the summary output was missing in - case an error occurred. This has been fixed. + When using `--json` output it happened from time to time that the summary output + was missing in case an error occurred. This has been fixed. https://github.com/restic/restic/pull/3305 * Bugfix #3334: Print `created new cache` message only on a terminal - The message `created new cache` was printed even when the output wasn't a terminal. That broke - piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now - only printed on a terminal. + The message `created new cache` was printed even when the output wasn't a + terminal. That broke piping `restic dump` output to tar or zip if cache + directory didn't exist. The message is now only printed on a terminal. https://github.com/restic/restic/issues/3334 https://github.com/restic/restic/pull/3343 * Bugfix #3380: Fix crash of `backup --exclude='**'` - The exclude filter `**`, which excludes all files, caused restic to crash. This has been - corrected. + The exclude filter `**`, which excludes all files, caused restic to crash. This + has been corrected. https://github.com/restic/restic/issues/3380 https://github.com/restic/restic/pull/3393 * Bugfix #3439: Correctly handle download errors during `restore` - Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download - errors and only printed a warning. This has been fixed by retrying incomplete data downloads. + Due to a regression in restic 0.12.0, the `restore` command in some cases did + not retry download errors and only printed a warning. This has been fixed by + retrying incomplete data downloads. https://github.com/restic/restic/issues/3439 https://github.com/restic/restic/pull/3449 * Change #3247: Empty files now have size of 0 in `ls --json` output - The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size - of zero explicitly for regular files, while omitting the size field for all other types. + The `ls --json` command used to omit the sizes of empty files in its output. It + now reports a size of zero explicitly for regular files, while omitting the size + field for all other types. https://github.com/restic/restic/issues/3247 https://github.com/restic/restic/pull/3257 @@ -2302,9 +3110,9 @@ restic users. The changes are ordered by importance. * Enhancement #3167: Allow specifying limit of `snapshots` list - The `--last` option allowed limiting the output of the `snapshots` command to the latest - snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n` - snapshots. + The `--last` option allowed limiting the output of the `snapshots` command to + the latest snapshot for each host. The new `--latest n` option allows limiting + the output to the latest `n` snapshots. This change deprecates the option `--last` in favour of `--latest 1`. @@ -2312,13 +3120,15 @@ restic users. The changes are ordered by importance. * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command - The `init` and `copy` command can now be used with the `--repository-file2` option or the - `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the - `--repo2` flag and allow you to read the destination repository from a file. + The `init` and `copy` command can now be used with the `--repository-file2` + option or the `$RESTIC_REPOSITORY_FILE2` environment variable. These to options + are in addition to the `--repo2` flag and allow you to read the destination + repository from a file. - Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or - `init` command. The handling of this combination of options has been fixed. A workaround for - this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`. + Using both `--repository-file` and `--repo2` options resulted in an error for + the `copy` or `init` command. The handling of this combination of options has + been fixed. A workaround for this issue is to only use `--repo` or `-r` and + `--repo2` for `init` or `copy`. https://github.com/restic/restic/issues/3293 https://github.com/restic/restic/pull/3294 @@ -2331,9 +3141,9 @@ restic users. The changes are ordered by importance. * Enhancement #3336: SFTP backend now checks for disk space - Backing up over SFTP previously spewed multiple generic "failure" messages when the remote - disk was full. It now checks for disk space before writing a file and fails immediately with a "no - space left on device" message. + Backing up over SFTP previously spewed multiple generic "failure" messages when + the remote disk was full. It now checks for disk space before writing a file and + fails immediately with a "no space left on device" message. https://github.com/restic/restic/issues/3336 https://github.com/restic/restic/pull/3345 @@ -2347,15 +3157,17 @@ restic users. The changes are ordered by importance. * Enhancement #3414: Add `--keep-within-hourly` option to restic forget - The `forget` command allowed keeping a given number of hourly backups or to keep all backups - within a given interval, but it was not possible to specify keeping hourly backups within a - given interval. + The `forget` command allowed keeping a given number of hourly backups or to keep + all backups within a given interval, but it was not possible to specify keeping + hourly backups within a given interval. - The new `--keep-within-hourly` option now offers this functionality. Similar options for - daily/weekly/monthly/yearly are also implemented, the new options are: + The new `--keep-within-hourly` option now offers this functionality. Similar + options for daily/weekly/monthly/yearly are also implemented, the new options + are: - --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly - <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h> + --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> + --keep-within-weekly <1y2m3d4h> --keep-within-monthly <1y2m3d4h> + --keep-within-yearly <1y2m3d4h> https://github.com/restic/restic/issues/3414 https://github.com/restic/restic/pull/3416 @@ -2363,30 +3175,32 @@ restic users. The changes are ordered by importance. * Enhancement #3426: Optimize read performance of mount command - Reading large files in a mounted repository may be up to five times faster. This improvement - primarily applies to repositories stored at a backend that can be accessed with low latency, - like e.g. the local backend. + Reading large files in a mounted repository may be up to five times faster. This + improvement primarily applies to repositories stored at a backend that can be + accessed with low latency, like e.g. the local backend. https://github.com/restic/restic/pull/3426 * Enhancement #3427: `find --pack` fallback to index if data file is missing - When investigating a repository with missing data files, it might be useful to determine - affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id` - returned no data as it required accessing the data file. Now, if the necessary data is still - available in the repository index, it gets retrieved from there. + When investigating a repository with missing data files, it might be useful to + determine affected snapshots before running `rebuild-index`. Previously, `find + --pack pack-id` returned no data as it required accessing the data file. Now, if + the necessary data is still available in the repository index, it gets retrieved + from there. - The command now also supports looking up multiple pack files in a single `find` run. + The command now also supports looking up multiple pack files in a single `find` + run. https://github.com/restic/restic/pull/3427 https://forum.restic.net/t/missing-packs-not-found/2600 * Enhancement #3456: Support filtering and specifying untagged snapshots - It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag` - options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches - snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only - operate on untagged snapshots. + It was previously not possible to specify an empty tag with the `--tag` and + `--keep-tag` options. This has now been fixed, such that `--tag ''` and + `--keep-tag ''` now matches snapshots without tags. This allows e.g. the + `snapshots` and `forget` commands to only operate on untagged snapshots. https://github.com/restic/restic/issues/3456 https://github.com/restic/restic/pull/3457 @@ -2410,7 +3224,7 @@ restic users. The changes are ordered by importance. * Fix #3151: Don't create invalid snapshots when `backup` is interrupted * Fix #3152: Do not hang until foregrounded when completed in background * Fix #3166: Improve error handling in the `restore` command - * Fix #3232: Correct statistics for overlapping targets + * Fix #3232: Correct statistics for overlapping backup sources * Fix #3249: Improve error handling in `gs` backend * Chg #3095: Deleting files on Google Drive now moves them to the trash * Enh #909: Back up mountpoints as empty directories @@ -2438,28 +3252,28 @@ restic users. The changes are ordered by importance. * Bugfix #1681: Make `mount` not create missing mount point directory - When specifying a non-existent directory as mount point for the `mount` command, restic used - to create the specified directory automatically. + When specifying a non-existent directory as mount point for the `mount` command, + restic used to create the specified directory automatically. - This has now changed such that restic instead gives an error when the specified directory for - the mount point does not exist. + This has now changed such that restic instead gives an error when the specified + directory for the mount point does not exist. https://github.com/restic/restic/issues/1681 https://github.com/restic/restic/pull/3008 * Bugfix #1800: Ignore `no data available` filesystem error during backup - Restic was unable to backup files on some filesystems, for example certain configurations of - CIFS on Linux which return a `no data available` error when reading extended attributes. These - errors are now ignored. + Restic was unable to backup files on some filesystems, for example certain + configurations of CIFS on Linux which return a `no data available` error when + reading extended attributes. These errors are now ignored. https://github.com/restic/restic/issues/1800 https://github.com/restic/restic/pull/3034 * Bugfix #2563: Report the correct owner of directories in FUSE mounts - Restic 0.10.0 changed the FUSE mount to always report the current user as the owner of - directories within the FUSE mount, which is incorrect. + Restic 0.10.0 changed the FUSE mount to always report the current user as the + owner of directories within the FUSE mount, which is incorrect. This is now changed back to reporting the correct owner of a directory. @@ -2468,30 +3282,31 @@ restic users. The changes are ordered by importance. * Bugfix #2688: Make `backup` and `tag` commands separate tags by comma - Running `restic backup --tag foo,bar` previously created snapshots with one single tag - containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`). + Running `restic backup --tag foo,bar` previously created snapshots with one + single tag containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`). - Similarly, the `tag` command's `--set`, `--add` and `--remove` options would treat - `foo,bar` as one tag instead of two tags. This was inconsistent with other commands and often - unexpected when one intended `foo,bar` to mean two tags. + Similarly, the `tag` command's `--set`, `--add` and `--remove` options would + treat `foo,bar` as one tag instead of two tags. This was inconsistent with other + commands and often unexpected when one intended `foo,bar` to mean two tags. - To be consistent in all commands, restic now interprets `foo,bar` to mean two separate tags - (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, including in the `backup` and - `tag` commands. + To be consistent in all commands, restic now interprets `foo,bar` to mean two + separate tags (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, + including in the `backup` and `tag` commands. - NOTE: This change might result in unexpected behavior in cases where you use the `forget` - command and filter on tags like `foo,bar`. Snapshots previously backed up with `--tag - foo,bar` will still not match that filter, but snapshots saved from now on will match that - filter. + NOTE: This change might result in unexpected behavior in cases where you use the + `forget` command and filter on tags like `foo,bar`. Snapshots previously backed + up with `--tag foo,bar` will still not match that filter, but snapshots saved + from now on will match that filter. - To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can first generate a - list of the relevant snapshots using a command like: + To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can + first generate a list of the relevant snapshots using a command like: - Restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id' + Restic snapshots --json --quiet | jq '.[] | select(contains({tags: + ["foo,bar"]})) | .id' - And then use `restic tag --set foo --set bar snapshotID [...]` to set the new tags. Please adjust - the commands to include real tag names and any additional tags, as well as the list of snapshots - to process. + And then use `restic tag --set foo --set bar snapshotID [...]` to set the new + tags. Please adjust the commands to include real tag names and any additional + tags, as well as the list of snapshots to process. https://github.com/restic/restic/issues/2688 https://github.com/restic/restic/pull/2690 @@ -2505,14 +3320,14 @@ restic users. The changes are ordered by importance. * Bugfix #3014: Fix sporadic stream reset between rclone and restic - Sometimes when using restic with the `rclone` backend, an error message similar to the - following would be printed: + Sometimes when using restic with the `rclone` backend, an error message similar + to the following would be printed: Didn't finish writing GET request (wrote 0/xxx): http2: stream closed - It was found that this was caused by restic closing the connection to rclone to soon when - downloading data. A workaround has been added which waits for the end of the download before - closing the connection. + It was found that this was caused by restic closing the connection to rclone to + soon when downloading data. A workaround has been added which waits for the end + of the download before closing the connection. https://github.com/rclone/rclone/issues/2598 https://github.com/restic/restic/pull/3014 @@ -2530,125 +3345,130 @@ restic users. The changes are ordered by importance. * Bugfix #3100: Do not require gs bucket permissions when running `init` - Restic used to require bucket level permissions for the `gs` backend in order to initialize a - restic repository. + Restic used to require bucket level permissions for the `gs` backend in order to + initialize a restic repository. - It now allows a `gs` service account to initialize a repository if the bucket does exist and the - service account has permissions to write/read to that bucket. + It now allows a `gs` service account to initialize a repository if the bucket + does exist and the service account has permissions to write/read to that bucket. https://github.com/restic/restic/issues/3100 * Bugfix #3111: Correctly detect output redirection for `backup` command on Windows - On Windows, since restic 0.10.0 the `backup` command did not properly detect when the output - was redirected to a file. This caused restic to output terminal control characters. This has - been fixed by correcting the terminal detection. + On Windows, since restic 0.10.0 the `backup` command did not properly detect + when the output was redirected to a file. This caused restic to output terminal + control characters. This has been fixed by correcting the terminal detection. https://github.com/restic/restic/issues/3111 https://github.com/restic/restic/pull/3150 * Bugfix #3151: Don't create invalid snapshots when `backup` is interrupted - When canceling a backup run at a certain moment it was possible that restic created a snapshot - with an invalid "null" tree. This caused `check` and other operations to fail. The `backup` - command now properly handles interruptions and never saves a snapshot when interrupted. + When canceling a backup run at a certain moment it was possible that restic + created a snapshot with an invalid "null" tree. This caused `check` and other + operations to fail. The `backup` command now properly handles interruptions and + never saves a snapshot when interrupted. https://github.com/restic/restic/issues/3151 https://github.com/restic/restic/pull/3164 * Bugfix #3152: Do not hang until foregrounded when completed in background - On Linux, when running in the background restic failed to stop the terminal output of the - `backup` command after it had completed. This caused restic to hang until moved to the - foreground. This has now been fixed. + On Linux, when running in the background restic failed to stop the terminal + output of the `backup` command after it had completed. This caused restic to + hang until moved to the foreground. This has now been fixed. https://github.com/restic/restic/pull/3152 https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334 * Bugfix #3166: Improve error handling in the `restore` command - The `restore` command used to not print errors while downloading file contents from the - repository. It also incorrectly exited with a zero error code even when there were errors - during the restore process. This has all been fixed and `restore` now returns with a non-zero - exit code when there's an error. + The `restore` command used to not print errors while downloading file contents + from the repository. It also incorrectly exited with a zero error code even when + there were errors during the restore process. This has all been fixed and + `restore` now returns with a non-zero exit code when there's an error. https://github.com/restic/restic/issues/3166 https://github.com/restic/restic/pull/3207 - * Bugfix #3232: Correct statistics for overlapping targets + * Bugfix #3232: Correct statistics for overlapping backup sources - A user reported that restic's statistics and progress information during backup was not - correctly calculated when the backup targets (files/dirs to save) overlap. For example, - consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic - backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the - completeness percentage as well as the number of files was wrong. This is now corrected. + A user reported that restic's statistics and progress information during backup + was not correctly calculated when the backup sources (files/dirs to save) + overlap. For example, consider a directory `foo` which contains (among others) a + file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the + size of the file `foo/bar` twice, so the completeness percentage as well as the + number of files was wrong. This is now corrected. https://github.com/restic/restic/issues/3232 https://github.com/restic/restic/pull/3243 * Bugfix #3249: Improve error handling in `gs` backend - The `gs` backend did not notice when the last step of completing a file upload failed. Under rare - circumstances, this could cause missing files in the backup repository. This has now been - fixed. + The `gs` backend did not notice when the last step of completing a file upload + failed. Under rare circumstances, this could cause missing files in the backup + repository. This has now been fixed. https://github.com/restic/restic/pull/3249 * Change #3095: Deleting files on Google Drive now moves them to the trash - When deleting files on Google Drive via the `rclone` backend, restic used to bypass the trash - folder required that one used the `-o rclone.args` option to enable usage of the trash folder. - This ensured that deleted files in Google Drive were not kept indefinitely in the trash folder. - However, since Google Drive's trash retention policy changed to deleting trashed files after - 30 days, this is no longer needed. + When deleting files on Google Drive via the `rclone` backend, restic used to + bypass the trash folder required that one used the `-o rclone.args` option to + enable usage of the trash folder. This ensured that deleted files in Google + Drive were not kept indefinitely in the trash folder. However, since Google + Drive's trash retention policy changed to deleting trashed files after 30 days, + this is no longer needed. - Restic now leaves it up to rclone and its configuration to use or not use the trash folder when - deleting files. The default is to use the trash folder, as of rclone 1.53.2. To re-enable the - restic 0.11 behavior, set the `RCLONE_DRIVE_USE_TRASH` environment variable or change the - rclone configuration. See the rclone documentation for more details. + Restic now leaves it up to rclone and its configuration to use or not use the + trash folder when deleting files. The default is to use the trash folder, as of + rclone 1.53.2. To re-enable the restic 0.11 behavior, set the + `RCLONE_DRIVE_USE_TRASH` environment variable or change the rclone + configuration. See the rclone documentation for more details. https://github.com/restic/restic/issues/3095 https://github.com/restic/restic/pull/3102 * Enhancement #909: Back up mountpoints as empty directories - When the `--one-file-system` option is specified to `restic backup`, it ignores all file - systems mounted below one of the target directories. This means that when a snapshot is - restored, users needed to manually recreate the mountpoint directories. + When the `--one-file-system` option is specified to `restic backup`, it ignores + all file systems mounted below one of the target directories. This means that + when a snapshot is restored, users needed to manually recreate the mountpoint + directories. - Restic now backs up mountpoints as empty directories and therefore implements the same - approach as `tar`. + Restic now backs up mountpoints as empty directories and therefore implements + the same approach as `tar`. https://github.com/restic/restic/issues/909 https://github.com/restic/restic/pull/3119 * Enhancement #2186: Allow specifying percentage in `check --read-data-subset` - We've enhanced the `check` command's `--read-data-subset` option to also accept a - percentage (e.g. `2.5%` or `10%`). This will check the given percentage of pack files (which - are randomly selected on each run). + We've enhanced the `check` command's `--read-data-subset` option to also accept + a percentage (e.g. `2.5%` or `10%`). This will check the given percentage of + pack files (which are randomly selected on each run). https://github.com/restic/restic/issues/2186 https://github.com/restic/restic/pull/3038 * Enhancement #2433: Make the `dump` command support `zip` format - Previously, restic could dump the contents of a whole folder structure only in the `tar` - format. The `dump` command now has a new flag to change output format to `zip`. Just pass - `--archive zip` as an option to `restic dump`. + Previously, restic could dump the contents of a whole folder structure only in + the `tar` format. The `dump` command now has a new flag to change output format + to `zip`. Just pass `--archive zip` as an option to `restic dump`. https://github.com/restic/restic/pull/2433 https://github.com/restic/restic/pull/3081 * Enhancement #2453: Report permanent/fatal backend errors earlier - When encountering errors in reading from or writing to storage backends, restic retries the - failing operation up to nine times (for a total of ten attempts). It used to retry all backend - operations, but now detects some permanent error conditions so that it can report fatal errors - earlier. + When encountering errors in reading from or writing to storage backends, restic + retries the failing operation up to nine times (for a total of ten attempts). It + used to retry all backend operations, but now detects some permanent error + conditions so that it can report fatal errors earlier. - Permanent failures include local disks being full, SSH connections dropping and permission - errors. + Permanent failures include local disks being full, SSH connections dropping and + permission errors. https://github.com/restic/restic/issues/2453 https://github.com/restic/restic/issues/3180 @@ -2657,23 +3477,26 @@ restic users. The changes are ordered by importance. * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime - The `backup` command used to require that both `ctime` and `mtime` of a file matched with a - previously backed up version to determine that the file was unchanged. In other words, if - either `ctime` or `mtime` of the file had changed, it would be considered changed and restic - would read the file's content again to back up the relevant (changed) parts of it. - - The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes - for a file does not cause restic to read the file's contents again. - - The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more - reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned - out to often be expensive because it made restic read file contents even if only the metadata - (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user - restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned + The `backup` command used to require that both `ctime` and `mtime` of a file + matched with a previously backed up version to determine that the file was + unchanged. In other words, if either `ctime` or `mtime` of the file had changed, + it would be considered changed and restic would read the file's content again to + back up the relevant (changed) parts of it. + + The new option `--ignore-ctime` makes restic look at `mtime` only, such that + `ctime` changes for a file does not cause restic to read the file's contents + again. + + The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make + backups more reliable in the face of programs that reset `mtime` (some Unix + archivers do that), but it turned out to often be expensive because it made + restic read file contents even if only the metadata (owner, permissions) of a + file had changed. The new `--ignore-ctime` option lets the user restore the + 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned off this behavior, but also removed a different check. - Please note that changes in files' metadata are still recorded, regardless of the command line - options provided to the backup command. + Please note that changes in files' metadata are still recorded, regardless of + the command line options provided to the backup command. https://github.com/restic/restic/issues/2495 https://github.com/restic/restic/issues/2558 @@ -2682,20 +3505,21 @@ restic users. The changes are ordered by importance. * Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend - A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the - `s3` backend. The option can be set to one of the following values: + A new extended option `s3.bucket-lookup` has been added to support + Alibaba/Aliyun OSS in the `s3` backend. The option can be set to one of the + following values: - - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - Use path style - bucket access + - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - + Use path style bucket access - To make the `s3` backend work with Alibaba/Aliyun OSS you must set `s3.bucket-lookup` to `dns` - and set the `s3.region` parameter. For example: + To make the `s3` backend work with Alibaba/Aliyun OSS you must set + `s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For example: Restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init - Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up and it seems that - Alibaba doesn't support that properly. + Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up + and it seems that Alibaba doesn't support that properly. https://github.com/restic/restic/issues/2528 https://github.com/restic/restic/pull/2535 @@ -2704,14 +3528,14 @@ restic users. The changes are ordered by importance. The `backup`, `check` and `prune` commands never printed any progress reports on non-interactive terminals. This behavior is now configurable using the - `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for an update - every second, or `0.01666` for an update every minute. + `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for + an update every second, or `0.01666` for an update every minute. - The `backup` command now also prints the current progress when restic receives a `SIGUSR1` - signal. + The `backup` command now also prints the current progress when restic receives a + `SIGUSR1` signal. - Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` signal - prints a status report even when `--quiet` was specified. + Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` + signal prints a status report even when `--quiet` was specified. https://github.com/restic/restic/issues/2706 https://github.com/restic/restic/issues/3194 @@ -2719,21 +3543,22 @@ restic users. The changes are ordered by importance. * Enhancement #2718: Improve `prune` performance and make it more customizable - The `prune` command is now much faster. This is especially the case for remote repositories or - repositories with not much data to remove. Also the memory usage of the `prune` command is now - reduced. + The `prune` command is now much faster. This is especially the case for remote + repositories or repositories with not much data to remove. Also the memory usage + of the `prune` command is now reduced. - Restic used to rebuild the index from scratch after pruning. This could lead to missing packs in - the index in some cases for eventually consistent backends such as e.g. AWS S3. This behavior is - now changed and the index rebuilding uses the information already known by `prune`. + Restic used to rebuild the index from scratch after pruning. This could lead to + missing packs in the index in some cases for eventually consistent backends such + as e.g. AWS S3. This behavior is now changed and the index rebuilding uses the + information already known by `prune`. - By default, the `prune` command no longer removes all unused data. This behavior can be - fine-tuned by new options, like the acceptable amount of unused space or the maximum size of - data to reorganize. For more details, please see + By default, the `prune` command no longer removes all unused data. This behavior + can be fine-tuned by new options, like the acceptable amount of unused space or + the maximum size of data to reorganize. For more details, please see https://restic.readthedocs.io/en/stable/060_forget.html . - Moreover, `prune` now accepts the `--dry-run` option and also running `forget --dry-run - --prune` will show what `prune` would do. + Moreover, `prune` now accepts the `--dry-run` option and also running `forget + --dry-run --prune` will show what `prune` would do. This enhancement also fixes several open issues, e.g.: - https://github.com/restic/restic/issues/1140 - @@ -2748,68 +3573,74 @@ restic users. The changes are ordered by importance. * Enhancement #2941: Speed up the repacking step of the `prune` command - The repack step of the `prune` command, which moves still used file parts into new pack files - such that the old ones can be garbage collected later on, now processes multiple pack files in - parallel. This is especially beneficial for high latency backends or when using a fast network - connection. + The repack step of the `prune` command, which moves still used file parts into + new pack files such that the old ones can be garbage collected later on, now + processes multiple pack files in parallel. This is especially beneficial for + high latency backends or when using a fast network connection. https://github.com/restic/restic/pull/2941 * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}` - The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of - files to back up from a file. Unlike the existing `--files-from` option, these options do not - interpret the listed filenames as glob patterns; instead, whitespace in filenames is - preserved as-is and no pattern expansion is done. Please see the documentation for specifics. + The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a + list of files to back up from a file. Unlike the existing `--files-from` option, + these options do not interpret the listed filenames as glob patterns; instead, + whitespace in filenames is preserved as-is and no pattern expansion is done. + Please see the documentation for specifics. - These new options are highly recommended over `--files-from`, when using a script to generate - the list of files to back up. + These new options are highly recommended over `--files-from`, when using a + script to generate the list of files to back up. https://github.com/restic/restic/issues/2944 https://github.com/restic/restic/issues/3013 * Enhancement #3006: Speed up the `rebuild-index` command - We've optimized the `rebuild-index` command. Now, existing index entries are used to - minimize the number of pack files that must be read. This speeds up the index rebuild a lot. + We've optimized the `rebuild-index` command. Now, existing index entries are + used to minimize the number of pack files that must be read. This speeds up the + index rebuild a lot. - Additionally, the option `--read-all-packs` has been added, implementing the previous - behavior. + Additionally, the option `--read-all-packs` has been added, implementing the + previous behavior. https://github.com/restic/restic/pull/3006 https://github.com/restic/restic/issue/2547 * Enhancement #3048: Add more checks for index and pack files in the `check` command - The `check` command run with the `--read-data` or `--read-data-subset` options used to only - verify only the pack file content - it did not check if the blobs within the pack are correctly - contained in the index. + The `check` command run with the `--read-data` or `--read-data-subset` options + used to only verify only the pack file content - it did not check if the blobs + within the pack are correctly contained in the index. A check for the latter is now in place, which can print the following error: Blob ID is not contained in index or position is incorrect - Another test is also added, which compares pack file sizes computed from the index and the pack - header with the actual file size. This test is able to detect truncated pack files. + Another test is also added, which compares pack file sizes computed from the + index and the pack header with the actual file size. This test is able to detect + truncated pack files. - If the index is not correct, it can be rebuilt by using the `rebuild-index` command. + If the index is not correct, it can be rebuilt by using the `rebuild-index` + command. - Having added these tests, `restic check` is now able to detect non-existing blobs which are - wrongly referenced in the index. This situation could have lead to missing data. + Having added these tests, `restic check` is now able to detect non-existing + blobs which are wrongly referenced in the index. This situation could have lead + to missing data. https://github.com/restic/restic/pull/3048 https://github.com/restic/restic/pull/3082 * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API - Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2` - implementation which causes problems for restic when using their API endpoints. When a broken - server implementation is used, restic prints errors similar to the following: + Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken + `ListObjectsV2` implementation which causes problems for restic when using their + API endpoints. When a broken server implementation is used, restic prints errors + similar to the following: List() returned error: Truncated response should have continuation token set - As a temporary workaround, restic now allows using the older `ListObjects` endpoint by - setting the `s3.list-objects-v1` extended option, for instance: + As a temporary workaround, restic now allows using the older `ListObjects` + endpoint by setting the `s3.list-objects-v1` extended option, for instance: Restic -o s3.list-objects-v1=true snapshots @@ -2820,28 +3651,30 @@ restic users. The changes are ordered by importance. * Enhancement #3099: Reduce memory usage of `check` command - The `check` command now requires less memory if it is run without the `--check-unused` option. + The `check` command now requires less memory if it is run without the + `--check-unused` option. https://github.com/restic/restic/pull/3099 * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune` - The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find - used data. This snapshot traversal is now parallized which can speed up this step several - times. + The `copy` and `prune` commands used to traverse the directories of snapshots + one by one to find used data. This snapshot traversal is now parallelized which + can speed up this step several times. - In addition the `check` command now reports how many snapshots have already been processed. + In addition the `check` command now reports how many snapshots have already been + processed. https://github.com/restic/restic/pull/3106 * Enhancement #3130: Parallelize reading of locks and snapshots - Restic used to read snapshots sequentially. For repositories containing many snapshots this - slowed down commands which have to read all snapshots. + Restic used to read snapshots sequentially. For repositories containing many + snapshots this slowed down commands which have to read all snapshots. - Now the reading of snapshots is parallelized. This speeds up for example `prune`, `backup` and - other commands that search for snapshots with certain properties or which have to find the - `latest` snapshot. + Now the reading of snapshots is parallelized. This speeds up for example + `prune`, `backup` and other commands that search for snapshots with certain + properties or which have to find the `latest` snapshot. The speed up also applies to locks stored in the backup repository. @@ -2850,37 +3683,39 @@ restic users. The changes are ordered by importance. * Enhancement #3147: Support additional environment variables for Swift authentication - The `swift` backend now supports the following additional environment variables for passing - authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`, + The `swift` backend now supports the following additional environment variables + for passing authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`, `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID` - Depending on the `openrc` configuration file these might be required when the user and project - domains differ from one another. + Depending on the `openrc` configuration file these might be required when the + user and project domains differ from one another. https://github.com/restic/restic/issues/3147 https://github.com/restic/restic/pull/3158 * Enhancement #3191: Add release binaries for MIPS architectures - We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`, - `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems. + We've added a few new architectures for Linux to the release binaries: `mips`, + `mipsle`, `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded + systems. https://github.com/restic/restic/issues/3191 https://github.com/restic/restic/pull/3208 * Enhancement #3250: Add several more error checks - We've added a lot more error checks in places where errors were previously ignored (as hinted by - the static analysis program `errcheck` via `golangci-lint`). + We've added a lot more error checks in places where errors were previously + ignored (as hinted by the static analysis program `errcheck` via + `golangci-lint`). https://github.com/restic/restic/pull/3250 * Enhancement #3254: Enable HTTP/2 for backend connections - Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on - what the server supports. But for compatibility this mechanism is disabled if DialContext is - used (which is the case for restic). This change allows restic's HTTP client to negotiate - HTTP/2 if supported by the server. + Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 + depending on what the server supports. But for compatibility this mechanism is + disabled if DialContext is used (which is the case for restic). This change + allows restic's HTTP client to negotiate HTTP/2 if supported by the server. https://github.com/restic/restic/pull/3254 @@ -2911,11 +3746,11 @@ restic users. The changes are ordered by importance. * Bugfix #1212: Restore timestamps and permissions on intermediate directories - When using the `--include` option of the restore command, restic restored timestamps and - permissions only on directories selected by the include pattern. Intermediate directories, - which are necessary to restore files located in sub- directories, were created with default - permissions. We've fixed the restore command to restore timestamps and permissions for these - directories as well. + When using the `--include` option of the restore command, restic restored + timestamps and permissions only on directories selected by the include pattern. + Intermediate directories, which are necessary to restore files located in sub- + directories, were created with default permissions. We've fixed the restore + command to restore timestamps and permissions for these directories as well. https://github.com/restic/restic/issues/1212 https://github.com/restic/restic/issues/1402 @@ -2923,13 +3758,14 @@ restic users. The changes are ordered by importance. * Bugfix #1756: Mark repository files as read-only when using the local backend - Files stored in a local repository were marked as writeable on the filesystem for non-Windows - systems, which did not prevent accidental file modifications outside of restic. In addition, - the local backend did not work with certain filesystems and network mounts which do not permit - modifications of file permissions. + Files stored in a local repository were marked as writable on the filesystem for + non-Windows systems, which did not prevent accidental file modifications outside + of restic. In addition, the local backend did not work with certain filesystems + and network mounts which do not permit modifications of file permissions. - Restic now marks files stored in a local repository as read-only on the filesystem on - non-Windows systems. The error handling is improved to support more filesystems. + Restic now marks files stored in a local repository as read-only on the + filesystem on non-Windows systems. The error handling is improved to support + more filesystems. https://github.com/restic/restic/issues/1756 https://github.com/restic/restic/issues/2157 @@ -2937,8 +3773,9 @@ restic users. The changes are ordered by importance. * Bugfix #2241: Hide password in REST backend repository URLs - When using a password in the REST backend repository URL, the password could in some cases be - included in the output from restic, e.g. when initializing a repo or during an error. + When using a password in the REST backend repository URL, the password could in + some cases be included in the output from restic, e.g. when initializing a repo + or during an error. The password is now replaced with "***" where applicable. @@ -2947,10 +3784,11 @@ restic users. The changes are ordered by importance. * Bugfix #2319: Correctly dump directories into tar files - The dump command previously wrote directories in a tar file in a way which can cause - compatibility problems. This caused, for example, 7zip on Windows to not open tar files - containing directories. In addition it was not possible to dump directories with extended - attributes. These compatibility problems are now corrected. + The dump command previously wrote directories in a tar file in a way which can + cause compatibility problems. This caused, for example, 7zip on Windows to not + open tar files containing directories. In addition it was not possible to dump + directories with extended attributes. These compatibility problems are now + corrected. In addition, a tar file now includes the name of the owner and group of a file. @@ -2959,17 +3797,18 @@ restic users. The changes are ordered by importance. * Bugfix #2491: Don't require `self-update --output` placeholder file - `restic self-update --output /path/to/new-restic` used to require that new-restic was an - existing file, to be overwritten. Now it's possible to download an updated restic binary to a - new path, without first having to create a placeholder file. + `restic self-update --output /path/to/new-restic` used to require that + new-restic was an existing file, to be overwritten. Now it's possible to + download an updated restic binary to a new path, without first having to create + a placeholder file. https://github.com/restic/restic/issues/2491 https://github.com/restic/restic/pull/2937 * Bugfix #2834: Fix rare cases of backup command hanging forever - We've fixed an issue with the backup progress reporting which could cause restic to hang - forever right before finishing a backup. + We've fixed an issue with the backup progress reporting which could cause restic + to hang forever right before finishing a backup. https://github.com/restic/restic/issues/2834 https://github.com/restic/restic/pull/2963 @@ -2983,47 +3822,50 @@ restic users. The changes are ordered by importance. * Bugfix #2942: Make --exclude-larger-than handle disappearing files - There was a small bug in the backup command's --exclude-larger-than option where files that - disappeared between scanning and actually backing them up to the repository caused a panic. - This is now fixed. + There was a small bug in the backup command's --exclude-larger-than option where + files that disappeared between scanning and actually backing them up to the + repository caused a panic. This is now fixed. https://github.com/restic/restic/issues/2942 * Bugfix #2951: Restic generate, help and self-update no longer check passwords - The commands `restic cache`, `generate`, `help` and `self-update` don't need passwords, but - they previously did run the RESTIC_PASSWORD_COMMAND (if set in the environment), prompting - users to authenticate for no reason. They now skip running the password command. + The commands `restic cache`, `generate`, `help` and `self-update` don't need + passwords, but they previously did run the RESTIC_PASSWORD_COMMAND (if set in + the environment), prompting users to authenticate for no reason. They now skip + running the password command. https://github.com/restic/restic/issues/2951 https://github.com/restic/restic/pull/2987 * Bugfix #2979: Make snapshots --json output [] instead of null when no snapshots - Restic previously output `null` instead of `[]` for the `--json snapshots` command, when - there were no snapshots in the repository. This caused some minor problems when parsing the - output, but is now fixed such that `[]` is output when the list of snapshots is empty. + Restic previously output `null` instead of `[]` for the `--json snapshots` + command, when there were no snapshots in the repository. This caused some minor + problems when parsing the output, but is now fixed such that `[]` is output when + the list of snapshots is empty. https://github.com/restic/restic/issues/2979 https://github.com/restic/restic/pull/2984 * Enhancement #340: Add support for Volume Shadow Copy Service (VSS) on Windows - Volume Shadow Copy Service allows read access to files that are locked by another process using - an exclusive lock through a filesystem snapshot. Restic was unable to backup those files - before. This update enables backing up these files. + Volume Shadow Copy Service allows read access to files that are locked by + another process using an exclusive lock through a filesystem snapshot. Restic + was unable to backup those files before. This update enables backing up these + files. - This needs to be enabled explicitely using the --use-fs-snapshot option of the backup - command. + This needs to be enabled explicitly using the --use-fs-snapshot option of the + backup command. https://github.com/restic/restic/issues/340 https://github.com/restic/restic/pull/2274 * Enhancement #1458: New option --repository-file - We've added a new command-line option --repository-file as an alternative to -r. This allows - to read the repository URL from a file in order to prevent certain types of information leaks, - especially for URLs containing credentials. + We've added a new command-line option --repository-file as an alternative to -r. + This allows to read the repository URL from a file in order to prevent certain + types of information leaks, especially for URLs containing credentials. https://github.com/restic/restic/issues/1458 https://github.com/restic/restic/issues/2900 @@ -3031,27 +3873,29 @@ restic users. The changes are ordered by importance. * Enhancement #2849: Authenticate to Google Cloud Storage with access token - When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens - instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable. + When using the GCS backend, it is now possible to authenticate with OAuth2 + access tokens instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN + environment variable. https://github.com/restic/restic/pull/2849 * Enhancement #2969: Optimize check for unchanged files during backup - During a backup restic skips processing files which have not changed since the last backup run. - Previously this required opening each file once which can be slow on network filesystems. The - backup command now checks for file changes before opening a file. This considerably reduces - the time to create a backup on network filesystems. + During a backup restic skips processing files which have not changed since the + last backup run. Previously this required opening each file once which can be + slow on network filesystems. The backup command now checks for file changes + before opening a file. This considerably reduces the time to create a backup on + network filesystems. https://github.com/restic/restic/issues/2969 https://github.com/restic/restic/pull/2970 * Enhancement #2978: Warn if parent snapshot cannot be loaded during backup - During a backup restic uses the parent snapshot to check whether a file was changed and has to be - backed up again. For this check the backup has to read the directories contained in the old - snapshot. If a tree blob cannot be loaded, restic now warns about this problem with the backup - repository. + During a backup restic uses the parent snapshot to check whether a file was + changed and has to be backed up again. For this check the backup has to read the + directories contained in the old snapshot. If a tree blob cannot be loaded, + restic now warns about this problem with the backup repository. https://github.com/restic/restic/pull/2978 @@ -3111,15 +3955,16 @@ restic users. The changes are ordered by importance. * Bugfix #1863: Report correct number of directories processed by backup - The directory statistics calculation was fixed to report the actual number of processed - directories instead of always zero. + The directory statistics calculation was fixed to report the actual number of + processed directories instead of always zero. https://github.com/restic/restic/issues/1863 * Bugfix #2254: Fix tar issues when dumping `/` - We've fixed an issue with dumping either `/` or files on the first sublevel e.g. `/foo` to tar. - This also fixes tar dumping issues on Windows where this issue could also happen. + We've fixed an issue with dumping either `/` or files on the first sublevel e.g. + `/foo` to tar. This also fixes tar dumping issues on Windows where this issue + could also happen. https://github.com/restic/restic/issues/2254 https://github.com/restic/restic/issues/2357 @@ -3127,59 +3972,63 @@ restic users. The changes are ordered by importance. * Bugfix #2281: Handle format verbs like '%' properly in `find` output - The JSON or "normal" output of the `find` command can now deal with file names that contain - substrings which the Golang `fmt` package considers "format verbs" like `%s`. + The JSON or "normal" output of the `find` command can now deal with file names + that contain substrings which the Golang `fmt` package considers "format verbs" + like `%s`. https://github.com/restic/restic/issues/2281 * Bugfix #2298: Do not hang when run as a background job - Restic did hang on exit while restoring the terminal configuration when it was started as a - background job, for example using `restic ... &`. This has been fixed by only restoring the - terminal configuration when restic is interrupted while reading a password from the - terminal. + Restic did hang on exit while restoring the terminal configuration when it was + started as a background job, for example using `restic ... &`. This has been + fixed by only restoring the terminal configuration when restic is interrupted + while reading a password from the terminal. https://github.com/restic/restic/issues/2298 * Bugfix #2389: Fix mangled json output of backup command - We've fixed a race condition in the json output of the backup command that could cause multiple - lines to get mixed up. We've also ensured that the backup summary is printed last. + We've fixed a race condition in the json output of the backup command that could + cause multiple lines to get mixed up. We've also ensured that the backup summary + is printed last. https://github.com/restic/restic/issues/2389 https://github.com/restic/restic/pull/2545 * Bugfix #2390: Refresh lock timestamp - Long-running operations did not refresh lock timestamp, resulting in locks becoming stale. - This is now fixed. + Long-running operations did not refresh lock timestamp, resulting in locks + becoming stale. This is now fixed. https://github.com/restic/restic/issues/2390 * Bugfix #2429: Backup --json reports total_bytes_processed as 0 - We've fixed the json output of total_bytes_processed. The non-json output was already fixed - with pull request #2138 but left the json output untouched. + We've fixed the json output of total_bytes_processed. The non-json output was + already fixed with pull request #2138 but left the json output untouched. https://github.com/restic/restic/issues/2429 * Bugfix #2469: Fix incorrect bytes stats in `diff` command - In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported by the `diff` - command. This is now fixed. + In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported + by the `diff` command. This is now fixed. https://github.com/restic/restic/issues/2469 * Bugfix #2518: Do not crash with Synology NAS sftp server - It was found that when restic is used to store data on an sftp server on a Synology NAS with a - relative path (one which does not start with a slash), it may go into an endless loop trying to - create directories on the server. We've fixed this bug by using a function in the sftp library - instead of our own implementation. + It was found that when restic is used to store data on an sftp server on a + Synology NAS with a relative path (one which does not start with a slash), it + may go into an endless loop trying to create directories on the server. We've + fixed this bug by using a function in the sftp library instead of our own + implementation. - The bug was discovered because the Synology sftp server behaves erratic with non-absolute - path (e.g. `home/restic-repo`). This can be resolved by just using an absolute path instead - (`/home/restic-repo`). We've also added a paragraph in the FAQ. + The bug was discovered because the Synology sftp server behaves erratic with + non-absolute path (e.g. `home/restic-repo`). This can be resolved by just using + an absolute path instead (`/home/restic-repo`). We've also added a paragraph in + the FAQ. https://github.com/restic/restic/issues/2518 https://github.com/restic/restic/issues/2363 @@ -3187,84 +4036,90 @@ restic users. The changes are ordered by importance. * Bugfix #2531: Fix incorrect size calculation in `stats --mode restore-size` - The restore-size mode of stats was counting hard-linked files as if they were independent. + The restore-size mode of stats was counting hard-linked files as if they were + independent. https://github.com/restic/restic/issues/2531 * Bugfix #2537: Fix incorrect file counts in `stats --mode restore-size` - The restore-size mode of stats was failing to count empty directories and some files with hard - links. + The restore-size mode of stats was failing to count empty directories and some + files with hard links. https://github.com/restic/restic/issues/2537 * Bugfix #2592: SFTP backend supports IPv6 addresses - The SFTP backend now supports IPv6 addresses natively, without relying on aliases in the - external SSH configuration. + The SFTP backend now supports IPv6 addresses natively, without relying on + aliases in the external SSH configuration. https://github.com/restic/restic/pull/2592 * Bugfix #2607: Honor RESTIC_CACHE_DIR environment variable on Mac and Windows - On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored. This variable can - now be used on all platforms to set the directory where restic stores caches. + On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored. This + variable can now be used on all platforms to set the directory where restic + stores caches. https://github.com/restic/restic/pull/2607 * Bugfix #2668: Don't abort the stats command when data blobs are missing - Runing the stats command in the blobs-per-file mode on a repository with missing data blobs - previously resulted in a crash. + Running the stats command in the blobs-per-file mode on a repository with + missing data blobs previously resulted in a crash. https://github.com/restic/restic/pull/2668 * Bugfix #2674: Add stricter prune error checks - Additional checks were added to the prune command in order to improve resiliency to backend, - hardware and/or networking issues. The checks now detect a few more cases where such outside - factors could potentially cause data loss. + Additional checks were added to the prune command in order to improve resiliency + to backend, hardware and/or networking issues. The checks now detect a few more + cases where such outside factors could potentially cause data loss. https://github.com/restic/restic/pull/2674 * Bugfix #2899: Fix possible crash in the progress bar of check --read-data - We've fixed a possible crash while displaying the progress bar for the check --read-data - command. The crash occurred when the length of the progress bar status exceeded the terminal - width, which only happened for very narrow terminal windows. + We've fixed a possible crash while displaying the progress bar for the check + --read-data command. The crash occurred when the length of the progress bar + status exceeded the terminal width, which only happened for very narrow terminal + windows. https://github.com/restic/restic/pull/2899 https://forum.restic.net/t/restic-rclone-pcloud-connection-issues/2963/15 * Change #1597: Honor the --no-lock flag in the mount command - The mount command now does not lock the repository if given the --no-lock flag. This allows to - mount repositories which are archived on a read only backend/filesystem. + The mount command now does not lock the repository if given the --no-lock flag. + This allows to mount repositories which are archived on a read only + backend/filesystem. https://github.com/restic/restic/issues/1597 https://github.com/restic/restic/pull/2821 * Change #2482: Remove vendored dependencies - We've removed the vendored dependencies (in the subdir `vendor/`). When building restic, the - Go compiler automatically fetches the dependencies. It will also cryptographically verify - that the correct code has been fetched by using the hashes in `go.sum` (see the link to the - documentation below). + We've removed the vendored dependencies (in the subdir `vendor/`). When building + restic, the Go compiler automatically fetches the dependencies. It will also + cryptographically verify that the correct code has been fetched by using the + hashes in `go.sum` (see the link to the documentation below). https://github.com/restic/restic/issues/2482 https://golang.org/cmd/go/#hdr-Module_downloading_and_verification * Change #2546: Return exit code 3 when failing to backup all source data - The backup command used to return a zero exit code as long as a snapshot could be created - successfully, even if some of the source files could not be read (in which case the snapshot - would contain the rest of the files). + The backup command used to return a zero exit code as long as a snapshot could + be created successfully, even if some of the source files could not be read (in + which case the snapshot would contain the rest of the files). - This made it hard for automation/scripts to detect failures/incomplete backups by looking at - the exit code. Restic now returns the following exit codes for the backup command: + This made it hard for automation/scripts to detect failures/incomplete backups + by looking at the exit code. Restic now returns the following exit codes for the + backup command: - - 0 when the command was successful - 1 when there was a fatal error (no snapshot created) - 3 when - some source data could not be read (incomplete snapshot created) + - 0 when the command was successful - 1 when there was a fatal error (no + snapshot created) - 3 when some source data could not be read (incomplete + snapshot created) https://github.com/restic/restic/issues/956 https://github.com/restic/restic/issues/2064 @@ -3274,12 +4129,12 @@ restic users. The changes are ordered by importance. * Change #2600: Update dependencies, require Go >= 1.13 - Restic now requires Go to be at least 1.13. This allows simplifications in the build process and - removing workarounds. + Restic now requires Go to be at least 1.13. This allows simplifications in the + build process and removing workarounds. - This is also probably the last version of restic still supporting mounting repositories via - fuse on macOS. The library we're using for fuse does not support macOS any more and osxfuse is not - open source any more. + This is also probably the last version of restic still supporting mounting + repositories via fuse on macOS. The library we're using for fuse does not + support macOS any more and osxfuse is not open source any more. https://github.com/bazil/fuse/issues/224 https://github.com/osxfuse/osxfuse/issues/590 @@ -3289,17 +4144,20 @@ restic users. The changes are ordered by importance. * Enhancement #323: Add command for copying snapshots between repositories - We've added a copy command, allowing you to copy snapshots from one repository to another. + We've added a copy command, allowing you to copy snapshots from one repository + to another. - Note that this process will have to read (download) and write (upload) the entire snapshot(s) - due to the different encryption keys used on the source and destination repository. Also, the - transferred files are not re-chunked, which may break deduplication between files already - stored in the destination repo and files copied there using this command. + Note that this process will have to read (download) and write (upload) the + entire snapshot(s) due to the different encryption keys used on the source and + destination repository. Also, the transferred files are not re-chunked, which + may break deduplication between files already stored in the destination repo and + files copied there using this command. - To fully support deduplication between repositories when the copy command is used, the init - command now supports the `--copy-chunker-params` option, which initializes the new - repository with identical parameters for splitting files into chunks as an already existing - repository. This allows copied snapshots to be equally deduplicated in both repositories. + To fully support deduplication between repositories when the copy command is + used, the init command now supports the `--copy-chunker-params` option, which + initializes the new repository with identical parameters for splitting files + into chunks as an already existing repository. This allows copied snapshots to + be equally deduplicated in both repositories. https://github.com/restic/restic/issues/323 https://github.com/restic/restic/pull/2606 @@ -3307,29 +4165,29 @@ restic users. The changes are ordered by importance. * Enhancement #551: Use optimized library for hash calculation of file chunks - We've switched the library used to calculate the hashes of file chunks, which are used for - deduplication, to the optimized Minio SHA-256 implementation. + We've switched the library used to calculate the hashes of file chunks, which + are used for deduplication, to the optimized Minio SHA-256 implementation. - Depending on the CPU it improves the hashing throughput by 10-30%. Modern x86 CPUs with the SHA - Extension should be about two to three times faster. + Depending on the CPU it improves the hashing throughput by 10-30%. Modern x86 + CPUs with the SHA Extension should be about two to three times faster. https://github.com/restic/restic/issues/551 https://github.com/restic/restic/pull/2709 * Enhancement #1570: Support specifying multiple host flags for various commands - Previously commands didn't take more than one `--host` or `-H` argument into account, which - could be limiting with e.g. the `forget` command. + Previously commands didn't take more than one `--host` or `-H` argument into + account, which could be limiting with e.g. the `forget` command. - The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag` - commands will now take into account multiple `--host` and `-H` flags. + The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and + `tag` commands will now take into account multiple `--host` and `-H` flags. https://github.com/restic/restic/issues/1570 * Enhancement #1680: Optimize `restic mount` - We've optimized the FUSE implementation used within restic. `restic mount` is now more - responsive and uses less memory. + We've optimized the FUSE implementation used within restic. `restic mount` is + now more responsive and uses less memory. https://github.com/restic/restic/issues/1680 https://github.com/restic/restic/pull/2587 @@ -3343,10 +4201,11 @@ restic users. The changes are ordered by importance. * Enhancement #2175: Allow specifying user and host when creating keys - When adding a new key to the repository, the username and hostname for the new key can be - specified on the command line. This allows overriding the defaults, for example if you would - prefer to use the FQDN to identify the host or if you want to add keys for several different hosts - without having to run the key add command on those hosts. + When adding a new key to the repository, the username and hostname for the new + key can be specified on the command line. This allows overriding the defaults, + for example if you would prefer to use the FQDN to identify the host or if you + want to add keys for several different hosts without having to run the key add + command on those hosts. https://github.com/restic/restic/issues/2175 @@ -3360,15 +4219,16 @@ restic users. The changes are ordered by importance. Fixes "not enough cache capacity" error during restore: https://github.com/restic/restic/issues/2244 - NOTE: This new implementation does not guarantee order in which blobs are written to the target - files and, for example, the last blob of a file can be written to the file before any of the - preceeding file blobs. It is therefore possible to have gaps in the data written to the target - files if restore fails or interrupted by the user. + NOTE: This new implementation does not guarantee order in which blobs are + written to the target files and, for example, the last blob of a file can be + written to the file before any of the preceding file blobs. It is therefore + possible to have gaps in the data written to the target files if restore fails + or interrupted by the user. - The implementation will try to preallocate space for the restored files on the filesystem to - prevent file fragmentation. This ensures good read performance for large files, like for - example VM images. If preallocating space is not supported by the filesystem, then this step is - silently skipped. + The implementation will try to preallocate space for the restored files on the + filesystem to prevent file fragmentation. This ensures good read performance for + large files, like for example VM images. If preallocating space is not supported + by the filesystem, then this step is silently skipped. https://github.com/restic/restic/pull/2195 https://github.com/restic/restic/pull/2893 @@ -3381,69 +4241,73 @@ restic users. The changes are ordered by importance. * Enhancement #2328: Improve speed of check command - We've improved the check command to traverse trees only once independent of whether they are - contained in multiple snapshots. The check command is now much faster for repositories with a - large number of snapshots. + We've improved the check command to traverse trees only once independent of + whether they are contained in multiple snapshots. The check command is now much + faster for repositories with a large number of snapshots. https://github.com/restic/restic/issues/2284 https://github.com/restic/restic/pull/2328 * Enhancement #2395: Ignore sync errors when operation not supported by local filesystem - The local backend has been modified to work with filesystems which doesn't support the `sync` - operation. This operation is normally used by restic to ensure that data files are fully - written to disk before continuing. + The local backend has been modified to work with filesystems which doesn't + support the `sync` operation. This operation is normally used by restic to + ensure that data files are fully written to disk before continuing. - For these limited filesystems, saving a file in the backend would previously fail with an - "operation not supported" error. This error is now ignored, which means that e.g. an SMB mount - on macOS can now be used as storage location for a repository. + For these limited filesystems, saving a file in the backend would previously + fail with an "operation not supported" error. This error is now ignored, which + means that e.g. an SMB mount on macOS can now be used as storage location for a + repository. https://github.com/restic/restic/issues/2395 https://forum.restic.net/t/sync-errors-on-mac-over-smb/1859 * Enhancement #2423: Support user@domain parsing as user - Added the ability for user@domain-like users to be authenticated over SFTP servers. + Added the ability for user@domain-like users to be authenticated over SFTP + servers. https://github.com/restic/restic/pull/2423 * Enhancement #2427: Add flag `--iexclude-file` to backup command - The backup command now supports the flag `--iexclude-file` which is a case-insensitive - version of `--exclude-file`. + The backup command now supports the flag `--iexclude-file` which is a + case-insensitive version of `--exclude-file`. https://github.com/restic/restic/issues/2427 https://github.com/restic/restic/pull/2898 * Enhancement #2569: Support excluding files by their size - The `backup` command now supports the `--exclude-larger-than` option to exclude files which - are larger than the specified maximum size. This can for example be useful to exclude - unimportant files with a large file size. + The `backup` command now supports the `--exclude-larger-than` option to exclude + files which are larger than the specified maximum size. This can for example be + useful to exclude unimportant files with a large file size. https://github.com/restic/restic/issues/2569 https://github.com/restic/restic/pull/2914 * Enhancement #2571: Self-heal missing file parts during backup of unchanged files - We've improved the resilience of restic to certain types of repository corruption. + We've improved the resilience of restic to certain types of repository + corruption. - For files that are unchanged since the parent snapshot, the backup command now verifies that - all parts of the files still exist in the repository. Parts that are missing, e.g. from a damaged - repository, are backed up again. This verification was already run for files that were - modified since the parent snapshot, but is now also done for unchanged files. + For files that are unchanged since the parent snapshot, the backup command now + verifies that all parts of the files still exist in the repository. Parts that + are missing, e.g. from a damaged repository, are backed up again. This + verification was already run for files that were modified since the parent + snapshot, but is now also done for unchanged files. - Note that restic will not backup file parts that are referenced in the index but where the actual - data is not present on disk, as this situation can only be detected by restic check. Please - ensure that you run `restic check` regularly. + Note that restic will not backup file parts that are referenced in the index but + where the actual data is not present on disk, as this situation can only be + detected by restic check. Please ensure that you run `restic check` regularly. https://github.com/restic/restic/issues/2571 https://github.com/restic/restic/pull/2827 * Enhancement #2576: Improve the chunking algorithm - We've updated the chunker library responsible for splitting files into smaller blocks. It - should improve the chunking throughput by 5-15% depending on the CPU. + We've updated the chunker library responsible for splitting files into smaller + blocks. It should improve the chunking throughput by 5-15% depending on the CPU. https://github.com/restic/restic/issues/2820 https://github.com/restic/restic/pull/2576 @@ -3451,65 +4315,68 @@ restic users. The changes are ordered by importance. * Enhancement #2598: Improve speed of diff command - We've improved the performance of the diff command when comparing snapshots with similar - content. It should run up to twice as fast as before. + We've improved the performance of the diff command when comparing snapshots with + similar content. It should run up to twice as fast as before. https://github.com/restic/restic/pull/2598 * Enhancement #2599: Slightly reduce memory usage of prune and stats commands - The prune and the stats command kept directory identifiers in memory twice while searching for - used blobs. + The prune and the stats command kept directory identifiers in memory twice while + searching for used blobs. https://github.com/restic/restic/pull/2599 * Enhancement #2733: S3 backend: Add support for WebIdentityTokenFile - We've added support for EKS IAM roles for service accounts feature to the S3 backend. + We've added support for EKS IAM roles for service accounts feature to the S3 + backend. https://github.com/restic/restic/issues/2703 https://github.com/restic/restic/pull/2733 * Enhancement #2773: Optimize handling of new index entries - Restic now uses less memory for backups which add a lot of data, e.g. large initial backups. In - addition, we've improved the stability in some edge cases. + Restic now uses less memory for backups which add a lot of data, e.g. large + initial backups. In addition, we've improved the stability in some edge cases. https://github.com/restic/restic/pull/2773 * Enhancement #2781: Reduce memory consumption of in-memory index - We've improved how the index is stored in memory. This change can reduce memory usage for large - repositories by up to 50% (depending on the operation). + We've improved how the index is stored in memory. This change can reduce memory + usage for large repositories by up to 50% (depending on the operation). https://github.com/restic/restic/pull/2781 https://github.com/restic/restic/pull/2812 * Enhancement #2786: Optimize `list blobs` command - We've changed the implementation of `list blobs` which should be now a bit faster and consume - almost no memory even for large repositories. + We've changed the implementation of `list blobs` which should be now a bit + faster and consume almost no memory even for large repositories. https://github.com/restic/restic/pull/2786 * Enhancement #2790: Optimized file access in restic mount - Reading large (> 100GiB) files from restic mountpoints is now faster, and the speedup is - greater for larger files. + Reading large (> 100GiB) files from restic mountpoints is now faster, and the + speedup is greater for larger files. https://github.com/restic/restic/pull/2790 * Enhancement #2840: Speed-up file deletion in forget, prune and rebuild-index - We've sped up the file deletion for the commands forget, prune and rebuild-index, especially - for remote repositories. Deletion was sequential before and is now run in parallel. + We've sped up the file deletion for the commands forget, prune and + rebuild-index, especially for remote repositories. Deletion was sequential + before and is now run in parallel. https://github.com/restic/restic/pull/2840 * Enhancement #2858: Support filtering snapshots by tag and path in the stats command - We've added filtering snapshots by `--tag tagList` and by `--path path` to the `stats` - command. This includes filtering of only 'latest' snapshots or all snapshots in a repository. + We've added filtering snapshots by `--tag tagList` and by `--path path` to the + `stats` command. This includes filtering of only 'latest' snapshots or all + snapshots in a repository. https://github.com/restic/restic/issues/2858 https://github.com/restic/restic/pull/2859 @@ -3536,81 +4403,85 @@ restic users. The changes are ordered by importance. * Bugfix #2063: Allow absolute path for filename when backing up from stdin - When backing up from stdin, handle directory path for `--stdin-filename`. This can be used to - specify the full path for the backed-up file. + When backing up from stdin, handle directory path for `--stdin-filename`. This + can be used to specify the full path for the backed-up file. https://github.com/restic/restic/issues/2063 * Bugfix #2174: Save files with invalid timestamps - When restic reads invalid timestamps (year is before 0000 or after 9999) it refused to read and - archive the file. We've changed the behavior and will now save modified timestamps with the - year set to either 0000 or 9999, the rest of the timestamp stays the same, so the file will be saved - (albeit with a bogus timestamp). + When restic reads invalid timestamps (year is before 0000 or after 9999) it + refused to read and archive the file. We've changed the behavior and will now + save modified timestamps with the year set to either 0000 or 9999, the rest of + the timestamp stays the same, so the file will be saved (albeit with a bogus + timestamp). https://github.com/restic/restic/issues/2174 https://github.com/restic/restic/issues/1173 * Bugfix #2249: Read fresh metadata for unmodified files - Restic took all metadata for files which were detected as unmodified, not taking into account - changed metadata (ownership, mode). This is now corrected. + Restic took all metadata for files which were detected as unmodified, not taking + into account changed metadata (ownership, mode). This is now corrected. https://github.com/restic/restic/issues/2249 https://github.com/restic/restic/pull/2252 * Bugfix #2301: Add upper bound for t in --read-data-subset=n/t - 256 is the effective maximum for t, but restic would allow larger values, leading to strange - behavior. + 256 is the effective maximum for t, but restic would allow larger values, + leading to strange behavior. https://github.com/restic/restic/issues/2301 https://github.com/restic/restic/pull/2304 * Bugfix #2321: Check errors when loading index files - Restic now checks and handles errors which occur when loading index files, the missing check - leads to odd errors (and a stack trace printed to users) later. This was reported in the forum. + Restic now checks and handles errors which occur when loading index files, the + missing check leads to odd errors (and a stack trace printed to users) later. + This was reported in the forum. https://github.com/restic/restic/pull/2321 https://forum.restic.net/t/check-rebuild-index-prune/1848/13 * Enhancement #2179: Use ctime when checking for file changes - Previously, restic only checked a file's mtime (along with other non-timestamp metadata) to - decide if a file has changed. This could cause restic to not notice that a file has changed (and - therefore continue to store the old version, as opposed to the modified version) if something - edits the file and then resets the timestamp. Restic now also checks the ctime of files, so any - modifications to a file should be noticed, and the modified file will be backed up. The ctime - check will be disabled if the --ignore-inode flag was given. + Previously, restic only checked a file's mtime (along with other non-timestamp + metadata) to decide if a file has changed. This could cause restic to not notice + that a file has changed (and therefore continue to store the old version, as + opposed to the modified version) if something edits the file and then resets the + timestamp. Restic now also checks the ctime of files, so any modifications to a + file should be noticed, and the modified file will be backed up. The ctime check + will be disabled if the --ignore-inode flag was given. - If this change causes problems for you, please open an issue, and we can look in to adding a - seperate flag to disable just the ctime check. + If this change causes problems for you, please open an issue, and we can look in + to adding a separate flag to disable just the ctime check. https://github.com/restic/restic/issues/2179 https://github.com/restic/restic/pull/2212 * Enhancement #2306: Allow multiple retries for interactive password input - Restic used to quit if the repository password was typed incorrectly once. Restic will now ask - the user again for the repository password if typed incorrectly. The user will now get three - tries to input the correct password before restic quits. + Restic used to quit if the repository password was typed incorrectly once. + Restic will now ask the user again for the repository password if typed + incorrectly. The user will now get three tries to input the correct password + before restic quits. https://github.com/restic/restic/issues/2306 * Enhancement #2330: Make `--group-by` accept both singular and plural - One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` interchangeably - in the `--group-by` argument. + One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` + interchangeably in the `--group-by` argument. https://github.com/restic/restic/issues/2330 * Enhancement #2350: Add option to configure S3 region - We've added a new option for setting the region when accessing an S3-compatible service. For - some providers, it is required to set this to a valid value. You can do that either by setting the - environment variable `AWS_DEFAULT_REGION` or using the option `s3.region`, e.g. like this: - `-o s3.region="us-east-1"`. + We've added a new option for setting the region when accessing an S3-compatible + service. For some providers, it is required to set this to a valid value. You + can do that either by setting the environment variable `AWS_DEFAULT_REGION` or + using the option `s3.region`, e.g. like this: `-o s3.region="us-east-1"`. https://github.com/restic/restic/pull/2350 @@ -3639,10 +4510,11 @@ restic users. The changes are ordered by importance. * Bugfix #2135: Return error when no bytes could be read from stdin - We assume that users reading backup data from stdin want to know when no data could be read, so now - restic returns an error when `backup --stdin` is called but no bytes could be read. Usually, - this means that an earlier command in a pipe has failed. The documentation was amended and now - recommends setting the `pipefail` option (`set -o pipefail`). + We assume that users reading backup data from stdin want to know when no data + could be read, so now restic returns an error when `backup --stdin` is called + but no bytes could be read. Usually, this means that an earlier command in a + pipe has failed. The documentation was amended and now recommends setting the + `pipefail` option (`set -o pipefail`). https://github.com/restic/restic/pull/2135 https://github.com/restic/restic/pull/2139 @@ -3653,84 +4525,88 @@ restic users. The changes are ordered by importance. * Bugfix #2203: Fix reading passwords from stdin - Passwords for the `init`, `key add`, and `key passwd` commands can now be read from - non-terminal stdin. + Passwords for the `init`, `key add`, and `key passwd` commands can now be read + from non-terminal stdin. https://github.com/restic/restic/issues/2203 * Bugfix #2224: Don't abort the find command when a tree can't be loaded - Change the find command so that missing trees don't result in a crash. Instead, the error is - logged to the debug log, and the tree ID is displayed along with the snapshot it belongs to. This - makes it possible to recover repositories that are missing trees by forgetting the snapshots - they are used in. + Change the find command so that missing trees don't result in a crash. Instead, + the error is logged to the debug log, and the tree ID is displayed along with + the snapshot it belongs to. This makes it possible to recover repositories that + are missing trees by forgetting the snapshots they are used in. https://github.com/restic/restic/issues/2224 * Enhancement #1895: Add case insensitive include & exclude options - The backup and restore commands now have --iexclude and --iinclude flags as case insensitive - variants of --exclude and --include. + The backup and restore commands now have --iexclude and --iinclude flags as case + insensitive variants of --exclude and --include. https://github.com/restic/restic/issues/1895 https://github.com/restic/restic/pull/2032 * Enhancement #1937: Support streaming JSON output for backup - We've added support for getting machine-readable status output during backup, just pass the - flag `--json` for `restic backup` and restic will output a stream of JSON objects which contain - the current progress. + We've added support for getting machine-readable status output during backup, + just pass the flag `--json` for `restic backup` and restic will output a stream + of JSON objects which contain the current progress. https://github.com/restic/restic/issues/1937 https://github.com/restic/restic/pull/1944 * Enhancement #2037: Add group-by option to snapshots command - We have added an option to group the output of the snapshots command, similar to the output of the - forget command. The option has been called "--group-by" and accepts any combination of the - values "host", "paths" and "tags", separated by commas. Default behavior (not specifying - --group-by) has not been changed. We have added support of the grouping to the JSON output. + We have added an option to group the output of the snapshots command, similar to + the output of the forget command. The option has been called "--group-by" and + accepts any combination of the values "host", "paths" and "tags", separated by + commas. Default behavior (not specifying --group-by) has not been changed. We + have added support of the grouping to the JSON output. https://github.com/restic/restic/issues/2037 https://github.com/restic/restic/pull/2087 * Enhancement #2124: Ability to dump folders to tar via stdout - We've added the ability to dump whole folders to stdout via the `dump` command. Restic now - requires at least Go 1.10 due to a limitation of the standard library for Go <= 1.9. + We've added the ability to dump whole folders to stdout via the `dump` command. + Restic now requires at least Go 1.10 due to a limitation of the standard library + for Go <= 1.9. https://github.com/restic/restic/issues/2123 https://github.com/restic/restic/pull/2124 * Enhancement #2139: Return error if no bytes could be read for `backup --stdin` - When restic is used to backup the output of a program, like `mysqldump | restic backup --stdin`, - it now returns an error if no bytes could be read at all. This catches the failure case when - `mysqldump` failed for some reason and did not output any data to stdout. + When restic is used to backup the output of a program, like `mysqldump | restic + backup --stdin`, it now returns an error if no bytes could be read at all. This + catches the failure case when `mysqldump` failed for some reason and did not + output any data to stdout. https://github.com/restic/restic/pull/2139 * Enhancement #2155: Add Openstack application credential auth for Swift - Since Openstack Queens Identity (auth V3) service supports an application credential auth - method. It allows to create a technical account with the limited roles. This commit adds an - application credential authentication method for the Swift backend. + Since Openstack Queens Identity (auth V3) service supports an application + credential auth method. It allows to create a technical account with the limited + roles. This commit adds an application credential authentication method for the + Swift backend. https://github.com/restic/restic/issues/2155 * Enhancement #2184: Add --json support to forget command - The forget command now supports the --json argument, outputting the information about what is - (or would-be) kept and removed from the repository. + The forget command now supports the --json argument, outputting the information + about what is (or would-be) kept and removed from the repository. https://github.com/restic/restic/issues/2184 https://github.com/restic/restic/pull/2185 * Enhancement #2205: Add --ignore-inode option to backup cmd - This option handles backup of virtual filesystems that do not keep fixed inodes for files, like - Fuse-based, pCloud, etc. Ignoring inode changes allows to consider the file as unchanged if - last modification date and size are unchanged. + This option handles backup of virtual filesystems that do not keep fixed inodes + for files, like Fuse-based, pCloud, etc. Ignoring inode changes allows to + consider the file as unchanged if last modification date and size are unchanged. https://github.com/restic/restic/issues/1631 https://github.com/restic/restic/pull/2205 @@ -3738,16 +4614,17 @@ restic users. The changes are ordered by importance. * Enhancement #2220: Add config option to set S3 storage class - The `s3.storage-class` option can be passed to restic (using `-o`) to specify the storage - class to be used for S3 objects created by restic. + The `s3.storage-class` option can be passed to restic (using `-o`) to specify + the storage class to be used for S3 objects created by restic. - The storage class is passed as-is to S3, so it needs to be understood by the API. On AWS, it can be - one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING` and - `REDUCED_REDUNDANCY`. If unspecified, the default storage class is used (`STANDARD` on - AWS). + The storage class is passed as-is to S3, so it needs to be understood by the + API. On AWS, it can be one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, + `INTELLIGENT_TIERING` and `REDUCED_REDUNDANCY`. If unspecified, the default + storage class is used (`STANDARD` on AWS). - You can mix storage classes in the same bucket, and the setting isn't stored in the restic - repository, so be sure to specify it with each command that writes to S3. + You can mix storage classes in the same bucket, and the setting isn't stored in + the restic repository, so be sure to specify it with each command that writes to + S3. https://github.com/restic/restic/issues/706 https://github.com/restic/restic/pull/2220 @@ -3775,19 +4652,19 @@ restic users. The changes are ordered by importance. * Bugfix #1989: Google Cloud Storage: Respect bandwidth limit - The GCS backend did not respect the bandwidth limit configured, a previous commit - accidentally removed support for it. + The GCS backend did not respect the bandwidth limit configured, a previous + commit accidentally removed support for it. https://github.com/restic/restic/issues/1989 https://github.com/restic/restic/pull/2100 * Bugfix #2040: Add host name filter shorthand flag for `stats` command - The default value for `--host` flag was set to 'H' (the shorthand version of the flag), this - caused the lookup for the latest snapshot to fail. + The default value for `--host` flag was set to 'H' (the shorthand version of the + flag), this caused the lookup for the latest snapshot to fail. - Add shorthand flag `-H` for `--host` (with empty default so if these flags are not specified the - latest snapshot will not filter by host name). + Add shorthand flag `-H` for `--host` (with empty default so if these flags are + not specified the latest snapshot will not filter by host name). Also add shorthand `-H` for `backup` command. @@ -3795,17 +4672,17 @@ restic users. The changes are ordered by importance. * Bugfix #2068: Correctly return error loading data - In one case during `prune` and `check`, an error loading data from the backend is not returned - properly. This is now corrected. + In one case during `prune` and `check`, an error loading data from the backend + is not returned properly. This is now corrected. https://github.com/restic/restic/issues/1999#issuecomment-433737921 https://github.com/restic/restic/pull/2068 * Bugfix #2095: Consistently use local time for snapshots times - By default snapshots created with restic backup were set to local time, but when the --time flag - was used the provided timestamp was parsed as UTC. With this change all snapshots times are set - to local time. + By default snapshots created with restic backup were set to local time, but when + the --time flag was used the provided timestamp was parsed as UTC. With this + change all snapshots times are set to local time. https://github.com/restic/restic/pull/2095 @@ -3814,65 +4691,70 @@ restic users. The changes are ordered by importance. This change significantly improves restore performance, especially when using high-latency remote repositories like B2. - The implementation now uses several concurrent threads to download and process multiple - remote files concurrently. To further reduce restore time, each remote file is downloaded - using a single repository request. + The implementation now uses several concurrent threads to download and process + multiple remote files concurrently. To further reduce restore time, each remote + file is downloaded using a single repository request. https://github.com/restic/restic/issues/1605 https://github.com/restic/restic/pull/1719 * Enhancement #2017: Mount: Enforce FUSE Unix permissions with allow-other - The fuse mount (`restic mount`) now lets the kernel check the permissions of the files within - snapshots (this is done through the `DefaultPermissions` FUSE option) when the option - `--allow-other` is specified. + The fuse mount (`restic mount`) now lets the kernel check the permissions of the + files within snapshots (this is done through the `DefaultPermissions` FUSE + option) when the option `--allow-other` is specified. - To restore the old behavior, we've added the `--no-default-permissions` option. This allows - all users that have access to the mount point to access all files within the snapshots. + To restore the old behavior, we've added the `--no-default-permissions` option. + This allows all users that have access to the mount point to access all files + within the snapshots. https://github.com/restic/restic/pull/2017 * Enhancement #2070: Make all commands display timestamps in local time - Restic used to drop the timezone information from displayed timestamps, it now converts - timestamps to local time before printing them so the times can be easily compared to. + Restic used to drop the timezone information from displayed timestamps, it now + converts timestamps to local time before printing them so the times can be + easily compared to. https://github.com/restic/restic/pull/2070 * Enhancement #2085: Allow --files-from to be specified multiple times - Before, restic took only the last file specified with `--files-from` into account, this is now - corrected. + Before, restic took only the last file specified with `--files-from` into + account, this is now corrected. https://github.com/restic/restic/issues/2085 https://github.com/restic/restic/pull/2086 * Enhancement #2089: Increase granularity of the "keep within" retention policy - The `keep-within` option of the `forget` command now accepts time ranges with an hourly - granularity. For example, running `restic forget --keep-within 3d12h` will keep all the - snapshots made within three days and twelve hours from the time of the latest snapshot. + The `keep-within` option of the `forget` command now accepts time ranges with an + hourly granularity. For example, running `restic forget --keep-within 3d12h` + will keep all the snapshots made within three days and twelve hours from the + time of the latest snapshot. https://github.com/restic/restic/issues/2089 https://github.com/restic/restic/pull/2090 * Enhancement #2094: Run command to get password - We've added the `--password-command` option which allows specifying a command that restic - runs every time the password for the repository is needed, so it can be integrated with a - password manager or keyring. The option can also be set via the environment variable - `$RESTIC_PASSWORD_COMMAND`. + We've added the `--password-command` option which allows specifying a command + that restic runs every time the password for the repository is needed, so it can + be integrated with a password manager or keyring. The option can also be set via + the environment variable `$RESTIC_PASSWORD_COMMAND`. https://github.com/restic/restic/pull/2094 * Enhancement #2097: Add key hinting - Added a new option `--key-hint` and corresponding environment variable `RESTIC_KEY_HINT`. - The key hint is a key ID to try decrypting first, before other keys in the repository. + Added a new option `--key-hint` and corresponding environment variable + `RESTIC_KEY_HINT`. The key hint is a key ID to try decrypting first, before + other keys in the repository. - This change will benefit repositories with many keys; if the correct key hint is supplied then - restic only needs to check one key. If the key hint is incorrect (the key does not exist, or the - password is incorrect) then restic will check all keys, as usual. + This change will benefit repositories with many keys; if the correct key hint is + supplied then restic only needs to check one key. If the key hint is incorrect + (the key does not exist, or the password is incorrect) then restic will check + all keys, as usual. https://github.com/restic/restic/issues/2097 @@ -3902,29 +4784,31 @@ restic users. The changes are ordered by importance. * Bugfix #1935: Remove truncated files from cache - When a file in the local cache is truncated, and restic tries to access data beyond the end of the - (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are - removed and the data is fetched directly from the backend. + When a file in the local cache is truncated, and restic tries to access data + beyond the end of the (cached) file, it used to return an error "EOF". This is + now fixed, such truncated files are removed and the data is fetched directly + from the backend. https://github.com/restic/restic/issues/1935 * Bugfix #1978: Do not return an error when the scanner is slower than backup - When restic makes a backup, there's a background task called "scanner" which collects - information on how many files and directories are to be saved, in order to display progress - information to the user. When the backup finishes faster than the scanner, it is aborted - because the result is not needed any more. This logic contained a bug, where quitting the - scanner process was treated as an error, and caused restic to print an unhelpful error message - ("context canceled"). + When restic makes a backup, there's a background task called "scanner" which + collects information on how many files and directories are to be saved, in order + to display progress information to the user. When the backup finishes faster + than the scanner, it is aborted because the result is not needed any more. This + logic contained a bug, where quitting the scanner process was treated as an + error, and caused restic to print an unhelpful error message ("context + canceled"). https://github.com/restic/restic/issues/1978 https://github.com/restic/restic/pull/1991 * Enhancement #1766: Restore: suppress lchown errors when not running as root - Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files - during restore if it is run as root, on non-Windows operating systems. On Windows, the error - is reported as usual. + Like "cp" and "rsync" do, restic now only reports errors for changing the + ownership of files during restore if it is run as root, on non-Windows + operating systems. On Windows, the error is reported as usual. https://github.com/restic/restic/issues/1766 @@ -3932,113 +4816,118 @@ restic users. The changes are ordered by importance. We've updated the `find` command to support multiple patterns. - `restic find` is now able to list the snapshots containing a specific tree or blob, or even the - snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they - all have the same type. + `restic find` is now able to list the snapshots containing a specific tree or + blob, or even the snapshots that contain blobs belonging to a given pack. A list + of IDs can be given, as long as they all have the same type. - The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id` - flag is provided. + The command `find` can also display the pack IDs the blobs belong to, if the + `--show-pack-id` flag is provided. https://github.com/restic/restic/issues/1777 https://github.com/restic/restic/pull/1780 * Enhancement #1876: Display reason why forget keeps snapshots - We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a - particuliar snapshot. This makes debugging policies for forget much easier. Please remember - to always try things out with `--dry-run`! + We've added a column to the list of snapshots `forget` keeps which details the + reasons to keep a particular snapshot. This makes debugging policies for forget + much easier. Please remember to always try things out with `--dry-run`! https://github.com/restic/restic/pull/1876 * Enhancement #1891: Accept glob in paths loaded via --files-from - Before that, behaviour was different if paths were appended to command line or from a file, - because wild card characters were expanded by shell if appended to command line, but not - expanded if loaded from file. + Before that, behaviour was different if paths were appended to command line or + from a file, because wild card characters were expanded by shell if appended to + command line, but not expanded if loaded from file. https://github.com/restic/restic/issues/1891 * Enhancement #1909: Reject files/dirs by name first - The current scanner/archiver code had an architectural limitation: it always ran the - `lstat()` system call on all files and directories before a decision to include/exclude the - file/dir was made. This lead to a lot of unnecessary system calls for items that could have been - rejected by their name or path only. + The current scanner/archiver code had an architectural limitation: it always ran + the `lstat()` system call on all files and directories before a decision to + include/exclude the file/dir was made. This lead to a lot of unnecessary system + calls for items that could have been rejected by their name or path only. - We've changed the archiver/scanner implementation so that it now first rejects by name/path, - and only runs the system call on the remaining items. This reduces the number of `lstat()` - system calls a lot (depending on the exclude settings). + We've changed the archiver/scanner implementation so that it now first rejects + by name/path, and only runs the system call on the remaining items. This reduces + the number of `lstat()` system calls a lot (depending on the exclude settings). https://github.com/restic/restic/issues/1909 https://github.com/restic/restic/pull/1912 * Enhancement #1920: Vendor dependencies with Go 1.11 Modules - Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules. - For users this does not change much, only if you want to compile restic without downloading - anything with Go 1.11, then you need to run: `go build -mod=vendor build.go` + Until now, we've used `dep` for managing dependencies, we've now switch to using + Go modules. For users this does not change much, only if you want to compile + restic without downloading anything with Go 1.11, then you need to run: `go + build -mod=vendor build.go` https://github.com/restic/restic/pull/1920 * Enhancement #1940: Add directory filter to ls command - The ls command can now be filtered by directories, so that only files in the given directories - will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list - their files as well. + The ls command can now be filtered by directories, so that only files in the + given directories will be shown. If the --recursive flag is specified, then ls + will traverse subfolders and list their files as well. - It used to be possible to specify multiple snapshots, but that has been replaced by only one - snapshot and the possibility of specifying multiple directories. + It used to be possible to specify multiple snapshots, but that has been replaced + by only one snapshot and the possibility of specifying multiple directories. - Specifying directories constrains the walk, which can significantly speed up the listing. + Specifying directories constrains the walk, which can significantly speed up the + listing. https://github.com/restic/restic/issues/1940 https://github.com/restic/restic/pull/1941 * Enhancement #1949: Add new command `self-update` - We have added a new command called `self-update` which downloads the latest released version - of restic from GitHub and replaces the current binary with it. It does not rely on any external - program (so it'll work everywhere), but still verifies the GPG signature using the embedded - GPG public key. + We have added a new command called `self-update` which downloads the latest + released version of restic from GitHub and replaces the current binary with it. + It does not rely on any external program (so it'll work everywhere), but still + verifies the GPG signature using the embedded GPG public key. - By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is - only set when restic is built using `build.go` (including official releases). The reason for - this is that downstream distributions will then not include the command by default, so users - are encouraged to use the platform-specific distribution mechanism. + By default, the `self-update` command is hidden behind the `selfupdate` built + tag, which is only set when restic is built using `build.go` (including official + releases). The reason for this is that downstream distributions will then not + include the command by default, so users are encouraged to use the + platform-specific distribution mechanism. https://github.com/restic/restic/pull/1949 * Enhancement #1953: Ls: Add JSON output support for restic ls cmd - We've implemented listing files in the repository with JSON as output, just pass `--json` as an - option to `restic ls`. This makes the output of the command machine readable. + We've implemented listing files in the repository with JSON as output, just pass + `--json` as an option to `restic ls`. This makes the output of the command + machine readable. https://github.com/restic/restic/pull/1953 * Enhancement #1962: Stream JSON output for ls command - The `ls` command now supports JSON output with the global `--json` flag, and this change - streams out JSON messages one object at a time rather than en entire array buffered in memory - before encoding. The advantage is it allows large listings to be handled efficiently. + The `ls` command now supports JSON output with the global `--json` flag, and + this change streams out JSON messages one object at a time rather than en entire + array buffered in memory before encoding. The advantage is it allows large + listings to be handled efficiently. - Two message types are printed: snapshots and nodes. A snapshot object will precede node - objects which belong to that snapshot. The `struct_type` field can be used to determine which - kind of message an object is. + Two message types are printed: snapshots and nodes. A snapshot object will + precede node objects which belong to that snapshot. The `struct_type` field can + be used to determine which kind of message an object is. https://github.com/restic/restic/pull/1962 * Enhancement #1967: Use `--host` everywhere - We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g. - for `restic backup`) still works, but will print a deprecation warning. Also, add the short - option `-H` where possible. + We now use the flag `--host` for all commands which need a host name, using + `--hostname` (e.g. for `restic backup`) still works, but will print a + deprecation warning. Also, add the short option `-H` where possible. https://github.com/restic/restic/issues/1967 * Enhancement #2028: Display size of cache directories - The `cache` command now by default shows the size of the individual cache directories. It can be - disabled with `--no-size`. + The `cache` command now by default shows the size of the individual cache + directories. It can be disabled with `--no-size`. https://github.com/restic/restic/issues/2028 https://github.com/restic/restic/pull/2033 @@ -4066,23 +4955,25 @@ restic users. The changes are ordered by importance. * Bugfix #1854: Allow saving files/dirs on different fs with `--one-file-system` - Restic now allows saving files/dirs on a different file system in a subdir correctly even when - `--one-file-system` is specified. + Restic now allows saving files/dirs on a different file system in a subdir + correctly even when `--one-file-system` is specified. The first thing the restic archiver code does is to build a tree of the target - files/directories. If it detects that a parent directory is already included (e.g. `restic - backup /foo /foo/bar/baz`), it'll ignore the latter argument. + files/directories. If it detects that a parent directory is already included + (e.g. `restic backup /foo /foo/bar/baz`), it'll ignore the latter argument. - Without `--one-file-system`, that's perfectly valid: If `/foo` is to be archived, it will - include `/foo/bar/baz`. But with `--one-file-system`, `/foo/bar/baz` may reside on a - different file system, so it won't be included with `/foo`. + Without `--one-file-system`, that's perfectly valid: If `/foo` is to be + archived, it will include `/foo/bar/baz`. But with `--one-file-system`, + `/foo/bar/baz` may reside on a different file system, so it won't be included + with `/foo`. https://github.com/restic/restic/issues/1854 https://github.com/restic/restic/pull/1855 * Bugfix #1861: Fix case-insensitive search with restic find - We've fixed the behavior for `restic find -i PATTERN`, which was broken in v0.9.1. + We've fixed the behavior for `restic find -i PATTERN`, which was broken in + v0.9.1. https://github.com/restic/restic/pull/1861 @@ -4095,21 +4986,22 @@ restic users. The changes are ordered by importance. * Bugfix #1880: Use `--cache-dir` argument for `check` command - `check` command now uses a temporary sub-directory of the specified directory if set using the - `--cache-dir` argument. If not set, the cache directory is created in the default temporary - directory as before. In either case a temporary cache is used to ensure the actual repository is - checked (rather than a local copy). + `check` command now uses a temporary sub-directory of the specified directory if + set using the `--cache-dir` argument. If not set, the cache directory is created + in the default temporary directory as before. In either case a temporary cache + is used to ensure the actual repository is checked (rather than a local copy). - The `--cache-dir` argument was not used by the `check` command, instead a cache directory was - created in the temporary directory. + The `--cache-dir` argument was not used by the `check` command, instead a cache + directory was created in the temporary directory. https://github.com/restic/restic/issues/1880 * Bugfix #1893: Return error when exclude file cannot be read - A bug was found: when multiple exclude files were passed to restic and one of them could not be - read, an error was printed and restic continued, ignoring even the existing exclude files. - Now, an error message is printed and restic aborts when an exclude file cannot be read. + A bug was found: when multiple exclude files were passed to restic and one of + them could not be read, an error was printed and restic continued, ignoring even + the existing exclude files. Now, an error message is printed and restic aborts + when an exclude file cannot be read. https://github.com/restic/restic/issues/1893 @@ -4120,9 +5012,9 @@ restic users. The changes are ordered by importance. * Enhancement #1477: S3 backend: accept AWS_SESSION_TOKEN - Before, it was not possible to use s3 backend with AWS temporary security credentials(with - AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials - provider. + Before, it was not possible to use s3 backend with AWS temporary security + credentials(with AWS_SESSION_TOKEN). This change gives higher priority to + credentials.EnvAWS credentials provider. https://github.com/restic/restic/issues/1477 https://github.com/restic/restic/pull/1479 @@ -4130,33 +5022,33 @@ restic users. The changes are ordered by importance. * Enhancement #1772: Add restore --verify to verify restored file content - Restore will print error message if restored file content does not match expected SHA256 - checksum + Restore will print error message if restored file content does not match + expected SHA256 checksum https://github.com/restic/restic/pull/1772 * Enhancement #1853: Add JSON output support to `restic key list` - This PR enables users to get the output of `restic key list` in JSON in addition to the existing - table format. + This PR enables users to get the output of `restic key list` in JSON in addition + to the existing table format. https://github.com/restic/restic/pull/1853 * Enhancement #1901: Update the Backblaze B2 library - We've updated the library we're using for accessing the Backblaze B2 service to 0.5.0 to - include support for upcoming so-called "application keys". With this feature, you can create - access credentials for B2 which are restricted to e.g. a single bucket or even a sub-directory - of a bucket. + We've updated the library we're using for accessing the Backblaze B2 service to + 0.5.0 to include support for upcoming so-called "application keys". With this + feature, you can create access credentials for B2 which are restricted to e.g. a + single bucket or even a sub-directory of a bucket. https://github.com/restic/restic/pull/1901 https://github.com/kurin/blazer * Enhancement #1906: Add support for B2 application keys - Restic can now use so-called "application keys" which can be created in the B2 dashboard and - were only introduced recently. In contrast to the "master key", such keys can be restricted to a - specific bucket and/or path. + Restic can now use so-called "application keys" which can be created in the B2 + dashboard and were only introduced recently. In contrast to the "master key", + such keys can be restricted to a specific bucket and/or path. https://github.com/restic/restic/issues/1906 https://github.com/restic/restic/pull/1914 @@ -4178,48 +5070,51 @@ restic users. The changes are ordered by importance. * Bugfix #1801: Add limiting bandwidth to the rclone backend - The rclone backend did not respect `--limit-upload` or `--limit-download`. Oftentimes it's - not necessary to use this, as the limiting in rclone itself should be used because it gives much - better results, but in case a remote instance of rclone is used (e.g. called via ssh), it is still - relevant to limit the bandwidth from restic to rclone. + The rclone backend did not respect `--limit-upload` or `--limit-download`. + Oftentimes it's not necessary to use this, as the limiting in rclone itself + should be used because it gives much better results, but in case a remote + instance of rclone is used (e.g. called via ssh), it is still relevant to limit + the bandwidth from restic to rclone. https://github.com/restic/restic/issues/1801 * Bugfix #1822: Allow uploading large files to MS Azure - Sometimes, restic creates files to be uploaded to the repository which are quite large, e.g. - when saving directories with many entries or very large files. The MS Azure API does not allow - uploading files larger that 256MiB directly, rather restic needs to upload them in blocks of - 100MiB. This is now implemented. + Sometimes, restic creates files to be uploaded to the repository which are quite + large, e.g. when saving directories with many entries or very large files. The + MS Azure API does not allow uploading files larger that 256MiB directly, rather + restic needs to upload them in blocks of 100MiB. This is now implemented. https://github.com/restic/restic/issues/1822 * Bugfix #1825: Correct `find` to not skip snapshots - Under certain circumstances, the `find` command was found to skip snapshots containing - directories with files to look for when the directories haven't been modified at all, and were - already printed as part of a different snapshot. This is now corrected. + Under certain circumstances, the `find` command was found to skip snapshots + containing directories with files to look for when the directories haven't been + modified at all, and were already printed as part of a different snapshot. This + is now corrected. - In addition, we've switched to our own matching/pattern implementation, so now things like - `restic find "/home/user/foo/**/main.go"` are possible. + In addition, we've switched to our own matching/pattern implementation, so now + things like `restic find "/home/user/foo/**/main.go"` are possible. https://github.com/restic/restic/issues/1825 https://github.com/restic/restic/issues/1823 * Bugfix #1833: Fix caching files on error - During `check` it may happen that different threads access the same file in the backend, which - is then downloaded into the cache only once. When that fails, only the thread which is - responsible for downloading the file signals the correct error. The other threads just assume - that the file has been downloaded successfully and then get an error when they try to access the - cached file. + During `check` it may happen that different threads access the same file in the + backend, which is then downloaded into the cache only once. When that fails, + only the thread which is responsible for downloading the file signals the + correct error. The other threads just assume that the file has been downloaded + successfully and then get an error when they try to access the cached file. https://github.com/restic/restic/issues/1833 * Bugfix #1834: Resolve deadlock - When the "scanning" process restic runs to find out how much data there is does not finish before - the backup itself is done, restic stops doing anything. This is resolved now. + When the "scanning" process restic runs to find out how much data there is does + not finish before the backup itself is done, restic stops doing anything. This + is resolved now. https://github.com/restic/restic/issues/1834 https://github.com/restic/restic/pull/1835 @@ -4247,7 +5142,7 @@ restic users. The changes are ordered by importance. * Enh #1665: Improve cache handling for `restic check` * Enh #1709: Improve messages `restic check` prints * Enh #1721: Add `cache` command to list cache dirs - * Enh #1735: Allow keeping a time range of snaphots + * Enh #1735: Allow keeping a time range of snapshots * Enh #1758: Allow saving OneDrive folders in Windows * Enh #1782: Use default AWS credentials chain for S3 backend @@ -4255,77 +5150,81 @@ restic users. The changes are ordered by importance. * Bugfix #1608: Respect time stamp for new backup when reading from stdin - When reading backups from stdin (via `restic backup --stdin`), restic now uses the time stamp - for the new backup passed in `--time`. + When reading backups from stdin (via `restic backup --stdin`), restic now uses + the time stamp for the new backup passed in `--time`. https://github.com/restic/restic/issues/1608 https://github.com/restic/restic/pull/1703 * Bugfix #1652: Ignore/remove invalid lock files - This corrects a bug introduced recently: When an invalid lock file in the repo is encountered - (e.g. if the file is empty), the code used to ignore that, but now returns the error. Now, invalid - files are ignored for the normal lock check, and removed when `restic unlock --remove-all` is - run. + This corrects a bug introduced recently: When an invalid lock file in the repo + is encountered (e.g. if the file is empty), the code used to ignore that, but + now returns the error. Now, invalid files are ignored for the normal lock check, + and removed when `restic unlock --remove-all` is run. https://github.com/restic/restic/issues/1652 https://github.com/restic/restic/pull/1653 * Bugfix #1684: Fix backend tests for rest-server - The REST server for restic now requires an explicit parameter (`--no-auth`) if no - authentication should be allowed. This is fixed in the tests. + The REST server for restic now requires an explicit parameter (`--no-auth`) if + no authentication should be allowed. This is fixed in the tests. https://github.com/restic/restic/pull/1684 * Bugfix #1730: Ignore sockets for restore - We've received a report and correct the behavior in which the restore code aborted restoring a - directory when a socket was encountered. Unix domain socket files cannot be restored (they are - created on the fly once a process starts listening). The error handling was corrected, and in - addition we're now ignoring sockets during restore. + We've received a report and correct the behavior in which the restore code + aborted restoring a directory when a socket was encountered. Unix domain socket + files cannot be restored (they are created on the fly once a process starts + listening). The error handling was corrected, and in addition we're now ignoring + sockets during restore. https://github.com/restic/restic/issues/1730 https://github.com/restic/restic/pull/1731 * Bugfix #1745: Correctly parse the argument to --tls-client-cert - Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) instead of the - argument that was passed to it. This has been corrected. + Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) + instead of the argument that was passed to it. This has been corrected. https://github.com/restic/restic/issues/1745 https://github.com/restic/restic/pull/1746 * Enhancement #549: Rework archiver code - The core archiver code and the complementary code for the `backup` command was rewritten - completely. This resolves very annoying issues such as 549. The first backup with this release - of restic will likely result in all files being re-read locally, so it will take a lot longer. The - next backup after that will be fast again. - - Basically, with the old code, restic took the last path component of each to-be-saved file or - directory as the top-level file/directory within the snapshot. This meant that when called as - `restic backup /home/user/foo`, the snapshot would contain the files in the directory - `/home/user/foo` as `/foo`. - - This is not the case any more with the new archiver code. Now, restic works very similar to what - `tar` does: When restic is called with an absolute path to save, then it'll preserve the - directory structure within the snapshot. For the example above, the snapshot would contain - the files in the directory within `/home/user/foo` in the snapshot. For relative - directories, it only preserves the relative path components. So `restic backup user/foo` - will save the files as `/user/foo` in the snapshot. - - While we were at it, the status display and notification system was completely rewritten. By - default, restic now shows which files are currently read (unless `--quiet` is specified) in a - multi-line status display. - - The `backup` command also gained a new option: `--verbose`. It can be specified once (which - prints a bit more detail what restic is doing) or twice (which prints a line for each - file/directory restic encountered, together with some statistics). - - Another issue that was resolved is the new code only reads two files at most. The old code would - read way too many files in parallel, thereby slowing down the backup process on spinning discs a - lot. + The core archiver code and the complementary code for the `backup` command was + rewritten completely. This resolves very annoying issues such as 549. The first + backup with this release of restic will likely result in all files being re-read + locally, so it will take a lot longer. The next backup after that will be fast + again. + + Basically, with the old code, restic took the last path component of each + to-be-saved file or directory as the top-level file/directory within the + snapshot. This meant that when called as `restic backup /home/user/foo`, the + snapshot would contain the files in the directory `/home/user/foo` as `/foo`. + + This is not the case any more with the new archiver code. Now, restic works very + similar to what `tar` does: When restic is called with an absolute path to save, + then it'll preserve the directory structure within the snapshot. For the example + above, the snapshot would contain the files in the directory within + `/home/user/foo` in the snapshot. For relative directories, it only preserves + the relative path components. So `restic backup user/foo` will save the files as + `/user/foo` in the snapshot. + + While we were at it, the status display and notification system was completely + rewritten. By default, restic now shows which files are currently read (unless + `--quiet` is specified) in a multi-line status display. + + The `backup` command also gained a new option: `--verbose`. It can be specified + once (which prints a bit more detail what restic is doing) or twice (which + prints a line for each file/directory restic encountered, together with some + statistics). + + Another issue that was resolved is the new code only reads two files at most. + The old code would read way too many files in parallel, thereby slowing down the + backup process on spinning discs a lot. https://github.com/restic/restic/issues/549 https://github.com/restic/restic/issues/1286 @@ -4347,11 +5246,11 @@ restic users. The changes are ordered by importance. * Enhancement #1433: Support UTF-16 encoding and process Byte Order Mark - On Windows, text editors commonly leave a Byte Order Mark at the beginning of the file to define - which encoding is used (oftentimes UTF-16). We've added code to support processing the BOMs in - text files, like the exclude files, the password file and the file passed via `--files-from`. - This does not apply to any file being saved in a backup, those are not touched and archived as they - are. + On Windows, text editors commonly leave a Byte Order Mark at the beginning of + the file to define which encoding is used (oftentimes UTF-16). We've added code + to support processing the BOMs in text files, like the exclude files, the + password file and the file passed via `--files-from`. This does not apply to any + file being saved in a backup, those are not touched and archived as they are. https://github.com/restic/restic/issues/1433 https://github.com/restic/restic/issues/1738 @@ -4359,9 +5258,9 @@ restic users. The changes are ordered by importance. * Enhancement #1477: Accept AWS_SESSION_TOKEN for the s3 backend - Before, it was not possible to use s3 backend with AWS temporary security credentials(with - AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials - provider. + Before, it was not possible to use s3 backend with AWS temporary security + credentials(with AWS_SESSION_TOKEN). This change gives higher priority to + credentials.EnvAWS credentials provider. https://github.com/restic/restic/issues/1477 https://github.com/restic/restic/pull/1479 @@ -4369,23 +5268,24 @@ restic users. The changes are ordered by importance. * Enhancement #1552: Use Google Application Default credentials - Google provide libraries to generate appropriate credentials with various fallback - sources. This change uses the library to generate our GCS client, which allows us to make use of - these extra methods. + Google provide libraries to generate appropriate credentials with various + fallback sources. This change uses the library to generate our GCS client, which + allows us to make use of these extra methods. - This should be backward compatible with previous restic behaviour while adding the - additional capabilities to auth from Google's internal metadata endpoints. For users - running restic in GCP this can make authentication far easier than it was before. + This should be backward compatible with previous restic behaviour while adding + the additional capabilities to auth from Google's internal metadata endpoints. + For users running restic in GCP this can make authentication far easier than it + was before. https://github.com/restic/restic/pull/1552 https://developers.google.com/identity/protocols/application-default-credentials * Enhancement #1561: Allow using rclone to access other services - We've added the ability to use rclone to store backup data on all backends that it supports. This - was done in collaboration with Nick, the author of rclone. You can now use it to first configure a - service, then restic manages the rest (starting and stopping rclone). For details, please see - the manual. + We've added the ability to use rclone to store backup data on all backends that + it supports. This was done in collaboration with Nick, the author of rclone. You + can now use it to first configure a service, then restic manages the rest + (starting and stopping rclone). For details, please see the manual. https://github.com/restic/restic/issues/1561 https://github.com/restic/restic/pull/1657 @@ -4393,9 +5293,9 @@ restic users. The changes are ordered by importance. * Enhancement #1648: Ignore AWS permission denied error when creating a repository - It's not possible to use s3 backend scoped to a subdirectory(with specific permissions). - Restic doesn't try to create repository in a subdirectory, when 'bucket exists' of parent - directory check fails due to permission issues. + It's not possible to use s3 backend scoped to a subdirectory(with specific + permissions). Restic doesn't try to create repository in a subdirectory, when + 'bucket exists' of parent directory check fails due to permission issues. https://github.com/restic/restic/pull/1648 @@ -4405,25 +5305,27 @@ restic users. The changes are ordered by importance. * Enhancement #1665: Improve cache handling for `restic check` - For safety reasons, restic does not use a local metadata cache for the `restic check` command, - so that data is loaded from the repository and restic can check it's in good condition. When the - cache is disabled, restic will fetch each tiny blob needed for checking the integrity using a - separate backend request. For non-local backends, that will take a long time, and depending on - the backend (e.g. B2) may also be much more expensive. + For safety reasons, restic does not use a local metadata cache for the `restic + check` command, so that data is loaded from the repository and restic can check + it's in good condition. When the cache is disabled, restic will fetch each tiny + blob needed for checking the integrity using a separate backend request. For + non-local backends, that will take a long time, and depending on the backend + (e.g. B2) may also be much more expensive. This PR adds a few commits which will change the behavior as follows: - * When `restic check` is called without any additional parameters, it will build a new cache in a - temporary directory, which is removed at the end of the check. This way, we'll get readahead for - metadata files (so restic will fetch the whole file when the first blob from the file is - requested), but all data is freshly fetched from the storage backend. This is the default - behavior and will work for almost all users. + * When `restic check` is called without any additional parameters, it will build + a new cache in a temporary directory, which is removed at the end of the check. + This way, we'll get readahead for metadata files (so restic will fetch the whole + file when the first blob from the file is requested), but all data is freshly + fetched from the storage backend. This is the default behavior and will work for + almost all users. - * When `restic check` is called with `--with-cache`, the default on-disc cache is used. This - behavior hasn't changed since the cache was introduced. + * When `restic check` is called with `--with-cache`, the default on-disc cache + is used. This behavior hasn't changed since the cache was introduced. - * When `--no-cache` is specified, restic falls back to the old behavior, and read all tiny blobs - in separate requests. + * When `--no-cache` is specified, restic falls back to the old behavior, and + read all tiny blobs in separate requests. https://github.com/restic/restic/issues/1665 https://github.com/restic/restic/issues/1694 @@ -4431,44 +5333,45 @@ restic users. The changes are ordered by importance. * Enhancement #1709: Improve messages `restic check` prints - Some messages `restic check` prints are not really errors, so from now on restic does not treat - them as errors any more and exits cleanly. + Some messages `restic check` prints are not really errors, so from now on restic + does not treat them as errors any more and exits cleanly. https://github.com/restic/restic/pull/1709 https://forum.restic.net/t/what-is-the-standard-procedure-to-follow-if-a-backup-or-restore-is-interrupted/571/2 * Enhancement #1721: Add `cache` command to list cache dirs - The command `cache` was added, it allows listing restic's cache directoriers together with - the last usage. It also allows removing old cache dirs without having to access a repo, via - `restic cache --cleanup` + The command `cache` was added, it allows listing restic's cache directoriers + together with the last usage. It also allows removing old cache dirs without + having to access a repo, via `restic cache --cleanup` https://github.com/restic/restic/issues/1721 https://github.com/restic/restic/pull/1749 - * Enhancement #1735: Allow keeping a time range of snaphots + * Enhancement #1735: Allow keeping a time range of snapshots - We've added the `--keep-within` option to the `forget` command. It instructs restic to keep - all snapshots within the given duration since the newest snapshot. For example, running - `restic forget --keep-within 5m7d` will keep all snapshots which have been made in the five - months and seven days since the latest snapshot. + We've added the `--keep-within` option to the `forget` command. It instructs + restic to keep all snapshots within the given duration since the newest + snapshot. For example, running `restic forget --keep-within 5m7d` will keep all + snapshots which have been made in the five months and seven days since the + latest snapshot. https://github.com/restic/restic/pull/1735 * Enhancement #1758: Allow saving OneDrive folders in Windows - Restic now contains a bugfix to two libraries, which allows saving OneDrive folders in - Windows. In order to use the newer versions of the libraries, the minimal version required to - compile restic is now Go 1.9. + Restic now contains a bugfix to two libraries, which allows saving OneDrive + folders in Windows. In order to use the newer versions of the libraries, the + minimal version required to compile restic is now Go 1.9. https://github.com/restic/restic/issues/1758 https://github.com/restic/restic/pull/1765 * Enhancement #1782: Use default AWS credentials chain for S3 backend - Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), and reorders - the credentials chain for the S3 backend to match AWS's standard, which is static credentials, - env vars, credentials file, and finally remote. + Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), + and reorders the credentials chain for the S3 backend to match AWS's standard, + which is static credentials, env vars, credentials file, and finally remote. https://github.com/restic/restic/pull/1782 @@ -4491,32 +5394,34 @@ restic users. The changes are ordered by importance. * Bugfix #1633: Fixed unexpected 'pack file cannot be listed' error - Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` commands failed to - read pack files with size of 587, 588, 589 or 590 bytes. + Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` + commands failed to read pack files with size of 587, 588, 589 or 590 bytes. https://github.com/restic/restic/issues/1633 https://github.com/restic/restic/pull/1635 * Bugfix #1638: Handle errors listing files in the backend - A user reported in the forum that restic completes a backup although a concurrent `prune` - operation was running. A few error messages were printed, but the backup was attempted and - completed successfully. No error code was returned. + A user reported in the forum that restic completes a backup although a + concurrent `prune` operation was running. A few error messages were printed, but + the backup was attempted and completed successfully. No error code was returned. - This should not happen: The repository is exclusively locked during `prune`, so when `restic - backup` is run in parallel, it should abort and return an error code instead. + This should not happen: The repository is exclusively locked during `prune`, so + when `restic backup` is run in parallel, it should abort and return an error + code instead. - It was found that the bug was in the code introduced only recently, which retries a List() - operation on the backend should that fail. It is now corrected. + It was found that the bug was in the code introduced only recently, which + retries a List() operation on the backend should that fail. It is now corrected. https://github.com/restic/restic/pull/1638 https://forum.restic.net/t/restic-backup-returns-0-exit-code-when-already-locked/484 * Bugfix #1641: Ignore files with invalid names in the repo - The release 0.8.2 introduced a bug: when restic encounters files in the repo which do not have a - valid name, it tries to load a file with a name of lots of zeroes instead of ignoring it. This is now - resolved, invalid file names are just ignored. + The release 0.8.2 introduced a bug: when restic encounters files in the repo + which do not have a valid name, it tries to load a file with a name of lots of + zeroes instead of ignoring it. This is now resolved, invalid file names are just + ignored. https://github.com/restic/restic/issues/1641 https://github.com/restic/restic/pull/1643 @@ -4524,8 +5429,9 @@ restic users. The changes are ordered by importance. * Enhancement #1497: Add --read-data-subset flag to check command - This change introduces ability to check integrity of a subset of repository data packs. This - can be used to spread integrity check of larger repositories over a period of time. + This change introduces ability to check integrity of a subset of repository data + packs. This can be used to spread integrity check of larger repositories over a + period of time. https://github.com/restic/restic/issues/1497 https://github.com/restic/restic/pull/1556 @@ -4538,21 +5444,22 @@ restic users. The changes are ordered by importance. * Enhancement #1623: Don't check for presence of files in the backend before writing - Before, all backend implementations were required to return an error if the file that is to be - written already exists in the backend. For most backends, that means making a request (e.g. via - HTTP) and returning an error when the file already exists. + Before, all backend implementations were required to return an error if the file + that is to be written already exists in the backend. For most backends, that + means making a request (e.g. via HTTP) and returning an error when the file + already exists. - This is not accurate, the file could have been created between the HTTP request testing for it, - and when writing starts, so we've relaxed this requeriment, which saves one additional HTTP - request per newly added file. + This is not accurate, the file could have been created between the HTTP request + testing for it, and when writing starts, so we've relaxed this requirement, + which saves one additional HTTP request per newly added file. https://github.com/restic/restic/pull/1623 * Enhancement #1634: Upgrade B2 client library, reduce HTTP requests - We've upgraded the B2 client library restic uses to access BackBlaze B2. This reduces the - number of HTTP requests needed to upload a new file from two to one, which should improve - throughput to B2. + We've upgraded the B2 client library restic uses to access BackBlaze B2. This + reduces the number of HTTP requests needed to upload a new file from two to one, + which should improve throughput to B2. https://github.com/restic/restic/pull/1634 @@ -4563,7 +5470,7 @@ restic users. The changes are ordered by importance. ## Summary - * Fix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends + * Fix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends * Fix #1512: Restore directory permissions as the last step * Fix #1528: Correctly create missing subdirs in data/ * Fix #1589: Complete intermediate index upload @@ -4583,17 +5490,17 @@ restic users. The changes are ordered by importance. ## Details - * Bugfix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends + * Bugfix #1506: Limit bandwidth at the http.RoundTripper for HTTP based backends https://github.com/restic/restic/issues/1506 https://github.com/restic/restic/pull/1511 * Bugfix #1512: Restore directory permissions as the last step - This change allows restoring into directories that were not writable during backup. Before, - restic created the directory, set the read-only mode and then failed to create files in the - directory. This change now restores the directory (with its permissions) as the very last - step. + This change allows restoring into directories that were not writable during + backup. Before, restic created the directory, set the read-only mode and then + failed to create files in the directory. This change now restores the directory + (with its permissions) as the very last step. https://github.com/restic/restic/issues/1512 https://github.com/restic/restic/pull/1536 @@ -4605,43 +5512,47 @@ restic users. The changes are ordered by importance. * Bugfix #1589: Complete intermediate index upload - After a user posted a comprehensive report of what he observed, we were able to find a bug and - correct it: During backup, restic uploads so-called "intermediate" index files. When the - backup finishes during a transfer of such an intermediate index, the upload is cancelled, but - the backup is finished without an error. This leads to an inconsistent state, where the - snapshot references data that is contained in the repo, but is not referenced in any index. + After a user posted a comprehensive report of what he observed, we were able to + find a bug and correct it: During backup, restic uploads so-called + "intermediate" index files. When the backup finishes during a transfer of such + an intermediate index, the upload is cancelled, but the backup is finished + without an error. This leads to an inconsistent state, where the snapshot + references data that is contained in the repo, but is not referenced in any + index. - The situation can be resolved by building a new index with `rebuild-index`, but looks very - confusing at first. Since all the data got uploaded to the repo successfully, there was no risk - of data loss, just minor inconvenience for our users. + The situation can be resolved by building a new index with `rebuild-index`, but + looks very confusing at first. Since all the data got uploaded to the repo + successfully, there was no risk of data loss, just minor inconvenience for our + users. https://github.com/restic/restic/pull/1589 https://forum.restic.net/t/error-loading-tree-check-prune-and-forget-gives-error-b2-backend/406 * Bugfix #1590: Strip spaces for lines read via --files-from - Leading and trailing spaces in lines read via `--files-from` are now stripped, so it behaves - the same as with lines read via `--exclude-file`. + Leading and trailing spaces in lines read via `--files-from` are now stripped, + so it behaves the same as with lines read via `--exclude-file`. https://github.com/restic/restic/issues/1590 https://github.com/restic/restic/pull/1613 * Bugfix #1594: Google Cloud Storage: Use generic HTTP transport - It was discovered that the Google Cloud Storage backend did not use the generic HTTP transport, - so things such as bandwidth limiting with `--limit-upload` did not work. This is resolved now. + It was discovered that the Google Cloud Storage backend did not use the generic + HTTP transport, so things such as bandwidth limiting with `--limit-upload` did + not work. This is resolved now. https://github.com/restic/restic/pull/1594 * Bugfix #1595: Backup: Remove bandwidth display - This commit removes the bandwidth displayed during backup process. It is misleading and - seldomly correct, because it's neither the "read bandwidth" (only for the very first backup) - nor the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, - #1591 + This commit removes the bandwidth displayed during backup process. It is + misleading and seldom correct, because it's neither the "read bandwidth" (only + for the very first backup) nor the "upload bandwidth". Many users are confused + about (and rightly so), c.f. #1581, #1033, #1591 - We'll eventually replace this display with something more relevant when the new archiver code - is ready. + We'll eventually replace this display with something more relevant when the new + archiver code is ready. https://github.com/restic/restic/pull/1595 @@ -4651,59 +5562,61 @@ restic users. The changes are ordered by importance. * Enhancement #1522: Add support for TLS client certificate authentication - Support has been added for using a TLS client certificate for authentication to HTTP based - backend. A file containing the PEM encoded private key and certificate can be set using the - `--tls-client-cert` option. + Support has been added for using a TLS client certificate for authentication to + HTTP based backend. A file containing the PEM encoded private key and + certificate can be set using the `--tls-client-cert` option. https://github.com/restic/restic/issues/1522 https://github.com/restic/restic/pull/1524 * Enhancement #1538: Reduce memory allocations for querying the index - This change reduces the internal memory allocations when the index data structures in memory - are queried if a blob (part of a file) already exists in the repo. It should speed up backup a bit, - and maybe even reduce RAM usage. + This change reduces the internal memory allocations when the index data + structures in memory are queried if a blob (part of a file) already exists in + the repo. It should speed up backup a bit, and maybe even reduce RAM usage. https://github.com/restic/restic/pull/1538 * Enhancement #1541: Reduce number of remote requests during repository check - This change eliminates redundant remote repository calls and significantly improves - repository check time. + This change eliminates redundant remote repository calls and significantly + improves repository check time. https://github.com/restic/restic/issues/1541 https://github.com/restic/restic/pull/1548 * Enhancement #1549: Speed up querying across indices and scanning existing files - This change increases the whenever a blob (part of a file) is searched for in a restic - repository. This will reduce cpu usage some when backing up files already backed up by restic. - Cpu usage is further decreased when scanning files. + This change increases the whenever a blob (part of a file) is searched for in a + restic repository. This will reduce cpu usage some when backing up files already + backed up by restic. Cpu usage is further decreased when scanning files. https://github.com/restic/restic/pull/1549 * Enhancement #1554: Fuse/mount: Correctly handle EOF, add template option - We've added the `--snapshot-template` string, which can be used to specify a template for a - snapshot directory. In addition, accessing data after the end of a file via the fuse mount is now - handled correctly. + We've added the `--snapshot-template` string, which can be used to specify a + template for a snapshot directory. In addition, accessing data after the end of + a file via the fuse mount is now handled correctly. https://github.com/restic/restic/pull/1554 * Enhancement #1564: Don't terminate ssh on SIGINT - We've reworked the code which runs the `ssh` login for the sftp backend so that it can prompt for a - password (if needed) but does not exit when the user presses CTRL+C (SIGINT) e.g. during - backup. This allows restic to properly shut down when it receives SIGINT and remove the lock - file from the repo, afterwards exiting the `ssh` process. + We've reworked the code which runs the `ssh` login for the sftp backend so that + it can prompt for a password (if needed) but does not exit when the user presses + CTRL+C (SIGINT) e.g. during backup. This allows restic to properly shut down + when it receives SIGINT and remove the lock file from the repo, afterwards + exiting the `ssh` process. https://github.com/restic/restic/pull/1564 https://github.com/restic/restic/pull/1588 * Enhancement #1567: Reduce number of backend requests for rebuild-index and prune - We've found a way to reduce then number of backend requests for the `rebuild-index` and `prune` - operations. This significantly speeds up the operations for high-latency backends. + We've found a way to reduce then number of backend requests for the + `rebuild-index` and `prune` operations. This significantly speeds up the + operations for high-latency backends. https://github.com/restic/restic/issues/1567 https://github.com/restic/restic/pull/1574 @@ -4715,10 +5628,11 @@ restic users. The changes are ordered by importance. * Enhancement #1584: Limit index file size - Before, restic would create a single new index file on `prune` or `rebuild-index`, this may - lead to memory problems when this huge index is created and loaded again. We're now limiting the - size of the index file, and split newly created index files into several smaller ones. This - allows restic to be more memory-efficient. + Before, restic would create a single new index file on `prune` or + `rebuild-index`, this may lead to memory problems when this huge index is + created and loaded again. We're now limiting the size of the index file, and + split newly created index files into several smaller ones. This allows restic to + be more memory-efficient. https://github.com/restic/restic/issues/1412 https://github.com/restic/restic/issues/979 @@ -4744,8 +5658,8 @@ restic users. The changes are ordered by importance. * Bugfix #1454: Correct cache dir location for Windows and Darwin - The cache directory on Windows and Darwin was not correct, instead the directory `.cache` was - used. + The cache directory on Windows and Darwin was not correct, instead the directory + `.cache` was used. https://github.com/restic/restic/pull/1454 @@ -4756,9 +5670,9 @@ restic users. The changes are ordered by importance. * Bugfix #1459: Disable handling SIGPIPE - We've disabled handling SIGPIPE again. Turns out, writing to broken TCP connections also - raised SIGPIPE, so restic exits on the first write to a broken connection. Instead, restic - should retry the request. + We've disabled handling SIGPIPE again. Turns out, writing to broken TCP + connections also raised SIGPIPE, so restic exits on the first write to a broken + connection. Instead, restic should retry the request. https://github.com/restic/restic/issues/1457 https://github.com/restic/restic/issues/1466 @@ -4766,16 +5680,18 @@ restic users. The changes are ordered by importance. * Change #1452: Do not save atime by default - By default, the access time for files and dirs is not saved any more. It is not possible to - reliably disable updating the access time during a backup, so for the next backup the access - time is different again. This means a lot of metadata is saved. If you want to save the access time - anyway, pass `--with-atime` to the `backup` command. + By default, the access time for files and dirs is not saved any more. It is not + possible to reliably disable updating the access time during a backup, so for + the next backup the access time is different again. This means a lot of metadata + is saved. If you want to save the access time anyway, pass `--with-atime` to the + `backup` command. https://github.com/restic/restic/pull/1452 * Enhancement #11: Add the `diff` command - The command `diff` was added, it allows comparing two snapshots and listing all differences. + The command `diff` was added, it allows comparing two snapshots and listing all + differences. https://github.com/restic/restic/issues/11 https://github.com/restic/restic/issues/1460 @@ -4783,17 +5699,18 @@ restic users. The changes are ordered by importance. * Enhancement #1436: Add code to detect old cache directories - We've added code to detect old cache directories of repositories that haven't been used in a - long time, restic now prints a note when it detects that such dirs exist. Also, the option - `--cleanup-cache` was added to automatically remove such directories. That's not a problem - because the cache will be rebuild once a repo is accessed again. + We've added code to detect old cache directories of repositories that haven't + been used in a long time, restic now prints a note when it detects that such + dirs exist. Also, the option `--cleanup-cache` was added to automatically remove + such directories. That's not a problem because the cache will be rebuild once a + repo is accessed again. https://github.com/restic/restic/pull/1436 * Enhancement #1439: Improve cancellation logic - The cancellation logic was improved, restic can now shut down cleanly when requested to do so - (e.g. via ctrl+c). + The cancellation logic was improved, restic can now shut down cleanly when + requested to do so (e.g. via ctrl+c). https://github.com/restic/restic/pull/1439 @@ -4828,17 +5745,18 @@ restic users. The changes are ordered by importance. * Security #1445: Prevent writing outside the target directory during restore - A vulnerability was found in the restic restorer, which allowed attackers in special - circumstances to restore files to a location outside of the target directory. Due to the - circumstances we estimate this to be a low-risk vulnerability, but urge all users to upgrade to - the latest version of restic. + A vulnerability was found in the restic restorer, which allowed attackers in + special circumstances to restore files to a location outside of the target + directory. Due to the circumstances we estimate this to be a low-risk + vulnerability, but urge all users to upgrade to the latest version of restic. - Exploiting the vulnerability requires a Linux/Unix system which saves backups via restic and - a Windows systems which restores files from the repo. In addition, the attackers need to be able - to create files with arbitrary names which are then saved to the restic repo. For example, by - creating a file named "..\test.txt" (which is a perfectly legal filename on Linux) and - restoring a snapshot containing this file on Windows, it would be written to the parent of the - target directory. + Exploiting the vulnerability requires a Linux/Unix system which saves backups + via restic and a Windows systems which restores files from the repo. In + addition, the attackers need to be able to create files with arbitrary names + which are then saved to the restic repo. For example, by creating a file named + "..\test.txt" (which is a perfectly legal filename on Linux) and restoring a + snapshot containing this file on Windows, it would be written to the parent of + the target directory. We'd like to thank Tyler Spivey for reporting this responsibly! @@ -4846,34 +5764,36 @@ restic users. The changes are ordered by importance. * Bugfix #1256: Re-enable workaround for S3 backend - We've re-enabled a workaround for `minio-go` (the library we're using to access s3 backends), - this reduces memory usage. + We've re-enabled a workaround for `minio-go` (the library we're using to access + s3 backends), this reduces memory usage. https://github.com/restic/restic/issues/1256 https://github.com/restic/restic/pull/1267 * Bugfix #1291: Reuse backend TCP connections to BackBlaze B2 - A bug was discovered in the library we're using to access Backblaze, it now reuses already - established TCP connections which should be a lot faster and not cause network failures any - more. + A bug was discovered in the library we're using to access Backblaze, it now + reuses already established TCP connections which should be a lot faster and not + cause network failures any more. https://github.com/restic/restic/issues/1291 https://github.com/restic/restic/pull/1301 * Bugfix #1317: Run prune when `forget --prune` is called with just snapshot IDs - A bug in the `forget` command caused `prune` not to be run when `--prune` was specified without a - policy, e.g. when only snapshot IDs that should be forgotten are listed manually. + A bug in the `forget` command caused `prune` not to be run when `--prune` was + specified without a policy, e.g. when only snapshot IDs that should be forgotten + are listed manually. https://github.com/restic/restic/pull/1317 * Bugfix #1437: Remove implicit path `/restic` for the s3 backend - The s3 backend used the subdir `restic` within a bucket if no explicit path after the bucket name - was specified. Since this version, restic does not use this default path any more. If you - created a repo on s3 in a bucket without specifying a path within the bucket, you need to add - `/restic` at the end of the repository specification to access your repo: + The s3 backend used the subdir `restic` within a bucket if no explicit path + after the bucket name was specified. Since this version, restic does not use + this default path any more. If you created a repo on s3 in a bucket without + specifying a path within the bucket, you need to add `/restic` at the end of the + repository specification to access your repo: `s3:s3.amazonaws.com/bucket/restic` https://github.com/restic/restic/issues/1292 @@ -4881,32 +5801,35 @@ restic users. The changes are ordered by importance. * Enhancement #448: Sftp backend prompts for password - The sftp backend now prompts for the password if a password is necessary for login. + The sftp backend now prompts for the password if a password is necessary for + login. https://github.com/restic/restic/issues/448 https://github.com/restic/restic/pull/1270 * Enhancement #510: Add `dump` command - We've added the `dump` command which prints a file from a snapshot to stdout. This can e.g. be - used to restore files read with `backup --stdin`. + We've added the `dump` command which prints a file from a snapshot to stdout. + This can e.g. be used to restore files read with `backup --stdin`. https://github.com/restic/restic/issues/510 https://github.com/restic/restic/pull/1346 * Enhancement #1040: Add local metadata cache - We've added a local cache for metadata so that restic doesn't need to load all metadata - (snapshots, indexes, ...) from the repo each time it starts. By default the cache is active, but - there's a new global option `--no-cache` that can be used to disable the cache. By deafult, the - cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache - will automatically populate, indexes and snapshots are saved as they are loaded. Cache - directories for repos that haven't been used recently can automatically be removed by restic + We've added a local cache for metadata so that restic doesn't need to load all + metadata (snapshots, indexes, ...) from the repo each time it starts. By default + the cache is active, but there's a new global option `--no-cache` that can be + used to disable the cache. By default, the cache a standard cache folder for the + OS, which can be overridden with `--cache-dir`. The cache will automatically + populate, indexes and snapshots are saved as they are loaded. Cache directories + for repos that haven't been used recently can automatically be removed by restic with the `--cleanup-cache` option. - A related change was to by default create pack files in the repo that contain either data or - metadata, not both mixed together. This allows easy caching of only the metadata files. The - next run of `restic prune` will untangle mixed files automatically. + A related change was to by default create pack files in the repo that contain + either data or metadata, not both mixed together. This allows easy caching of + only the metadata files. The next run of `restic prune` will untangle mixed + files automatically. https://github.com/restic/restic/issues/29 https://github.com/restic/restic/issues/738 @@ -4918,8 +5841,8 @@ restic users. The changes are ordered by importance. * Enhancement #1102: Add subdirectory `ids` to fuse mount - The fuse mount now has an `ids` subdirectory which contains the snapshots below their (short) - IDs. + The fuse mount now has an `ids` subdirectory which contains the snapshots below + their (short) IDs. https://github.com/restic/restic/issues/1102 https://github.com/restic/restic/pull/1299 @@ -4927,17 +5850,17 @@ restic users. The changes are ordered by importance. * Enhancement #1114: Add `--cacert` to specify TLS certificates to check against - We've added the `--cacert` option which can be used to pass one (or more) CA certificates to - restic. These are used in addition to the system CA certificates to verify HTTPS certificates - (e.g. for the REST backend). + We've added the `--cacert` option which can be used to pass one (or more) CA + certificates to restic. These are used in addition to the system CA certificates + to verify HTTPS certificates (e.g. for the REST backend). https://github.com/restic/restic/issues/1114 https://github.com/restic/restic/pull/1276 * Enhancement #1216: Add upload/download limiting - We've added support for rate limiting through `--limit-upload` and `--limit-download` - flags. + We've added support for rate limiting through `--limit-upload` and + `--limit-download` flags. https://github.com/restic/restic/issues/1216 https://github.com/restic/restic/pull/1336 @@ -4945,15 +5868,15 @@ restic users. The changes are ordered by importance. * Enhancement #1249: Add `latest` symlink in fuse mount - The directory structure in the fuse mount now exposes a symlink `latest` which points to the - latest snapshot in that particular directory. + The directory structure in the fuse mount now exposes a symlink `latest` which + points to the latest snapshot in that particular directory. https://github.com/restic/restic/pull/1249 * Enhancement #1269: Add `--compact` to `forget` command - The option `--compact` was added to the `forget` command to provide the same compact view as the - `snapshots` command. + The option `--compact` was added to the `forget` command to provide the same + compact view as the `snapshots` command. https://github.com/restic/restic/pull/1269 @@ -4966,25 +5889,26 @@ restic users. The changes are ordered by importance. * Enhancement #1274: Add `generate` command, replaces `manpage` and `autocomplete` - The `generate` command has been added, which replaces the now removed commands `manpage` and - `autocomplete`. This release of restic contains the most recent manpages in `doc/man` and the - auto-completion files for bash and zsh in `doc/bash-completion.sh` and - `doc/zsh-completion.zsh` + The `generate` command has been added, which replaces the now removed commands + `manpage` and `autocomplete`. This release of restic contains the most recent + manpages in `doc/man` and the auto-completion files for bash and zsh in + `doc/bash-completion.sh` and `doc/zsh-completion.zsh` https://github.com/restic/restic/issues/1274 https://github.com/restic/restic/pull/1282 * Enhancement #1281: Google Cloud Storage backend needs less permissions - The Google Cloud Storage backend no longer requires the service account to have the - `storage.buckets.get` permission ("Storage Admin" role) in `restic init` if the bucket - already exists. + The Google Cloud Storage backend no longer requires the service account to have + the `storage.buckets.get` permission ("Storage Admin" role) in `restic init` if + the bucket already exists. https://github.com/restic/restic/pull/1281 * Enhancement #1319: Make `check` print `no errors found` explicitly - The `check` command now explicetly prints `No errors were found` when no errors could be found. + The `check` command now explicitly prints `No errors were found` when no errors + could be found. https://github.com/restic/restic/issues/1303 https://github.com/restic/restic/pull/1319 @@ -4995,8 +5919,8 @@ restic users. The changes are ordered by importance. * Enhancement #1367: Allow comments in files read from via `--file-from` - When the list of files/dirs to be saved is read from a file with `--files-from`, comment lines - (starting with `#`) are now ignored. + When the list of files/dirs to be saved is read from a file with `--files-from`, + comment lines (starting with `#`) are now ignored. https://github.com/restic/restic/issues/1367 https://github.com/restic/restic/pull/1368 @@ -5014,9 +5938,10 @@ restic users. The changes are ordered by importance. * Bugfix #1246: List all files stored in Google Cloud Storage - For large backups stored in Google Cloud Storage, the `prune` command fails because listing - only returns the first 1000 files. This has been corrected, no data is lost in the process. In - addition, a plausibility check was added to `prune`. + For large backups stored in Google Cloud Storage, the `prune` command fails + because listing only returns the first 1000 files. This has been corrected, no + data is lost in the process. In addition, a plausibility check was added to + `prune`. https://github.com/restic/restic/issues/1246 https://github.com/restic/restic/pull/1247 @@ -5054,26 +5979,28 @@ restic users. The changes are ordered by importance. * Bugfix #1167: Do not create a local repo unless `init` is used - When a restic command other than `init` is used with a local repository and the repository - directory does not exist, restic creates the directory structure. That's an error, only the - `init` command should create the dir. + When a restic command other than `init` is used with a local repository and the + repository directory does not exist, restic creates the directory structure. + That's an error, only the `init` command should create the dir. https://github.com/restic/restic/issues/1167 https://github.com/restic/restic/pull/1182 * Bugfix #1191: Make sure to write profiling files on interrupt - Since a few releases restic had the ability to write profiling files for memory and CPU usage - when `debug` is enabled. It was discovered that when restic is interrupted (ctrl+c is - pressed), the proper shutdown hook is not run. This is now corrected. + Since a few releases restic had the ability to write profiling files for memory + and CPU usage when `debug` is enabled. It was discovered that when restic is + interrupted (ctrl+c is pressed), the proper shutdown hook is not run. This is + now corrected. https://github.com/restic/restic/pull/1191 * Enhancement #317: Add `--exclude-caches` and `--exclude-if-present` - A new option `--exclude-caches` was added that allows excluding cache directories (that are - tagged as such). This is a special case of a more generic option `--exclude-if-present` which - excludes a directory if a file with a specific name (and contents) is present. + A new option `--exclude-caches` was added that allows excluding cache + directories (that are tagged as such). This is a special case of a more generic + option `--exclude-if-present` which excludes a directory if a file with a + specific name (and contents) is present. https://github.com/restic/restic/issues/317 https://github.com/restic/restic/pull/1170 @@ -5094,16 +6021,17 @@ restic users. The changes are ordered by importance. * Enhancement #1126: Use the standard Go git repository layout, use `dep` for vendoring - The git repository layout was changed to resemble the layout typically used in Go projects, - we're not using `gb` for building restic any more and vendoring the dependencies is now taken - care of by `dep`. + The git repository layout was changed to resemble the layout typically used in + Go projects, we're not using `gb` for building restic any more and vendoring the + dependencies is now taken care of by `dep`. https://github.com/restic/restic/pull/1126 * Enhancement #1132: Make `key` command always prompt for a password - The `key` command now prompts for a password even if the original password to access a repo has - been specified via the `RESTIC_PASSWORD` environment variable or a password file. + The `key` command now prompts for a password even if the original password to + access a repo has been specified via the `RESTIC_PASSWORD` environment variable + or a password file. https://github.com/restic/restic/issues/1132 https://github.com/restic/restic/pull/1133 @@ -5120,8 +6048,8 @@ restic users. The changes are ordered by importance. * Enhancement #1149: Add support for storing backups on Microsoft Azure Blob Storage - The library we're using to access the service requires Go 1.8, so restic now needs at least Go - 1.8. + The library we're using to access the service requires Go 1.8, so restic now + needs at least Go 1.8. https://github.com/restic/restic/issues/609 https://github.com/restic/restic/pull/1149 @@ -5147,8 +6075,8 @@ restic users. The changes are ordered by importance. * Enhancement #1218: Add `--compact` to `snapshots` command - The option `--compact` was added to the `snapshots` command to get a better overview of the - snapshots in a repo. It limits each snapshot to a single line. + The option `--compact` was added to the `snapshots` command to get a better + overview of the snapshots in a repo. It limits each snapshot to a single line. https://github.com/restic/restic/issues/1218 https://github.com/restic/restic/pull/1223 @@ -5172,18 +6100,19 @@ restic users. The changes are ordered by importance. * Bugfix #1115: Fix `prune`, only include existing files in indexes - A bug was found (and corrected) in the index rebuilding after prune, which led to indexes which - include blobs that were not present in the repo any more. There were already checks in place - which detected this situation and aborted with an error message. A new run of either `prune` or - `rebuild-index` corrected the index files. This is now fixed and a test has been added to detect - this. + A bug was found (and corrected) in the index rebuilding after prune, which led + to indexes which include blobs that were not present in the repo any more. There + were already checks in place which detected this situation and aborted with an + error message. A new run of either `prune` or `rebuild-index` corrected the + index files. This is now fixed and a test has been added to detect this. https://github.com/restic/restic/pull/1115 * Enhancement #1055: Create subdirs below `data/` for local/sftp backends - The local and sftp backends now create the subdirs below `data/` on open/init. This way, restic - makes sure that they always exist. This is connected to an issue for the sftp server. + The local and sftp backends now create the subdirs below `data/` on open/init. + This way, restic makes sure that they always exist. This is connected to an + issue for the sftp server. https://github.com/restic/restic/issues/1055 https://github.com/restic/rest-server/pull/11#issuecomment-309879710 @@ -5192,17 +6121,18 @@ restic users. The changes are ordered by importance. * Enhancement #1067: Allow loading credentials for s3 from IAM - When no S3 credentials are specified in the environment variables, restic now tries to load - credentials from an IAM instance profile when the s3 backend is used. + When no S3 credentials are specified in the environment variables, restic now + tries to load credentials from an IAM instance profile when the s3 backend is + used. https://github.com/restic/restic/issues/1067 https://github.com/restic/restic/pull/1086 * Enhancement #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout - The `migrate` command for changing the `s3legacy` layout to the `default` layout for s3 - backends has been improved: It can now be restarted with `restic migrate --force s3_layout` - and automatically retries operations on error. + The `migrate` command for changing the `s3legacy` layout to the `default` layout + for s3 backends has been improved: It can now be restarted with `restic migrate + --force s3_layout` and automatically retries operations on error. https://github.com/restic/restic/issues/1073 https://github.com/restic/restic/pull/1075 @@ -5242,18 +6172,18 @@ restic users. The changes are ordered by importance. * Bugfix #965: Switch to `default` repo layout for the s3 backend - The default layout for the s3 backend is now `default` (instead of `s3legacy`). Also, there's a - new `migrate` command to convert an existing repo, it can be run like this: `restic migrate - s3_layout` + The default layout for the s3 backend is now `default` (instead of `s3legacy`). + Also, there's a new `migrate` command to convert an existing repo, it can be run + like this: `restic migrate s3_layout` https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/1004 * Bugfix #1013: Switch back to using the high-level minio-go API for s3 - For the s3 backend we're back to using the high-level API the s3 client library for uploading - data, a few users reported dropped connections (which the library will automatically retry - now). + For the s3 backend we're back to using the high-level API the s3 client library + for uploading data, a few users reported dropped connections (which the library + will automatically retry now). https://github.com/restic/restic/issues/1013 https://github.com/restic/restic/issues/1023 @@ -5266,9 +6196,10 @@ restic users. The changes are ordered by importance. * Enhancement #636: Add dirs `tags` and `hosts` to fuse mount - The fuse mount now has two more directories: `tags` contains a subdir for each tag, which in turn - contains only the snapshots that have this tag. The subdir `hosts` contains a subdir for each - host that has a snapshot, and the subdir contains the snapshots for that host. + The fuse mount now has two more directories: `tags` contains a subdir for each + tag, which in turn contains only the snapshots that have this tag. The subdir + `hosts` contains a subdir for each host that has a snapshot, and the subdir + contains the snapshots for that host. https://github.com/restic/restic/issues/636 https://github.com/restic/restic/pull/1050 @@ -5280,8 +6211,9 @@ restic users. The changes are ordered by importance. * Enhancement #989: Improve performance of the `find` command - Improved performance for the `find` command: Restic recognizes paths it has already checked - for the files in question, so the number of backend requests is reduced a lot. + Improved performance for the `find` command: Restic recognizes paths it has + already checked for the files in question, so the number of backend requests is + reduced a lot. https://github.com/restic/restic/issues/989 https://github.com/restic/restic/pull/993 @@ -5294,16 +6226,17 @@ restic users. The changes are ordered by importance. * Enhancement #1021: Detect invalid backend name and print error - Restic now tries to detect when an invalid/unknown backend is used and returns an error - message. + Restic now tries to detect when an invalid/unknown backend is used and returns + an error message. https://github.com/restic/restic/issues/1021 https://github.com/restic/restic/pull/1070 * Enhancement #1029: Remove invalid pack files when `prune` is run - The `prune` command has been improved and will now remove invalid pack files, for example files - that have not been uploaded completely because a backup was interrupted. + The `prune` command has been improved and will now remove invalid pack files, + for example files that have not been uploaded completely because a backup was + interrupted. https://github.com/restic/restic/issues/1029 https://github.com/restic/restic/pull/1036 @@ -5323,24 +6256,24 @@ restic users. The changes are ordered by importance. * Enhancement #974: Remove regular status reports - Regular status report: We've removed the status report that was printed every 10 seconds when - restic is run non-interactively. You can still force reporting the current status by sending a - `USR1` signal to the process. + Regular status report: We've removed the status report that was printed every 10 + seconds when restic is run non-interactively. You can still force reporting the + current status by sending a `USR1` signal to the process. https://github.com/restic/restic/pull/974 * Enhancement #981: Remove temporary path from binary in `build.go` - The `build.go` now strips the temporary directory used for compilation from the binary. This - is the first step in enabling reproducible builds. + The `build.go` now strips the temporary directory used for compilation from the + binary. This is the first step in enabling reproducible builds. https://github.com/restic/restic/pull/981 * Enhancement #985: Allow multiple parallel idle HTTP connections - Backends based on HTTP now allow several idle connections in parallel. This is especially - important for the REST backend, which (when used with a local server) may create a lot - connections and exhaust available ports quickly. + Backends based on HTTP now allow several idle connections in parallel. This is + especially important for the REST backend, which (when used with a local server) + may create a lot connections and exhaust available ports quickly. https://github.com/restic/restic/issues/985 https://github.com/restic/restic/pull/986 @@ -5360,21 +6293,22 @@ restic users. The changes are ordered by importance. * Enhancement #957: Make `forget` consistent - The `forget` command was corrected to be more consistent in which snapshots are to be - forgotten. It is possible that the new code removes more snapshots than before, so please - review what would be deleted by using the `--dry-run` option. + The `forget` command was corrected to be more consistent in which snapshots are + to be forgotten. It is possible that the new code removes more snapshots than + before, so please review what would be deleted by using the `--dry-run` option. https://github.com/restic/restic/issues/953 https://github.com/restic/restic/pull/957 * Enhancement #962: Improve memory and runtime for the s3 backend - We've updated the library used for accessing s3, switched to using a lower level API and added - caching for some requests. This lead to a decrease in memory usage and a great speedup. In - addition, we added benchmark functions for all backends, so we can track improvements over - time. The Continuous Integration test service we're using (Travis) now runs the s3 backend - tests not only against a Minio server, but also against the Amazon s3 live service, so we should - be notified of any regressions much sooner. + We've updated the library used for accessing s3, switched to using a lower level + API and added caching for some requests. This lead to a decrease in memory usage + and a great speedup. In addition, we added benchmark functions for all backends, + so we can track improvements over time. The Continuous Integration test service + we're using (Travis) now runs the s3 backend tests not only against a Minio + server, but also against the Amazon s3 live service, so we should be notified of + any regressions much sooner. https://github.com/restic/restic/pull/962 https://github.com/restic/restic/pull/960 @@ -5384,11 +6318,12 @@ restic users. The changes are ordered by importance. * Enhancement #966: Unify repository layout for all backends - Up to now the s3 backend used a special repository layout. We've decided to unify the repository - layout and implemented the default layout also for the s3 backend. For creating a new - repository on s3 with the default layout, use `restic -o s3.layout=default init`. For further - commands the option is not necessary any more, restic will automatically detect the correct - layout to use. A future version will switch to the default layout for new repositories. + Up to now the s3 backend used a special repository layout. We've decided to + unify the repository layout and implemented the default layout also for the s3 + backend. For creating a new repository on s3 with the default layout, use + `restic -o s3.layout=default init`. For further commands the option is not + necessary any more, restic will automatically detect the correct layout to use. + A future version will switch to the default layout for new repositories. https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/966 diff --git a/mover-restic/restic/CONTRIBUTING.md b/mover-restic/restic/CONTRIBUTING.md index 39a829337..dc278fa3a 100644 --- a/mover-restic/restic/CONTRIBUTING.md +++ b/mover-restic/restic/CONTRIBUTING.md @@ -6,7 +6,8 @@ Ways to Help Out Thank you for your contribution! Please **open an issue first** (or add a comment to an existing issue) if you plan to work on any code or add a new feature. This way, duplicate work is prevented and we can discuss your ideas -and design first. +and design first. Small bugfixes are an exception to this rule, just open a +pull request in this case. There are several ways you can help us out. First of all code contributions and bug fixes are most welcome. However even "minor" details as fixing spelling @@ -61,7 +62,7 @@ uploading it somewhere or post only the parts that are really relevant. If restic gets stuck, please also include a stacktrace in the description. On non-Windows systems, you can send a SIGQUIT signal to restic or press `Ctrl-\` to achieve the same result. This causes restic to print a stacktrace -and then exit immediatelly. This will not damage your repository, however, +and then exit immediately. This will not damage your repository, however, it might be necessary to manually clean up stale lock files using `restic unlock`. diff --git a/mover-restic/restic/README.md b/mover-restic/restic/README.md index ad6b13cef..ef12f3e1b 100644 --- a/mover-restic/restic/README.md +++ b/mover-restic/restic/README.md @@ -10,8 +10,7 @@ For detailed usage and installation instructions check out the [documentation](h You can ask questions in our [Discourse forum](https://forum.restic.net). -Quick start ------------ +## Quick start Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start off with creating a repository for your backups: @@ -59,7 +58,7 @@ Therefore, restic supports the following backends for storing backups natively: Restic is a program that does backups right and was designed with the following principles in mind: -- **Easy:** Doing backups should be a frictionless process, otherwise +- **Easy**: Doing backups should be a frictionless process, otherwise you might be tempted to skip it. Restic should be easy to configure and use, so that, in the event of a data loss, you can just restore it. Likewise, restoring data should not be complicated. @@ -92,20 +91,17 @@ reproduce a byte identical version from the source code for that release. Instructions on how to do that are contained in the [builder repository](https://github.com/restic/builder). -News ----- +## News -You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to +You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or subscribe to the [project blog](https://restic.net/blog/). -License -------- +## License Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the -complete text in [``LICENSE``](LICENSE). +complete text in [`LICENSE`](LICENSE). -Sponsorship ------------ +## Sponsorship Backend integration tests for Google Cloud Storage and Microsoft Azure Blob Storage are sponsored by [AppsCode](https://appscode.com)! diff --git a/mover-restic/restic/VERSION b/mover-restic/restic/VERSION index 19270385e..c5523bd09 100644 --- a/mover-restic/restic/VERSION +++ b/mover-restic/restic/VERSION @@ -1 +1 @@ -0.16.5 +0.17.0 diff --git a/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2195 b/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2195 index a139aa4e1..7898568fa 100644 --- a/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2195 +++ b/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2195 @@ -10,7 +10,7 @@ https://github.com/restic/restic/issues/2244 NOTE: This new implementation does not guarantee order in which blobs are written to the target files and, for example, the last blob of a -file can be written to the file before any of the preceeding file blobs. +file can be written to the file before any of the preceding file blobs. It is therefore possible to have gaps in the data written to the target files if restore fails or interrupted by the user. diff --git a/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2668 b/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2668 index 94a661c05..dd95587ce 100644 --- a/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2668 +++ b/mover-restic/restic/changelog/0.10.0_2020-09-19/pull-2668 @@ -1,6 +1,6 @@ Bugfix: Don't abort the stats command when data blobs are missing -Runing the stats command in the blobs-per-file mode on a repository with +Running the stats command in the blobs-per-file mode on a repository with missing data blobs previously resulted in a crash. https://github.com/restic/restic/pull/2668 diff --git a/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-1756 b/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-1756 index f735cf1f9..c182c1a6c 100644 --- a/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-1756 +++ b/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-1756 @@ -1,6 +1,6 @@ Bugfix: Mark repository files as read-only when using the local backend -Files stored in a local repository were marked as writeable on the +Files stored in a local repository were marked as writable on the filesystem for non-Windows systems, which did not prevent accidental file modifications outside of restic. In addition, the local backend did not work with certain filesystems and network mounts which do not permit modifications diff --git a/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-340 b/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-340 index 84c67f145..d688ee0db 100644 --- a/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-340 +++ b/mover-restic/restic/changelog/0.11.0_2020-11-05/issue-340 @@ -5,7 +5,7 @@ another process using an exclusive lock through a filesystem snapshot. Restic was unable to backup those files before. This update enables backing up these files. -This needs to be enabled explicitely using the --use-fs-snapshot option of the +This needs to be enabled explicitly using the --use-fs-snapshot option of the backup command. https://github.com/restic/restic/issues/340 diff --git a/mover-restic/restic/changelog/0.12.0_2021-02-14/issue-3232 b/mover-restic/restic/changelog/0.12.0_2021-02-14/issue-3232 index 7d9f5c3b7..30b9ee293 100644 --- a/mover-restic/restic/changelog/0.12.0_2021-02-14/issue-3232 +++ b/mover-restic/restic/changelog/0.12.0_2021-02-14/issue-3232 @@ -1,7 +1,7 @@ -Bugfix: Correct statistics for overlapping targets +Bugfix: Correct statistics for overlapping backup sources A user reported that restic's statistics and progress information during backup -was not correctly calculated when the backup targets (files/dirs to save) +was not correctly calculated when the backup sources (files/dirs to save) overlap. For example, consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the completeness percentage as well as the diff --git a/mover-restic/restic/changelog/0.12.0_2021-02-14/pull-3106 b/mover-restic/restic/changelog/0.12.0_2021-02-14/pull-3106 index 2d5857de7..f0cb54df0 100644 --- a/mover-restic/restic/changelog/0.12.0_2021-02-14/pull-3106 +++ b/mover-restic/restic/changelog/0.12.0_2021-02-14/pull-3106 @@ -2,7 +2,7 @@ Enhancement: Parallelize scan of snapshot content in `copy` and `prune` The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find used data. This snapshot traversal is -now parallized which can speed up this step several times. +now parallelized which can speed up this step several times. In addition the `check` command now reports how many snapshots have already been processed. diff --git a/mover-restic/restic/changelog/0.16.0_2023-07-31/issue-3941 b/mover-restic/restic/changelog/0.16.0_2023-07-31/issue-3941 index ff56d52cc..f1f02db93 100644 --- a/mover-restic/restic/changelog/0.16.0_2023-07-31/issue-3941 +++ b/mover-restic/restic/changelog/0.16.0_2023-07-31/issue-3941 @@ -1,7 +1,7 @@ Enhancement: Support `--group-by` for backup parent selection Previously, the `backup` command by default selected the parent snapshot based -on the hostname and the backup targets. When the backup path list changed, the +on the hostname and the backup paths. When the backup path list changed, the `backup` command was unable to determine a suitable parent snapshot and had to read all files again. diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-1786 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-1786 new file mode 100644 index 000000000..41517f5db --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-1786 @@ -0,0 +1,20 @@ +Enhancement: Support repositories with empty password + +Restic previously required a password to create or operate on repositories. +Using the new option `--insecure-no-password` it is now possible to disable +this requirement. Restic will not prompt for a password when using this option. + +For security reasons, the option must always be specified when operating on +repositories with an empty password, and specifying `--insecure-no-password` +while also passing a password to restic via a CLI option or environment +variable results in an error. + +The `init` and `copy` commands add the related `--from-insecure-no-password` +option, which applies to the source repository. The `key add` and `key passwd` +commands add the `--new-insecure-no-password` option to add or set an empty +password. + +https://github.com/restic/restic/issues/1786 +https://github.com/restic/restic/issues/4326 +https://github.com/restic/restic/pull/4698 +https://github.com/restic/restic/pull/4808 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-2348 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-2348 new file mode 100644 index 000000000..c329ae0a2 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-2348 @@ -0,0 +1,12 @@ +Enhancement: Add `--delete` option to `restore` command + +The `restore` command now supports a `--delete` option that allows removing +files and directories from the target directory that do not exist in the +snapshot. This option also allows files in the snapshot to replace non-empty +directories having the same name. + +To check that only expected files are deleted, add the `--dry-run --verbose=2` +options. + +https://github.com/restic/restic/issues/2348 +https://github.com/restic/restic/pull/4881 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3600 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3600 new file mode 100644 index 000000000..b972ecc64 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3600 @@ -0,0 +1,11 @@ +Bugfix: Handle unreadable xattrs in folders above `backup` source + +When backup sources are specified using absolute paths, `backup` also includes +information about the parent folders of the backup sources in the snapshot. + +If the extended attributes for some of these folders could not be read due to +missing permissions, this caused the backup to fail. This has now been fixed. + +https://github.com/restic/restic/issues/3600 +https://github.com/restic/restic/pull/4668 +https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3806 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3806 new file mode 100644 index 000000000..6b0663c9f --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-3806 @@ -0,0 +1,12 @@ +Enhancement: Optimize and make `prune` command resumable + +Previously, if the `prune` command was interrupted, a later `prune` run would +start repacking pack files from the start, as `prune` did not update the index +while repacking. + +The `prune` command now supports resuming interrupted prune runs. The update +of the repository index has also been optimized to use less memory and only +rewrite parts of the index that have changed. + +https://github.com/restic/restic/issues/3806 +https://github.com/restic/restic/pull/4812 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4048 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4048 new file mode 100644 index 000000000..3b9c61d20 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4048 @@ -0,0 +1,6 @@ +Enhancement: Add support for FUSE-T with `mount` on macOS + +The restic `mount` command now supports creating FUSE mounts using FUSE-T on macOS. + +https://github.com/restic/restic/issues/4048 +https://github.com/restic/restic/pull/4825 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4209 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4209 new file mode 100644 index 000000000..04eb8ef18 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4209 @@ -0,0 +1,7 @@ +Bugfix: Fix slow SFTP upload performance + +Since restic 0.12.1, the upload speed of the sftp backend to a remote server +has regressed significantly. This has now been fixed. + +https://github.com/restic/restic/issues/4209 +https://github.com/restic/restic/pull/4782 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4251 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4251 new file mode 100644 index 000000000..5541f2d7e --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4251 @@ -0,0 +1,16 @@ +Enhancement: Support reading backup from a command's standard output + +The `backup` command now supports the `--stdin-from-command` option. When using +this option, the arguments to `backup` are interpreted as a command instead of +paths to back up. `backup` then executes the given command and stores the +standard output from it in the backup, similar to the what the `--stdin` option +does. This also enables restic to verify that the command completes with exit +code zero. A non-zero exit code causes the backup to fail. + +Note that the `--stdin` option does not have to be specified at the same time, +and that the `--stdin-filename` option also applies to `--stdin-from-command`. + +Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump [...]` + +https://github.com/restic/restic/issues/4251 +https://github.com/restic/restic/pull/4410 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4287 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4287 new file mode 100644 index 000000000..cd25a8dee --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4287 @@ -0,0 +1,12 @@ +Enhancement: Support connection to rest-server using unix socket + +Restic now supports using a unix socket to connect to a rest-server +version 0.13.0 or later. This allows running restic as follows: + +``` +rest-server --listen unix:/tmp/rest.socket --data /path/to/data & +restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...] +``` + +https://github.com/restic/restic/issues/4287 +https://github.com/restic/restic/pull/4655 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4437 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4437 new file mode 100644 index 000000000..bc76c0983 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4437 @@ -0,0 +1,11 @@ +Enhancement: Make `check` command create non-existent cache directory + +Previously, if a custom cache directory was specified for the `check` command, +but the directory did not exist, `check` continued with the cache disabled. + +The `check` command now attempts to create the cache directory before +initializing the cache. + +https://github.com/restic/restic/issues/4437 +https://github.com/restic/restic/pull/4805 +https://github.com/restic/restic/pull/4883 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4472 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4472 new file mode 100644 index 000000000..beb3612b8 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4472 @@ -0,0 +1,18 @@ +Enhancement: Support AWS Assume Role for S3 backend + +Previously only credentials discovered via the Minio discovery methods +were used to authenticate. + +However, there are many circumstances where the discovered credentials have +lower permissions and need to assume a specific role. This is now possible +using the following new environment variables: + +- RESTIC_AWS_ASSUME_ROLE_ARN +- RESTIC_AWS_ASSUME_ROLE_SESSION_NAME +- RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID +- RESTIC_AWS_ASSUME_ROLE_REGION (defaults to us-east-1) +- RESTIC_AWS_ASSUME_ROLE_POLICY +- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT + +https://github.com/restic/restic/issues/4472 +https://github.com/restic/restic/pull/4474 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4540 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4540 new file mode 100644 index 000000000..25358c332 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4540 @@ -0,0 +1,7 @@ +Change: Require at least ARMv6 for ARM binaries + +The official release binaries of restic now require +at least ARMv6 support for ARM platforms. + +https://github.com/restic/restic/issues/4540 +https://github.com/restic/restic/pull/4542 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4547 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4547 new file mode 100644 index 000000000..bb69a59e6 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4547 @@ -0,0 +1,7 @@ +Enhancement: Add `--json` option to `version` command + +Restic now supports outputting restic version along with the Go version, OS +and architecture used to build restic in JSON format using `version --json`. + +https://github.com/restic/restic/issues/4547 +https://github.com/restic/restic/pull/4553 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4549 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4549 new file mode 100644 index 000000000..245ed484a --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4549 @@ -0,0 +1,11 @@ +Enhancement: Add `--ncdu` option to `ls` command + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. It has +an option to save a directory tree and analyse it later. + +The `ls` command now supports outputting snapshot information in the NCDU format +using the `--ncdu` option. Example usage: `restic ls latest --ncdu | ncdu -f -` + +https://github.com/restic/restic/issues/4549 +https://github.com/restic/restic/pull/4550 +https://github.com/restic/restic/pull/4911 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4568 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4568 new file mode 100644 index 000000000..00394fc44 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4568 @@ -0,0 +1,16 @@ +Bugfix: Prevent `forget --keep-tags ` from deleting all snapshots + +Running `forget --keep-tags `, where `` is a tag that does +not exist in the repository, would remove all snapshots. This is especially +problematic if the tag name contains a typo. + +The `forget` command now fails with an error if all snapshots in a snapshot +group would be deleted. This prevents the above example from deleting all +snapshots. + +It is possible to temporarily disable the new check by setting the environment +variable `RESTIC_FEATURES=safe-forget-keep-tags=false`. Note that this feature +flag will be removed in the next minor restic version. + +https://github.com/restic/restic/pull/4568 +https://github.com/restic/restic/pull/4764 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4583 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4583 new file mode 100644 index 000000000..bc1d030cc --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4583 @@ -0,0 +1,13 @@ +Enhancement: Ignore `s3.storage-class` archive tiers for metadata + +Restic used to store all files on S3 using the specified `s3.storage-class`. + +Now, restic will only use non-archive storage tiers for metadata, to avoid +problems when accessing a repository. To restore any data, it is still +necessary to manually warm up the required data beforehand. + +NOTE: There is no official cold storage support in restic, use this option at +your own risk. + +https://github.com/restic/restic/issues/4583 +https://github.com/restic/restic/pull/4584 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4601 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4601 new file mode 100644 index 000000000..8efeba47f --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4601 @@ -0,0 +1,9 @@ +Enhancement: Add support for feature flags + +Restic now supports feature flags that can be used to enable and disable +experimental features. The flags can be set using the environment variable +`RESTIC_FEATURES`. To get a list of currently supported feature flags, use +the `features` command. + +https://github.com/restic/restic/issues/4601 +https://github.com/restic/restic/pull/4666 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4602 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4602 new file mode 100644 index 000000000..3fe19db79 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4602 @@ -0,0 +1,22 @@ +Change: Deprecate legacy index format and `s3legacy` repository layout + +Support for the legacy index format used by restic before version 0.2.0 has +been deprecated and will be removed in the next minor restic version. You can +use `restic repair index` to update the index to the current format. + +It is possible to temporarily reenable support for the legacy index format by +setting the environment variable `RESTIC_FEATURES=deprecate-legacy-index=false`. +Note that this feature flag will be removed in the next minor restic version. + +Support for the `s3legacy` repository layout used for the S3 backend before +restic 0.7.0 has been deprecated and will be removed in the next minor restic +version. You can migrate your S3 repository to the current layout using +`RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`. + +It is possible to temporarily reenable support for the `s3legacy` layout by +setting the environment variable `RESTIC_FEATURES=deprecate-s3-legacy-layout=false`. +Note that this feature flag will be removed in the next minor restic version. + +https://github.com/restic/restic/issues/4602 +https://github.com/restic/restic/pull/4724 +https://github.com/restic/restic/pull/4743 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4627 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4627 new file mode 100644 index 000000000..87a185604 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4627 @@ -0,0 +1,33 @@ +Change: Redesign backend error handling to improve reliability + +Restic now downloads pack files in large chunks instead of using a streaming +download. This prevents failures due to interrupted streams. The `restore` +command now also retries downloading individual blobs that could not be +retrieved. + +HTTP requests that are stuck for more than two minutes while uploading or +downloading are now forcibly interrupted. This ensures that stuck requests are +retried after a short timeout. + +Attempts to access a missing or truncated file will no longer be retried. This +avoids unnecessary retries in those cases. All other backend requests are +retried for up to 15 minutes. This ensures that temporarily interrupted network +connections can be tolerated. + +If a download yields a corrupt file or blob, then the download will be retried +once. + +Most parts of the new backend error handling can temporarily be disabled by +setting the environment variable `RESTIC_FEATURES=backend-error-redesign=false`. +Note that this feature flag will be removed in the next minor restic version. + +https://github.com/restic/restic/issues/4627 +https://github.com/restic/restic/issues/4193 +https://github.com/restic/restic/pull/4605 +https://github.com/restic/restic/pull/4792 +https://github.com/restic/restic/issues/4515 +https://github.com/restic/restic/issues/1523 +https://github.com/restic/restic/pull/4520 +https://github.com/restic/restic/pull/4800 +https://github.com/restic/restic/pull/4784 +https://github.com/restic/restic/pull/4844 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4656 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4656 new file mode 100644 index 000000000..ef8c1e12a --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4656 @@ -0,0 +1,7 @@ +Bugfix: Properly report ID of newly added keys + +`restic key add` now reports the ID of the newly added key. This simplifies +selecting a specific key using the `--key-hint key` option. + +https://github.com/restic/restic/issues/4656 +https://github.com/restic/restic/pull/4657 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4676 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4676 new file mode 100644 index 000000000..ecea79361 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4676 @@ -0,0 +1,8 @@ +Enhancement: Make `key` command's actions separate sub-commands + +Each of the `add`, `list`, `remove` and `passwd` actions provided by the `key` +command is now a separate sub-command and have its own documentation which can +be invoked using `restic key --help`. + +https://github.com/restic/restic/issues/4676 +https://github.com/restic/restic/pull/4685 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4678 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4678 new file mode 100644 index 000000000..401449bd2 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4678 @@ -0,0 +1,8 @@ +Enhancement: Add `--target` option to the `dump` command + +Restic `dump` always printed to the standard output. It now supports specifying +a `--target` file to write its output to. + +https://github.com/restic/restic/issues/4678 +https://github.com/restic/restic/pull/4682 +https://github.com/restic/restic/pull/4692 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4707 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4707 new file mode 100644 index 000000000..3c8f1a2f3 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4707 @@ -0,0 +1,14 @@ +Change: Disable S3 anonymous authentication by default + +When using the S3 backend with anonymous authentication, it continuously +tried to retrieve new authentication credentials, causing bad performance. + +Now, to use anonymous authentication, it is necessary to pass the extended +option `-o s3.unsafe-anonymous-auth=true` to restic. + +It is possible to temporarily revert to the old behavior by setting the +environment variable `RESTIC_FEATURES=explicit-s3-anonymous-auth=false`. Note +that this feature flag will be removed in the next minor restic version. + +https://github.com/restic/restic/issues/4707 +https://github.com/restic/restic/pull/4908 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4733 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4733 new file mode 100644 index 000000000..fb5a072d6 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4733 @@ -0,0 +1,12 @@ +Enhancement: Allow specifying `--host` via environment variable + +Restic commands that operate on snapshots, such as `restic backup` and +`restic snapshots`, support the `--host` option to specify the hostname +for grouping snapshots. + +Such commands now also support specifying the hostname via the environment +variable `RESTIC_HOST`. Note that `--host` still takes precedence over the +environment variable. + +https://github.com/restic/restic/issues/4733 +https://github.com/restic/restic/pull/4734 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4744 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4744 new file mode 100644 index 000000000..b5c759bed --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4744 @@ -0,0 +1,9 @@ +Change: Include full key ID in JSON output of `key list` + +The JSON output of the `key list` command has changed to include the full key +ID instead of just a shortened version of the ID, as the latter can be ambiguous +in some rare cases. To derive the short ID, please truncate the full ID down to +eight characters. + +https://github.com/restic/restic/issues/4744 +https://github.com/restic/restic/pull/4745 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4760 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4760 new file mode 100644 index 000000000..e56f41a44 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4760 @@ -0,0 +1,8 @@ +Bugfix: Fix possible error on concurrent cache cleanup + +If multiple restic processes concurrently cleaned up no longer existing files +from the cache, this could cause some of the processes to fail with an `no such +file or directory` error. This has now been fixed. + +https://github.com/restic/restic/issues/4760 +https://github.com/restic/restic/pull/4761 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4768 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4768 new file mode 100644 index 000000000..9fb1a29de --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4768 @@ -0,0 +1,8 @@ +Enhancement: Allow specifying custom User-Agent for outgoing requests + +Restic now supports setting a custom `User-Agent` for outgoing HTTP requests +using the global option `--http-user-agent` or the `RESTIC_HTTP_USER_AGENT` +environment variable. + +https://github.com/restic/restic/issues/4768 +https://github.com/restic/restic/pull/4810 \ No newline at end of file diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4781 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4781 new file mode 100644 index 000000000..2c9584d77 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4781 @@ -0,0 +1,8 @@ +Enhancement: Add `restore` options to read include/exclude patterns from files + +Restic now supports reading include and exclude patterns from files using the +`--include-file`, `--exclude-file`, `--iinclude-file` and `--iexclude-file` +options of the `restore` command. + +https://github.com/restic/restic/issues/4781 +https://github.com/restic/restic/pull/4811 \ No newline at end of file diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4817 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4817 new file mode 100644 index 000000000..83c682775 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4817 @@ -0,0 +1,26 @@ +Enhancement: Make overwrite behavior of `restore` customizable + +The `restore` command now supports an `--overwrite` option to configure whether +already existing files are overwritten. The overwrite behavior can be configured +using the following option values: + +- `--overwrite always` (default): Always overwrites already existing files. + The `restore` command will verify the existing file content and only restore + mismatching parts to minimize downloads. Updates the metadata of all files. +- `--overwrite if-changed`: Like `always`, but speeds up the file content check + by assuming that files with matching size and modification time (mtime) are + already up to date. In case of a mismatch, the full file content is verified + like with `always`. Updates the metadata of all files. +- `--overwrite if-newer`: Like `always`, but only overwrites existing files + when the file in the snapshot has a newer modification time (mtime) than the + existing file. +- `--overwrite never`: Never overwrites existing files. + +https://github.com/restic/restic/issues/4817 +https://github.com/restic/restic/issues/200 +https://github.com/restic/restic/issues/407 +https://github.com/restic/restic/issues/2662 +https://github.com/restic/restic/pull/4837 +https://github.com/restic/restic/pull/4838 +https://github.com/restic/restic/pull/4864 +https://github.com/restic/restic/pull/4921 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4850 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4850 new file mode 100644 index 000000000..b04edd159 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4850 @@ -0,0 +1,8 @@ +Bugfix: Handle UTF-16 password files in `key` command correctly + +Previously, `key add` and `key passwd` did not properly decode UTF-16 +encoded passwords read from a password file. This has now been fixed +to correctly match the encoding when opening a repository. + +https://github.com/restic/restic/issues/4850 +https://github.com/restic/restic/pull/4851 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4902 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4902 new file mode 100644 index 000000000..507d8abbe --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-4902 @@ -0,0 +1,8 @@ +Bugfix: Update snapshot summary on `rewrite` + +Restic previously did not recalculate the total number of files and bytes +processed when files were excluded from a snapshot by the `rewrite` command. +This has now been fixed. + +https://github.com/restic/restic/issues/4902 +https://github.com/restic/restic/pull/4905 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-662 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-662 new file mode 100644 index 000000000..9fd2f27d0 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-662 @@ -0,0 +1,11 @@ +Enhancement: Optionally skip snapshot creation if nothing changed + +The `backup` command always created a snapshot even if nothing in the +backup set changed compared to the parent snapshot. + +Restic now supports the `--skip-if-unchanged` option for the `backup` +command, which omits creating a snapshot if the new snapshot's content +would be identical to that of the parent snapshot. + +https://github.com/restic/restic/issues/662 +https://github.com/restic/restic/pull/4816 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-693 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-693 new file mode 100644 index 000000000..4a8c766a4 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-693 @@ -0,0 +1,13 @@ +Enhancement: Include snapshot size in `snapshots` output + +The `snapshots` command now prints the size for snapshots created using this +or a future restic version. To achieve this, the `backup` command now stores +the backup summary statistics in the snapshot. + +The text output of the `snapshots` command only shows the snapshot size. The +other statistics are only included in the JSON output. To inspect these +statistics use `restic snapshots --json` or `restic cat snapshot `. + +https://github.com/restic/restic/issues/693 +https://github.com/restic/restic/pull/4705 +https://github.com/restic/restic/pull/4913 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-828 b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-828 new file mode 100644 index 000000000..72d66dae0 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/issue-828 @@ -0,0 +1,11 @@ +Enhancement: Improve features of the `repair packs` command + +The `repair packs` command has been improved to also be able to process +truncated pack files. The `check` and `check --read-data` command will provide +instructions on using the command if necessary to repair a repository. See the +guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html for +further instructions. + +https://github.com/restic/restic/issues/828 +https://github.com/restic/restic/pull/4644 +https://github.com/restic/restic/pull/4882 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-3067 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-3067 new file mode 100644 index 000000000..9ecec4838 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-3067 @@ -0,0 +1,25 @@ +Enhancement: Add extended options to configure Windows Shadow Copy Service + +Previous, restic always used a 120 seconds timeout and unconditionally created +VSS snapshots for all volume mount points on disk. This behavior can now be +fine-tuned by the following new extended options (available only on Windows): + +- `-o vss.timeout`: Time that VSS can spend creating snapshot before timing out (default: 120s) +- `-o vss.exclude-all-mount-points`: Exclude mountpoints from snapshotting on all volumes (default: false) +- `-o vss.exclude-volumes`: Semicolon separated list of volumes to exclude from snapshotting +- `-o vss.provider`: VSS provider identifier which will be used for snapshotting + +For example, change VSS timeout to five minutes and disable snapshotting of +mount points on all volumes: + + restic backup --use-fs-snapshot -o vss.timeout=5m -o vss.exclude-all-mount-points=true + +Exclude drive `d:`, mount point `c:\mnt` and a specific volume from snapshotting: + + restic backup --use-fs-snapshot -o vss.exclude-volumes="d:\;c:\mnt\;\\?\Volume{e2e0315d-9066-4f97-8343-eb5659b35762}" + +Uses 'Microsoft Software Shadow Copy provider 1.0' instead of the default provider: + + restic backup --use-fs-snapshot -o vss.provider={b5946137-7b9f-4925-af80-51abd60b20d5} + +https://github.com/restic/restic/pull/3067 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4006 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4006 new file mode 100644 index 000000000..3bfacb8a0 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4006 @@ -0,0 +1,15 @@ +Enhancement: (alpha) Store deviceID only for hardlinks + +Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature. +The feature flag will be removed after repository format version 3 becomes +available or be replaced with a different solution. + +When creating backups from a filesystem snapshot, for example created using +BTRFS subvolumes, the deviceID of the filesystem changes compared to previous +snapshots. This prevented restic from deduplicating the directory metadata of +a snapshot. + +When this alpha feature is enabled, the deviceID is only stored for hardlinks, +which significantly reduces the metadata duplication for most backups. + +https://github.com/restic/restic/pull/4006 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4354 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4354 new file mode 100644 index 000000000..d3cf33249 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4354 @@ -0,0 +1,7 @@ +Enhancement: Significantly reduce `prune` memory usage + +The `prune` command has been optimized to use up to 60% less memory. +The memory usage should now be roughly similar to creating a backup. + +https://github.com/restic/restic/pull/4354 +https://github.com/restic/restic/pull/4812 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4503 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4503 new file mode 100644 index 000000000..549aa9f53 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4503 @@ -0,0 +1,8 @@ +Bugfix: Correct hardlink handling in `stats` command + +If files on different devices had the same inode ID, the `stats` command +did not correctly calculate the snapshot size. This has now been fixed. + +https://forum.restic.net/t/possible-bug-in-stats/6461/8 +https://github.com/restic/restic/pull/4503 +https://github.com/restic/restic/pull/4006 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4526 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4526 new file mode 100644 index 000000000..4d0fee691 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4526 @@ -0,0 +1,12 @@ +Enhancement: Add bitrot detection to `diff` command + +The output of the `diff` command now includes the modifier `?` for files to +indicate bitrot in backed up files. The `?` will appear whenever there is a +difference in content while the metadata is exactly the same. + +Since files with unchanged metadata are normally not read again when creating +a backup, the detection is only effective when the right-hand side of the diff +has been created with `backup --force`. + +https://github.com/restic/restic/issues/805 +https://github.com/restic/restic/pull/4526 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4573 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4573 new file mode 100644 index 000000000..36fc727be --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4573 @@ -0,0 +1,6 @@ +Enhancement: Support rewriting host and time metadata in snapshots + +The `rewrite` command now supports rewriting the host and/or time metadata of +a snapshot using the new `--new-host` and `--new-time` options. + +https://github.com/restic/restic/pull/4573 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4590 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4590 new file mode 100644 index 000000000..7904c18af --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4590 @@ -0,0 +1,6 @@ +Enhancement: Speed up `mount` command's error detection + +The `mount` command now checks for the existence of the mountpoint before +opening the repository, leading to quicker error detection. + +https://github.com/restic/restic/pull/4590 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4611 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4611 new file mode 100644 index 000000000..426ed590f --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4611 @@ -0,0 +1,9 @@ +Enhancement: Back up more file metadata on Windows + +Previously, restic did not back up all common Windows-specific metadata. + +Restic now stores file creation time and file attributes like the hidden, +read-only and encrypted flags when backing up files and folders on Windows. + +https://github.com/restic/restic/pull/4611 + diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4615 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4615 new file mode 100644 index 000000000..a8916df3c --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4615 @@ -0,0 +1,6 @@ +Bugfix: Make `find` not sometimes ignore directories + +In some cases, the `find` command ignored empty or moved directories. This has +now been fixed. + +https://github.com/restic/restic/pull/4615 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4664 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4664 new file mode 100644 index 000000000..655ccd082 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4664 @@ -0,0 +1,10 @@ +Enhancement: Make `ls` use `message_type` field in JSON output + +The `ls` command was the only restic command that used the `struct_type` field +in its JSON output format to specify the message type. + +The JSON output of the `ls` command now also includes the `message_type` field, +which is consistent with other commands. The `struct_type` field is still +included, but now deprecated. + +https://github.com/restic/restic/pull/4664 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4703 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4703 new file mode 100644 index 000000000..178842c6c --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4703 @@ -0,0 +1,11 @@ +Bugfix: Shutdown cleanly when receiving SIGTERM + +Previously, when restic received the SIGTERM signal it would terminate +immediately, skipping cleanup and potentially causing issues like stale locks +being left behind. This primarily effected containerized restic invocations +that use SIGTERM, but could also be triggered via a simple `killall restic`. + +This has now been fixed, such that restic shuts down cleanly when receiving +the SIGTERM signal. + +https://github.com/restic/restic/pull/4703 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4708 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4708 new file mode 100644 index 000000000..16bf33e57 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4708 @@ -0,0 +1,13 @@ +Enhancement: Back up and restore SecurityDescriptors on Windows + +Restic now backs up and restores SecurityDescriptors for files and folders on +Windows which includes owner, group, discretionary access control list (DACL) +and system access control list (SACL). + +This requires the user to be a member of backup operators or the application +must be run as admin. If that is not the case, only the current user's owner, +group and DACL will be backed up, and during restore only the DACL of the +backed up file will be restored, with the current user's owner and group +being set on the restored file. + +https://github.com/restic/restic/pull/4708 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4709 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4709 new file mode 100644 index 000000000..62be8b54b --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4709 @@ -0,0 +1,10 @@ +Bugfix: Correct `--no-lock` handling of `ls` and `tag` commands + +The `ls` command never locked the repository. This has now been fixed, with the +old behavior still being supported using `ls --no-lock`. The latter invocation +also works with older restic versions. + +The `tag` command erroneously accepted the `--no-lock` command. This command +now always requires an exclusive lock. + +https://github.com/restic/restic/pull/4709 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4737 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4737 new file mode 100644 index 000000000..bf528237d --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4737 @@ -0,0 +1,6 @@ +Enhancement: Include snapshot ID in `reason` field of `forget` JSON output + +The JSON output of the `forget` command now includes `id` and `short_id` of +snapshots in the `reason` field. + +https://github.com/restic/restic/pull/4737 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4764 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4764 new file mode 100644 index 000000000..d85eadbc3 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4764 @@ -0,0 +1,10 @@ +Enhancement: Support forgetting all snapshots + +The `forget` command now supports the `--unsafe-allow-remove-all` option, which +removes all snapshots in the repository. + +This option must always be combined with a snapshot filter (by host, path or +tag). For example, the command `forget --tag example --unsafe-allow-remove-all` +removes all snapshots with the tag "example". + +https://github.com/restic/restic/pull/4764 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4796 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4796 new file mode 100644 index 000000000..2729c635e --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4796 @@ -0,0 +1,8 @@ +Enhancement: Improve `dump` performance for large files + +The `dump` command now retrieves the data chunks for a file in +parallel. This improves the download performance by up to as many +times as the configured number of parallel backend connections. + +https://github.com/restic/restic/issues/3406 +https://github.com/restic/restic/pull/4796 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4807 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4807 new file mode 100644 index 000000000..b5e5cd7fd --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4807 @@ -0,0 +1,6 @@ +Enhancement: Support Extended Attributes on Windows NTFS + +Restic now backs up and restores Extended Attributes for files +and folders on Windows NTFS. + +https://github.com/restic/restic/pull/4807 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4839 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4839 new file mode 100644 index 000000000..672ac2e69 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4839 @@ -0,0 +1,7 @@ +Enhancement: Add dry-run support to `restore` command + +The `restore` command now supports the `--dry-run` option to perform +a dry run. Pass the `--verbose=2` option to see which files would +remain unchanged, and which would be updated or freshly restored. + +https://github.com/restic/restic/pull/4839 diff --git a/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4884 b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4884 new file mode 100644 index 000000000..3a7e0d342 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.0_2024-07-26/pull-4884 @@ -0,0 +1,11 @@ +Change: Return exit code 10 and 11 for non-existing and locked repository + +If a repository does not exist or cannot be locked, restic previously always +returned exit code 1. This made it difficult to distinguish these cases from +other errors. + +Restic now returns exit code 10 if the repository does not exist, and exit code +11 if the repository could be not locked due to a conflicting lock. + +https://github.com/restic/restic/issues/956 +https://github.com/restic/restic/pull/4884 diff --git a/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1040 b/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1040 index b39ee2fee..190ed01f8 100644 --- a/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1040 +++ b/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1040 @@ -3,7 +3,7 @@ Enhancement: Add local metadata cache We've added a local cache for metadata so that restic doesn't need to load all metadata (snapshots, indexes, ...) from the repo each time it starts. By default the cache is active, but there's a new global option `--no-cache` -that can be used to disable the cache. By deafult, the cache a standard +that can be used to disable the cache. By default, the cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache will automatically populate, indexes and snapshots are saved as they are loaded. Cache directories for repos that haven't been used recently can diff --git a/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1319 b/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1319 index d74a3f947..efc2e2c8a 100644 --- a/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1319 +++ b/mover-restic/restic/changelog/0.8.0_2017-11-26/pull-1319 @@ -1,6 +1,6 @@ Enhancement: Make `check` print `no errors found` explicitly -The `check` command now explicetly prints `No errors were found` when no errors +The `check` command now explicitly prints `No errors were found` when no errors could be found. https://github.com/restic/restic/pull/1319 diff --git a/mover-restic/restic/changelog/0.8.2_2018-02-17/issue-1506 b/mover-restic/restic/changelog/0.8.2_2018-02-17/issue-1506 index 5f0122529..aca77c458 100644 --- a/mover-restic/restic/changelog/0.8.2_2018-02-17/issue-1506 +++ b/mover-restic/restic/changelog/0.8.2_2018-02-17/issue-1506 @@ -1,4 +1,4 @@ -Bugfix: Limit bandwith at the http.RoundTripper for HTTP based backends +Bugfix: Limit bandwidth at the http.RoundTripper for HTTP based backends https://github.com/restic/restic/issues/1506 https://github.com/restic/restic/pull/1511 diff --git a/mover-restic/restic/changelog/0.8.2_2018-02-17/pull-1595 b/mover-restic/restic/changelog/0.8.2_2018-02-17/pull-1595 index 81e0a8748..3dbea73ce 100644 --- a/mover-restic/restic/changelog/0.8.2_2018-02-17/pull-1595 +++ b/mover-restic/restic/changelog/0.8.2_2018-02-17/pull-1595 @@ -1,7 +1,7 @@ Bugfix: backup: Remove bandwidth display This commit removes the bandwidth displayed during backup process. It is -misleading and seldomly correct, because it's neither the "read +misleading and seldom correct, because it's neither the "read bandwidth" (only for the very first backup) nor the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, #1591 diff --git a/mover-restic/restic/changelog/0.8.3_2018-02-26/pull-1623 b/mover-restic/restic/changelog/0.8.3_2018-02-26/pull-1623 index 0e03ee776..1579a9ebc 100644 --- a/mover-restic/restic/changelog/0.8.3_2018-02-26/pull-1623 +++ b/mover-restic/restic/changelog/0.8.3_2018-02-26/pull-1623 @@ -6,7 +6,7 @@ that means making a request (e.g. via HTTP) and returning an error when the file already exists. This is not accurate, the file could have been created between the HTTP request -testing for it, and when writing starts, so we've relaxed this requeriment, +testing for it, and when writing starts, so we've relaxed this requirement, which saves one additional HTTP request per newly added file. https://github.com/restic/restic/pull/1623 diff --git a/mover-restic/restic/changelog/0.9.0_2018-05-21/pull-1735 b/mover-restic/restic/changelog/0.9.0_2018-05-21/pull-1735 index 2cfd115d8..fbf6135a6 100644 --- a/mover-restic/restic/changelog/0.9.0_2018-05-21/pull-1735 +++ b/mover-restic/restic/changelog/0.9.0_2018-05-21/pull-1735 @@ -1,4 +1,4 @@ -Enhancement: Allow keeping a time range of snaphots +Enhancement: Allow keeping a time range of snapshots We've added the `--keep-within` option to the `forget` command. It instructs restic to keep all snapshots within the given duration since the newest diff --git a/mover-restic/restic/changelog/0.9.3_2018-10-13/pull-1876 b/mover-restic/restic/changelog/0.9.3_2018-10-13/pull-1876 index 2fb1a8ea8..aa92b24e8 100644 --- a/mover-restic/restic/changelog/0.9.3_2018-10-13/pull-1876 +++ b/mover-restic/restic/changelog/0.9.3_2018-10-13/pull-1876 @@ -1,7 +1,7 @@ Enhancement: Display reason why forget keeps snapshots We've added a column to the list of snapshots `forget` keeps which details the -reasons to keep a particuliar snapshot. This makes debugging policies for +reasons to keep a particular snapshot. This makes debugging policies for forget much easier. Please remember to always try things out with `--dry-run`! https://github.com/restic/restic/pull/1876 diff --git a/mover-restic/restic/changelog/0.9.6_2019-11-22/issue-2179 b/mover-restic/restic/changelog/0.9.6_2019-11-22/issue-2179 index e87778d17..96589f9cf 100644 --- a/mover-restic/restic/changelog/0.9.6_2019-11-22/issue-2179 +++ b/mover-restic/restic/changelog/0.9.6_2019-11-22/issue-2179 @@ -9,7 +9,7 @@ file should be noticed, and the modified file will be backed up. The ctime check will be disabled if the --ignore-inode flag was given. If this change causes problems for you, please open an issue, and we can look in -to adding a seperate flag to disable just the ctime check. +to adding a separate flag to disable just the ctime check. https://github.com/restic/restic/issues/2179 https://github.com/restic/restic/pull/2212 diff --git a/mover-restic/restic/cmd/restic/cleanup.go b/mover-restic/restic/cmd/restic/cleanup.go index 75933fe96..90ea93b92 100644 --- a/mover-restic/restic/cmd/restic/cleanup.go +++ b/mover-restic/restic/cmd/restic/cleanup.go @@ -1,89 +1,41 @@ package main import ( + "context" "os" "os/signal" - "sync" "syscall" "github.com/restic/restic/internal/debug" ) -var cleanupHandlers struct { - sync.Mutex - list []func(code int) (int, error) - done bool - ch chan os.Signal -} - -func init() { - cleanupHandlers.ch = make(chan os.Signal, 1) - go CleanupHandler(cleanupHandlers.ch) - signal.Notify(cleanupHandlers.ch, syscall.SIGINT) -} +func createGlobalContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) -// AddCleanupHandler adds the function f to the list of cleanup handlers so -// that it is executed when all the cleanup handlers are run, e.g. when SIGINT -// is received. -func AddCleanupHandler(f func(code int) (int, error)) { - cleanupHandlers.Lock() - defer cleanupHandlers.Unlock() + ch := make(chan os.Signal, 1) + go cleanupHandler(ch, cancel) + signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM) - // reset the done flag for integration tests - cleanupHandlers.done = false - - cleanupHandlers.list = append(cleanupHandlers.list, f) + return ctx } -// RunCleanupHandlers runs all registered cleanup handlers -func RunCleanupHandlers(code int) int { - cleanupHandlers.Lock() - defer cleanupHandlers.Unlock() - - if cleanupHandlers.done { - return code - } - cleanupHandlers.done = true +// cleanupHandler handles the SIGINT and SIGTERM signals. +func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) { + s := <-c + debug.Log("signal %v received, cleaning up", s) + Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s) - for _, f := range cleanupHandlers.list { - var err error - code, err = f(code) - if err != nil { - Warnf("error in cleanup handler: %v\n", err) - } + if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" { + _, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n") + _, _ = os.Stderr.WriteString(debug.DumpStacktrace()) + _, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n") } - cleanupHandlers.list = nil - return code -} - -// CleanupHandler handles the SIGINT signals. -func CleanupHandler(c <-chan os.Signal) { - for s := range c { - debug.Log("signal %v received, cleaning up", s) - Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s) - - if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" { - _, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n") - _, _ = os.Stderr.WriteString(debug.DumpStacktrace()) - _, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n") - } - code := 0 - - if s == syscall.SIGINT { - code = 130 - } else { - code = 1 - } - - Exit(code) - } + cancel() } -// Exit runs the cleanup handlers and then terminates the process with the -// given exit code. +// Exit terminates the process with the given exit code. func Exit(code int) { - code = RunCleanupHandlers(code) debug.Log("exiting with status code %d", code) os.Exit(code) } diff --git a/mover-restic/restic/cmd/restic/cmd_backup.go b/mover-restic/restic/cmd/restic/cmd_backup.go index e476ae7b8..9957b5784 100644 --- a/mover-restic/restic/cmd/restic/cmd_backup.go +++ b/mover-restic/restic/cmd/restic/cmd_backup.go @@ -12,7 +12,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "github.com/spf13/cobra" @@ -25,7 +24,6 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/textfile" - "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/backup" "github.com/restic/restic/internal/ui/termstatus" ) @@ -43,8 +41,10 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, - PreRun: func(cmd *cobra.Command, args []string) { + PreRun: func(_ *cobra.Command, _ []string) { if backupOptions.Host == "" { hostname, err := os.Hostname() if err != nil { @@ -56,31 +56,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // use the terminal for stdout/stderr - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runBackup(ctx, backupOptions, globalOptions, term, args) + term, cancel := setupTermstatus() + defer cancel() + return runBackup(cmd.Context(), backupOptions, globalOptions, term, args) }, } @@ -97,6 +75,7 @@ type BackupOptions struct { ExcludeLargerThan string Stdin bool StdinFilename string + StdinCommand bool Tags restic.TagLists Host string FilesFrom []string @@ -110,6 +89,7 @@ type BackupOptions struct { DryRun bool ReadConcurrency uint NoScan bool + SkipIfUnchanged bool } var backupOptions BackupOptions @@ -124,7 +104,7 @@ func init() { f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)") backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true} f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')") - f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`) + f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`) initExcludePatternOptions(f, &backupOptions.excludePatternOptions) @@ -134,9 +114,10 @@ func init() { f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin") + f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout") f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)") f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)") - f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag") + f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag") f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually") err := f.MarkDeprecated("hostname", "use --host") if err != nil { @@ -148,17 +129,23 @@ func init() { f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)") f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)") f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories") - f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files") + f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files") f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files") f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done") f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup") if runtime.GOOS == "windows" { f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)") } + f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot") // parse read concurrency from env, on error the default value will be used readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32) backupOptions.ReadConcurrency = uint(readConcurrency) + + // parse host from env, if not exists or empty the default value will be used + if host := os.Getenv("RESTIC_HOST"); host != "" { + backupOptions.Host = host + } } // filterExisting returns a slice of all existing items, or an error if no @@ -175,7 +162,7 @@ func filterExisting(items []string) (result []string, err error) { } if len(result) == 0 { - return nil, errors.Fatal("all target directories/files do not exist") + return nil, errors.Fatal("all source directories/files do not exist") } return @@ -274,7 +261,7 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) { // Check returns an error when an invalid combination of options was set. func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { - if gopts.password == "" { + if gopts.password == "" && !gopts.InsecureNoPassword { if opts.Stdin { return errors.Fatal("cannot read both password and data from stdin") } @@ -287,7 +274,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { } } - if opts.Stdin { + if opts.Stdin || opts.StdinCommand { if len(opts.FilesFrom) > 0 { return errors.Fatal("--stdin and --files-from cannot be used together") } @@ -298,7 +285,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error { return errors.Fatal("--stdin and --files-from-raw cannot be used together") } - if len(args) > 0 { + if len(args) > 0 && !opts.StdinCommand { return errors.Fatal("--stdin was specified and files/dirs were listed as arguments") } } @@ -366,7 +353,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc, // collectTargets returns a list of target files/dirs from several sources. func collectTargets(opts BackupOptions, args []string) (targets []string, err error) { - if opts.Stdin { + if opts.Stdin || opts.StdinCommand { return nil, nil } @@ -420,7 +407,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er // and have the ability to use both files-from and args at the same time. targets = append(targets, args...) if len(targets) == 0 && !opts.Stdin { - return nil, errors.Fatal("nothing to backup, please specify target files/dirs") + return nil, errors.Fatal("nothing to backup, please specify source files/dirs") } targets, err = filterExisting(targets) @@ -433,7 +420,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er // parent returns the ID of the parent snapshot. If there is none, nil is // returned. -func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { +func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { if opts.Force { return nil, nil } @@ -453,7 +440,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup f.Tags = []restic.TagList{opts.Tags.Flatten()} } - sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName) + sn, _, err := f.FindLatest(ctx, repo, repo, snName) // Snapshot not found is ok if no explicit parent was set if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) { err = nil @@ -462,7 +449,16 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup } func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { - err := opts.Check(gopts, args) + var vsscfg fs.VSSConfig + var err error + + if runtime.GOOS == "windows" { + if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil { + return err + } + } + + err = opts.Check(gopts, args) if err != nil { return err } @@ -473,6 +469,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } timeStamp := time.Now() + backupStart := timeStamp if opts.TimeStamp != "" { timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local) if err != nil { @@ -484,10 +481,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter Verbosef("open repository\n") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun) if err != nil { return err } + defer unlock() var progressPrinter backup.ProgressPrinter if gopts.JSON { @@ -499,22 +497,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter calculateProgressInterval(!gopts.Quiet, gopts.JSON)) defer progressReporter.Done() - if opts.DryRun { - repo.SetDryRun() - } - - if !gopts.JSON { - progressPrinter.V("lock repository") - } - if !opts.DryRun { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - // rejectByNameFuncs collect functions that can reject items from the backup based on path only rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo) if err != nil { @@ -578,8 +560,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter return err } - errorHandler := func(item string, err error) error { - return progressReporter.Error(item, err) + errorHandler := func(item string, err error) { + _ = progressReporter.Error(item, err) } messageHandler := func(msg string, args ...interface{}) { @@ -588,20 +570,28 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } } - localVss := fs.NewLocalVss(errorHandler, messageHandler) + localVss := fs.NewLocalVss(errorHandler, messageHandler, vsscfg) defer localVss.DeleteSnapshots() targetFS = localVss } - if opts.Stdin { + + if opts.Stdin || opts.StdinCommand { if !gopts.JSON { progressPrinter.V("read data from stdin") } filename := path.Join("/", opts.StdinFilename) + var source io.ReadCloser = os.Stdin + if opts.StdinCommand { + source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr) + if err != nil { + return err + } + } targetFS = &fs.Reader{ ModTime: timeStamp, Name: filename, Mode: 0644, - ReadCloser: os.Stdin, + ReadCloser: source, } targets = []string{filename} } @@ -623,14 +613,20 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) } - arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency}) + arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter arch.WithAtime = opts.WithAtime success := true arch.Error = func(item string, err error) error { success = false - return progressReporter.Error(item, err) + reterr := progressReporter.Error(item, err) + // If we receive a fatal error during the execution of the snapshot, + // we abort the snapshot. + if reterr == nil && errors.IsFatal(err) { + reterr = err + } + return reterr } arch.CompleteItem = progressReporter.CompleteItem arch.StartFile = progressReporter.StartFile @@ -646,18 +642,20 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } snapshotOpts := archiver.SnapshotOptions{ - Excludes: opts.Excludes, - Tags: opts.Tags.Flatten(), - Time: timeStamp, - Hostname: opts.Host, - ParentSnapshot: parentSnapshot, - ProgramVersion: "restic " + version, + Excludes: opts.Excludes, + Tags: opts.Tags.Flatten(), + BackupStart: backupStart, + Time: timeStamp, + Hostname: opts.Host, + ParentSnapshot: parentSnapshot, + ProgramVersion: "restic " + version, + SkipIfUnchanged: opts.SkipIfUnchanged, } if !gopts.JSON { progressPrinter.V("start backup on %v", targets) } - _, id, err := arch.Snapshot(ctx, targets, snapshotOpts) + _, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts) // cleanly shutdown all running goroutines cancel() @@ -671,10 +669,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter } // Report finished execution - progressReporter.Finish(id, opts.DryRun) - if !gopts.JSON && !opts.DryRun { - progressPrinter.P("snapshot %s saved\n", id.Str()) - } + progressReporter.Finish(id, summary, opts.DryRun) if !success { return ErrInvalidSourceData } diff --git a/mover-restic/restic/cmd/restic/cmd_backup_integration_test.go b/mover-restic/restic/cmd/restic/cmd_backup_integration_test.go index 76c227e3d..5e00b84b0 100644 --- a/mover-restic/restic/cmd/restic/cmd_backup_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_backup_integration_test.go @@ -249,29 +249,18 @@ func TestBackupTreeLoadError(t *testing.T) { opts := BackupOptions{} // Backup a subdirectory first, such that we can remove the tree pack for the subdirectory testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts) - - r, err := OpenRepository(context.TODO(), env.gopts) - rtest.OK(t, err) - rtest.OK(t, r.LoadIndex(context.TODO(), nil)) - treePacks := restic.NewIDSet() - r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { - if pb.Type == restic.TreeBlob { - treePacks.Insert(pb.PackID) - } - }) + treePacks := listTreePacks(env.gopts, t) testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) testRunCheck(t, env.gopts) // delete the subdirectory pack first - for id := range treePacks { - rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) - } + removePacks(env.gopts, t, treePacks) testRunRebuildIndex(t, env.gopts) // now the repo is missing the tree blob in the index; check should report this testRunCheckMustFail(t, env.gopts) // second backup should report an error but "heal" this situation - err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory") testRunCheck(t, env.gopts) @@ -405,6 +394,7 @@ func TestIncrementalBackup(t *testing.T) { t.Logf("repository grown by %d bytes", stat3.size-stat2.size) } +// nolint: staticcheck // false positive nil pointer dereference check func TestBackupTags(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() @@ -440,6 +430,7 @@ func TestBackupTags(t *testing.T) { "expected parent to be %v, got %v", parent.ID, newest.Parent) } +// nolint: staticcheck // false positive nil pointer dereference check func TestBackupProgramVersion(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() @@ -567,3 +558,101 @@ func linkEqual(source, dest []string) bool { return true } + +func TestStdinFromCommand(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts) + testListSnapshots(t, env.gopts, 1) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandNoOutput(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(0)"}, opts, env.gopts) + rtest.Assert(t, err != nil && err.Error() == "at least one source file could not be read", "No data error expected") + testListSnapshots(t, env.gopts, 1) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandFailExitCode(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('test'); sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} + +func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{ + StdinCommand: true, + StdinFilename: "stdin", + } + + err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts) + rtest.Assert(t, err != nil, "Expected error while backing up") + + testListSnapshots(t, env.gopts, 0) + + testRunCheck(t, env.gopts) +} + +func TestBackupEmptyPassword(t *testing.T) { + // basic sanity test that empty passwords work + env, cleanup := withTestEnvironment(t) + defer cleanup() + + env.gopts.password = "" + env.gopts.InsecureNoPassword = true + + testSetupBackupData(t, env) + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts) + testListSnapshots(t, env.gopts, 1) + testRunCheck(t, env.gopts) +} + +func TestBackupSkipIfUnchanged(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{SkipIfUnchanged: true} + + for i := 0; i < 3; i++ { + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + testListSnapshots(t, env.gopts, 1) + } + + testRunCheck(t, env.gopts) +} diff --git a/mover-restic/restic/cmd/restic/cmd_cache.go b/mover-restic/restic/cmd/restic/cmd_cache.go index 4a10d1027..e71d38365 100644 --- a/mover-restic/restic/cmd/restic/cmd_cache.go +++ b/mover-restic/restic/cmd/restic/cmd_cache.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/restic/restic/internal/cache" + "github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/ui" @@ -25,10 +25,11 @@ The "cache" command allows listing and cleaning local cache directories. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return runCache(cacheOptions, globalOptions, args) }, } diff --git a/mover-restic/restic/cmd/restic/cmd_cat.go b/mover-restic/restic/cmd/restic/cmd_cat.go index 238614cd0..693c26790 100644 --- a/mover-restic/restic/cmd/restic/cmd_cat.go +++ b/mover-restic/restic/cmd/restic/cmd_cat.go @@ -7,7 +7,6 @@ import ( "github.com/spf13/cobra" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -22,7 +21,10 @@ The "cat" command is used to print internal objects to stdout. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -64,19 +66,11 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return err } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() tpe := args[0] @@ -106,7 +100,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { Println(string(buf)) return nil case "snapshot": - sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) + sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v\n", err) } @@ -154,9 +148,9 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return nil case "pack": - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) - if err != nil { + buf, err := repo.LoadRaw(ctx, restic.PackFile, id) + // allow returning broken pack files + if buf == nil { return err } @@ -176,8 +170,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { } for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} { - bh := restic.BlobHandle{ID: id, Type: t} - if !repo.Index().Has(bh) { + if _, ok := repo.LookupBlobSize(t, id); !ok { continue } @@ -193,7 +186,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatal("blob not found") case "tree": - sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1]) + sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1]) if err != nil { return errors.Fatalf("could not find snapshot: %v\n", err) } diff --git a/mover-restic/restic/cmd/restic/cmd_check.go b/mover-restic/restic/cmd/restic/cmd_check.go index fd512c7e7..9cccc0609 100644 --- a/mover-restic/restic/cmd/restic/cmd_check.go +++ b/mover-restic/restic/cmd/restic/cmd_check.go @@ -11,12 +11,15 @@ import ( "github.com/spf13/cobra" - "github.com/restic/restic/internal/cache" + "github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" + "github.com/restic/restic/internal/ui/termstatus" ) var cmdCheck = &cobra.Command{ @@ -32,13 +35,18 @@ repository and not use a local cache. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runCheck(cmd.Context(), checkOptions, globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runCheck(cmd.Context(), checkOptions, globalOptions, args, term) }, - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(_ *cobra.Command, _ []string) error { return checkFlags(checkOptions) }, } @@ -154,7 +162,7 @@ func parsePercentage(s string) (float64, error) { // - if the user explicitly requested --no-cache, we don't use any cache // - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check // - by default, we use a cache in a temporary directory that is deleted after the check -func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) { +func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress.Printer) (cleanup func()) { cleanup = func() {} if opts.WithCache { // use the default cache, no setup needed @@ -171,53 +179,54 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) cachedir = cache.EnvDir() } - // use a cache in a temporary directory + if cachedir != "" { + // use a cache in a temporary directory + err := os.MkdirAll(cachedir, 0755) + if err != nil { + Warnf("unable to create cache directory %s, disabling cache: %v\n", cachedir, err) + gopts.NoCache = true + return cleanup + } + } tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-") if err != nil { // if an error occurs, don't use any cache - Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err) + printer.E("unable to create temporary directory for cache during check, disabling cache: %v\n", err) gopts.NoCache = true return cleanup } gopts.CacheDir = tempdir - Verbosef("using temporary cache in %v\n", tempdir) + printer.P("using temporary cache in %v\n", tempdir) cleanup = func() { err := fs.RemoveAll(tempdir) if err != nil { - Warnf("error removing temporary cache directory: %v\n", err) + printer.E("error removing temporary cache directory: %v\n", err) } } return cleanup } -func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error { +func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error { if len(args) != 0 { return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags") } - cleanup := prepareCheckCache(opts, &gopts) - AddCleanupHandler(func(code int) (int, error) { - cleanup() - return code, nil - }) + printer := newTerminalProgressPrinter(gopts.verbosity, term) - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } + cleanup := prepareCheckCache(opts, &gopts, printer) + defer cleanup() if !gopts.NoLock { - Verbosef("create exclusive lock for repository\n") - var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } + printer.P("create exclusive lock for repository\n") } + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock) + if err != nil { + return err + } + defer unlock() chkr := checker.New(repo, opts.CheckUnused) err = chkr.LoadSnapshots(ctx) @@ -225,71 +234,99 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args return err } - Verbosef("load indexes\n") - bar := newIndexProgress(gopts.Quiet, gopts.JSON) + printer.P("load indexes\n") + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) hints, errs := chkr.LoadIndex(ctx, bar) + if ctx.Err() != nil { + return ctx.Err() + } errorsFound := false suggestIndexRebuild := false + suggestLegacyIndexRebuild := false mixedFound := false for _, hint := range hints { switch hint.(type) { - case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat: - Printf("%v\n", hint) + case *checker.ErrDuplicatePacks: + term.Print(hint.Error()) suggestIndexRebuild = true + case *checker.ErrOldIndexFormat: + printer.E("error: %v\n", hint) + suggestLegacyIndexRebuild = true + errorsFound = true case *checker.ErrMixedPack: - Printf("%v\n", hint) + term.Print(hint.Error()) mixedFound = true default: - Warnf("error: %v\n", hint) + printer.E("error: %v\n", hint) errorsFound = true } } if suggestIndexRebuild { - Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n") + term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n") + } + if suggestLegacyIndexRebuild { + printer.E("error: Found indexes using the legacy format, you must run `restic repair index' to correct this.\n") } if mixedFound { - Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") + term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n") } if len(errs) > 0 { for _, err := range errs { - Warnf("error: %v\n", err) + printer.E("error: %v\n", err) } - return errors.Fatal("LoadIndex returned errors") + + printer.E("\nThe repository index is damaged and must be repaired. You must run `restic repair index' to correct this.\n\n") + return errors.Fatal("repository contains errors") } orphanedPacks := 0 errChan := make(chan error) + salvagePacks := restic.NewIDSet() - Verbosef("check all packs\n") + printer.P("check all packs\n") go chkr.Packs(ctx, errChan) for err := range errChan { - if checker.IsOrphanedPack(err) { - orphanedPacks++ - Verbosef("%v\n", err) + var packErr *checker.PackError + if errors.As(err, &packErr) { + if packErr.Orphaned { + orphanedPacks++ + printer.V("%v\n", err) + } else { + if packErr.Truncated { + salvagePacks.Insert(packErr.ID) + } + errorsFound = true + printer.E("%v\n", err) + } } else if err == checker.ErrLegacyLayout { - Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n") + errorsFound = true + printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n") } else { errorsFound = true - Warnf("%v\n", err) + printer.E("%v\n", err) } } - if orphanedPacks > 0 { - Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks) + if orphanedPacks > 0 && !errorsFound { + // hide notice if repository is damaged + printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks) + } + if ctx.Err() != nil { + return ctx.Err() } - Verbosef("check snapshots, trees and blobs\n") + printer.P("check snapshots, trees and blobs\n") errChan = make(chan error) var wg sync.WaitGroup wg.Add(1) go func() { defer wg.Done() - bar := newProgressMax(!gopts.Quiet, 0, "snapshots") + bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term) defer bar.Done() chkr.Structure(ctx, bar, errChan) }() @@ -297,16 +334,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args for err := range errChan { errorsFound = true if e, ok := err.(*checker.TreeError); ok { - var clean string - if stdoutCanUpdateStatus() { - clean = clearLine(0) - } - Warnf(clean+"error for tree %v:\n", e.ID.Str()) + printer.E("error for tree %v:\n", e.ID.Str()) for _, treeErr := range e.Errors { - Warnf(" %v\n", treeErr) + printer.E(" %v\n", treeErr) } } else { - Warnf("error: %v\n", err) + printer.E("error: %v\n", err) } } @@ -314,10 +347,17 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args // Must happen after `errChan` is read from in the above loop to avoid // deadlocking in the case of errors. wg.Wait() + if ctx.Err() != nil { + return ctx.Err() + } if opts.CheckUnused { - for _, id := range chkr.UnusedBlobs(ctx) { - Verbosef("unused blob %v\n", id) + unused, err := chkr.UnusedBlobs(ctx) + if err != nil { + return err + } + for _, id := range unused { + printer.P("unused blob %v\n", id) errorsFound = true } } @@ -325,38 +365,24 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args doReadData := func(packs map[restic.ID]int64) { packCount := uint64(len(packs)) - p := newProgressMax(!gopts.Quiet, packCount, "packs") + p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term) errChan := make(chan error) go chkr.ReadPacks(ctx, packs, p, errChan) - var salvagePacks restic.IDs - for err := range errChan { errorsFound = true - Warnf("%v\n", err) - if err, ok := err.(*checker.ErrPackData); ok { - if strings.Contains(err.Error(), "wrong data returned, hash is") { - salvagePacks = append(salvagePacks, err.PackID) - } + printer.E("%v\n", err) + if err, ok := err.(*repository.ErrPackData); ok { + salvagePacks.Insert(err.PackID) } } p.Done() - - if len(salvagePacks) > 0 { - Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n") - var strIds []string - for _, id := range salvagePacks { - strIds = append(strIds, id.String()) - } - Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) - Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") - } } switch { case opts.ReadData: - Verbosef("read all data\n") + printer.P("read all data\n") doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1)) case opts.ReadDataSubset != "": var packs map[restic.ID]int64 @@ -366,12 +392,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args totalBuckets := dataSubset[1] packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets) packCount := uint64(len(packs)) - Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets) + printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets) } else if strings.HasSuffix(opts.ReadDataSubset, "%") { percentage, err := parsePercentage(opts.ReadDataSubset) if err == nil { packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage) - Verbosef("read %.1f%% of data packs\n", percentage) + printer.P("read %.1f%% of data packs\n", percentage) } } else { repoSize := int64(0) @@ -387,7 +413,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args subsetSize = repoSize } packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize) - Verbosef("read %d bytes of data packs\n", subsetSize) + printer.P("read %d bytes of data packs\n", subsetSize) } if packs == nil { return errors.Fatal("internal error: failed to select packs to check") @@ -395,11 +421,27 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args doReadData(packs) } + if len(salvagePacks) > 0 { + printer.E("\nThe repository contains damaged pack files. These damaged files must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") + var strIDs []string + for id := range salvagePacks { + strIDs = append(strIDs, id.String()) + } + printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) + printer.E("Damaged pack files can be caused by backend problems, hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") + } + + if ctx.Err() != nil { + return ctx.Err() + } + if errorsFound { + if len(salvagePacks) == 0 { + printer.E("\nThe repository is damaged and must be repaired. Please follow the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html .\n\n") + } return errors.Fatal("repository contains errors") } - - Verbosef("no errors were found\n") + printer.P("no errors were found\n") return nil } @@ -417,7 +459,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint return packs } -// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen. +// selectRandomPacksByPercentage selects the given percentage of packs which are randomly chosen. func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 { packCount := len(allPacks) packsToCheck := int(float64(packCount) * (percentage / 100.0)) diff --git a/mover-restic/restic/cmd/restic/cmd_check_integration_test.go b/mover-restic/restic/cmd/restic/cmd_check_integration_test.go index 9eb4fec62..f1e6517e0 100644 --- a/mover-restic/restic/cmd/restic/cmd_check_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_check_integration_test.go @@ -1,10 +1,12 @@ package main import ( + "bytes" "context" "testing" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) func testRunCheck(t testing.TB, gopts GlobalOptions) { @@ -23,12 +25,14 @@ func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) { } func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) { - buf, err := withCaptureStdout(func() error { + buf := bytes.NewBuffer(nil) + gopts.stdout = buf + err := withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { opts := CheckOptions{ ReadData: true, CheckUnused: checkUnused, } - return runCheck(context.TODO(), opts, gopts, nil) + return runCheck(context.TODO(), opts, gopts, nil, term) }) return buf.String(), err } diff --git a/mover-restic/restic/cmd/restic/cmd_check_test.go b/mover-restic/restic/cmd/restic/cmd_check_test.go index fb61f8420..18d607a14 100644 --- a/mover-restic/restic/cmd/restic/cmd_check_test.go +++ b/mover-restic/restic/cmd/restic/cmd_check_test.go @@ -1,12 +1,17 @@ package main import ( + "io/fs" "math" + "os" "reflect" + "strings" "testing" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" ) func TestParsePercentage(t *testing.T) { @@ -71,7 +76,7 @@ func TestSelectPacksByBucket(t *testing.T) { var testPacks = make(map[restic.ID]int64) for i := 1; i <= 10; i++ { id := restic.NewRandomID() - // ensure relevant part of generated id is reproducable + // ensure relevant part of generated id is reproducible id[0] = byte(i) testPacks[id] = 0 } @@ -124,7 +129,7 @@ func TestSelectRandomPacksByPercentage(t *testing.T) { } func TestSelectNoRandomPacksByPercentage(t *testing.T) { - // that the a repository without pack files works + // that the repository without pack files works var testPacks = make(map[restic.ID]int64) selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0) rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs") @@ -158,8 +163,84 @@ func TestSelectRandomPacksByFileSize(t *testing.T) { } func TestSelectNoRandomPacksByFileSize(t *testing.T) { - // that the a repository without pack files works + // that the repository without pack files works var testPacks = make(map[restic.ID]int64) selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500) rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs") } + +func checkIfFileWithSimilarNameExists(files []fs.DirEntry, fileName string) bool { + found := false + for _, file := range files { + if file.IsDir() { + dirName := file.Name() + if strings.Contains(dirName, fileName) { + found = true + } + } + } + return found +} + +func TestPrepareCheckCache(t *testing.T) { + // Create a temporary directory for the cache + tmpDirBase := t.TempDir() + + testCases := []struct { + opts CheckOptions + withValidCache bool + }{ + {CheckOptions{WithCache: true}, true}, // Shouldn't create temp directory + {CheckOptions{WithCache: false}, true}, // Should create temp directory + {CheckOptions{WithCache: false}, false}, // Should create cache directory first, then temp directory + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + if !testCase.withValidCache { + // remove tmpDirBase to simulate non-existing cache directory + err := os.Remove(tmpDirBase) + rtest.OK(t, err) + } + gopts := GlobalOptions{CacheDir: tmpDirBase} + cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{}) + files, err := os.ReadDir(tmpDirBase) + rtest.OK(t, err) + + if !testCase.opts.WithCache { + // If using a temporary cache directory, the cache directory should exist + // listing all directories inside tmpDirBase (cacheDir) + // one directory should be tmpDir created by prepareCheckCache with 'restic-check-cache-' in path + found := checkIfFileWithSimilarNameExists(files, "restic-check-cache-") + if !found { + t.Errorf("Expected temporary directory to exist, but it does not") + } + } else { + // If not using the cache, the temp directory should not exist + rtest.Assert(t, len(files) == 0, "expected cache directory not to exist, but it does: %v", files) + } + + // Call the cleanup function to remove the temporary cache directory + cleanup() + + // Verify that the cache directory has been removed + files, err = os.ReadDir(tmpDirBase) + rtest.OK(t, err) + rtest.Assert(t, len(files) == 0, "Expected cache directory to be removed, but it still exists: %v", files) + }) + } +} + +func TestPrepareDefaultCheckCache(t *testing.T) { + gopts := GlobalOptions{CacheDir: ""} + cleanup := prepareCheckCache(CheckOptions{}, &gopts, &progress.NoopPrinter{}) + _, err := os.ReadDir(gopts.CacheDir) + rtest.OK(t, err) + + // Call the cleanup function to remove the temporary cache directory + cleanup() + + // Verify that the cache directory has been removed + _, err = os.ReadDir(gopts.CacheDir) + rtest.Assert(t, errors.Is(err, os.ErrNotExist), "Expected cache directory to be removed, but it still exists") +} diff --git a/mover-restic/restic/cmd/restic/cmd_copy.go b/mover-restic/restic/cmd/restic/cmd_copy.go index f31c17adb..d7761174a 100644 --- a/mover-restic/restic/cmd/restic/cmd_copy.go +++ b/mover-restic/restic/cmd/restic/cmd_copy.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -31,6 +30,14 @@ This means that copied files, which existed in both the source and destination repository, /may occupy up to twice their space/ in the destination repository. This can be mitigated by the "--copy-chunker-params" option when initializing a new destination repository using the "init" command. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, RunE: func(cmd *cobra.Command, args []string) error { return runCopy(cmd.Context(), copyOptions, globalOptions, args) @@ -54,7 +61,7 @@ func init() { } func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error { - secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination") + secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination") if err != nil { return err } @@ -63,37 +70,24 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] gopts, secondaryGopts = secondaryGopts, gopts } - srcRepo, err := OpenRepository(ctx, gopts) + ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } + defer unlock() - dstRepo, err := OpenRepository(ctx, secondaryGopts) - if err != nil { - return err - } - - if !gopts.NoLock { - var srcLock *restic.Lock - srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(srcLock) - if err != nil { - return err - } - } - - dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(dstLock) + ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false) if err != nil { return err } + defer unlock() - srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile) + srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile) if err != nil { return err } - dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile) + dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile) if err != nil { return err } @@ -117,6 +111,9 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] // also consider identical snapshot copies dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn) } + if ctx.Err() != nil { + return ctx.Err() + } // remember already processed trees across all snapshots visitedTrees := restic.NewIDSet() @@ -127,11 +124,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] if sn.Original != nil { srcOriginal = *sn.Original } + if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok { isCopy := false for _, originalSn := range originalSns { if similarSnapshots(originalSn, sn) { - Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verboseff("\n%v\n", sn) Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str()) isCopy = true break @@ -141,7 +139,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] continue } } - Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\n%v\n", sn) Verbosef(" copy started, this may take a while...\n") if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil { return err @@ -160,7 +158,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] } Verbosef("snapshot %s saved\n", newID.Str()) } - return nil + return ctx.Err() } func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool { @@ -197,7 +195,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep packList := restic.NewIDSet() enqueue := func(h restic.BlobHandle) { - pb := srcRepo.Index().Lookup(h) + pb := srcRepo.LookupBlob(h.Type, h.ID) copyBlobs.Insert(h) for _, p := range pb { packList.Insert(p.PackID) @@ -212,7 +210,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep // Do we already have this tree blob? treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob} - if !dstRepo.Index().Has(treeHandle) { + if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok { // copy raw tree bytes to avoid problems if the serialization changes enqueue(treeHandle) } @@ -222,7 +220,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep // Copy the blobs for this file. for _, blobID := range entry.Content { h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID} - if !dstRepo.Index().Has(h) { + if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok { enqueue(h) } } diff --git a/mover-restic/restic/cmd/restic/cmd_copy_integration_test.go b/mover-restic/restic/cmd/restic/cmd_copy_integration_test.go index 1c8837690..704615870 100644 --- a/mover-restic/restic/cmd/restic/cmd_copy_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_copy_integration_test.go @@ -13,10 +13,12 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) { gopts := srcGopts gopts.Repo = dstGopts.Repo gopts.password = dstGopts.password + gopts.InsecureNoPassword = dstGopts.InsecureNoPassword copyOpts := CopyOptions{ secondaryRepoOptions: secondaryRepoOptions{ - Repo: srcGopts.Repo, - password: srcGopts.password, + Repo: srcGopts.Repo, + password: srcGopts.password, + InsecureNoPassword: srcGopts.InsecureNoPassword, }, } @@ -134,3 +136,22 @@ func TestCopyUnstableJSON(t *testing.T) { testRunCheck(t, env2.gopts) testListSnapshots(t, env2.gopts, 1) } + +func TestCopyToEmptyPassword(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + env2, cleanup2 := withTestEnvironment(t) + defer cleanup2() + env2.gopts.password = "" + env2.gopts.InsecureNoPassword = true + + testSetupBackupData(t, env) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, BackupOptions{}, env.gopts) + + testRunInit(t, env2.gopts) + testRunCopy(t, env.gopts, env2.gopts) + + testListSnapshots(t, env.gopts, 1) + testListSnapshots(t, env2.gopts, 1) + testRunCheck(t, env2.gopts) +} diff --git a/mover-restic/restic/cmd/restic/cmd_debug.go b/mover-restic/restic/cmd/restic/cmd_debug.go index bf05c448d..74c21df24 100644 --- a/mover-restic/restic/cmd/restic/cmd_debug.go +++ b/mover-restic/restic/cmd/restic/cmd_debug.go @@ -20,12 +20,11 @@ import ( "github.com/spf13/cobra" "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" ) @@ -44,7 +43,10 @@ is used for debugging purposes only. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -52,19 +54,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } -var tryRepair bool -var repairByte bool -var extractPack bool -var reuploadBlobs bool +type DebugExamineOptions struct { + TryRepair bool + RepairByte bool + ExtractPack bool + ReuploadBlobs bool +} + +var debugExamineOpts DebugExamineOptions func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) - cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory") - cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") - cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") - cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -78,7 +84,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error { } func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error { - return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { + return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error { if err != nil { return err } @@ -107,7 +113,7 @@ type Blob struct { func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error { var m sync.Mutex - return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error { blobs, _, err := repo.ListPack(ctx, id, size) if err != nil { Warnf("error for pack %v: %v\n", id.Str(), err) @@ -133,8 +139,8 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) }) } -func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { - return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { +func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error { + return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { return err @@ -149,19 +155,11 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error return errors.Fatal("type not specified") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() tpe := args[0] @@ -196,7 +194,7 @@ var cmdDebugExamine = &cobra.Command{ Short: "Examine a pack file", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugExamine(cmd.Context(), globalOptions, args) + return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args) }, } @@ -290,7 +288,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by }) err := wg.Wait() if err != nil { - panic("all go rountines can only return nil") + panic("all go routines can only return nil") } if !found { @@ -315,39 +313,32 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error { +func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error { dec, err := zstd.NewReader(nil) if err != nil { panic(err) } - be := repo.Backend() - h := restic.Handle{ - Name: packID.String(), - Type: restic.PackFile, + + pack, err := repo.LoadRaw(ctx, restic.PackFile, packID) + // allow processing broken pack files + if pack == nil { + return err } wg, ctx := errgroup.WithContext(ctx) - if reuploadBlobs { + if opts.ReuploadBlobs { repo.StartPackUploader(ctx, wg) } wg.Go(func() error { for _, blob := range list { Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length) - buf := make([]byte, blob.Length) - err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error { - n, err := io.ReadFull(rd, buf) - if err != nil { - return fmt.Errorf("read error after %d bytes: %v", n, err) - } - return nil - }) - if err != nil { - Warnf("error read: %v\n", err) + if int(blob.Offset+blob.Length) > len(pack) { + Warnf("skipping truncated blob\n") continue } - + buf := pack[blob.Offset : blob.Offset+blob.Length] key := repo.Key() nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():] @@ -356,8 +347,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li filePrefix := "" if err != nil { Warnf("error decrypting blob: %v\n", err) - if tryRepair || repairByte { - plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte) + if opts.TryRepair || opts.RepairByte { + plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte) } if plaintext != nil { outputPrefix = "repaired " @@ -391,13 +382,13 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id) prefix = "correct-" } - if extractPack { + if opts.ExtractPack { err = storePlainBlob(id, filePrefix+prefix, plaintext) if err != nil { return err } } - if reuploadBlobs { + if opts.ReuploadBlobs { _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) if err != nil { return err @@ -406,7 +397,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li } } - if reuploadBlobs { + if opts.ReuploadBlobs { return repo.Flush(ctx) } return nil @@ -437,17 +428,22 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { return nil } -func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error { - repo, err := OpenRepository(ctx, gopts) +func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error { + if opts.ExtractPack && gopts.NoLock { + return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive") + } + + ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock) if err != nil { return err } + defer unlock() ids := make([]restic.ID, 0) for _, name := range args { id, err := restic.ParseID(name) if err != nil { - id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name) + id, err = restic.Find(ctx, repo, restic.PackFile, name) if err != nil { Warnf("error: %v\n", err) continue @@ -460,15 +456,6 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er return errors.Fatal("no pack files to examine") } - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) if err != nil { @@ -476,7 +463,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } for _, id := range ids { - err := examinePack(ctx, repo, id) + err := examinePack(ctx, opts, repo, id) if err != nil { Warnf("error: %v\n", err) } @@ -487,23 +474,15 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er return nil } -func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { +func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error { Printf("examine %v\n", id) - h := restic.Handle{ - Type: restic.PackFile, - Name: id.String(), - } - fi, err := repo.Backend().Stat(ctx, h) - if err != nil { - return err - } - Printf(" file size is %v\n", fi.Size) - - buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h) - if err != nil { + buf, err := repo.LoadRaw(ctx, restic.PackFile, id) + // also process damaged pack files + if buf == nil { return err } + Printf(" file size is %v\n", len(buf)) gotID := restic.Hash(buf) if !id.Equal(gotID) { Printf(" wanted hash %v, got %v\n", id, gotID) @@ -516,15 +495,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro blobsLoaded := false // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) { + for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) { blobs := b.Blobs if len(blobs) == 0 { continue } - checkPackSize(blobs, fi.Size) + checkPackSize(blobs, len(buf)) - err = loadBlobs(ctx, repo, id, blobs) + err = loadBlobs(ctx, opts, repo, id, blobs) if err != nil { Warnf("error: %v\n", err) } else { @@ -535,19 +514,19 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro Printf(" ========================================\n") Printf(" inspect the pack itself\n") - blobs, _, err := repo.ListPack(ctx, id, fi.Size) + blobs, _, err := repo.ListPack(ctx, id, int64(len(buf))) if err != nil { return fmt.Errorf("pack %v: %v", id.Str(), err) } - checkPackSize(blobs, fi.Size) + checkPackSize(blobs, len(buf)) if !blobsLoaded { - return loadBlobs(ctx, repo, id, blobs) + return loadBlobs(ctx, opts, repo, id, blobs) } return nil } -func checkPackSize(blobs []restic.Blob, fileSize int64) { +func checkPackSize(blobs []restic.Blob, fileSize int) { // track current size and offset var size, offset uint64 diff --git a/mover-restic/restic/cmd/restic/cmd_diff.go b/mover-restic/restic/cmd/restic/cmd_diff.go index 125904068..6488a7c35 100644 --- a/mover-restic/restic/cmd/restic/cmd_diff.go +++ b/mover-restic/restic/cmd/restic/cmd_diff.go @@ -7,7 +7,6 @@ import ( "reflect" "sort" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -28,15 +27,22 @@ directory: * U The metadata (access mode, timestamps, ...) for the item was updated * M The file's content was modified * T The type was changed, e.g. a file was made a symlink +* ? Bitrot detected: The file's content has changed but all metadata is the same + +Metadata comparison will likely not work if a backup was created using the +'--ignore-inode' or '--ignore-ctime' option. To only compare files in specific subfolders, you can use the -":" syntax, where "subfolder" is a path within the +"snapshotID:subfolder" syntax, where "subfolder" is a path within the snapshot. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -58,7 +64,7 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) { sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) if err != nil { return nil, "", errors.Fatal(err.Error()) @@ -68,7 +74,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, // Comparer collects all things needed to compare two snapshots. type Comparer struct { - repo restic.Repository + repo restic.BlobLoader opts DiffOptions printChange func(change *Change) } @@ -144,7 +150,7 @@ type DiffStatsContainer struct { } // updateBlobs updates the blob counters in the stats struct. -func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) { +func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) { for h := range blobs { switch h.Type { case restic.DataBlob: @@ -153,7 +159,7 @@ func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) stats.TreeBlobs++ } - size, found := repo.LookupBlobSize(h.ID, h.Type) + size, found := repo.LookupBlobSize(h.Type, h.ID) if !found { Warnf("unable to find blob size for %v\n", h) continue @@ -273,6 +279,16 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref !reflect.DeepEqual(node1.Content, node2.Content) { mod += "M" stats.ChangedFiles++ + + node1NilContent := *node1 + node2NilContent := *node2 + node1NilContent.Content = nil + node2NilContent.Content = nil + // the bitrot detection may not work if `backup --ignore-inode` or `--ignore-ctime` were used + if node1NilContent.Equals(node2NilContent) { + // probable bitrot detected + mod += "?" + } } else if c.opts.ShowMetadata && !node1.Equals(*node2) { mod += "U" } @@ -331,22 +347,14 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] return errors.Fatalf("specify two snapshot IDs") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() // cache snapshots listing - be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -388,7 +396,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] c := &Comparer{ repo: repo, - opts: diffOptions, + opts: opts, printChange: func(change *Change) { Printf("%-5s%v\n", change.Modifier, change.Path) }, @@ -405,7 +413,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] } if gopts.Quiet { - c.printChange = func(change *Change) {} + c.printChange = func(_ *Change) {} } stats := &DiffStatsContainer{ diff --git a/mover-restic/restic/cmd/restic/cmd_dump.go b/mover-restic/restic/cmd/restic/cmd_dump.go index 8b9fa9624..7e1efa3ae 100644 --- a/mover-restic/restic/cmd/restic/cmd_dump.go +++ b/mover-restic/restic/cmd/restic/cmd_dump.go @@ -28,13 +28,16 @@ The special snapshotID "latest" can be used to use the latest snapshot in the repository. To include the folder content at the root of the archive, you can use the -":" syntax, where "subfolder" is a path within the +"snapshotID:subfolder" syntax, where "subfolder" is a path within the snapshot. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -46,6 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er type DumpOptions struct { restic.SnapshotFilter Archive string + Target string } var dumpOptions DumpOptions @@ -56,6 +60,7 @@ func init() { flags := cmdDump.Flags() initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") + flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`") } func splitPath(p string) []string { @@ -67,11 +72,11 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error { +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { - if err := checkStdoutArchive(); err != nil { + if err := canWriteArchiveFunc(); err != nil { return err } return d.DumpTree(ctx, tree, "/") @@ -91,9 +96,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d) + return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc) case dump.IsDir(node): - if err := checkStdoutArchive(); err != nil { + if err := canWriteArchiveFunc(); err != nil { return err } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) @@ -129,25 +134,17 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] splittedPath := splitPath(path.Clean(pathToPrint)) - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, - }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) + }).FindLatest(ctx, repo, repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } @@ -168,8 +165,24 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) } - d := dump.New(opts.Archive, repo, os.Stdout) - err = printFromTree(ctx, tree, repo, "/", splittedPath, d) + outputFileWriter := os.Stdout + canWriteArchiveFunc := checkStdoutArchive + + if opts.Target != "" { + file, err := os.Create(opts.Target) + if err != nil { + return fmt.Errorf("cannot dump to file: %w", err) + } + defer func() { + _ = file.Close() + }() + + outputFileWriter = file + canWriteArchiveFunc = func() error { return nil } + } + + d := dump.New(opts.Archive, repo, outputFileWriter) + err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) } diff --git a/mover-restic/restic/cmd/restic/cmd_features.go b/mover-restic/restic/cmd/restic/cmd_features.go new file mode 100644 index 000000000..497013696 --- /dev/null +++ b/mover-restic/restic/cmd/restic/cmd_features.go @@ -0,0 +1,59 @@ +package main + +import ( + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" + "github.com/restic/restic/internal/ui/table" + + "github.com/spf13/cobra" +) + +var featuresCmd = &cobra.Command{ + Use: "features", + Short: "Print list of feature flags", + Long: ` +The "features" command prints a list of supported feature flags. + +To pass feature flags to restic, set the RESTIC_FEATURES environment variable +to "featureA=true,featureB=false". Specifying an unknown feature flag is an error. + +A feature can either be in alpha, beta, stable or deprecated state. +An _alpha_ feature is disabled by default and may change in arbitrary ways between restic versions or be removed. +A _beta_ feature is enabled by default, but still can change in minor ways or be removed. +A _stable_ feature is always enabled and cannot be disabled. The flag will be removed in a future restic version. +A _deprecated_ feature is always disabled and cannot be enabled. The flag will be removed in a future restic version. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +`, + Hidden: true, + DisableAutoGenTag: true, + RunE: func(_ *cobra.Command, args []string) error { + if len(args) != 0 { + return errors.Fatal("the feature command expects no arguments") + } + + fmt.Printf("All Feature Flags:\n") + flags := feature.Flag.List() + + tab := table.New() + tab.AddColumn("Name", "{{ .Name }}") + tab.AddColumn("Type", "{{ .Type }}") + tab.AddColumn("Default", "{{ .Default }}") + tab.AddColumn("Description", "{{ .Description }}") + + for _, flag := range flags { + tab.AddRow(flag) + } + return tab.Write(globalOptions.stdout) + }, +} + +func init() { + cmdRoot.AddCommand(featuresCmd) +} diff --git a/mover-restic/restic/cmd/restic/cmd_find.go b/mover-restic/restic/cmd/restic/cmd_find.go index abcf4f829..4f9549ca4 100644 --- a/mover-restic/restic/cmd/restic/cmd_find.go +++ b/mover-restic/restic/cmd/restic/cmd_find.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/filter" @@ -34,7 +33,10 @@ restic find --pack 025c1d06 EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -127,6 +129,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { // Make the following attributes disappear Name byte `json:"name,omitempty"` ExtendedAttributes byte `json:"extended_attributes,omitempty"` + GenericAttributes byte `json:"generic_attributes,omitempty"` Device byte `json:"device,omitempty"` Content byte `json:"content,omitempty"` Subtree byte `json:"subtree,omitempty"` @@ -245,13 +248,12 @@ func (s *statefulOutput) Finish() { // Finder bundles information needed to find a file or directory. type Finder struct { - repo restic.Repository - pat findPattern - out statefulOutput - ignoreTrees restic.IDSet - blobIDs map[string]struct{} - treeIDs map[string]struct{} - itemsFound int + repo restic.Repository + pat findPattern + out statefulOutput + blobIDs map[string]struct{} + treeIDs map[string]struct{} + itemsFound int } func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error { @@ -262,17 +264,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID()) - return false, walker.ErrSkipNode + return walker.ErrSkipNode } if node == nil { - return false, nil + return nil } normalizedNodepath := nodepath @@ -285,7 +287,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error for _, pat := range f.pat.pattern { found, err := filter.Match(pat, normalizedNodepath) if err != nil { - return false, err + return err } if found { foundMatch = true @@ -293,16 +295,13 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } } - var ( - ignoreIfNoMatch = true - errIfNoMatch error - ) + var errIfNoMatch error if node.Type == "dir" { var childMayMatch bool for _, pat := range f.pat.pattern { mayMatch, err := filter.ChildMatch(pat, normalizedNodepath) if err != nil { - return false, err + return err } if mayMatch { childMayMatch = true @@ -311,31 +310,28 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } if !childMayMatch { - ignoreIfNoMatch = true errIfNoMatch = walker.ErrSkipNode - } else { - ignoreIfNoMatch = false } } if !foundMatch { - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) { debug.Log(" ModTime is older than %s\n", f.pat.oldest) - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) { debug.Log(" ModTime is newer than %s\n", f.pat.newest) - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } debug.Log(" found match\n") f.out.PrintPattern(nodepath, node) - return false, nil - }) + return nil + }}) } func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { @@ -346,17 +342,17 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID()) - return false, walker.ErrSkipNode + return walker.ErrSkipNode } if node == nil { - return false, nil + return nil } if node.Type == "dir" && f.treeIDs != nil { @@ -374,7 +370,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { // looking for blobs) if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil { // Return an error to terminate the Walk - return true, errors.New("OK") + return errors.New("OK") } } } @@ -395,8 +391,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } } - return false, nil - }) + return nil + }}) } var errAllPacksFound = errors.New("all packs found") @@ -446,7 +442,10 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { if err != errAllPacksFound { // try to resolve unknown pack ids from the index - packIDs = f.indexPacksToBlobs(ctx, packIDs) + packIDs, err = f.indexPacksToBlobs(ctx, packIDs) + if err != nil { + return err + } } if len(packIDs) > 0 { @@ -463,13 +462,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error { return nil } -func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} { +func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) (map[string]struct{}, error) { wctx, cancel := context.WithCancel(ctx) defer cancel() // remember which packs were found in the index indexPackIDs := make(map[string]struct{}) - f.repo.Index().Each(wctx, func(pb restic.PackedBlob) { + err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) { idStr := pb.PackID.String() // keep entry in packIDs as Each() returns individual index entries matchingID := false @@ -488,6 +487,9 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc indexPackIDs[idStr] = struct{}{} } }) + if err != nil { + return nil, err + } for id := range indexPackIDs { delete(packIDs, id) @@ -500,19 +502,17 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc } Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list) } - return packIDs + return packIDs, nil } func (f *Finder) findObjectPack(id string, t restic.BlobType) { - idx := f.repo.Index() - rid, err := restic.ParseID(id) if err != nil { Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err) return } - blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t}) + blobs := f.repo.LookupBlob(t, rid) if len(blobs) == 0 { Printf("Object %s not found in the index\n", rid.Str()) return @@ -570,21 +570,13 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] return errors.Fatal("cannot have several ID types") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } + defer unlock() - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -594,10 +586,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } f := &Finder{ - repo: repo, - pat: pat, - out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, - ignoreTrees: restic.NewIDSet(), + repo: repo, + pat: pat, + out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, } if opts.BlobID { @@ -624,6 +615,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) { filteredSnapshots = append(filteredSnapshots, sn) } + if ctx.Err() != nil { + return ctx.Err() + } sort.Slice(filteredSnapshots, func(i, j int) bool { return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time) diff --git a/mover-restic/restic/cmd/restic/cmd_forget.go b/mover-restic/restic/cmd/restic/cmd_forget.go index 22398b806..87738b518 100644 --- a/mover-restic/restic/cmd/restic/cmd_forget.go +++ b/mover-restic/restic/cmd/restic/cmd_forget.go @@ -3,11 +3,14 @@ package main import ( "context" "encoding/json" + "fmt" "io" "strconv" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" ) @@ -18,6 +21,9 @@ var cmdForget = &cobra.Command{ The "forget" command removes snapshots according to a policy. All snapshots are first divided into groups according to "--group-by", and after that the policy specified by the "--keep-*" options is applied to each group individually. +If there are not enough snapshots to keep one for each duration related +"--keep-{within-,}*" option, the oldest snapshot in the group is kept +additionally. Please note that this command really only deletes the snapshot object in the repository, which is a reference to data stored there. In order to remove the @@ -29,11 +35,16 @@ security considerations. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runForget(cmd.Context(), forgetOptions, globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args) }, } @@ -88,6 +99,8 @@ type ForgetOptions struct { WithinYearly restic.Duration KeepTags restic.TagLists + UnsafeAllowRemoveAll bool + restic.SnapshotFilter Compact bool @@ -98,6 +111,7 @@ type ForgetOptions struct { } var forgetOptions ForgetOptions +var forgetPruneOptions PruneOptions func init() { cmdRoot.AddCommand(cmdForget) @@ -116,6 +130,7 @@ func init() { f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") + f.BoolVar(&forgetOptions.UnsafeAllowRemoveAll, "unsafe-allow-remove-all", false, "allow deleting all snapshots of a snapshot group") initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false) f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)") @@ -132,7 +147,7 @@ func init() { f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") f.SortFlags = false - addPruneOptions(cmdForget) + addPruneOptions(cmdForget, &forgetPruneOptions) } func verifyForgetOptions(opts *ForgetOptions) error { @@ -151,7 +166,7 @@ func verifyForgetOptions(opts *ForgetOptions) error { return nil } -func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error { +func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { err := verifyForgetOptions(&opts) if err != nil { return err @@ -162,30 +177,31 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg return err } - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } - if gopts.NoLock && !opts.DryRun { return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command") } - if !opts.DryRun || !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock) + if err != nil { + return err } + defer unlock() + + verbosity := gopts.verbosity + if gopts.JSON { + verbosity = 0 + } + printer := newTerminalProgressPrinter(verbosity, term) var snapshots restic.Snapshots removeSnIDs := restic.NewIDSet() - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } + if ctx.Err() != nil { + return ctx.Err() + } var jsonGroups []*ForgetGroup @@ -216,72 +232,87 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg Tags: opts.KeepTags, } - if policy.Empty() && len(args) == 0 { - if !gopts.JSON { - Verbosef("no policy was specified, no snapshots will be removed\n") + if policy.Empty() { + if opts.UnsafeAllowRemoveAll { + if opts.SnapshotFilter.Empty() { + return errors.Fatal("--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified") + } + // UnsafeAllowRemoveAll together with snapshot filter is fine + } else { + return errors.Fatal("no policy was specified, no snapshots will be removed") } } - if !policy.Empty() { - if !gopts.JSON { - Verbosef("Applying Policy: %v\n", policy) - } - - for k, snapshotGroup := range snapshotGroups { - if gopts.Verbose >= 1 && !gopts.JSON { - err = PrintSnapshotGroupHeader(globalOptions.stdout, k) - if err != nil { - return err - } - } + printer.P("Applying Policy: %v\n", policy) - var key restic.SnapshotGroupKey - if json.Unmarshal([]byte(k), &key) != nil { + for k, snapshotGroup := range snapshotGroups { + if gopts.Verbose >= 1 && !gopts.JSON { + err = PrintSnapshotGroupHeader(globalOptions.stdout, k) + if err != nil { return err } + } - var fg ForgetGroup - fg.Tags = key.Tags - fg.Host = key.Hostname - fg.Paths = key.Paths + var key restic.SnapshotGroupKey + if json.Unmarshal([]byte(k), &key) != nil { + return err + } - keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) + var fg ForgetGroup + fg.Tags = key.Tags + fg.Host = key.Hostname + fg.Paths = key.Paths - if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { - Printf("keep %d snapshots:\n", len(keep)) - PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact) - Printf("\n") - } - addJSONSnapshots(&fg.Keep, keep) + keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) - if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { - Printf("remove %d snapshots:\n", len(remove)) - PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact) - Printf("\n") - } - addJSONSnapshots(&fg.Remove, remove) + if feature.Flag.Enabled(feature.SafeForgetKeepTags) && !policy.Empty() && len(keep) == 0 { + return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String()) + } + if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { + printer.P("keep %d snapshots:\n", len(keep)) + PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact) + printer.P("\n") + } + fg.Keep = asJSONSnapshots(keep) + + if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { + printer.P("remove %d snapshots:\n", len(remove)) + PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact) + printer.P("\n") + } + fg.Remove = asJSONSnapshots(remove) - fg.Reasons = reasons + fg.Reasons = asJSONKeeps(reasons) - jsonGroups = append(jsonGroups, &fg) + jsonGroups = append(jsonGroups, &fg) - for _, sn := range remove { - removeSnIDs.Insert(*sn.ID()) - } + for _, sn := range remove { + removeSnIDs.Insert(*sn.ID()) } } } + if ctx.Err() != nil { + return ctx.Err() + } + if len(removeSnIDs) > 0 { if !opts.DryRun { - err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile) + bar := printer.NewCounter("files deleted") + err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error { + if err != nil { + printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id) + } else { + printer.VV("removed %v/%v\n", restic.SnapshotFile, id) + } + return nil + }, bar) + bar.Done() if err != nil { return err } } else { - if !gopts.JSON { - Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs) - } + printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs) } } @@ -293,15 +324,13 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg } if len(removeSnIDs) > 0 && opts.Prune { - if !gopts.JSON { - if opts.DryRun { - Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs)) - } else { - Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs)) - } + if opts.DryRun { + printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs)) + } else { + printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs)) } pruneOptions.DryRun = opts.DryRun - return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs) + return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term) } return nil @@ -309,23 +338,47 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg // ForgetGroup helps to print what is forgotten in JSON. type ForgetGroup struct { - Tags []string `json:"tags"` - Host string `json:"host"` - Paths []string `json:"paths"` - Keep []Snapshot `json:"keep"` - Remove []Snapshot `json:"remove"` - Reasons []restic.KeepReason `json:"reasons"` + Tags []string `json:"tags"` + Host string `json:"host"` + Paths []string `json:"paths"` + Keep []Snapshot `json:"keep"` + Remove []Snapshot `json:"remove"` + Reasons []KeepReason `json:"reasons"` } -func addJSONSnapshots(js *[]Snapshot, list restic.Snapshots) { +func asJSONSnapshots(list restic.Snapshots) []Snapshot { + var resultList []Snapshot for _, sn := range list { k := Snapshot{ Snapshot: sn, ID: sn.ID(), ShortID: sn.ID().Str(), } - *js = append(*js, k) + resultList = append(resultList, k) + } + return resultList +} + +// KeepReason helps to print KeepReasons as JSON with Snapshots with their ID included. +type KeepReason struct { + Snapshot Snapshot `json:"snapshot"` + Matches []string `json:"matches"` +} + +func asJSONKeeps(list []restic.KeepReason) []KeepReason { + var resultList []KeepReason + for _, keep := range list { + k := KeepReason{ + Snapshot: Snapshot{ + Snapshot: keep.Snapshot, + ID: keep.Snapshot.ID(), + ShortID: keep.Snapshot.ID().Str(), + }, + Matches: keep.Matches, + } + resultList = append(resultList, k) } + return resultList } func printJSONForget(stdout io.Writer, forgets []*ForgetGroup) error { diff --git a/mover-restic/restic/cmd/restic/cmd_forget_integration_test.go b/mover-restic/restic/cmd/restic/cmd_forget_integration_test.go index 8908d5a5f..96dd7c63e 100644 --- a/mover-restic/restic/cmd/restic/cmd_forget_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_forget_integration_test.go @@ -2,12 +2,65 @@ package main import ( "context" + "path/filepath" + "strings" "testing" + "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) -func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) { - opts := ForgetOptions{} - rtest.OK(t, runForget(context.TODO(), opts, gopts, args)) +func testRunForgetMayFail(gopts GlobalOptions, opts ForgetOptions, args ...string) error { + pruneOpts := PruneOptions{ + MaxUnused: "5%", + } + return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runForget(context.TODO(), opts, pruneOpts, gopts, term, args) + }) +} + +func testRunForget(t testing.TB, gopts GlobalOptions, opts ForgetOptions, args ...string) { + rtest.OK(t, testRunForgetMayFail(gopts, opts, args...)) +} + +func TestRunForgetSafetyNet(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + + opts := BackupOptions{ + Host: "example", + } + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) + testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts) + testListSnapshots(t, env.gopts, 2) + + // --keep-tags invalid + err := testRunForgetMayFail(env.gopts, ForgetOptions{ + KeepTags: restic.TagLists{restic.TagList{"invalid"}}, + GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true}, + }) + rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err) + + // disallow `forget --unsafe-allow-remove-all` + err = testRunForgetMayFail(env.gopts, ForgetOptions{ + UnsafeAllowRemoveAll: true, + }) + rtest.Assert(t, strings.Contains(err.Error(), `--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified`), "wrong error message got %v", err) + + // disallow `forget` without options + err = testRunForgetMayFail(env.gopts, ForgetOptions{}) + rtest.Assert(t, strings.Contains(err.Error(), `no policy was specified, no snapshots will be removed`), "wrong error message got %v", err) + + // `forget --host example --unsafe-allow-remove-all` should work + testRunForget(t, env.gopts, ForgetOptions{ + UnsafeAllowRemoveAll: true, + GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true}, + SnapshotFilter: restic.SnapshotFilter{ + Hosts: []string{opts.Host}, + }, + }) + testListSnapshots(t, env.gopts, 0) } diff --git a/mover-restic/restic/cmd/restic/cmd_generate.go b/mover-restic/restic/cmd/restic/cmd_generate.go index b284767ca..b5c7cecb5 100644 --- a/mover-restic/restic/cmd/restic/cmd_generate.go +++ b/mover-restic/restic/cmd/restic/cmd_generate.go @@ -18,10 +18,13 @@ and the auto-completion files for bash, fish and zsh). EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. `, DisableAutoGenTag: true, - RunE: runGenerate, + RunE: func(_ *cobra.Command, args []string) error { + return runGenerate(genOpts, args) + }, } type generateOptions struct { @@ -90,48 +93,48 @@ func writePowerShellCompletion(file string) error { return cmdRoot.GenPowerShellCompletionFile(file) } -func runGenerate(_ *cobra.Command, args []string) error { +func runGenerate(opts generateOptions, args []string) error { if len(args) > 0 { return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags") } - if genOpts.ManDir != "" { - err := writeManpages(genOpts.ManDir) + if opts.ManDir != "" { + err := writeManpages(opts.ManDir) if err != nil { return err } } - if genOpts.BashCompletionFile != "" { - err := writeBashCompletion(genOpts.BashCompletionFile) + if opts.BashCompletionFile != "" { + err := writeBashCompletion(opts.BashCompletionFile) if err != nil { return err } } - if genOpts.FishCompletionFile != "" { - err := writeFishCompletion(genOpts.FishCompletionFile) + if opts.FishCompletionFile != "" { + err := writeFishCompletion(opts.FishCompletionFile) if err != nil { return err } } - if genOpts.ZSHCompletionFile != "" { - err := writeZSHCompletion(genOpts.ZSHCompletionFile) + if opts.ZSHCompletionFile != "" { + err := writeZSHCompletion(opts.ZSHCompletionFile) if err != nil { return err } } - if genOpts.PowerShellCompletionFile != "" { - err := writePowerShellCompletion(genOpts.PowerShellCompletionFile) + if opts.PowerShellCompletionFile != "" { + err := writePowerShellCompletion(opts.PowerShellCompletionFile) if err != nil { return err } } var empty generateOptions - if genOpts == empty { + if opts == empty { return errors.Fatal("nothing to do, please specify at least one output file/dir") } diff --git a/mover-restic/restic/cmd/restic/cmd_init.go b/mover-restic/restic/cmd/restic/cmd_init.go index 7154279e8..3c0319e55 100644 --- a/mover-restic/restic/cmd/restic/cmd_init.go +++ b/mover-restic/restic/cmd/restic/cmd_init.go @@ -23,7 +23,8 @@ The "init" command initializes a new repository. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -80,7 +81,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - gopts.password, err = ReadPasswordTwice(gopts, + gopts.password, err = ReadPasswordTwice(ctx, gopts, "enter password for new repository: ", "enter password again: ") if err != nil { @@ -131,7 +132,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) { if opts.CopyChunkerParameters { - otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary") + otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary") if err != nil { return nil, err } diff --git a/mover-restic/restic/cmd/restic/cmd_key.go b/mover-restic/restic/cmd/restic/cmd_key.go index ab41b4be3..c687eca53 100644 --- a/mover-restic/restic/cmd/restic/cmd_key.go +++ b/mover-restic/restic/cmd/restic/cmd_key.go @@ -1,265 +1,18 @@ package main import ( - "context" - "encoding/json" - "os" - "strings" - "sync" - - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/table" - "github.com/spf13/cobra" ) var cmdKey = &cobra.Command{ - Use: "key [flags] [list|add|remove|passwd] [ID]", + Use: "key", Short: "Manage keys (passwords)", Long: ` -The "key" command manages keys (passwords) for accessing the repository. - -EXIT STATUS -=========== - -Exit status is 0 if the command was successful, and non-zero if there was any error. -`, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - return runKey(cmd.Context(), globalOptions, args) - }, +The "key" command allows you to set multiple access keys or passwords +per repository. + `, } -var ( - newPasswordFile string - keyUsername string - keyHostname string -) - func init() { cmdRoot.AddCommand(cmdKey) - - flags := cmdKey.Flags() - flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") - flags.StringVarP(&keyUsername, "user", "", "", "the username for new keys") - flags.StringVarP(&keyHostname, "host", "", "", "the hostname for new keys") -} - -func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { - type keyInfo struct { - Current bool `json:"current"` - ID string `json:"id"` - UserName string `json:"userName"` - HostName string `json:"hostName"` - Created string `json:"created"` - } - - var m sync.Mutex - var keys []keyInfo - - err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { - k, err := repository.LoadKey(ctx, s, id) - if err != nil { - Warnf("LoadKey() failed: %v\n", err) - return nil - } - - key := keyInfo{ - Current: id == s.KeyID(), - ID: id.Str(), - UserName: k.Username, - HostName: k.Hostname, - Created: k.Created.Local().Format(TimeFormat), - } - - m.Lock() - defer m.Unlock() - keys = append(keys, key) - return nil - }) - - if err != nil { - return err - } - - if gopts.JSON { - return json.NewEncoder(globalOptions.stdout).Encode(keys) - } - - tab := table.New() - tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}") - tab.AddColumn("User", "{{ .UserName }}") - tab.AddColumn("Host", "{{ .HostName }}") - tab.AddColumn("Created", "{{ .Created }}") - - for _, key := range keys { - tab.AddRow(key) - } - - return tab.Write(globalOptions.stdout) -} - -// testKeyNewPassword is used to set a new password during integration testing. -var testKeyNewPassword string - -func getNewPassword(gopts GlobalOptions) (string, error) { - if testKeyNewPassword != "" { - return testKeyNewPassword, nil - } - - if newPasswordFile != "" { - return loadPasswordFromFile(newPasswordFile) - } - - // Since we already have an open repository, temporary remove the password - // to prompt the user for the passwd. - newopts := gopts - newopts.password = "" - - return ReadPasswordTwice(newopts, - "enter new password: ", - "enter password again: ") -} - -func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { - pw, err := getNewPassword(gopts) - if err != nil { - return err - } - - id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key()) - if err != nil { - return errors.Fatalf("creating new key failed: %v\n", err) - } - - err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) - if err != nil { - return err - } - - Verbosef("saved new key as %s\n", id) - - return nil -} - -func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error { - if id == repo.KeyID() { - return errors.Fatal("refusing to remove key currently used to access repository") - } - - h := restic.Handle{Type: restic.KeyFile, Name: id.String()} - err := repo.Backend().Remove(ctx, h) - if err != nil { - return err - } - - Verbosef("removed key %v\n", id) - return nil -} - -func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { - pw, err := getNewPassword(gopts) - if err != nil { - return err - } - - id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) - if err != nil { - return errors.Fatalf("creating new key failed: %v\n", err) - } - oldID := repo.KeyID() - - err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) - if err != nil { - return err - } - - h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()} - err = repo.Backend().Remove(ctx, h) - if err != nil { - return err - } - - Verbosef("saved new key as %s\n", id) - - return nil -} - -func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error { - // Verify new key to make sure it really works. A broken key can render the - // whole repository inaccessible - err := repo.SearchKey(ctx, pw, 0, key.ID().String()) - if err != nil { - // the key is invalid, try to remove it - h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()} - _ = repo.Backend().Remove(ctx, h) - return errors.Fatalf("failed to access repository with new key: %v", err) - } - return nil -} - -func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { - if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { - return errors.Fatal("wrong number of arguments") - } - - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } - - switch args[0] { - case "list": - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - return listKeys(ctx, repo, gopts) - case "add": - lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return addKey(ctx, repo, gopts) - case "remove": - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1]) - if err != nil { - return err - } - - return deleteKey(ctx, repo, id) - case "passwd": - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return changePassword(ctx, repo, gopts) - } - - return nil -} - -func loadPasswordFromFile(pwdFile string) (string, error) { - s, err := os.ReadFile(pwdFile) - if os.IsNotExist(err) { - return "", errors.Fatalf("%s does not exist", pwdFile) - } - return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") } diff --git a/mover-restic/restic/cmd/restic/cmd_key_add.go b/mover-restic/restic/cmd/restic/cmd_key_add.go new file mode 100644 index 000000000..c9f0ef233 --- /dev/null +++ b/mover-restic/restic/cmd/restic/cmd_key_add.go @@ -0,0 +1,137 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var cmdKeyAdd = &cobra.Command{ + Use: "add", + Short: "Add a new key (password) to the repository; returns the new key ID", + Long: ` +The "add" sub-command creates a new key and validates the key. Returns the new key ID. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + `, + DisableAutoGenTag: true, +} + +type KeyAddOptions struct { + NewPasswordFile string + InsecureNoPassword bool + Username string + Hostname string +} + +func (opts *KeyAddOptions) Add(flags *pflag.FlagSet) { + flags.StringVarP(&opts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") + flags.BoolVar(&opts.InsecureNoPassword, "new-insecure-no-password", false, "add an empty password for the repository (insecure)") + flags.StringVarP(&opts.Username, "user", "", "", "the username for new key") + flags.StringVarP(&opts.Hostname, "host", "", "", "the hostname for new key") +} + +func init() { + cmdKey.AddCommand(cmdKeyAdd) + + var keyAddOpts KeyAddOptions + keyAddOpts.Add(cmdKeyAdd.Flags()) + cmdKeyAdd.RunE = func(cmd *cobra.Command, args []string) error { + return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args) + } +} + +func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags") + } + + ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false) + if err != nil { + return err + } + defer unlock() + + return addKey(ctx, repo, gopts, opts) +} + +func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error { + pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword) + if err != nil { + return err + } + + id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key()) + if err != nil { + return errors.Fatalf("creating new key failed: %v\n", err) + } + + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) + if err != nil { + return err + } + + Verbosef("saved new key with ID %s\n", id.ID()) + + return nil +} + +// testKeyNewPassword is used to set a new password during integration testing. +var testKeyNewPassword string + +func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string, insecureNoPassword bool) (string, error) { + if testKeyNewPassword != "" { + return testKeyNewPassword, nil + } + + if insecureNoPassword { + if newPasswordFile != "" { + return "", fmt.Errorf("only either --new-password-file or --new-insecure-no-password may be specified") + } + return "", nil + } + + if newPasswordFile != "" { + password, err := loadPasswordFromFile(newPasswordFile) + if err != nil { + return "", err + } + if password == "" { + return "", fmt.Errorf("an empty password is not allowed by default. Pass the flag `--new-insecure-no-password` to restic to disable this check") + } + return password, nil + } + + // Since we already have an open repository, temporary remove the password + // to prompt the user for the passwd. + newopts := gopts + newopts.password = "" + // empty passwords are already handled above + newopts.InsecureNoPassword = false + + return ReadPasswordTwice(ctx, newopts, + "enter new password: ", + "enter password again: ") +} + +func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error { + // Verify new key to make sure it really works. A broken key can render the + // whole repository inaccessible + err := repo.SearchKey(ctx, pw, 0, key.ID().String()) + if err != nil { + // the key is invalid, try to remove it + _ = repository.RemoveKey(ctx, repo, key.ID()) + return errors.Fatalf("failed to access repository with new key: %v", err) + } + return nil +} diff --git a/mover-restic/restic/cmd/restic/cmd_key_integration_test.go b/mover-restic/restic/cmd/restic/cmd_key_integration_test.go index 9ea5795ba..0b4533887 100644 --- a/mover-restic/restic/cmd/restic/cmd_key_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_key_integration_test.go @@ -3,17 +3,20 @@ package main import ( "bufio" "context" + "os" + "path/filepath" "regexp" + "strings" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string { buf, err := withCaptureStdout(func() error { - return runKey(context.TODO(), gopts, []string{"list"}) + return runKeyList(context.TODO(), gopts, []string{}) }) rtest.OK(t, err) @@ -36,21 +39,20 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) + rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{})) } func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { testKeyNewPassword = "john's geheimnis" defer func() { testKeyNewPassword = "" - keyUsername = "" - keyHostname = "" }() - rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"})) - t.Log("adding key for john@example.com") - rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) + rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{ + Username: "john", + Hostname: "example.com", + }, []string{})) repo, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) @@ -67,13 +69,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) { testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"})) + rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{})) } func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) { t.Logf("remove %d keys: %q\n", len(IDs), IDs) for _, id := range IDs { - rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id})) + rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id})) } } @@ -103,18 +105,55 @@ func TestKeyAddRemove(t *testing.T) { env.gopts.password = passwordList[len(passwordList)-1] t.Logf("testing access with last password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) + rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{})) testRunCheck(t, env.gopts) testRunKeyAddNewKeyUserHost(t, env.gopts) } +func TestKeyAddInvalid(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + testRunInit(t, env.gopts) + + err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{ + NewPasswordFile: "some-file", + InsecureNoPassword: true, + }, []string{}) + rtest.Assert(t, strings.Contains(err.Error(), "only either"), "unexpected error message, got %q", err) + + pwfile := filepath.Join(t.TempDir(), "pwfile") + rtest.OK(t, os.WriteFile(pwfile, []byte{}, 0o666)) + + err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{ + NewPasswordFile: pwfile, + }, []string{}) + rtest.Assert(t, strings.Contains(err.Error(), "an empty password is not allowed by default"), "unexpected error message, got %q", err) +} + +func TestKeyAddEmpty(t *testing.T) { + env, cleanup := withTestEnvironment(t) + // must list keys more than once + env.gopts.backendTestHook = nil + defer cleanup() + testRunInit(t, env.gopts) + + rtest.OK(t, runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{ + InsecureNoPassword: true, + }, []string{})) + + env.gopts.password = "" + env.gopts.InsecureNoPassword = true + + testRunCheck(t, env.gopts) +} + type emptySaveBackend struct { - restic.Backend + backend.Backend } -func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error { - return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) +func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error { + return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil)) } func TestKeyProblems(t *testing.T) { @@ -122,7 +161,7 @@ func TestKeyProblems(t *testing.T) { defer cleanup() testRunInit(t, env.gopts) - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return &emptySaveBackend{r}, nil } @@ -131,15 +170,45 @@ func TestKeyProblems(t *testing.T) { testKeyNewPassword = "" }() - err := runKey(context.TODO(), env.gopts, []string{"passwd"}) + err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{}) t.Log(err) rtest.Assert(t, err != nil, "expected passwd change to fail") - err = runKey(context.TODO(), env.gopts, []string{"add"}) + err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{}) t.Log(err) rtest.Assert(t, err != nil, "expected key adding to fail") t.Logf("testing access with initial password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) + rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{})) testRunCheck(t, env.gopts) } + +func TestKeyCommandInvalidArguments(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { + return &emptySaveBackend{r}, nil + } + + err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err) + + err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err) + + err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err) + + err = runKeyRemove(context.TODO(), env.gopts, []string{}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err) + + err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"}) + t.Log(err) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err) +} diff --git a/mover-restic/restic/cmd/restic/cmd_key_list.go b/mover-restic/restic/cmd/restic/cmd_key_list.go new file mode 100644 index 000000000..ae751a487 --- /dev/null +++ b/mover-restic/restic/cmd/restic/cmd_key_list.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/table" + "github.com/spf13/cobra" +) + +var cmdKeyList = &cobra.Command{ + Use: "list", + Short: "List keys (passwords)", + Long: ` +The "list" sub-command lists all the keys (passwords) associated with the repository. +Returns the key ID, username, hostname, created time and if it's the current key being +used to access the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyList(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdKey.AddCommand(cmdKeyList) +} + +func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags") + } + + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) + if err != nil { + return err + } + defer unlock() + + return listKeys(ctx, repo, gopts) +} + +func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { + type keyInfo struct { + Current bool `json:"current"` + ID string `json:"id"` + ShortID string `json:"-"` + UserName string `json:"userName"` + HostName string `json:"hostName"` + Created string `json:"created"` + } + + var m sync.Mutex + var keys []keyInfo + + err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error { + k, err := repository.LoadKey(ctx, s, id) + if err != nil { + Warnf("LoadKey() failed: %v\n", err) + return nil + } + + key := keyInfo{ + Current: id == s.KeyID(), + ID: id.String(), + ShortID: id.Str(), + UserName: k.Username, + HostName: k.Hostname, + Created: k.Created.Local().Format(TimeFormat), + } + + m.Lock() + defer m.Unlock() + keys = append(keys, key) + return nil + }) + + if err != nil { + return err + } + + if gopts.JSON { + return json.NewEncoder(globalOptions.stdout).Encode(keys) + } + + tab := table.New() + tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ShortID }}") + tab.AddColumn("User", "{{ .UserName }}") + tab.AddColumn("Host", "{{ .HostName }}") + tab.AddColumn("Created", "{{ .Created }}") + + for _, key := range keys { + tab.AddRow(key) + } + + return tab.Write(globalOptions.stdout) +} diff --git a/mover-restic/restic/cmd/restic/cmd_key_passwd.go b/mover-restic/restic/cmd/restic/cmd_key_passwd.go new file mode 100644 index 000000000..723acaaab --- /dev/null +++ b/mover-restic/restic/cmd/restic/cmd_key_passwd.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/spf13/cobra" +) + +var cmdKeyPasswd = &cobra.Command{ + Use: "passwd", + Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID", + Long: ` +The "passwd" sub-command creates a new key, validates the key and remove the old key ID. +Returns the new key ID. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + `, + DisableAutoGenTag: true, +} + +type KeyPasswdOptions struct { + KeyAddOptions +} + +func init() { + cmdKey.AddCommand(cmdKeyPasswd) + + var keyPasswdOpts KeyPasswdOptions + keyPasswdOpts.KeyAddOptions.Add(cmdKeyPasswd.Flags()) + cmdKeyPasswd.RunE = func(cmd *cobra.Command, args []string) error { + return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args) + } +} + +func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags") + } + + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) + if err != nil { + return err + } + defer unlock() + + return changePassword(ctx, repo, gopts, opts) +} + +func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error { + pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword) + if err != nil { + return err + } + + id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) + if err != nil { + return errors.Fatalf("creating new key failed: %v\n", err) + } + oldID := repo.KeyID() + + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) + if err != nil { + return err + } + + err = repository.RemoveKey(ctx, repo, oldID) + if err != nil { + return err + } + + Verbosef("saved new key as %s\n", id) + + return nil +} diff --git a/mover-restic/restic/cmd/restic/cmd_key_remove.go b/mover-restic/restic/cmd/restic/cmd_key_remove.go new file mode 100644 index 000000000..c4c24fdb7 --- /dev/null +++ b/mover-restic/restic/cmd/restic/cmd_key_remove.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/spf13/cobra" +) + +var cmdKeyRemove = &cobra.Command{ + Use: "remove [ID]", + Short: "Remove key ID (password) from the repository.", + Long: ` +The "remove" sub-command removes the selected key ID. The "remove" command does not allow +removing the current key being used to access the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyRemove(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdKey.AddCommand(cmdKeyRemove) +} + +func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error { + if len(args) != 1 { + return fmt.Errorf("key remove expects one argument as the key id") + } + + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) + if err != nil { + return err + } + defer unlock() + + return deleteKey(ctx, repo, args[0]) +} + +func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error { + id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix) + if err != nil { + return err + } + + if id == repo.KeyID() { + return errors.Fatal("refusing to remove key currently used to access repository") + } + + err = repository.RemoveKey(ctx, repo, id) + if err != nil { + return err + } + + Verbosef("removed key %v\n", id) + return nil +} diff --git a/mover-restic/restic/cmd/restic/cmd_list.go b/mover-restic/restic/cmd/restic/cmd_list.go index 5974da9ac..060bca871 100644 --- a/mover-restic/restic/cmd/restic/cmd_list.go +++ b/mover-restic/restic/cmd/restic/cmd_list.go @@ -4,7 +4,7 @@ import ( "context" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" @@ -19,11 +19,14 @@ The "list" command allows listing objects in the repository based on type. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runList(cmd.Context(), cmd, globalOptions, args) + return runList(cmd.Context(), globalOptions, args) }, } @@ -31,24 +34,16 @@ func init() { cmdRoot.AddCommand(cmdList) } -func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error { +func runList(ctx context.Context, gopts GlobalOptions, args []string) error { if len(args) != 1 { - return errors.Fatal("type not specified, usage: " + cmd.Use) + return errors.Fatal("type not specified") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks") if err != nil { return err } - - if !gopts.NoLock && args[0] != "locks" { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() var t restic.FileType switch args[0] { @@ -63,20 +58,19 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { return err } - idx.Each(ctx, func(blobs restic.PackedBlob) { + return idx.Each(ctx, func(blobs restic.PackedBlob) { Printf("%v %v\n", blobs.Type, blobs.ID) }) - return nil }) default: return errors.Fatal("invalid type") } - return repo.List(ctx, t, func(id restic.ID, size int64) error { + return repo.List(ctx, t, func(id restic.ID, _ int64) error { Printf("%s\n", id) return nil }) diff --git a/mover-restic/restic/cmd/restic/cmd_list_integration_test.go b/mover-restic/restic/cmd/restic/cmd_list_integration_test.go index 4140a3ea8..ef2b8bf8f 100644 --- a/mover-restic/restic/cmd/restic/cmd_list_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_list_integration_test.go @@ -12,7 +12,7 @@ import ( func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { buf, err := withCaptureStdout(func() error { - return runList(context.TODO(), cmdList, opts, []string{tpe}) + return runList(context.TODO(), opts, []string{tpe}) }) rtest.OK(t, err) return parseIDsFromReader(t, buf) diff --git a/mover-restic/restic/cmd/restic/cmd_ls.go b/mover-restic/restic/cmd/restic/cmd_ls.go index fa2f9fbc2..76e192b6c 100644 --- a/mover-restic/restic/cmd/restic/cmd_ls.go +++ b/mover-restic/restic/cmd/restic/cmd_ls.go @@ -3,13 +3,14 @@ package main import ( "context" "encoding/json" + "fmt" + "io" "os" "strings" "time" "github.com/spf13/cobra" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -38,7 +39,10 @@ a path separator); paths use the forward slash '/' as separator. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -52,6 +56,7 @@ type LsOptions struct { restic.SnapshotFilter Recursive bool HumanReadable bool + Ncdu bool } var lsOptions LsOptions @@ -64,16 +69,52 @@ func init() { flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") + flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')") } -type lsSnapshot struct { - *restic.Snapshot - ID *restic.ID `json:"id"` - ShortID string `json:"short_id"` - StructType string `json:"struct_type"` // "snapshot" +type lsPrinter interface { + Snapshot(sn *restic.Snapshot) + Node(path string, node *restic.Node, isPrefixDirectory bool) + LeaveDir(path string) + Close() +} + +type jsonLsPrinter struct { + enc *json.Encoder +} + +func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) { + type lsSnapshot struct { + *restic.Snapshot + ID *restic.ID `json:"id"` + ShortID string `json:"short_id"` + MessageType string `json:"message_type"` // "snapshot" + StructType string `json:"struct_type"` // "snapshot", deprecated + } + + err := p.enc.Encode(lsSnapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + MessageType: "snapshot", + StructType: "snapshot", + }) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } } // Print node in our custom JSON format, followed by a newline. +func (p *jsonLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) { + if isPrefixDirectory { + return + } + err := lsNodeJSON(p.enc, path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } +} + func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { n := &struct { Name string `json:"name"` @@ -88,7 +129,8 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { AccessTime time.Time `json:"atime,omitempty"` ChangeTime time.Time `json:"ctime,omitempty"` Inode uint64 `json:"inode,omitempty"` - StructType string `json:"struct_type"` // "node" + MessageType string `json:"message_type"` // "node" + StructType string `json:"struct_type"` // "node", deprecated size uint64 // Target for Size pointer. }{ @@ -104,6 +146,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { AccessTime: node.AccessTime, ChangeTime: node.ChangeTime, Inode: node.Inode, + MessageType: "node", StructType: "node", } // Always print size for regular files, even when empty, @@ -115,10 +158,126 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } +func (p *jsonLsPrinter) LeaveDir(_ string) {} +func (p *jsonLsPrinter) Close() {} + +type ncduLsPrinter struct { + out io.Writer + depth int +} + +// lsSnapshotNcdu prints a restic snapshot in Ncdu save format. +// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. +// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt +func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) { + const NcduMajorVer = 1 + const NcduMinorVer = 2 + + snapshotBytes, err := json.Marshal(sn) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + p.depth++ + fmt.Fprintf(p.out, "[%d, %d, %s, [{\"name\":\"/\"}", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) +} + +func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { + type NcduNode struct { + Name string `json:"name"` + Asize uint64 `json:"asize"` + Dsize uint64 `json:"dsize"` + Dev uint64 `json:"dev"` + Ino uint64 `json:"ino"` + NLink uint64 `json:"nlink"` + NotReg bool `json:"notreg"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + Mode uint16 `json:"mode"` + Mtime int64 `json:"mtime"` + } + + const blockSize = 512 + + outNode := NcduNode{ + Name: node.Name, + Asize: node.Size, + // round up to nearest full blocksize + Dsize: (node.Size + blockSize - 1) / blockSize * blockSize, + Dev: node.DeviceID, + Ino: node.Inode, + NLink: node.Links, + NotReg: node.Type != "dir" && node.Type != "file", + UID: node.UID, + GID: node.GID, + Mode: uint16(node.Mode & os.ModePerm), + Mtime: node.ModTime.Unix(), + } + // bits according to inode(7) manpage + if node.Mode&os.ModeSetuid != 0 { + outNode.Mode |= 0o4000 + } + if node.Mode&os.ModeSetgid != 0 { + outNode.Mode |= 0o2000 + } + if node.Mode&os.ModeSticky != 0 { + outNode.Mode |= 0o1000 + } + if outNode.Mtime < 0 { + // ncdu does not allow negative times + outNode.Mtime = 0 + } + + return json.Marshal(outNode) +} + +func (p *ncduLsPrinter) Node(path string, node *restic.Node, _ bool) { + out, err := lsNcduNode(path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + + if node.Type == "dir" { + fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out)) + p.depth++ + } else { + fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out)) + } +} + +func (p *ncduLsPrinter) LeaveDir(_ string) { + p.depth-- + fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth)) +} + +func (p *ncduLsPrinter) Close() { + fmt.Fprint(p.out, "\n]\n]\n") +} + +type textLsPrinter struct { + dirs []string + ListLong bool + HumanReadable bool +} + +func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) { + Verbosef("%v filtered by %v:\n", sn, p.dirs) +} +func (p *textLsPrinter) Node(path string, node *restic.Node, isPrefixDirectory bool) { + if !isPrefixDirectory { + Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) + } +} + +func (p *textLsPrinter) LeaveDir(_ string) {} +func (p *textLsPrinter) Close() {} + func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") } + if opts.Ncdu && gopts.JSON { + return errors.Fatal("only either '--json' or '--ncdu' can be specified") + } // extract any specific directories to walk var dirs []string @@ -165,12 +324,13 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return false } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } + defer unlock() - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -180,38 +340,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - var ( - printSnapshot func(sn *restic.Snapshot) - printNode func(path string, node *restic.Node) - ) + var printer lsPrinter if gopts.JSON { - enc := json.NewEncoder(globalOptions.stdout) - - printSnapshot = func(sn *restic.Snapshot) { - err := enc.Encode(lsSnapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - StructType: "snapshot", - }) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + printer = &jsonLsPrinter{ + enc: json.NewEncoder(globalOptions.stdout), } - - printNode = func(path string, node *restic.Node) { - err := lsNodeJSON(enc, path, node) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + } else if opts.Ncdu { + printer = &ncduLsPrinter{ + out: globalOptions.stdout, } } else { - printSnapshot = func(sn *restic.Snapshot) { - Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time) - } - printNode = func(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable)) + printer = &textLsPrinter{ + dirs: dirs, + ListLong: opts.ListLong, + HumanReadable: opts.HumanReadable, } } @@ -229,44 +372,65 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - printSnapshot(sn) + printer.Snapshot(sn) - err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { - return false, err + return err } if node == nil { - return false, nil + return nil } + printedDir := false if withinDir(nodepath) { - // if we're within a dir, print the node - printNode(nodepath, node) + // if we're within a target path, print the node + printer.Node(nodepath, node, false) + printedDir = true // if recursive listing is requested, signal the walker that it // should continue walking recursively if opts.Recursive { - return false, nil + return nil } } // if there's an upcoming match deeper in the tree (but we're not // there yet), signal the walker to descend into any subdirs if approachingMatchingTree(nodepath) { - return false, nil + // print node leading up to the target paths + if !printedDir { + printer.Node(nodepath, node, true) + } + return nil } // otherwise, signal the walker to not walk recursively into any // subdirs if node.Type == "dir" { - return false, walker.ErrSkipNode + // immediately generate leaveDir if the directory is skipped + if printedDir { + printer.LeaveDir(nodepath) + } + return walker.ErrSkipNode } - return false, nil + return nil + } + + err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ + ProcessNode: processNode, + LeaveDir: func(path string) { + // the root path `/` has no corresponding node and is thus also skipped by processNode + if path != "/" { + printer.LeaveDir(path) + } + }, }) if err != nil { return err } + printer.Close() return nil } diff --git a/mover-restic/restic/cmd/restic/cmd_ls_integration_test.go b/mover-restic/restic/cmd/restic/cmd_ls_integration_test.go index 39bf9c3b0..f5655bdff 100644 --- a/mover-restic/restic/cmd/restic/cmd_ls_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_ls_integration_test.go @@ -2,18 +2,50 @@ package main import ( "context" + "encoding/json" "strings" "testing" rtest "github.com/restic/restic/internal/test" ) -func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { +func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte { buf, err := withCaptureStdout(func() error { gopts.Quiet = true - opts := LsOptions{} - return runLs(context.TODO(), opts, gopts, []string{snapshotID}) + return runLs(context.TODO(), opts, gopts, args) }) rtest.OK(t, err) - return strings.Split(buf.String(), "\n") + return buf.Bytes() +} + +func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { + out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID}) + return strings.Split(string(out), "\n") +} + +func assertIsValidJSON(t *testing.T, data []byte) { + // Sanity check: output must be valid JSON. + var v []any + err := json.Unmarshal(data, &v) + rtest.OK(t, err) + rtest.Assert(t, len(v) == 4, "invalid ncdu output, expected 4 array elements, got %v", len(v)) +} + +func TestRunLsNcdu(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testSetupBackupData(t, env) + opts := BackupOptions{} + // backup such that there are multiple toplevel elements + testRunBackup(t, env.testdata+"/0", []string{"."}, opts, env.gopts) + + for _, paths := range [][]string{ + {"latest"}, + {"latest", "/0"}, + {"latest", "/0", "/0/9"}, + } { + ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, paths) + assertIsValidJSON(t, ncdu) + } } diff --git a/mover-restic/restic/cmd/restic/cmd_ls_test.go b/mover-restic/restic/cmd/restic/cmd_ls_test.go index 8a4fa51ee..a1fcd479b 100644 --- a/mover-restic/restic/cmd/restic/cmd_ls_test.go +++ b/mover-restic/restic/cmd/restic/cmd_ls_test.go @@ -11,78 +11,94 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func TestLsNodeJSON(t *testing.T) { - for _, c := range []struct { - path string - restic.Node - expect string - }{ - // Mode is omitted when zero. - // Permissions, by convention is "-" per mode bit - { - path: "/bar/baz", - Node: restic.Node{ - Name: "baz", - Type: "file", - Size: 12345, - UID: 10000000, - GID: 20000000, - - User: "nobody", - Group: "nobodies", - Links: 1, - }, - expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, +type lsTestNode struct { + path string + restic.Node +} + +var lsTestNodes = []lsTestNode{ + // Mode is omitted when zero. + // Permissions, by convention is "-" per mode bit + { + path: "/bar/baz", + Node: restic.Node{ + Name: "baz", + Type: "file", + Size: 12345, + UID: 10000000, + GID: 20000000, + + User: "nobody", + Group: "nobodies", + Links: 1, + }, + }, + + // Even empty files get an explicit size. + { + path: "/foo/empty", + Node: restic.Node{ + Name: "empty", + Type: "file", + Size: 0, + UID: 1001, + GID: 1001, + + User: "not printed", + Group: "not printed", + Links: 0xF00, }, + }, - // Even empty files get an explicit size. - { - path: "/foo/empty", - Node: restic.Node{ - Name: "empty", - Type: "file", - Size: 0, - UID: 1001, - GID: 1001, - - User: "not printed", - Group: "not printed", - Links: 0xF00, - }, - expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + // Non-regular files do not get a size. + // Mode is printed in decimal, including the type bits. + { + path: "/foo/link", + Node: restic.Node{ + Name: "link", + Type: "symlink", + Mode: os.ModeSymlink | 0777, + LinkTarget: "not printed", }, + }, - // Non-regular files do not get a size. - // Mode is printed in decimal, including the type bits. - { - path: "/foo/link", - Node: restic.Node{ - Name: "link", - Type: "symlink", - Mode: os.ModeSymlink | 0777, - LinkTarget: "not printed", - }, - expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + { + path: "/some/directory", + Node: restic.Node{ + Name: "directory", + Type: "dir", + Mode: os.ModeDir | 0755, + ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), + AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), + ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), }, + }, - { - path: "/some/directory", - Node: restic.Node{ - Name: "directory", - Type: "dir", - Mode: os.ModeDir | 0755, - ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), - AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), - ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), - }, - expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, + // Test encoding of setuid/setgid/sticky bit + { + path: "/some/sticky", + Node: restic.Node{ + Name: "sticky", + Type: "dir", + Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky, }, + }, +} + +func TestLsNodeJSON(t *testing.T) { + for i, expect := range []string{ + `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","message_type":"node","struct_type":"node"}`, + `{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, } { + c := lsTestNodes[i] buf := new(bytes.Buffer) enc := json.NewEncoder(buf) err := lsNodeJSON(enc, c.path, &c.Node) rtest.OK(t, err) - rtest.Equals(t, c.expect+"\n", buf.String()) + rtest.Equals(t, expect+"\n", buf.String()) // Sanity check: output must be valid JSON. var v interface{} @@ -90,3 +106,65 @@ func TestLsNodeJSON(t *testing.T) { rtest.OK(t, err) } } + +func TestLsNcduNode(t *testing.T) { + for i, expect := range []string{ + `{"name":"baz","asize":12345,"dsize":12800,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":0}`, + `{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":0}`, + `{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":0}`, + `{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`, + `{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":0}`, + } { + c := lsTestNodes[i] + out, err := lsNcduNode(c.path, &c.Node) + rtest.OK(t, err) + rtest.Equals(t, expect, string(out)) + + // Sanity check: output must be valid JSON. + var v interface{} + err = json.Unmarshal(out, &v) + rtest.OK(t, err) + } +} + +func TestLsNcdu(t *testing.T) { + var buf bytes.Buffer + printer := &ncduLsPrinter{ + out: &buf, + } + modTime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + + printer.Snapshot(&restic.Snapshot{ + Hostname: "host", + Paths: []string{"/example"}, + }) + printer.Node("/directory", &restic.Node{ + Type: "dir", + Name: "directory", + ModTime: modTime, + }, false) + printer.Node("/directory/data", &restic.Node{ + Type: "file", + Name: "data", + Size: 42, + ModTime: modTime, + }, false) + printer.LeaveDir("/directory") + printer.Node("/file", &restic.Node{ + Type: "file", + Name: "file", + Size: 12345, + ModTime: modTime, + }, false) + printer.Close() + + rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, [{"name":"/"}, + [ + {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":1577934245}, + {"name":"data","asize":42,"dsize":512,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":1577934245} + ], + {"name":"file","asize":12345,"dsize":12800,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":1577934245} +] +] +`, buf.String()) +} diff --git a/mover-restic/restic/cmd/restic/cmd_migrate.go b/mover-restic/restic/cmd/restic/cmd_migrate.go index fd2e762c0..e89980050 100644 --- a/mover-restic/restic/cmd/restic/cmd_migrate.go +++ b/mover-restic/restic/cmd/restic/cmd_migrate.go @@ -5,6 +5,8 @@ import ( "github.com/restic/restic/internal/migrations" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" ) @@ -20,11 +22,16 @@ names are specified, these migrations are applied. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runMigrate(cmd.Context(), migrateOptions, globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runMigrate(cmd.Context(), migrateOptions, globalOptions, args, term) }, } @@ -41,8 +48,8 @@ func init() { f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`) } -func checkMigrations(ctx context.Context, repo restic.Repository) error { - Printf("available migrations:\n") +func checkMigrations(ctx context.Context, repo restic.Repository, printer progress.Printer) error { + printer.P("available migrations:\n") found := false for _, m := range migrations.All { @@ -52,19 +59,19 @@ func checkMigrations(ctx context.Context, repo restic.Repository) error { } if ok { - Printf(" %v\t%v\n", m.Name(), m.Desc()) + printer.P(" %v\t%v\n", m.Name(), m.Desc()) found = true } } if !found { - Printf("no migrations found\n") + printer.P("no migrations found\n") } return nil } -func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error { +func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string, term *termstatus.Terminal, printer progress.Printer) error { var firsterr error for _, name := range args { for _, m := range migrations.All { @@ -79,36 +86,37 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio if reason == "" { reason = "check failed" } - Warnf("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason) + printer.E("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason) continue } - Warnf("check for migration %v failed, continuing anyway\n", m.Name()) + printer.E("check for migration %v failed, continuing anyway\n", m.Name()) } if m.RepoCheck() { - Printf("checking repository integrity...\n") + printer.P("checking repository integrity...\n") checkOptions := CheckOptions{} checkGopts := gopts // the repository is already locked checkGopts.NoLock = true - err = runCheck(ctx, checkOptions, checkGopts, []string{}) + + err = runCheck(ctx, checkOptions, checkGopts, []string{}, term) if err != nil { return err } } - Printf("applying migration %v...\n", m.Name()) + printer.P("applying migration %v...\n", m.Name()) if err = m.Apply(ctx, repo); err != nil { - Warnf("migration %v failed: %v\n", m.Name(), err) + printer.E("migration %v failed: %v\n", m.Name(), err) if firsterr == nil { firsterr = err } continue } - Printf("migration %v: success\n", m.Name()) + printer.P("migration %v: success\n", m.Name()) } } } @@ -116,21 +124,18 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio return firsterr } -func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error { - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } +func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error { + printer := newTerminalProgressPrinter(gopts.verbosity, term) - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) if err != nil { return err } + defer unlock() if len(args) == 0 { - return checkMigrations(ctx, repo) + return checkMigrations(ctx, repo, printer) } - return applyMigrations(ctx, opts, gopts, repo, args) + return applyMigrations(ctx, opts, gopts, repo, args, term, printer) } diff --git a/mover-restic/restic/cmd/restic/cmd_mount.go b/mover-restic/restic/cmd/restic/cmd_mount.go index 04c072daf..3e0b159be 100644 --- a/mover-restic/restic/cmd/restic/cmd_mount.go +++ b/mover-restic/restic/cmd/restic/cmd_mount.go @@ -64,7 +64,10 @@ The default path templates are: EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -113,22 +116,23 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args return errors.Fatal("wrong number of parameters") } + mountpoint := args[0] + + // Check the existence of the mount point at the earliest stage to + // prevent unnecessary computations while opening the repository. + if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) { + Verbosef("Mountpoint %s doesn't exist\n", mountpoint) + return err + } + debug.Log("start mount") defer debug.Log("finish mount") - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() bar := newIndexProgress(gopts.Quiet, gopts.JSON) err = repo.LoadIndex(ctx, bar) @@ -136,12 +140,6 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args return err } - mountpoint := args[0] - - if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) { - Verbosef("Mountpoint %s doesn't exist\n", mountpoint) - return err - } mountOptions := []systemFuse.MountOption{ systemFuse.ReadOnly(), systemFuse.FSName("restic"), @@ -157,28 +155,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args } } - AddCleanupHandler(func(code int) (int, error) { - debug.Log("running umount cleanup handler for mount at %v", mountpoint) - err := umount(mountpoint) - if err != nil { - Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err) - } - // replace error code of sigint - if code == 130 { - code = 0 - } - return code, nil - }) + systemFuse.Debug = func(msg interface{}) { + debug.Log("fuse: %v", msg) + } c, err := systemFuse.Mount(mountpoint, mountOptions...) if err != nil { return err } - systemFuse.Debug = func(msg interface{}) { - debug.Log("fuse: %v", msg) - } - cfg := fuse.Config{ OwnerIsRoot: opts.OwnerRoot, Filter: opts.SnapshotFilter, @@ -192,15 +177,26 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n") debug.Log("serving mount at %v", mountpoint) - err = fs.Serve(c, root) - if err != nil { - return err - } - <-c.Ready - return c.MountError -} + done := make(chan struct{}) + + go func() { + defer close(done) + err = fs.Serve(c, root) + }() + + select { + case <-ctx.Done(): + debug.Log("running umount cleanup handler for mount at %v", mountpoint) + err := systemFuse.Unmount(mountpoint) + if err != nil { + Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err) + } + + return ErrOK + case <-done: + // clean shutdown, nothing to do + } -func umount(mountpoint string) error { - return systemFuse.Unmount(mountpoint) + return err } diff --git a/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go b/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go index 1b069d582..d764b4e4f 100644 --- a/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go @@ -12,8 +12,7 @@ import ( "testing" "time" - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/repository" + systemFuse "github.com/anacrolix/fuse" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -67,7 +66,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGr func testRunUmount(t testing.TB, dir string) { var err error for i := 0; i < mountWait; i++ { - if err = umount(dir); err == nil { + if err = systemFuse.Unmount(dir); err == nil { t.Logf("directory %v umounted", dir) return } @@ -87,12 +86,12 @@ func listSnapshots(t testing.TB, dir string) []string { return names } -func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) { +func checkSnapshots(t testing.TB, gopts GlobalOptions, mountpoint string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) { t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs) var wg sync.WaitGroup wg.Add(1) - go testRunMount(t, global, mountpoint, &wg) + go testRunMount(t, gopts, mountpoint, &wg) waitForMount(t, mountpoint) defer wg.Wait() defer testRunUmount(t, mountpoint) @@ -101,7 +100,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit t.Fatal(`virtual directory "snapshots" doesn't exist`) } - ids := listSnapshots(t, repodir) + ids := listSnapshots(t, gopts.Repo) t.Logf("found %v snapshots in repo: %v", len(ids), ids) namesInSnapshots := listSnapshots(t, mountpoint) @@ -125,6 +124,10 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit } } + _, repo, unlock, err := openWithReadLock(context.TODO(), gopts, false) + rtest.OK(t, err) + defer unlock() + for _, id := range snapshotIDs { snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) @@ -160,11 +163,6 @@ func TestMount(t *testing.T) { t.Skip("Skipping fuse tests") } - debugEnabled := debug.TestLogToStderr(t) - if debugEnabled { - defer debug.TestDisableLog(t) - } - env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.backendTestHook = nil @@ -172,10 +170,7 @@ func TestMount(t *testing.T) { testRunInit(t, env.gopts) - repo, err := OpenRepository(context.TODO(), env.gopts) - rtest.OK(t, err) - - checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0) + checkSnapshots(t, env.gopts, env.mountpoint, []restic.ID{}, 0) rtest.SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz")) @@ -185,7 +180,7 @@ func TestMount(t *testing.T) { rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs) - checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 2) + checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 2) // second backup, implicit incremental testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts) @@ -193,7 +188,7 @@ func TestMount(t *testing.T) { rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) - checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 3) + checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 3) // third backup, explicit incremental bopts := BackupOptions{Parent: snapshotIDs[0].String()} @@ -202,7 +197,7 @@ func TestMount(t *testing.T) { rtest.Assert(t, len(snapshotIDs) == 3, "expected three snapshots, got %v", snapshotIDs) - checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 4) + checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4) } func TestMountSameTimestamps(t *testing.T) { @@ -217,14 +212,11 @@ func TestMountSameTimestamps(t *testing.T) { rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz")) - repo, err := OpenRepository(context.TODO(), env.gopts) - rtest.OK(t, err) - ids := []restic.ID{ restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"), restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"), restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"), } - checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, ids, 4) + checkSnapshots(t, env.gopts, env.mountpoint, ids, 4) } diff --git a/mover-restic/restic/cmd/restic/cmd_options.go b/mover-restic/restic/cmd/restic/cmd_options.go index 471319dfb..4cd574b68 100644 --- a/mover-restic/restic/cmd/restic/cmd_options.go +++ b/mover-restic/restic/cmd/restic/cmd_options.go @@ -17,11 +17,12 @@ The "options" command prints a list of extended options. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. `, Hidden: true, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { fmt.Printf("All Extended Options:\n") var maxLen int for _, opt := range options.List() { diff --git a/mover-restic/restic/cmd/restic/cmd_prune.go b/mover-restic/restic/cmd/restic/cmd_prune.go index 638a0de5e..7e706ccf8 100644 --- a/mover-restic/restic/cmd/restic/cmd_prune.go +++ b/mover-restic/restic/cmd/restic/cmd_prune.go @@ -4,25 +4,20 @@ import ( "context" "math" "runtime" - "sort" "strconv" "strings" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" ) -var errorIndexIncomplete = errors.Fatal("index is not complete") -var errorPacksMissing = errors.Fatal("packs from index missing in repo") -var errorSizeNotMatching = errors.Fatal("pack size does not match calculated size from index") - var cmdPrune = &cobra.Command{ Use: "prune [flags]", Short: "Remove unneeded data from the repository", @@ -33,11 +28,16 @@ referenced and therefore not needed any more. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - return runPrune(cmd.Context(), pruneOptions, globalOptions) + RunE: func(cmd *cobra.Command, _ []string) error { + term, cancel := setupTermstatus() + defer cancel() + return runPrune(cmd.Context(), pruneOptions, globalOptions, term) }, } @@ -54,9 +54,9 @@ type PruneOptions struct { MaxRepackSize string MaxRepackBytes uint64 - RepackCachableOnly bool - RepackSmall bool - RepackUncompressed bool + RepackCacheableOnly bool + RepackSmall bool + RepackUncompressed bool } var pruneOptions PruneOptions @@ -66,14 +66,14 @@ func init() { f := cmdPrune.Flags() f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done") f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.") - addPruneOptions(cmdPrune) + addPruneOptions(cmdPrune, &pruneOptions) } -func addPruneOptions(c *cobra.Command) { +func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) { f := c.Flags() f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')") f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)") - f.BoolVar(&pruneOptions.RepackCachableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable") + f.BoolVar(&pruneOptions.RepackCacheableOnly, "repack-cacheable-only", false, "only repack packs which are cacheable") f.BoolVar(&pruneOptions.RepackSmall, "repack-small", false, "repack pack files below 80% of target pack size") f.BoolVar(&pruneOptions.RepackUncompressed, "repack-uncompressed", false, "repack all uncompressed data") } @@ -100,7 +100,7 @@ func verifyPruneOptions(opts *PruneOptions) error { // parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes switch { case maxUnused == "unlimited": - opts.maxUnusedBytes = func(used uint64) uint64 { + opts.maxUnusedBytes = func(_ uint64) uint64 { return math.MaxUint64 } @@ -129,7 +129,7 @@ func verifyPruneOptions(opts *PruneOptions) error { return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err) } - opts.maxUnusedBytes = func(used uint64) uint64 { + opts.maxUnusedBytes = func(_ uint64) uint64 { return uint64(size) } } @@ -137,7 +137,7 @@ func verifyPruneOptions(opts *PruneOptions) error { return nil } -func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error { +func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error { err := verifyPruneOptions(&opts) if err != nil { return err @@ -147,18 +147,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive") } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) if err != nil { return err } - - if repo.Backend().Connections() < 2 { - return errors.Fatal("prune requires a backend connection limit of at least two") - } - - if repo.Config().Version < 2 && opts.RepackUncompressed { - return errors.Fatal("compression requires at least repository format version 2") - } + defer unlock() if opts.UnsafeNoSpaceRecovery != "" { repoID := repo.Config().ID @@ -168,649 +161,107 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error opts.unsafeRecovery = true } - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet()) + return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term) } -func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error { - // we do not need index updates while pruning! - repo.DisableAutoIndexUpdate() - +func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error { if repo.Cache == nil { Print("warning: running prune without a cache, this may be very slow!\n") } - Verbosef("loading indexes...\n") + printer := newTerminalProgressPrinter(gopts.verbosity, term) + + printer.P("loading indexes...\n") // loading the index before the snapshots is ok, as we use an exclusive lock here - bar := newIndexProgress(gopts.Quiet, gopts.JSON) + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) err := repo.LoadIndex(ctx, bar) if err != nil { return err } - plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet) - if err != nil { - return err - } - - if opts.DryRun { - Verbosef("\nWould have made the following changes:") - } - - err = printPruneStats(stats) - if err != nil { - return err - } - - // Trigger GC to reset garbage collection threshold - runtime.GC() + popts := repository.PruneOptions{ + DryRun: opts.DryRun, + UnsafeRecovery: opts.unsafeRecovery, - return doPrune(ctx, opts, gopts, repo, plan) -} + MaxUnusedBytes: opts.maxUnusedBytes, + MaxRepackBytes: opts.MaxRepackBytes, -type pruneStats struct { - blobs struct { - used uint - duplicate uint - unused uint - remove uint - repack uint - repackrm uint - } - size struct { - used uint64 - duplicate uint64 - unused uint64 - remove uint64 - repack uint64 - repackrm uint64 - unref uint64 - uncompressed uint64 - } - packs struct { - used uint - unused uint - partlyUsed uint - unref uint - keep uint - repack uint - remove uint - } -} - -type prunePlan struct { - removePacksFirst restic.IDSet // packs to remove first (unreferenced packs) - repackPacks restic.IDSet // packs to repack - keepBlobs restic.CountedBlobSet // blobs to keep during repacking - removePacks restic.IDSet // packs to remove - ignorePacks restic.IDSet // packs to ignore when rebuilding the index -} - -type packInfo struct { - usedBlobs uint - unusedBlobs uint - usedSize uint64 - unusedSize uint64 - tpe restic.BlobType - uncompressed bool -} - -type packInfoWithID struct { - ID restic.ID - packInfo - mustCompress bool -} - -// planPrune selects which files to rewrite and which to delete and which blobs to keep. -// Also some summary statistics are returned. -func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) { - var stats pruneStats - - usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet) - if err != nil { - return prunePlan{}, stats, err - } - - Verbosef("searching used packs...\n") - keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats) - if err != nil { - return prunePlan{}, stats, err + RepackCacheableOnly: opts.RepackCacheableOnly, + RepackSmall: opts.RepackSmall, + RepackUncompressed: opts.RepackUncompressed, } - Verbosef("collecting packs for deletion and repacking\n") - plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet) + plan, err := repository.PlanPrune(ctx, popts, repo, func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error { + return getUsedBlobs(ctx, repo, usedBlobs, ignoreSnapshots, printer) + }, printer) if err != nil { - return prunePlan{}, stats, err - } - - if len(plan.repackPacks) != 0 { - blobCount := keepBlobs.Len() - // when repacking, we do not want to keep blobs which are - // already contained in kept packs, so delete them from keepBlobs - repo.Index().Each(ctx, func(blob restic.PackedBlob) { - if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { - return - } - keepBlobs.Delete(blob.BlobHandle) - }) - - if keepBlobs.Len() < blobCount/2 { - // replace with copy to shrink map to necessary size if there's a chance to benefit - keepBlobs = keepBlobs.Copy() - } - } else { - // keepBlobs is only needed if packs are repacked - keepBlobs = nil - } - plan.keepBlobs = keepBlobs - - return plan, stats, nil -} - -func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) { - // iterate over all blobs in index to find out which blobs are duplicates - // The counter in usedBlobs describes how many instances of the blob exist in the repository index - // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist - idx.Each(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle - count, ok := usedBlobs[bh] - if ok { - if count < math.MaxUint8 { - // don't overflow, but saturate count at 255 - // this can lead to a non-optimal pack selection, but won't cause - // problems otherwise - count++ - } - - usedBlobs[bh] = count - } - }) - - // Check if all used blobs have been found in index - missingBlobs := restic.NewBlobSet() - for bh, count := range usedBlobs { - if count == 0 { - // blob does not exist in any pack files - missingBlobs.Insert(bh) - } - } - - if len(missingBlobs) != 0 { - Warnf("%v not found in the index\n\n"+ - "Integrity check failed: Data seems to be missing.\n"+ - "Will not start prune to prevent (additional) data loss!\n"+ - "Please report this error (along with the output of the 'prune' run) at\n"+ - "https://github.com/restic/restic/issues/new/choose\n", missingBlobs) - return nil, nil, errorIndexIncomplete - } - - indexPack := make(map[restic.ID]packInfo) - - // save computed pack header size - for pid, hdrSize := range pack.Size(ctx, idx, true) { - // initialize tpe with NumBlobTypes to indicate it's not set - indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)} - } - - hasDuplicates := false - // iterate over all blobs in index to generate packInfo - idx.Each(ctx, func(blob restic.PackedBlob) { - ip := indexPack[blob.PackID] - - // Set blob type if not yet set - if ip.tpe == restic.NumBlobTypes { - ip.tpe = blob.Type - } - - // mark mixed packs with "Invalid blob type" - if ip.tpe != blob.Type { - ip.tpe = restic.InvalidBlob - } - - bh := blob.BlobHandle - size := uint64(blob.Length) - dupCount := usedBlobs[bh] - switch { - case dupCount >= 2: - hasDuplicates = true - // mark as unused for now, we will later on select one copy - ip.unusedSize += size - ip.unusedBlobs++ - - // count as duplicate, will later on change one copy to be counted as used - stats.size.duplicate += size - stats.blobs.duplicate++ - case dupCount == 1: // used blob, not duplicate - ip.usedSize += size - ip.usedBlobs++ - - stats.size.used += size - stats.blobs.used++ - default: // unused blob - ip.unusedSize += size - ip.unusedBlobs++ - - stats.size.unused += size - stats.blobs.unused++ - } - if !blob.IsCompressed() { - ip.uncompressed = true - } - // update indexPack - indexPack[blob.PackID] = ip - }) - - // if duplicate blobs exist, those will be set to either "used" or "unused": - // - mark only one occurence of duplicate blobs as used - // - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used" - // - if there are no used blobs in a pack, possibly mark duplicates as "unused" - if hasDuplicates { - // iterate again over all blobs in index (this is pretty cheap, all in-mem) - idx.Each(ctx, func(blob restic.PackedBlob) { - bh := blob.BlobHandle - count, ok := usedBlobs[bh] - // skip non-duplicate, aka. normal blobs - // count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining - if !ok || count == 1 { - return - } - - ip := indexPack[blob.PackID] - size := uint64(blob.Length) - switch { - case ip.usedBlobs > 0, count == 0: - // other used blobs in pack or "last" occurence -> transition to used - ip.usedSize += size - ip.usedBlobs++ - ip.unusedSize -= size - ip.unusedBlobs-- - // same for the global statistics - stats.size.used += size - stats.blobs.used++ - stats.size.duplicate -= size - stats.blobs.duplicate-- - // let other occurences remain marked as unused - usedBlobs[bh] = 1 - default: - // remain unused and decrease counter - count-- - if count == 1 { - // setting count to 1 would lead to forgetting that this blob had duplicates - // thus use the special value zero. This will select the last instance of the blob for keeping. - count = 0 - } - usedBlobs[bh] = count - } - // update indexPack - indexPack[blob.PackID] = ip - }) + return err } - - // Sanity check. If no duplicates exist, all blobs have value 1. After handling - // duplicates, this also applies to duplicates. - for _, count := range usedBlobs { - if count != 1 { - panic("internal error during blob selection") - } + if ctx.Err() != nil { + return ctx.Err() } - return usedBlobs, indexPack, nil -} - -func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) { - removePacksFirst := restic.NewIDSet() - removePacks := restic.NewIDSet() - repackPacks := restic.NewIDSet() - - var repackCandidates []packInfoWithID - var repackSmallCandidates []packInfoWithID - repoVersion := repo.Config().Version - // only repack very small files by default - targetPackSize := repo.PackSize() / 25 - if opts.RepackSmall { - // consider files with at least 80% of the target size as large enough - targetPackSize = repo.PackSize() / 5 * 4 + if popts.DryRun { + printer.P("\nWould have made the following changes:") } - // loop over all packs and decide what to do - bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed") - err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error { - p, ok := indexPack[id] - if !ok { - // Pack was not referenced in index and is not used => immediately remove! - Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str()) - removePacksFirst.Insert(id) - stats.size.unref += uint64(packSize) - return nil - } - - if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 { - // Pack size does not fit and pack is needed => error - // If the pack is not needed, this is no error, the pack can - // and will be simply removed, see below. - Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n", - id.Str(), p.unusedSize+p.usedSize, packSize) - return errorSizeNotMatching - } - - // statistics - switch { - case p.usedBlobs == 0: - stats.packs.unused++ - case p.unusedBlobs == 0: - stats.packs.used++ - default: - stats.packs.partlyUsed++ - } - - if p.uncompressed { - stats.size.uncompressed += p.unusedSize + p.usedSize - } - mustCompress := false - if repoVersion >= 2 { - // repo v2: always repack tree blobs if uncompressed - // compress data blobs if requested - mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed - } - - // decide what to do - switch { - case p.usedBlobs == 0: - // All blobs in pack are no longer used => remove pack! - removePacks.Insert(id) - stats.blobs.remove += p.unusedBlobs - stats.size.remove += p.unusedSize - - case opts.RepackCachableOnly && p.tpe == restic.DataBlob: - // if this is a data pack and --repack-cacheable-only is set => keep pack! - stats.packs.keep++ - - case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress: - if packSize >= int64(targetPackSize) { - // All blobs in pack are used and not mixed => keep pack! - stats.packs.keep++ - } else { - repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress}) - } - - default: - // all other packs are candidates for repacking - repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress}) - } - - delete(indexPack, id) - bar.Add(1) - return nil - }) - bar.Done() + err = printPruneStats(printer, plan.Stats()) if err != nil { - return prunePlan{}, err - } - - // At this point indexPacks contains only missing packs! - - // missing packs that are not needed can be ignored - ignorePacks := restic.NewIDSet() - for id, p := range indexPack { - if p.usedBlobs == 0 { - ignorePacks.Insert(id) - stats.blobs.remove += p.unusedBlobs - stats.size.remove += p.unusedSize - delete(indexPack, id) - } - } - - if len(indexPack) != 0 { - Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack)) - for id := range indexPack { - Warnf(" %v\n", id) - } - return prunePlan{}, errorPacksMissing - } - if len(ignorePacks) != 0 { - Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n") - for id := range ignorePacks { - Warnf("will forget missing pack file %v\n", id) - } - } - - if len(repackSmallCandidates) < 10 { - // too few small files to be worth the trouble, this also prevents endlessly repacking - // if there is just a single pack file below the target size - stats.packs.keep += uint(len(repackSmallCandidates)) - } else { - repackCandidates = append(repackCandidates, repackSmallCandidates...) - } - - // Sort repackCandidates such that packs with highest ratio unused/used space are picked first. - // This is equivalent to sorting by unused / total space. - // Instead of unused[i] / used[i] > unused[j] / used[j] we use - // unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64 - // Moreover packs containing trees and too small packs are sorted to the beginning - sort.Slice(repackCandidates, func(i, j int) bool { - pi := repackCandidates[i].packInfo - pj := repackCandidates[j].packInfo - switch { - case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob: - return true - case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob: - return false - case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize): - return true - case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize): - return false - } - return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize - }) - - repack := func(id restic.ID, p packInfo) { - repackPacks.Insert(id) - stats.blobs.repack += p.unusedBlobs + p.usedBlobs - stats.size.repack += p.unusedSize + p.usedSize - stats.blobs.repackrm += p.unusedBlobs - stats.size.repackrm += p.unusedSize - if p.uncompressed { - stats.size.uncompressed -= p.unusedSize + p.usedSize - } - } - - // calculate limit for number of unused bytes in the repo after repacking - maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used) - - for _, p := range repackCandidates { - reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter) - reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes - packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize) - - switch { - case reachedRepackSize: - stats.packs.keep++ - - case p.tpe != restic.DataBlob, p.mustCompress: - // repacking non-data packs / uncompressed-trees is only limited by repackSize - repack(p.ID, p.packInfo) - - case reachedUnusedSizeAfter && packIsLargeEnough: - // for all other packs stop repacking if tolerated unused size is reached. - stats.packs.keep++ - - default: - repack(p.ID, p.packInfo) - } + return err } - stats.packs.unref = uint(len(removePacksFirst)) - stats.packs.repack = uint(len(repackPacks)) - stats.packs.remove = uint(len(removePacks)) - - if repo.Config().Version < 2 { - // compression not supported for repository format version 1 - stats.size.uncompressed = 0 - } + // Trigger GC to reset garbage collection threshold + runtime.GC() - return prunePlan{removePacksFirst: removePacksFirst, - removePacks: removePacks, - repackPacks: repackPacks, - ignorePacks: ignorePacks, - }, nil + return plan.Execute(ctx, printer) } // printPruneStats prints out the statistics -func printPruneStats(stats pruneStats) error { - Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used)) - if stats.blobs.duplicate > 0 { - Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate)) - } - Verboseff("unused: %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused)) - if stats.size.unref > 0 { - Verboseff("unreferenced: %s\n", ui.FormatBytes(stats.size.unref)) - } - totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate - totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref - unusedSize := stats.size.duplicate + stats.size.unused - Verboseff("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize)) - Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize)) - - Verbosef("\nto repack: %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack)) - Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm)) - Verbosef("to delete: %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref)) - totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref - Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize)) - if stats.size.uncompressed > 0 { - Verbosef("not yet compressed: %s\n", ui.FormatBytes(stats.size.uncompressed)) - } - Verbosef("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize)) - unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm - Verbosef("unused size after prune: %s (%s of remaining size)\n", +func printPruneStats(printer progress.Printer, stats repository.PruneStats) error { + printer.V("\nused: %10d blobs / %s\n", stats.Blobs.Used, ui.FormatBytes(stats.Size.Used)) + if stats.Blobs.Duplicate > 0 { + printer.V("duplicates: %10d blobs / %s\n", stats.Blobs.Duplicate, ui.FormatBytes(stats.Size.Duplicate)) + } + printer.V("unused: %10d blobs / %s\n", stats.Blobs.Unused, ui.FormatBytes(stats.Size.Unused)) + if stats.Size.Unref > 0 { + printer.V("unreferenced: %s\n", ui.FormatBytes(stats.Size.Unref)) + } + totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate + totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref + unusedSize := stats.Size.Duplicate + stats.Size.Unused + printer.V("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize)) + printer.V("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize)) + + printer.P("\nto repack: %10d blobs / %s\n", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack)) + printer.P("this removes: %10d blobs / %s\n", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm)) + printer.P("to delete: %10d blobs / %s\n", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref)) + totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref + printer.P("total prune: %10d blobs / %s\n", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize)) + if stats.Size.Uncompressed > 0 { + printer.P("not yet compressed: %s\n", ui.FormatBytes(stats.Size.Uncompressed)) + } + printer.P("remaining: %10d blobs / %s\n", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize)) + unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm + printer.P("unused size after prune: %s (%s of remaining size)\n", ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize)) - Verbosef("\n") - Verboseff("totally used packs: %10d\n", stats.packs.used) - Verboseff("partly used packs: %10d\n", stats.packs.partlyUsed) - Verboseff("unused packs: %10d\n\n", stats.packs.unused) - - Verboseff("to keep: %10d packs\n", stats.packs.keep) - Verboseff("to repack: %10d packs\n", stats.packs.repack) - Verboseff("to delete: %10d packs\n", stats.packs.remove) - if stats.packs.unref > 0 { - Verboseff("to delete: %10d unreferenced packs\n\n", stats.packs.unref) - } - return nil -} - -// doPrune does the actual pruning: -// - remove unreferenced packs first -// - repack given pack files while keeping the given blobs -// - rebuild the index while ignoring all files that will be deleted -// - delete the files -// plan.removePacks and plan.ignorePacks are modified in this function. -func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) { - if opts.DryRun { - if !gopts.JSON && gopts.verbosity >= 2 { - Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n") - if len(plan.removePacksFirst) > 0 { - Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst) - } - Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks) - Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks) - } - // Always quit here if DryRun was set! - return nil - } - - // unreferenced packs can be safely deleted first - if len(plan.removePacksFirst) != 0 { - Verbosef("deleting unreferenced packs\n") - DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile) - } - - if len(plan.repackPacks) != 0 { - Verbosef("repacking packs\n") - bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked") - _, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar) - bar.Done() - if err != nil { - return errors.Fatal(err.Error()) - } - - // Also remove repacked packs - plan.removePacks.Merge(plan.repackPacks) - - if len(plan.keepBlobs) != 0 { - Warnf("%v was not repacked\n\n"+ - "Integrity check failed.\n"+ - "Please report this error (along with the output of the 'prune' run) at\n"+ - "https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs) - return errors.Fatal("internal error: blobs were not repacked") - } - - // allow GC of the blob set - plan.keepBlobs = nil - } - - if len(plan.ignorePacks) == 0 { - plan.ignorePacks = plan.removePacks - } else { - plan.ignorePacks.Merge(plan.removePacks) - } - - if opts.unsafeRecovery { - Verbosef("deleting index files\n") - indexFiles := repo.Index().(*index.MasterIndex).IDs() - err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile) - if err != nil { - return errors.Fatalf("%s", err) - } - } else if len(plan.ignorePacks) != 0 { - err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) - if err != nil { - return errors.Fatalf("%s", err) - } - } - - if len(plan.removePacks) != 0 { - Verbosef("removing %d old packs\n", len(plan.removePacks)) - DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile) - } + printer.P("\n") + printer.V("totally used packs: %10d\n", stats.Packs.Used) + printer.V("partly used packs: %10d\n", stats.Packs.PartlyUsed) + printer.V("unused packs: %10d\n\n", stats.Packs.Unused) - if opts.unsafeRecovery { - _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) - if err != nil { - return errors.Fatalf("%s", err) - } + printer.V("to keep: %10d packs\n", stats.Packs.Keep) + printer.V("to repack: %10d packs\n", stats.Packs.Repack) + printer.V("to delete: %10d packs\n", stats.Packs.Remove) + if stats.Packs.Unref > 0 { + printer.V("to delete: %10d unreferenced packs\n\n", stats.Packs.Unref) } - - Verbosef("done\n") return nil } -func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { - Verbosef("rebuilding index\n") - - bar := newProgressMax(!gopts.Quiet, 0, "packs processed") - obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar) - bar.Done() - return obsoleteIndexes, err -} - -func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { - obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete) - if err != nil { - return err - } - - Verbosef("deleting obsolete index files\n") - return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile) -} - -func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) { +func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet, ignoreSnapshots restic.IDSet, printer progress.Printer) error { var snapshotTrees restic.IDs - Verbosef("loading all snapshots...\n") - err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots, + printer.P("loading all snapshots...\n") + err := restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { debug.Log("failed to load snapshot %v (error %v)", id, err) @@ -821,23 +272,14 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots r return nil }) if err != nil { - return nil, errors.Fatalf("failed loading snapshot: %v", err) + return errors.Fatalf("failed loading snapshot: %v", err) } - Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees)) + printer.P("finding data that is still in use for %d snapshots\n", len(snapshotTrees)) - usedBlobs = restic.NewCountedBlobSet() - - bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots") + bar := printer.NewCounter("snapshots") + bar.SetMax(uint64(len(snapshotTrees))) defer bar.Done() - err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar) - if err != nil { - if repo.Backend().IsNotExist(err) { - return nil, errors.Fatal("unable to load a tree from the repository: " + err.Error()) - } - - return nil, err - } - return usedBlobs, nil + return restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar) } diff --git a/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go b/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go index 2cd86d895..746eb5cc9 100644 --- a/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go @@ -6,17 +6,21 @@ import ( "path/filepath" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/repository" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) { oldHook := gopts.backendTestHook - gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } defer func() { gopts.backendTestHook = oldHook }() - rtest.OK(t, runPrune(context.TODO(), opts, gopts)) + rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runPrune(context.TODO(), opts, gopts, term) + })) } func TestPrune(t *testing.T) { @@ -31,7 +35,7 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) { } t.Run("0"+suffix, func(t *testing.T) { opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery} - checkOpts := CheckOptions{ReadData: true, CheckUnused: true} + checkOpts := CheckOptions{ReadData: true, CheckUnused: !unsafeNoSpaceRecovery} testPrune(t, opts, checkOpts) }) @@ -47,8 +51,8 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) { testPrune(t, opts, checkOpts) }) - t.Run("CachableOnly"+suffix, func(t *testing.T) { - opts := PruneOptions{MaxUnused: "5%", RepackCachableOnly: true, unsafeRecovery: unsafeNoSpaceRecovery} + t.Run("CacheableOnly"+suffix, func(t *testing.T) { + opts := PruneOptions{MaxUnused: "5%", RepackCacheableOnly: true, unsafeRecovery: unsafeNoSpaceRecovery} checkOpts := CheckOptions{ReadData: true} testPrune(t, opts, checkOpts) }) @@ -71,7 +75,7 @@ func createPrunableRepo(t *testing.T, env *testEnvironment) { testListSnapshots(t, env.gopts, 3) testRunForgetJSON(t, env.gopts) - testRunForget(t, env.gopts, firstSnapshot.String()) + testRunForget(t, env.gopts, ForgetOptions{}, firstSnapshot.String()) } func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { @@ -81,7 +85,12 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { DryRun: true, Last: 1, } - return runForget(context.TODO(), opts, gopts, args) + pruneOpts := PruneOptions{ + MaxUnused: "5%", + } + return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runForget(context.TODO(), opts, pruneOpts, gopts, term, args) + }) }) rtest.OK(t, err) @@ -102,7 +111,9 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) { createPrunableRepo(t, env) testRunPrune(t, env.gopts, pruneOpts) - rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) + rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runCheck(context.TODO(), checkOpts, env.gopts, nil, term) + })) } var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"} @@ -120,7 +131,7 @@ func TestPruneWithDamagedRepository(t *testing.T) { // create and delete snapshot to create unused blobs testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts) firstSnapshot := testListSnapshots(t, env.gopts, 1)[0] - testRunForget(t, env.gopts, firstSnapshot.String()) + testRunForget(t, env.gopts, ForgetOptions{}, firstSnapshot.String()) oldPacks := listPacks(env.gopts, t) @@ -130,12 +141,14 @@ func TestPruneWithDamagedRepository(t *testing.T) { removePacksExcept(env.gopts, t, oldPacks, false) oldHook := env.gopts.backendTestHook - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil } + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } defer func() { env.gopts.backendTestHook = oldHook }() // prune should fail - rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing, + rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term) + }) == repository.ErrPacksMissing, "prune should have reported index not complete error") } @@ -207,7 +220,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o if checkOK { testRunCheck(t, env.gopts) } else { - rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil, + rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runCheck(context.TODO(), optionsCheck, env.gopts, nil, term) + }) != nil, "check should have reported an error") } @@ -215,7 +230,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o testRunPrune(t, env.gopts, optionsPrune) testRunCheck(t, env.gopts) } else { - rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil, + rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runPrune(context.TODO(), optionsPrune, env.gopts, term) + }) != nil, "prune should have reported an error") } } diff --git a/mover-restic/restic/cmd/restic/cmd_recover.go b/mover-restic/restic/cmd/restic/cmd_recover.go index 63084dd5f..5e4744bb6 100644 --- a/mover-restic/restic/cmd/restic/cmd_recover.go +++ b/mover-restic/restic/cmd/restic/cmd_recover.go @@ -5,7 +5,6 @@ import ( "os" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" @@ -23,10 +22,13 @@ It can be used if, for example, a snapshot has been removed by accident with "fo EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runRecover(cmd.Context(), globalOptions) }, } @@ -41,18 +43,13 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { return err } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false) if err != nil { return err } + defer unlock() - lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -67,16 +64,22 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { // tree. If it is not referenced, we have a root tree. trees := make(map[restic.ID]bool) - repo.Index().Each(ctx, func(blob restic.PackedBlob) { + err = repo.ListBlobs(ctx, func(blob restic.PackedBlob) { if blob.Type == restic.TreeBlob { trees[blob.Blob.ID] = false } }) + if err != nil { + return err + } Verbosef("load %d trees\n", len(trees)) bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded") for id := range trees { tree, err := restic.LoadTree(ctx, repo, id) + if ctx.Err() != nil { + return ctx.Err() + } if err != nil { Warnf("unable to load tree %v: %v\n", id.Str(), err) continue @@ -92,7 +95,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { bar.Done() Verbosef("load snapshots\n") - err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { + err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error { trees[*sn.Tree] = true return nil }) @@ -159,7 +162,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { } -func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error { +func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error { sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now()) if err != nil { return errors.Fatalf("unable to save snapshot: %v", err) diff --git a/mover-restic/restic/cmd/restic/cmd_repair_index.go b/mover-restic/restic/cmd/restic/cmd_repair_index.go index 622c77801..e6b6e9fa5 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_index.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_index.go @@ -3,10 +3,8 @@ package main import ( "context" - "github.com/restic/restic/internal/index" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -21,11 +19,16 @@ repository. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions) + RunE: func(cmd *cobra.Command, _ []string) error { + term, cancel := setupTermstatus() + defer cancel() + return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions, term) }, } @@ -55,110 +58,22 @@ func init() { } } -func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error { - repo, err := OpenRepository(ctx, gopts) +func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error { + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) if err != nil { return err } + defer unlock() - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return rebuildIndex(ctx, opts, gopts, repo) -} - -func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository) error { - var obsoleteIndexes restic.IDs - packSizeFromList := make(map[restic.ID]int64) - packSizeFromIndex := make(map[restic.ID]int64) - removePacks := restic.NewIDSet() - - if opts.ReadAllPacks { - // get list of old index files but start with empty index - err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { - obsoleteIndexes = append(obsoleteIndexes, id) - return nil - }) - if err != nil { - return err - } - } else { - Verbosef("loading indexes...\n") - mi := index.NewMasterIndex() - err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { - if err != nil { - Warnf("removing invalid index %v: %v\n", id, err) - obsoleteIndexes = append(obsoleteIndexes, id) - return nil - } - - mi.Insert(idx) - return nil - }) - if err != nil { - return err - } - - err = mi.MergeFinalIndexes() - if err != nil { - return err - } - - err = repo.SetIndex(mi) - if err != nil { - return err - } - packSizeFromIndex = pack.Size(ctx, repo.Index(), false) - } - - Verbosef("getting pack files to read...\n") - err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error { - size, ok := packSizeFromIndex[id] - if !ok || size != packSize { - // Pack was not referenced in index or size does not match - packSizeFromList[id] = packSize - removePacks.Insert(id) - } - if !ok { - Warnf("adding pack file to index %v\n", id) - } else if size != packSize { - Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size) - } - delete(packSizeFromIndex, id) - return nil - }) - if err != nil { - return err - } - for id := range packSizeFromIndex { - // forget pack files that are referenced in the index but do not exist - // when rebuilding the index - removePacks.Insert(id) - Warnf("removing not found pack file %v\n", id) - } - - if len(packSizeFromList) > 0 { - Verbosef("reading pack files\n") - bar := newProgressMax(!gopts.Quiet, uint64(len(packSizeFromList)), "packs") - invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar) - bar.Done() - if err != nil { - return err - } - - for _, id := range invalidFiles { - Verboseff("skipped incomplete pack file: %v\n", id) - } - } + printer := newTerminalProgressPrinter(gopts.verbosity, term) - err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes) + err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{ + ReadAllPacks: opts.ReadAllPacks, + }, printer) if err != nil { return err } - Verbosef("done\n") + printer.P("done\n") return nil } diff --git a/mover-restic/restic/cmd/restic/cmd_repair_index_integration_test.go b/mover-restic/restic/cmd/restic/cmd_repair_index_integration_test.go index f451173a3..9bfc93b40 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_index_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_index_integration_test.go @@ -8,16 +8,20 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) { rtest.OK(t, withRestoreGlobalOptions(func() error { - globalOptions.stdout = io.Discard - return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts) + return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error { + globalOptions.stdout = io.Discard + return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, term) + }) })) } @@ -64,18 +68,18 @@ func TestRebuildIndexAlwaysFull(t *testing.T) { defer func() { index.IndexFull = indexFull }() - index.IndexFull = func(*index.Index, bool) bool { return true } + index.IndexFull = func(*index.Index) bool { return true } testRebuildIndex(t, nil) } // indexErrorBackend modifies the first index after reading. type indexErrorBackend struct { - restic.Backend + backend.Backend lock sync.Mutex hasErred bool } -func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { // protect hasErred b.lock.Lock() @@ -101,7 +105,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) { } func TestRebuildIndexDamage(t *testing.T) { - testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) { + testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) { return &indexErrorBackend{ Backend: r, }, nil @@ -109,11 +113,11 @@ func TestRebuildIndexDamage(t *testing.T) { } type appendOnlyBackend struct { - restic.Backend + backend.Backend } // called via repo.Backend().Remove() -func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error { +func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error { return errors.Errorf("Failed to remove %v", h) } @@ -125,12 +129,13 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) { rtest.SetupTarTestFixture(t, env.base, datafile) err := withRestoreGlobalOptions(func() error { - globalOptions.stdout = io.Discard - - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return &appendOnlyBackend{r}, nil } - return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts) + return withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + globalOptions.stdout = io.Discard + return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term) + }) }) if err == nil { diff --git a/mover-restic/restic/cmd/restic/cmd_repair_packs.go b/mover-restic/restic/cmd/restic/cmd_repair_packs.go index aadfe73be..b0afefb2d 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_packs.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_packs.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "context" "io" "os" @@ -8,27 +9,30 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" ) var cmdRepairPacks = &cobra.Command{ Use: "packs [packIDs...]", Short: "Salvage damaged pack files", Long: ` -WARNING: The CLI for this command is experimental and will likely change in the future! - The "repair packs" command extracts intact blobs from the specified pack files, rebuilds the index to remove the damaged pack files and removes the pack files from the repository. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepairPacks(cmd.Context(), globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runRepairPacks(cmd.Context(), globalOptions, term, args) }, } @@ -36,14 +40,7 @@ func init() { cmdRepair.AddCommand(cmdRepairPacks) } -func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { - // FIXME discuss and add proper feature flag mechanism - flag, _ := os.LookupEnv("RESTIC_FEATURES") - if flag != "repair-packs-v1" { - return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " + - "Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.") - } - +func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { ids := restic.NewIDSet() for _, arg := range args { id, err := restic.ParseID(arg) @@ -56,103 +53,46 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) err return errors.Fatal("no ids specified") } - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } - - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) if err != nil { return err } + defer unlock() - return repairPacks(ctx, gopts, repo, ids) -} + printer := newTerminalProgressPrinter(gopts.verbosity, term) -func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error { - bar := newIndexProgress(gopts.Quiet, gopts.JSON) - err := repo.LoadIndex(ctx, bar) + bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term) + err = repo.LoadIndex(ctx, bar) if err != nil { return errors.Fatalf("%s", err) } - Warnf("saving backup copies of pack files in current folder\n") + printer.P("saving backup copies of pack files to current folder") for id := range ids { + buf, err := repo.LoadRaw(ctx, restic.PackFile, id) + // corrupted data is fine + if buf == nil { + return err + } + f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) if err != nil { - return errors.Fatalf("%s", err) + return err } - - err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { - _, err := f.Seek(0, 0) - if err != nil { - return err - } - _, err = io.Copy(f, rd) + if _, err := io.Copy(f, bytes.NewReader(buf)); err != nil { + _ = f.Close() return err - }) - if err != nil { - return errors.Fatalf("%s", err) } - } - - wg, wgCtx := errgroup.WithContext(ctx) - repo.StartPackUploader(wgCtx, wg) - repo.DisableAutoIndexUpdate() - - Warnf("salvaging intact data from specified pack files\n") - bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files") - defer bar.Done() - - wg.Go(func() error { - // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(wgCtx, ids) { - blobs := b.Blobs - if len(blobs) == 0 { - Warnf("no blobs found for pack %v\n", b.PackID) - bar.Add(1) - continue - } - - err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - if err != nil { - // Fallback path - buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) - if err != nil { - Warnf("failed to load blob %v: %v\n", blob.ID, err) - return nil - } - } - id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) - if !id.Equal(blob.ID) { - panic("pack id mismatch during upload") - } - return err - }) - if err != nil { - return err - } - bar.Add(1) + if err := f.Close(); err != nil { + return err } - return repo.Flush(wgCtx) - }) - - if err := wg.Wait(); err != nil { - return errors.Fatalf("%s", err) } - bar.Done() - // remove salvaged packs from index - err = rebuildIndexFiles(ctx, gopts, repo, ids, nil) + err = repository.RepairPacks(ctx, repo, ids, printer) if err != nil { return errors.Fatalf("%s", err) } - // cleanup - Warnf("removing salvaged pack files\n") - DeleteFiles(ctx, gopts, repo, ids, restic.PackFile) - Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n") return nil } diff --git a/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go b/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go index 720523762..fc221ebea 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go @@ -3,7 +3,6 @@ package main import ( "context" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" @@ -38,7 +37,10 @@ snapshot! EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -67,24 +69,13 @@ func init() { } func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error { - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun) if err != nil { return err } + defer unlock() - if !opts.DryRun { - var lock *restic.Lock - var err error - lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } else { - repo.SetDryRun() - } - - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -109,7 +100,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt var newSize uint64 // check all contents and remove if not available for _, id := range node.Content { - if size, found := repo.LookupBlobSize(id, restic.DataBlob); !found { + if size, found := repo.LookupBlobSize(restic.DataBlob, id); !found { ok = false } else { newContent = append(newContent, id) @@ -126,7 +117,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt node.Size = newSize return node }, - RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { + RewriteFailedTree: func(_ restic.ID, path string, _ error) (restic.ID, error) { if path == "/" { Verbosef(" dir %q: not readable\n", path) // remove snapshots with invalid root node @@ -145,11 +136,11 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\n%v\n", sn) changed, err := filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, "repaired") + }, opts.DryRun, opts.Forget, nil, "repaired") if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) } @@ -157,6 +148,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt changedCount++ } } + if ctx.Err() != nil { + return ctx.Err() + } Verbosef("\n") if changedCount == 0 { diff --git a/mover-restic/restic/cmd/restic/cmd_repair_snapshots_integration_test.go b/mover-restic/restic/cmd/restic/cmd_repair_snapshots_integration_test.go index 34cd186d3..9f65c9328 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_snapshots_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_snapshots_integration_test.go @@ -62,7 +62,7 @@ func TestRepairSnapshotsWithLostData(t *testing.T) { testRunCheckMustFail(t, env.gopts) // repository must be ok after removing the broken snapshots - testRunForget(t, env.gopts, snapshotIDs[0].String(), snapshotIDs[1].String()) + testRunForget(t, env.gopts, ForgetOptions{}, snapshotIDs[0].String(), snapshotIDs[1].String()) testListSnapshots(t, env.gopts, 2) _, err := testRunCheckOutput(env.gopts, false) rtest.OK(t, err) @@ -86,7 +86,7 @@ func TestRepairSnapshotsWithLostTree(t *testing.T) { // remove tree for foo/bar and the now completely broken first snapshot removePacks(env.gopts, t, restic.NewIDSet(oldPacks...)) - testRunForget(t, env.gopts, oldSnapshot[0].String()) + testRunForget(t, env.gopts, ForgetOptions{}, oldSnapshot[0].String()) testRunCheckMustFail(t, env.gopts) // repair diff --git a/mover-restic/restic/cmd/restic/cmd_restore.go b/mover-restic/restic/cmd/restic/cmd_restore.go index 494c6b86a..89942f4cf 100644 --- a/mover-restic/restic/cmd/restic/cmd_restore.go +++ b/mover-restic/restic/cmd/restic/cmd_restore.go @@ -2,13 +2,11 @@ package main import ( "context" - "strings" - "sync" + "path/filepath" "time" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/ui" @@ -28,54 +26,36 @@ a directory. The special snapshotID "latest" can be used to restore the latest snapshot in the repository. -To only restore a specific subfolder, you can use the ":" +To only restore a specific subfolder, you can use the "snapshotID:subfolder" syntax, where "subfolder" is a path within the snapshot. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // allow usage of warnf / verbosef - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runRestore(ctx, restoreOptions, globalOptions, term, args) + term, cancel := setupTermstatus() + defer cancel() + return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args) }, } // RestoreOptions collects all options for the restore command. type RestoreOptions struct { - Exclude []string - InsensitiveExclude []string - Include []string - InsensitiveInclude []string - Target string + excludePatternOptions + includePatternOptions + Target string restic.SnapshotFilter - Sparse bool - Verify bool + DryRun bool + Sparse bool + Verify bool + Overwrite restorer.OverwriteBehavior + Delete bool } var restoreOptions RestoreOptions @@ -84,52 +64,34 @@ func init() { cmdRoot.AddCommand(cmdRestore) flags := cmdRestore.Flags() - flags.StringArrayVarP(&restoreOptions.Exclude, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") - flags.StringArrayVar(&restoreOptions.InsensitiveExclude, "iexclude", nil, "same as --exclude but ignores the casing of `pattern`") - flags.StringArrayVarP(&restoreOptions.Include, "include", "i", nil, "include a `pattern`, exclude everything else (can be specified multiple times)") - flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as --include but ignores the casing of `pattern`") flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to") + initExcludePatternOptions(flags, &restoreOptions.excludePatternOptions) + initIncludePatternOptions(flags, &restoreOptions.includePatternOptions) + initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter) + flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done") flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse") flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content") + flags.Var(&restoreOptions.Overwrite, "overwrite", "overwrite behavior, one of (always|if-changed|if-newer|never) (default: always)") + flags.BoolVar(&restoreOptions.Delete, "delete", false, "delete files from target directory if they do not exist in snapshot. Use '--dry-run -vv' to check what would be deleted") } func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { - hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0 - hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0 - - // Validate provided patterns - if len(opts.Exclude) > 0 { - if err := filter.ValidatePatterns(opts.Exclude); err != nil { - return errors.Fatalf("--exclude: %s", err) - } - } - if len(opts.InsensitiveExclude) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveExclude); err != nil { - return errors.Fatalf("--iexclude: %s", err) - } - } - if len(opts.Include) > 0 { - if err := filter.ValidatePatterns(opts.Include); err != nil { - return errors.Fatalf("--include: %s", err) - } - } - if len(opts.InsensitiveInclude) > 0 { - if err := filter.ValidatePatterns(opts.InsensitiveInclude); err != nil { - return errors.Fatalf("--iinclude: %s", err) - } + excludePatternFns, err := opts.excludePatternOptions.CollectPatterns() + if err != nil { + return err } - for i, str := range opts.InsensitiveExclude { - opts.InsensitiveExclude[i] = strings.ToLower(str) + includePatternFns, err := opts.includePatternOptions.CollectPatterns() + if err != nil { + return err } - for i, str := range opts.InsensitiveInclude { - opts.InsensitiveInclude[i] = strings.ToLower(str) - } + hasExcludes := len(excludePatternFns) > 0 + hasIncludes := len(includePatternFns) > 0 switch { case len(args) == 0: @@ -145,30 +107,29 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, if hasExcludes && hasIncludes { return errors.Fatal("exclude and include patterns are mutually exclusive") } + if opts.DryRun && opts.Verify { + return errors.Fatal("--dry-run and --verify are mutually exclusive") + } + + if opts.Delete && filepath.Clean(opts.Target) == "/" && !hasExcludes && !hasIncludes { + return errors.Fatal("'--target / --delete' must be combined with an include or exclude filter") + } snapshotIDString := args[0] debug.Log("restore %v to %v", snapshotIDString, opts.Target) - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() sn, subfolder, err := (&restic.SnapshotFilter{ Hosts: opts.Hosts, Paths: opts.Paths, Tags: opts.Tags, - }).FindLatest(ctx, repo.Backend(), repo, snapshotIDString) + }).FindLatest(ctx, repo, repo, snapshotIDString) if err != nil { return errors.Fatalf("failed to find snapshot: %v", err) } @@ -187,13 +148,19 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, msg := ui.NewMessage(term, gopts.verbosity) var printer restoreui.ProgressPrinter if gopts.JSON { - printer = restoreui.NewJSONProgress(term) + printer = restoreui.NewJSONProgress(term, gopts.verbosity) } else { - printer = restoreui.NewTextProgress(term) + printer = restoreui.NewTextProgress(term, gopts.verbosity) } progress := restoreui.NewProgress(printer, calculateProgressInterval(!gopts.Quiet, gopts.JSON)) - res := restorer.NewRestorer(repo, sn, opts.Sparse, progress) + res := restorer.NewRestorer(repo, sn, restorer.Options{ + DryRun: opts.DryRun, + Sparse: opts.Sparse, + Progress: progress, + Overwrite: opts.Overwrite, + Delete: opts.Delete, + }) totalErrors := 0 res.Error = func(location string, err error) error { @@ -201,45 +168,45 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, totalErrors++ return nil } + res.Warn = func(message string) { + msg.E("Warning: %s\n", message) + } - excludePatterns := filter.ParsePatterns(opts.Exclude) - insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) - selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - matched, err := filter.List(excludePatterns, item) - if err != nil { - msg.E("error for exclude pattern: %v", err) - } - - matchedInsensitive, err := filter.List(insensitiveExcludePatterns, strings.ToLower(item)) - if err != nil { - msg.E("error for iexclude pattern: %v", err) + selectExcludeFilter := func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { + matched := false + for _, rejectFn := range excludePatternFns { + matched = matched || rejectFn(item) + + // implementing a short-circuit here to improve the performance + // to prevent additional pattern matching once the first pattern + // matches. + if matched { + break + } } - // An exclude filter is basically a 'wildcard but foo', // so even if a childMayMatch, other children of a dir may not, // therefore childMayMatch does not matter, but we should not go down // unless the dir is selected for restore - selectedForRestore = !matched && !matchedInsensitive - childMayBeSelected = selectedForRestore && node.Type == "dir" + selectedForRestore = !matched + childMayBeSelected = selectedForRestore && isDir return selectedForRestore, childMayBeSelected } - includePatterns := filter.ParsePatterns(opts.Include) - insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude) - selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - matched, childMayMatch, err := filter.ListWithChild(includePatterns, item) - if err != nil { - msg.E("error for include pattern: %v", err) - } - - matchedInsensitive, childMayMatchInsensitive, err := filter.ListWithChild(insensitiveIncludePatterns, strings.ToLower(item)) - if err != nil { - msg.E("error for iexclude pattern: %v", err) + selectIncludeFilter := func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { + selectedForRestore = false + childMayBeSelected = false + for _, includeFn := range includePatternFns { + matched, childMayMatch := includeFn(item) + selectedForRestore = selectedForRestore || matched + childMayBeSelected = childMayBeSelected || childMayMatch + + if selectedForRestore && childMayBeSelected { + break + } } - - selectedForRestore = matched || matchedInsensitive - childMayBeSelected = (childMayMatch || childMayMatchInsensitive) && node.Type == "dir" + childMayBeSelected = childMayBeSelected && isDir return selectedForRestore, childMayBeSelected } diff --git a/mover-restic/restic/cmd/restic/cmd_restore_integration_test.go b/mover-restic/restic/cmd/restic/cmd_restore_integration_test.go index 2c7cbe1fb..b0543850b 100644 --- a/mover-restic/restic/cmd/restic/cmd_restore_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_restore_integration_test.go @@ -4,14 +4,15 @@ import ( "context" "fmt" "io" - mrand "math/rand" + "math/rand" "os" "path/filepath" + "strings" "syscall" "testing" "time" - "github.com/restic/restic/internal/filter" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/termstatus" @@ -23,9 +24,9 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) { opts := RestoreOptions{ - Target: dir, - Exclude: excludes, + Target: dir, } + opts.Excludes = excludes rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) } @@ -50,22 +51,132 @@ func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths [ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) { opts := RestoreOptions{ - Target: dir, - Include: includes, + Target: dir, + } + opts.Includes = includes + + rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) +} + +func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) { + opts := RestoreOptions{ + Target: dir, + } + opts.IncludeFiles = []string{includesFile} + + rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) +} + +func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) { + opts := RestoreOptions{ + Target: dir, } + opts.ExcludeFiles = []string{excludesFile} rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts)) } +func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + // Add both include and exclude patterns + includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"} + excludePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"} + + restoredir := filepath.Join(env.base, "restore") + + restoreOpts := RestoreOptions{ + Target: restoredir, + } + restoreOpts.Includes = includePatterns + restoreOpts.Excludes = excludePatterns + + err := testRunRestoreAssumeFailure("latest", restoreOpts, env.gopts) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "exclude and include patterns are mutually exclusive"), + "expected: %s error, got %v", "exclude and include patterns are mutually exclusive", err) +} + +func TestRestoreIncludes(t *testing.T) { + testfiles := []struct { + path string + size uint + include bool // Whether this file should be included in the restore + }{ + {"dir1/include_me.txt", 100, true}, + {"dir1/something_else.txt", 200, false}, + {"dir2/also_include_me.txt", 150, true}, + {"dir2/important_file.txt", 150, true}, + {"dir3/not_included.txt", 180, false}, + {"dir4/subdir/should_include_me.txt", 120, true}, + } + + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + // Create test files and directories + for _, testFile := range testfiles { + fullPath := filepath.Join(env.testdata, testFile.path) + rtest.OK(t, os.MkdirAll(filepath.Dir(fullPath), 0755)) + rtest.OK(t, appendRandomData(fullPath, testFile.size)) + } + + opts := BackupOptions{} + + // Perform backup + testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts) + testRunCheck(t, env.gopts) + + snapshotID := testListSnapshots(t, env.gopts, 1)[0] + + // Restore using includes + includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"} + restoredir := filepath.Join(env.base, "restore") + testRunRestoreIncludes(t, env.gopts, restoredir, snapshotID, includePatterns) + + testRestoreFileInclusions := func(t *testing.T) { + // Check that only the included files are restored + for _, testFile := range testfiles { + restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path) + _, err := os.Stat(restoredFilePath) + if testFile.include { + rtest.OK(t, err) + } else { + rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path) + } + } + } + + testRestoreFileInclusions(t) + + // Create an include file with some patterns + patternsFile := env.base + "/patternsFile" + fileErr := os.WriteFile(patternsFile, []byte(strings.Join(includePatterns, "\n")), 0644) + if fileErr != nil { + t.Fatalf("Could not write include file: %v", fileErr) + } + + restoredir = filepath.Join(env.base, "restore-include-from-file") + + testRunRestoreIncludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile) + + testRestoreFileInclusions(t) +} + func TestRestoreFilter(t *testing.T) { testfiles := []struct { - name string - size uint + name string + size uint + exclude bool }{ - {"testfile1.c", 100}, - {"testfile2.exe", 101}, - {"subdir1/subdir2/testfile3.docx", 102}, - {"subdir1/subdir2/testfile4.c", 102}, + {"testfile1.c", 100, true}, + {"testfile2.exe", 101, true}, + {"subdir1/subdir2/testfile3.docx", 102, true}, + {"subdir1/subdir2/testfile4.c", 102, false}, } env, cleanup := withTestEnvironment(t) @@ -92,19 +203,38 @@ func TestRestoreFilter(t *testing.T) { rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size))) } - for i, pat := range []string{"*.c", "*.exe", "*", "*file3*"} { - base := filepath.Join(env.base, fmt.Sprintf("restore%d", i+1)) - testRunRestoreExcludes(t, env.gopts, base, snapshotID, []string{pat}) + excludePatterns := []string{"testfile1.c", "*.exe", "*file3*"} + + // checks if the files are excluded correctly + testRestoredFileExclusions := func(t *testing.T, restoredir string) { for _, testFile := range testfiles { - err := testFileSize(filepath.Join(base, "testdata", testFile.name), int64(testFile.size)) - if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok { - rtest.OK(t, err) + restoredFilePath := filepath.Join(restoredir, "testdata", testFile.name) + _, err := os.Stat(restoredFilePath) + if testFile.exclude { + rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.name) } else { - rtest.Assert(t, os.IsNotExist(err), - "expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err) + rtest.OK(t, testFileSize(restoredFilePath, int64(testFile.size))) } } } + + // restore with excludes + restoredir := filepath.Join(env.base, "restore-with-excludes") + testRunRestoreExcludes(t, env.gopts, restoredir, snapshotID, excludePatterns) + testRestoredFileExclusions(t, restoredir) + + // Create an exclude file with some patterns + patternsFile := env.base + "/patternsFile" + fileErr := os.WriteFile(patternsFile, []byte(strings.Join(excludePatterns, "\n")), 0644) + if fileErr != nil { + t.Fatalf("Could not write include file: %v", fileErr) + } + + // restore with excludes from file + restoredir = filepath.Join(env.base, "restore-with-exclude-from-file") + testRunRestoreExcludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile) + + testRestoredFileExclusions(t, restoredir) } func TestRestore(t *testing.T) { @@ -116,7 +246,7 @@ func TestRestore(t *testing.T) { for i := 0; i < 10; i++ { p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i)) rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755)) - rtest.OK(t, appendRandomData(p, uint(mrand.Intn(2<<21)))) + rtest.OK(t, appendRandomData(p, uint(rand.Intn(2<<21)))) } opts := BackupOptions{} @@ -274,6 +404,7 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) { } func TestRestoreLocalLayout(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/mover-restic/restic/cmd/restic/cmd_rewrite.go b/mover-restic/restic/cmd/restic/cmd_rewrite.go index 9d9ab5d5d..463720ee1 100644 --- a/mover-restic/restic/cmd/restic/cmd_rewrite.go +++ b/mover-restic/restic/cmd/restic/cmd_rewrite.go @@ -3,11 +3,11 @@ package main import ( "context" "fmt" + "time" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -38,7 +38,10 @@ use the "prune" command. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -46,11 +49,42 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } +type snapshotMetadata struct { + Hostname string + Time *time.Time +} + +type snapshotMetadataArgs struct { + Hostname string + Time string +} + +func (sma snapshotMetadataArgs) empty() bool { + return sma.Hostname == "" && sma.Time == "" +} + +func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) { + if sma.empty() { + return nil, nil + } + + var timeStamp *time.Time + if sma.Time != "" { + t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local) + if err != nil { + return nil, errors.Fatalf("error in time option: %v\n", err) + } + timeStamp = &t + } + return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil +} + // RewriteOptions collects all options for the rewrite command. type RewriteOptions struct { Forget bool DryRun bool + Metadata snapshotMetadataArgs restic.SnapshotFilter excludePatternOptions } @@ -63,11 +97,15 @@ func init() { f := cmdRewrite.Flags() f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones") f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done") + f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname") + f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup") initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true) initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions) } +type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) + func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) { if sn.Tree == nil { return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str()) @@ -78,33 +116,59 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return false, err } - selectByName := func(nodepath string) bool { - for _, reject := range rejectByNameFuncs { - if reject(nodepath) { - return false + metadata, err := opts.Metadata.convert() + + if err != nil { + return false, err + } + + var filter rewriteFilterFunc + + if len(rejectByNameFuncs) > 0 { + selectByName := func(nodepath string) bool { + for _, reject := range rejectByNameFuncs { + if reject(nodepath) { + return false + } } + return true } - return true - } - rewriter := walker.NewTreeRewriter(walker.RewriteOpts{ - RewriteNode: func(node *restic.Node, path string) *restic.Node { + rewriteNode := func(node *restic.Node, path string) *restic.Node { if selectByName(path) { return node } Verbosef(fmt.Sprintf("excluding %s\n", path)) return nil - }, - DisableNodeCache: true, - }) + } + + rewriter, querySize := walker.NewSnapshotSizeRewriter(rewriteNode) + + filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + id, err := rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) + if err != nil { + return restic.ID{}, err + } + ss := querySize() + if sn.Summary != nil { + sn.Summary.TotalFilesProcessed = ss.FileCount + sn.Summary.TotalBytesProcessed = ss.FileSize + } + return id, err + } + + } else { + filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) { + return *sn.Tree, nil + } + } return filterAndReplaceSnapshot(ctx, repo, sn, - func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { - return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) - }, opts.DryRun, opts.Forget, "rewrite") + filter, opts.DryRun, opts.Forget, metadata, "rewrite") } -func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) { +func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, + filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) @@ -128,8 +192,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r if dryRun { Verbosef("would delete empty snapshot\n") } else { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} - if err = repo.Backend().Remove(ctx, h); err != nil { + if err = repo.RemoveUnpacked(ctx, restic.SnapshotFile, *sn.ID()); err != nil { return false, err } debug.Log("removed empty snapshot %v", sn.ID()) @@ -138,7 +201,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r return true, nil } - if filteredTree == *sn.Tree { + if filteredTree == *sn.Tree && newMetadata == nil { debug.Log("Snapshot %v not modified", sn) return false, nil } @@ -151,6 +214,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("would remove old snapshot\n") } + if newMetadata != nil && newMetadata.Time != nil { + Verbosef("would set time to %s\n", newMetadata.Time) + } + + if newMetadata != nil && newMetadata.Hostname != "" { + Verbosef("would set hostname to %s\n", newMetadata.Hostname) + } + return true, nil } @@ -162,6 +233,16 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r sn.AddTags([]string{addTag}) } + if newMetadata != nil && newMetadata.Time != nil { + Verbosef("setting time to %s\n", *newMetadata.Time) + sn.Time = *newMetadata.Time + } + + if newMetadata != nil && newMetadata.Hostname != "" { + Verbosef("setting host to %s\n", newMetadata.Hostname) + sn.Hostname = newMetadata.Hostname + } + // Save the new snapshot. id, err := restic.SaveSnapshot(ctx, repo, sn) if err != nil { @@ -170,8 +251,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r Verbosef("saved new snapshot %v\n", id.Str()) if forget { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} - if err = repo.Backend().Remove(ctx, h); err != nil { + if err = repo.RemoveUnpacked(ctx, restic.SnapshotFile, *sn.ID()); err != nil { return false, err } debug.Log("removed old snapshot %v", sn.ID()) @@ -181,33 +261,28 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error { - if opts.excludePatternOptions.Empty() { - return errors.Fatal("Nothing to do: no excludes provided") + if opts.excludePatternOptions.Empty() && opts.Metadata.empty() { + return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided") } - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } + var ( + repo *repository.Repository + unlock func() + err error + ) - if !opts.DryRun { - var lock *restic.Lock - var err error - if opts.Forget { - Verbosef("create exclusive lock for repository\n") - lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - } else { - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - } - defer unlockRepo(lock) - if err != nil { - return err - } + if opts.Forget { + Verbosef("create exclusive lock for repository\n") + ctx, repo, unlock, err = openWithExclusiveLock(ctx, gopts, opts.DryRun) } else { - repo.SetDryRun() + ctx, repo, unlock, err = openWithAppendLock(ctx, gopts, opts.DryRun) + } + if err != nil { + return err } + defer unlock() - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -219,7 +294,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\n%v\n", sn) changed, err := rewriteSnapshot(ctx, repo, sn, opts) if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) @@ -228,6 +303,9 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a changedCount++ } } + if ctx.Err() != nil { + return ctx.Err() + } Verbosef("\n") if changedCount == 0 { diff --git a/mover-restic/restic/cmd/restic/cmd_rewrite_integration_test.go b/mover-restic/restic/cmd/restic/cmd_rewrite_integration_test.go index e6007973b..781266184 100644 --- a/mover-restic/restic/cmd/restic/cmd_rewrite_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_rewrite_integration_test.go @@ -7,14 +7,16 @@ import ( "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" ) -func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) { +func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) { opts := RewriteOptions{ excludePatternOptions: excludePatternOptions{ Excludes: excludes, }, - Forget: forget, + Forget: forget, + Metadata: metadata, } rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil)) @@ -32,13 +34,31 @@ func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID { return snapshotIDs[0] } +func getSnapshot(t testing.TB, snapshotID restic.ID, env *testEnvironment) *restic.Snapshot { + t.Helper() + + ctx, repo, unlock, err := openWithReadLock(context.TODO(), env.gopts, false) + rtest.OK(t, err) + defer unlock() + + snapshots, err := restic.TestLoadAllSnapshots(ctx, repo, nil) + rtest.OK(t, err) + + for _, s := range snapshots { + if *s.ID() == snapshotID { + return s + } + } + return nil +} + func TestRewrite(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() createBasicRewriteRepo(t, env) // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, false) + testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""}) snapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs) testRunCheck(t, env.gopts) @@ -50,7 +70,7 @@ func TestRewriteUnchanged(t *testing.T) { snapshotID := createBasicRewriteRepo(t, env) // use an exclude that will not exclude anything - testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false) + testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""}) newSnapshotIDs := testRunList(t, "snapshots", env.gopts) rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly") @@ -62,12 +82,59 @@ func TestRewriteReplace(t *testing.T) { defer cleanup() snapshotID := createBasicRewriteRepo(t, env) + snapshot := getSnapshot(t, snapshotID, env) + // exclude some data - testRunRewriteExclude(t, env.gopts, []string{"3"}, true) - newSnapshotIDs := testRunList(t, "snapshots", env.gopts) - rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs) + testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""}) + bytesExcluded, err := ui.ParseBytes("16K") + rtest.OK(t, err) + + newSnapshotIDs := testListSnapshots(t, env.gopts, 1) rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed") + + newSnapshot := getSnapshot(t, newSnapshotIDs[0], env) + + rtest.Equals(t, snapshot.Summary.TotalFilesProcessed-1, newSnapshot.Summary.TotalFilesProcessed, "snapshot file count should have changed") + rtest.Equals(t, snapshot.Summary.TotalBytesProcessed-uint64(bytesExcluded), newSnapshot.Summary.TotalBytesProcessed, "snapshot size should have changed") + // check forbids unused blobs, thus remove them first testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"}) testRunCheck(t, env.gopts) } + +func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + createBasicRewriteRepo(t, env) + testRunRewriteExclude(t, env.gopts, []string{}, true, metadata) + + ctx, repo, unlock, err := openWithReadLock(context.TODO(), env.gopts, false) + rtest.OK(t, err) + defer unlock() + + snapshots, err := restic.TestLoadAllSnapshots(ctx, repo, nil) + rtest.OK(t, err) + rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots)) + newSnapshot := snapshots[0] + + if metadata.Time != "" { + rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time) + } + + if metadata.Hostname != "" { + rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname) + } +} + +func TestRewriteMetadata(t *testing.T) { + newHost := "new host" + newTime := "1999-01-01 11:11:11" + + for _, metadata := range []snapshotMetadataArgs{ + {Hostname: "", Time: newTime}, + {Hostname: newHost, Time: ""}, + {Hostname: newHost, Time: newTime}, + } { + testRewriteMetadata(t, metadata) + } +} diff --git a/mover-restic/restic/cmd/restic/cmd_self_update.go b/mover-restic/restic/cmd/restic/cmd_self_update.go index 4b86c416f..0fce41241 100644 --- a/mover-restic/restic/cmd/restic/cmd_self_update.go +++ b/mover-restic/restic/cmd/restic/cmd_self_update.go @@ -24,7 +24,10 @@ files. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_snapshots.go b/mover-restic/restic/cmd/restic/cmd_snapshots.go index 889ac5e20..9112e1b95 100644 --- a/mover-restic/restic/cmd/restic/cmd_snapshots.go +++ b/mover-restic/restic/cmd/restic/cmd_snapshots.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/table" "github.com/spf13/cobra" ) @@ -22,7 +23,10 @@ The "snapshots" command lists all snapshots stored in the repository. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -58,24 +62,19 @@ func init() { } func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error { - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } - - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } + defer unlock() var snapshots restic.Snapshots - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { snapshots = append(snapshots, sn) } + if ctx.Err() != nil { + return ctx.Err() + } snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) if err != nil { return err @@ -85,9 +84,9 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions if opts.Last { // This branch should be removed in the same time // that --last. - list = FilterLastestSnapshots(list, 1) + list = FilterLatestSnapshots(list, 1) } else if opts.Latest > 0 { - list = FilterLastestSnapshots(list, opts.Latest) + list = FilterLatestSnapshots(list, opts.Latest) } sort.Sort(sort.Reverse(list)) snapshotGroups[k] = list @@ -130,11 +129,11 @@ func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey { return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")} } -// FilterLastestSnapshots filters a list of snapshots to only return +// FilterLatestSnapshots filters a list of snapshots to only return // the limit last entries for each hostname and path. If the snapshot // contains multiple paths, they will be joined and treated as one // item. -func FilterLastestSnapshots(list restic.Snapshots, limit int) restic.Snapshots { +func FilterLatestSnapshots(list restic.Snapshots, limit int) restic.Snapshots { // Sort the snapshots so that the newer ones are listed first sort.SliceStable(list, func(i, j int) bool { return list[i].Time.After(list[j].Time) @@ -163,6 +162,11 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke keepReasons[*id] = reasons[i] } } + // check if any snapshot contains a summary + hasSize := false + for _, sn := range list { + hasSize = hasSize || (sn.Summary != nil) + } // always sort the snapshots so that the newer ones are listed last sort.SliceStable(list, func(i, j int) bool { @@ -189,6 +193,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke tab.AddColumn("Time", "{{ .Timestamp }}") tab.AddColumn("Host", "{{ .Hostname }}") tab.AddColumn("Tags ", `{{ join .Tags "\n" }}`) + if hasSize { + tab.AddColumn("Size", `{{ .Size }}`) + } } else { tab.AddColumn("ID", "{{ .ID }}") tab.AddColumn("Time", "{{ .Timestamp }}") @@ -198,6 +205,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`) } tab.AddColumn("Paths", `{{ join .Paths "\n" }}`) + if hasSize { + tab.AddColumn("Size", `{{ .Size }}`) + } } type snapshot struct { @@ -207,6 +217,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke Tags []string Reasons []string Paths []string + Size string } var multiline bool @@ -228,6 +239,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke multiline = true } + if sn.Summary != nil { + data.Size = ui.FormatBytes(sn.Summary.TotalBytesProcessed) + } + tab.AddRow(data) } @@ -290,7 +305,7 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { return nil } -// Snapshot helps to print Snaphots as JSON with their ID included. +// Snapshot helps to print Snapshots as JSON with their ID included. type Snapshot struct { *restic.Snapshot @@ -298,7 +313,7 @@ type Snapshot struct { ShortID string `json:"short_id"` } -// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included. +// SnapshotGroup helps to print SnapshotGroups as JSON with their GroupReasons included. type SnapshotGroup struct { GroupKey restic.SnapshotGroupKey `json:"group_key"` Snapshots []Snapshot `json:"snapshots"` diff --git a/mover-restic/restic/cmd/restic/cmd_stats.go b/mover-restic/restic/cmd/restic/cmd_stats.go index 30052c912..ab333e6ef 100644 --- a/mover-restic/restic/cmd/restic/cmd_stats.go +++ b/mover-restic/restic/cmd/restic/cmd_stats.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/restic/chunker" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/restorer" "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/table" "github.com/restic/restic/internal/walker" @@ -38,7 +38,7 @@ depending on what you are trying to calculate. The modes are: * restore-size: (default) Counts the size of the restored files. -* files-by-contents: Counts total size of files, where a file is +* files-by-contents: Counts total size of unique files, where a file is considered unique if it has unique contents. * raw-data: Counts the size of blobs in the repository, regardless of how many files reference them. @@ -49,7 +49,10 @@ Refer to the online manual for more details about each mode. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -80,21 +83,13 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return err } - repo, err := OpenRepository(ctx, gopts) + ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock) if err != nil { return err } + defer unlock() - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile) + snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile) if err != nil { return err } @@ -125,15 +120,14 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return fmt.Errorf("error walking snapshot: %v", err) } } - - if err != nil { - return err + if ctx.Err() != nil { + return ctx.Err() } if opts.countMode == countModeRawData { // the blob handles have been collected, but not yet counted for blobHandle := range stats.blobs { - pbs := repo.Index().Lookup(blobHandle) + pbs := repo.LookupBlob(blobHandle.Type, blobHandle.ID) if len(pbs) == 0 { return fmt.Errorf("blob %v not found", blobHandle) } @@ -189,7 +183,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return nil } -func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, opts StatsOptions, stats *statsContainer) error { +func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error { if snapshot.Tree == nil { return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) } @@ -202,8 +196,10 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return restic.FindUsedBlobs(ctx, repo, restic.IDs{*snapshot.Tree}, stats.blobs, nil) } - uniqueInodes := make(map[uint64]struct{}) - err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, opts, stats, uniqueInodes)) + hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() + err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{ + ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex), + }) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } @@ -211,13 +207,13 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return nil } -func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, uniqueInodes map[uint64]struct{}) walker.WalkFunc { - return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) { +func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { + return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error { if nodeErr != nil { - return true, nodeErr + return nodeErr } if node == nil { - return true, nil + return nil } if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile { @@ -245,9 +241,9 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai } if _, ok := stats.fileBlobs[nodePath][blobID]; !ok { // is always a data blob since we're accessing it via a file's Content array - blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob) + blobSize, found := repo.LookupBlobSize(restic.DataBlob, blobID) if !found { - return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID) + return fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID) } // count the blob's size, then add this blob by this @@ -268,17 +264,18 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai // will still be restored stats.TotalFileCount++ - // if inodes are present, only count each inode once - // (hard links do not increase restore size) - if _, ok := uniqueInodes[node.Inode]; !ok || node.Inode == 0 { - uniqueInodes[node.Inode] = struct{}{} + if node.Links == 1 || node.Type == "dir" { stats.TotalSize += node.Size + } else { + // if hardlinks are present only count each deviceID+inode once + if !hardLinkIndex.Has(node.Inode, node.DeviceID) || node.Inode == 0 { + hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{}) + stats.TotalSize += node.Size + } } - - return false, nil } - return true, nil + return nil } } @@ -357,7 +354,10 @@ func statsDebug(ctx context.Context, repo restic.Repository) error { Warnf("File Type: %v\n%v\n", t, hist) } - hist := statsDebugBlobs(ctx, repo) + hist, err := statsDebugBlobs(ctx, repo) + if err != nil { + return err + } for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} { Warnf("Blob Type: %v\n%v\n\n", t, hist[t]) } @@ -365,9 +365,9 @@ func statsDebug(ctx context.Context, repo restic.Repository) error { return nil } -func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic.FileType) (*sizeHistogram, error) { +func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.FileType) (*sizeHistogram, error) { hist := newSizeHistogram(2 * repository.MaxPackSize) - err := repo.List(ctx, tpe, func(id restic.ID, size int64) error { + err := repo.List(ctx, tpe, func(_ restic.ID, size int64) error { hist.Add(uint64(size)) return nil }) @@ -375,17 +375,17 @@ func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic. return hist, err } -func statsDebugBlobs(ctx context.Context, repo restic.Repository) [restic.NumBlobTypes]*sizeHistogram { +func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBlobTypes]*sizeHistogram, error) { var hist [restic.NumBlobTypes]*sizeHistogram for i := 0; i < len(hist); i++ { hist[i] = newSizeHistogram(2 * chunker.MaxSize) } - repo.Index().Each(ctx, func(pb restic.PackedBlob) { + err := repo.ListBlobs(ctx, func(pb restic.PackedBlob) { hist[pb.Type].Add(uint64(pb.Length)) }) - return hist + return hist, err } type sizeClass struct { diff --git a/mover-restic/restic/cmd/restic/cmd_tag.go b/mover-restic/restic/cmd/restic/cmd_tag.go index 1ad465faa..ea73955f0 100644 --- a/mover-restic/restic/cmd/restic/cmd_tag.go +++ b/mover-restic/restic/cmd/restic/cmd_tag.go @@ -25,7 +25,10 @@ When no snapshotID is given, all snapshots matching the host, tag and path filte EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -85,8 +88,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna debug.Log("new snapshot saved as %v", id) // Remove the old snapshot. - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} - if err = repo.Backend().Remove(ctx, h); err != nil { + if err = repo.RemoveUnpacked(ctx, restic.SnapshotFile, *sn.ID()); err != nil { return false, err } @@ -103,23 +105,15 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st return errors.Fatal("--set and --add/--remove cannot be given at the same time") } - repo, err := OpenRepository(ctx, gopts) + Verbosef("create exclusive lock for repository\n") + ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false) if err != nil { - return err - } - - if !gopts.NoLock { - Verbosef("create exclusive lock for repository\n") - var lock *restic.Lock - lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } + return nil } + defer unlock() changeCnt := 0 - for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) { + for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) { changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten()) if err != nil { Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err) @@ -129,6 +123,9 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st changeCnt++ } } + if ctx.Err() != nil { + return ctx.Err() + } if changeCnt == 0 { Verbosef("no snapshots were modified\n") } else { diff --git a/mover-restic/restic/cmd/restic/cmd_tag_integration_test.go b/mover-restic/restic/cmd/restic/cmd_tag_integration_test.go index 3b902c51e..6979f9c11 100644 --- a/mover-restic/restic/cmd/restic/cmd_tag_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_tag_integration_test.go @@ -12,6 +12,7 @@ func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) { rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{})) } +// nolint: staticcheck // false positive nil pointer dereference check func TestTag(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() diff --git a/mover-restic/restic/cmd/restic/cmd_unlock.go b/mover-restic/restic/cmd/restic/cmd_unlock.go index 7b449d949..96eef7e02 100644 --- a/mover-restic/restic/cmd/restic/cmd_unlock.go +++ b/mover-restic/restic/cmd/restic/cmd_unlock.go @@ -16,10 +16,11 @@ The "unlock" command removes stale locks that have been created by other restic EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runUnlock(cmd.Context(), unlockOptions, globalOptions) }, } diff --git a/mover-restic/restic/cmd/restic/cmd_version.go b/mover-restic/restic/cmd/restic/cmd_version.go index b17103706..cd32e2470 100644 --- a/mover-restic/restic/cmd/restic/cmd_version.go +++ b/mover-restic/restic/cmd/restic/cmd_version.go @@ -1,6 +1,7 @@ package main import ( + "encoding/json" "fmt" "runtime" @@ -17,12 +18,36 @@ and the version of this software. EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. `, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("restic %s compiled with %v on %v/%v\n", - version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + Run: func(_ *cobra.Command, _ []string) { + if globalOptions.JSON { + type jsonVersion struct { + Version string `json:"version"` + GoVersion string `json:"go_version"` + GoOS string `json:"go_os"` + GoArch string `json:"go_arch"` + } + + jsonS := jsonVersion{ + Version: version, + GoVersion: runtime.Version(), + GoOS: runtime.GOOS, + GoArch: runtime.GOARCH, + } + + err := json.NewEncoder(globalOptions.stdout).Encode(jsonS) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + return + } + } else { + fmt.Printf("restic %s compiled with %v on %v/%v\n", + version, runtime.Version(), runtime.GOOS, runtime.GOARCH) + } + }, } diff --git a/mover-restic/restic/cmd/restic/delete.go b/mover-restic/restic/cmd/restic/delete.go deleted file mode 100644 index 2046ccfde..000000000 --- a/mover-restic/restic/cmd/restic/delete.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "context" - - "golang.org/x/sync/errgroup" - - "github.com/restic/restic/internal/restic" -) - -// DeleteFiles deletes the given fileList of fileType in parallel -// it will print a warning if there is an error, but continue deleting the remaining files -func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) { - _ = deleteFiles(ctx, gopts, true, repo, fileList, fileType) -} - -// DeleteFilesChecked deletes the given fileList of fileType in parallel -// if an error occurs, it will cancel and return this error -func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { - return deleteFiles(ctx, gopts, false, repo, fileList, fileType) -} - -// deleteFiles deletes the given fileList of fileType in parallel -// if ignoreError=true, it will print a warning if there was an error, else it will abort. -func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { - totalCount := len(fileList) - fileChan := make(chan restic.ID) - wg, ctx := errgroup.WithContext(ctx) - wg.Go(func() error { - defer close(fileChan) - for id := range fileList { - select { - case fileChan <- id: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - }) - - bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted") - defer bar.Done() - // deleting files is IO-bound - workerCount := repo.Connections() - for i := 0; i < int(workerCount); i++ { - wg.Go(func() error { - for id := range fileChan { - h := restic.Handle{Type: fileType, Name: id.String()} - err := repo.Backend().Remove(ctx, h) - if err != nil { - if !gopts.JSON { - Warnf("unable to remove %v from the repository\n", h) - } - if !ignoreError { - return err - } - } - if !gopts.JSON && gopts.verbosity > 2 { - Verbosef("removed %v\n", h) - } - bar.Add(1) - } - return nil - }) - } - err := wg.Wait() - return err -} diff --git a/mover-restic/restic/cmd/restic/exclude.go b/mover-restic/restic/cmd/restic/exclude.go index 095944610..4657e4915 100644 --- a/mover-restic/restic/cmd/restic/exclude.go +++ b/mover-restic/restic/cmd/restic/exclude.go @@ -385,12 +385,12 @@ func rejectBySize(maxSizeStr string) (RejectFunc, error) { }, nil } -// readExcludePatternsFromFiles reads all exclude files and returns the list of -// exclude patterns. For each line, leading and trailing white space is removed +// readPatternsFromFiles reads all files and returns the list of +// patterns. For each line, leading and trailing white space is removed // and comment lines are ignored. For each remaining pattern, environment // variables are resolved. For adding a literal dollar sign ($), write $$ to // the file. -func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { +func readPatternsFromFiles(files []string) ([]string, error) { getenvOrDollar := func(s string) string { if s == "$" { return "$" @@ -398,8 +398,8 @@ func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { return os.Getenv(s) } - var excludes []string - for _, filename := range excludeFiles { + var patterns []string + for _, filename := range files { err := func() (err error) { data, err := textfile.Read(filename) if err != nil { @@ -421,15 +421,15 @@ func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { } line = os.Expand(line, getenvOrDollar) - excludes = append(excludes, line) + patterns = append(patterns, line) } return scanner.Err() }() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err) } } - return excludes, nil + return patterns, nil } type excludePatternOptions struct { @@ -454,7 +454,7 @@ func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) var fs []RejectByNameFunc // add patterns from file if len(opts.ExcludeFiles) > 0 { - excludePatterns, err := readExcludePatternsFromFiles(opts.ExcludeFiles) + excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles) if err != nil { return nil, err } @@ -467,7 +467,7 @@ func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) } if len(opts.InsensitiveExcludeFiles) > 0 { - excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles) + excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles) if err != nil { return nil, err } diff --git a/mover-restic/restic/cmd/restic/find.go b/mover-restic/restic/cmd/restic/find.go index 54d3563b1..faf7024e1 100644 --- a/mover-restic/restic/cmd/restic/find.go +++ b/mover-restic/restic/cmd/restic/find.go @@ -2,8 +2,8 @@ package main import ( "context" + "os" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" "github.com/spf13/pflag" ) @@ -15,17 +15,27 @@ func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter, if !addHostShorthand { hostShorthand = "" } - flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)") + flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times) (default: $RESTIC_HOST)") flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)") - flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)") + flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times, snapshots must include all specified paths)") + + // set default based on env if set + if host := os.Getenv("RESTIC_HOST"); host != "" { + filt.Hosts = []string{host} + } } // initSingleSnapshotFilter is used for commands that work on a single snapshot // MUST be combined with restic.FindFilteredSnapshot func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) { - flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)") + flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times) (default: $RESTIC_HOST)") flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)") - flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)") + flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times, snapshots must include all specified paths)") + + // set default based on env if set + if host := os.Getenv("RESTIC_HOST"); host != "" { + filt.Hosts = []string{host} + } } // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots. @@ -33,7 +43,7 @@ func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic. out := make(chan *restic.Snapshot) go func() { defer close(out) - be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile) + be, err := restic.MemorizeList(ctx, be, restic.SnapshotFile) if err != nil { Warnf("could not load snapshots: %v\n", err) return diff --git a/mover-restic/restic/cmd/restic/find_test.go b/mover-restic/restic/cmd/restic/find_test.go new file mode 100644 index 000000000..a98a14f04 --- /dev/null +++ b/mover-restic/restic/cmd/restic/find_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "testing" + + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" + "github.com/spf13/pflag" +) + +func TestSnapshotFilter(t *testing.T) { + for _, test := range []struct { + name string + args []string + expected []string + env string + }{ + { + "no value", + []string{}, + nil, + "", + }, + { + "args only", + []string{"--host", "abc"}, + []string{"abc"}, + "", + }, + { + "env default", + []string{}, + []string{"def"}, + "def", + }, + { + "both", + []string{"--host", "abc"}, + []string{"abc"}, + "def", + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Setenv("RESTIC_HOST", test.env) + + for _, mode := range []bool{false, true} { + set := pflag.NewFlagSet("test", pflag.PanicOnError) + flt := &restic.SnapshotFilter{} + if mode { + initMultiSnapshotFilter(set, flt, false) + } else { + initSingleSnapshotFilter(set, flt) + } + err := set.Parse(test.args) + rtest.OK(t, err) + + rtest.Equals(t, test.expected, flt.Hosts, "unexpected hosts") + } + }) + } +} diff --git a/mover-restic/restic/cmd/restic/global.go b/mover-restic/restic/cmd/restic/global.go index 9e2a8b261..080863da7 100644 --- a/mover-restic/restic/cmd/restic/global.go +++ b/mover-restic/restic/cmd/restic/global.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/azure" "github.com/restic/restic/internal/backend/b2" + "github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/backend/gs" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/local" @@ -27,7 +28,6 @@ import ( "github.com/restic/restic/internal/backend/sema" "github.com/restic/restic/internal/backend/sftp" "github.com/restic/restic/internal/backend/swift" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/options" @@ -43,31 +43,36 @@ import ( "golang.org/x/term" ) -var version = "0.16.5" +// ErrNoRepository is used to report if opening a repsitory failed due +// to a missing backend storage location or config file +var ErrNoRepository = errors.New("repository does not exist") + +var version = "0.17.0" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" -type backendWrapper func(r restic.Backend) (restic.Backend, error) +type backendWrapper func(r backend.Backend) (backend.Backend, error) // GlobalOptions hold all global options for restic. type GlobalOptions struct { - Repo string - RepositoryFile string - PasswordFile string - PasswordCommand string - KeyHint string - Quiet bool - Verbose int - NoLock bool - RetryLock time.Duration - JSON bool - CacheDir string - NoCache bool - CleanupCache bool - Compression repository.CompressionMode - PackSize uint - NoExtraVerify bool + Repo string + RepositoryFile string + PasswordFile string + PasswordCommand string + KeyHint string + Quiet bool + Verbose int + NoLock bool + RetryLock time.Duration + JSON bool + CacheDir string + NoCache bool + CleanupCache bool + Compression repository.CompressionMode + PackSize uint + NoExtraVerify bool + InsecureNoPassword bool backend.TransportOptions limiter.Limits @@ -96,9 +101,6 @@ var globalOptions = GlobalOptions{ stderr: os.Stderr, } -var isReadingPassword bool -var internalGlobalCtx context.Context - func init() { backends := location.NewRegistry() backends.Register(azure.NewFactory()) @@ -112,15 +114,6 @@ func init() { backends.Register(swift.NewFactory()) globalOptions.backends = backends - var cancel context.CancelFunc - internalGlobalCtx, cancel = context.WithCancel(context.Background()) - AddCleanupHandler(func(code int) (int, error) { - // Must be called before the unlock cleanup handler to ensure that the latter is - // not blocked due to limited number of backend connections, see #1434 - cancel() - return code, nil - }) - f := cmdRoot.PersistentFlags() f.StringVarP(&globalOptions.Repo, "repo", "r", "", "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)") f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", "", "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)") @@ -128,7 +121,7 @@ func init() { f.StringVarP(&globalOptions.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)") f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)") f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report") - // use empty paremeter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing + // use empty parameter name as `-v, --verbose n` instead of the correct `--verbose=n` is confusing f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2)") f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories") f.DurationVar(&globalOptions.RetryLock, "retry-lock", 0, "retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries)") @@ -137,6 +130,7 @@ func init() { f.BoolVar(&globalOptions.NoCache, "no-cache", false, "do not use a local cache") f.StringSliceVar(&globalOptions.RootCertFilenames, "cacert", nil, "`file` to load root certificates from (default: use system certificates or $RESTIC_CACERT)") f.StringVar(&globalOptions.TLSClientCertKeyFilename, "tls-client-cert", "", "path to a `file` containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT)") + f.BoolVar(&globalOptions.InsecureNoPassword, "insecure-no-password", false, "use an empty password for the repository, must be passed to every restic command (insecure)") f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") @@ -145,6 +139,7 @@ func init() { f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") + f.StringVar(&globalOptions.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests") // Use our "generate" command instead of the cobra provided "completion" command cmdRoot.CompletionOptions.DisableDefaultCmd = true @@ -166,7 +161,9 @@ func init() { targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32) globalOptions.PackSize = uint(targetPackSize) - restoreTerminal() + if os.Getenv("RESTIC_HTTP_USER_AGENT") != "" { + globalOptions.HTTPUserAgent = os.Getenv("RESTIC_HTTP_USER_AGENT") + } } func stdinIsTerminal() bool { @@ -191,40 +188,6 @@ func stdoutTerminalWidth() int { return w } -// restoreTerminal installs a cleanup handler that restores the previous -// terminal state on exit. This handler is only intended to restore the -// terminal configuration if restic exits after receiving a signal. A regular -// program execution must revert changes to the terminal configuration itself. -// The terminal configuration is only restored while reading a password. -func restoreTerminal() { - if !term.IsTerminal(int(os.Stdout.Fd())) { - return - } - - fd := int(os.Stdout.Fd()) - state, err := term.GetState(fd) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err) - return - } - - AddCleanupHandler(func(code int) (int, error) { - // Restoring the terminal configuration while restic runs in the - // background, causes restic to get stopped on unix systems with - // a SIGTTOU signal. Thus only restore the terminal settings if - // they might have been modified, which is the case while reading - // a password. - if !isReadingPassword { - return code, nil - } - err := term.Restore(fd, state) - if err != nil { - fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err) - } - return code, err - }) -} - // ClearLine creates a platform dependent string to clear the current // line, so it can be overwritten. // @@ -309,11 +272,7 @@ func resolvePassword(opts GlobalOptions, envStr string) (string, error) { return (strings.TrimSpace(string(output))), nil } if opts.PasswordFile != "" { - s, err := textfile.Read(opts.PasswordFile) - if errors.Is(err, os.ErrNotExist) { - return "", errors.Fatalf("%s does not exist", opts.PasswordFile) - } - return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") + return loadPasswordFromFile(opts.PasswordFile) } if pwd := os.Getenv(envStr); pwd != "" { @@ -323,34 +282,75 @@ func resolvePassword(opts GlobalOptions, envStr string) (string, error) { return "", nil } +// loadPasswordFromFile loads a password from a file while stripping a BOM and +// converting the password to UTF-8. +func loadPasswordFromFile(pwdFile string) (string, error) { + s, err := textfile.Read(pwdFile) + if errors.Is(err, os.ErrNotExist) { + return "", errors.Fatalf("%s does not exist", pwdFile) + } + return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") +} + // readPassword reads the password from the given reader directly. func readPassword(in io.Reader) (password string, err error) { sc := bufio.NewScanner(in) sc.Scan() - return sc.Text(), errors.Wrap(err, "Scan") + return sc.Text(), errors.WithStack(sc.Err()) } // readPasswordTerminal reads the password from the given reader which must be a // tty. Prompt is printed on the writer out before attempting to read the -// password. -func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) { - fmt.Fprint(out, prompt) - isReadingPassword = true - buf, err := term.ReadPassword(int(in.Fd())) - isReadingPassword = false - fmt.Fprintln(out) +// password. If the context is canceled, the function leaks the password reading +// goroutine. +func readPasswordTerminal(ctx context.Context, in *os.File, out *os.File, prompt string) (password string, err error) { + fd := int(out.Fd()) + state, err := term.GetState(fd) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err) + return "", err + } + + done := make(chan struct{}) + var buf []byte + + go func() { + defer close(done) + fmt.Fprint(out, prompt) + buf, err = term.ReadPassword(int(in.Fd())) + fmt.Fprintln(out) + }() + + select { + case <-ctx.Done(): + err := term.Restore(fd, state) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err) + } + return "", ctx.Err() + case <-done: + // clean shutdown, nothing to do + } + if err != nil { return "", errors.Wrap(err, "ReadPassword") } - password = string(buf) - return password, nil + return string(buf), nil } // ReadPassword reads the password from a password file, the environment -// variable RESTIC_PASSWORD or prompts the user. -func ReadPassword(opts GlobalOptions, prompt string) (string, error) { +// variable RESTIC_PASSWORD or prompts the user. If the context is canceled, +// the function leaks the password reading goroutine. +func ReadPassword(ctx context.Context, opts GlobalOptions, prompt string) (string, error) { + if opts.InsecureNoPassword { + if opts.password != "" { + return "", errors.Fatal("--insecure-no-password must not be specified together with providing a password via a cli option or environment variable") + } + return "", nil + } + if opts.password != "" { return opts.password, nil } @@ -361,7 +361,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) { ) if stdinIsTerminal() { - password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt) + password, err = readPasswordTerminal(ctx, os.Stdin, os.Stderr, prompt) } else { password, err = readPassword(os.Stdin) Verbosef("reading repository password from stdin\n") @@ -372,21 +372,22 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) { } if len(password) == 0 { - return "", errors.Fatal("an empty password is not a password") + return "", errors.Fatal("an empty password is not allowed by default. Pass the flag `--insecure-no-password` to restic to disable this check") } return password, nil } // ReadPasswordTwice calls ReadPassword two times and returns an error when the -// passwords don't match. -func ReadPasswordTwice(gopts GlobalOptions, prompt1, prompt2 string) (string, error) { - pw1, err := ReadPassword(gopts, prompt1) +// passwords don't match. If the context is canceled, the function leaks the +// password reading goroutine. +func ReadPasswordTwice(ctx context.Context, gopts GlobalOptions, prompt1, prompt2 string) (string, error) { + pw1, err := ReadPassword(ctx, gopts, prompt1) if err != nil { return "", err } if stdinIsTerminal() { - pw2, err := ReadPassword(gopts, prompt2) + pw2, err := ReadPassword(ctx, gopts, prompt2) if err != nil { return "", err } @@ -439,12 +440,16 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } report := func(msg string, err error, d time.Duration) { - Warnf("%v returned error, retrying after %v: %v\n", msg, d, err) + if d >= 0 { + Warnf("%v returned error, retrying after %v: %v\n", msg, d, err) + } else { + Warnf("%v failed: %v\n", msg, err) + } } success := func(msg string, retries int) { Warnf("%v operation successful after %d retries\n", msg, retries) } - be = retry.New(be, 10, report, success) + be = retry.New(be, 15*time.Minute, report, success) // wrap backend if a test specified a hook if opts.backendTestHook != nil { @@ -464,12 +469,15 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } passwordTriesLeft := 1 - if stdinIsTerminal() && opts.password == "" { + if stdinIsTerminal() && opts.password == "" && !opts.InsecureNoPassword { passwordTriesLeft = 3 } for ; passwordTriesLeft > 0; passwordTriesLeft-- { - opts.password, err = ReadPassword(opts, "enter password for repository: ") + opts.password, err = ReadPassword(ctx, opts, "enter password for repository: ") + if ctx.Err() != nil { + return nil, ctx.Err() + } if err != nil && passwordTriesLeft > 1 { opts.password = "" fmt.Printf("%s. Try again\n", err) @@ -556,7 +564,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi func parseConfig(loc location.Location, opts options.Options) (interface{}, error) { cfg := loc.Config - if cfg, ok := cfg.(restic.ApplyEnvironmenter); ok { + if cfg, ok := cfg.(backend.ApplyEnvironmenter); ok { cfg.ApplyEnvironment("") } @@ -570,16 +578,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro return cfg, nil } -// Open the backend specified by a location config. -func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { +func innerOpen(ctx context.Context, s string, gopts GlobalOptions, opts options.Options, create bool) (backend.Backend, error) { debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) loc, err := location.Parse(gopts.backends, s) if err != nil { return nil, errors.Fatalf("parsing repository location failed: %v", err) } - var be restic.Backend - cfg, err := parseConfig(loc, opts) if err != nil { return nil, err @@ -599,7 +604,16 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) } - be, err = factory.Open(ctx, cfg, rt, lim) + var be backend.Backend + if create { + be, err = factory.Create(ctx, cfg, rt, lim) + } else { + be, err = factory.Open(ctx, cfg, rt, lim) + } + + if errors.Is(err, backend.ErrNoRepository) { + return nil, fmt.Errorf("Fatal: %w at %v: %v", ErrNoRepository, location.StripPassword(gopts.backends, s), err) + } if err != nil { return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.backends, s), err) } @@ -615,46 +629,34 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio } } - // check if config is there - fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil { - return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) - } - - if fi.Size == 0 { - return nil, errors.New("config file has zero size, invalid repository?") - } - return be, nil } -// Create the backend specified by URI. -func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) { - debug.Log("parsing location %v", location.StripPassword(gopts.backends, s)) - loc, err := location.Parse(gopts.backends, s) - if err != nil { - return nil, err - } +// Open the backend specified by a location config. +func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) { - cfg, err := parseConfig(loc, opts) + be, err := innerOpen(ctx, s, gopts, opts, false) if err != nil { return nil, err } - rt, err := backend.Transport(globalOptions.TransportOptions) + // check if config is there + fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile}) + if be.IsNotExist(err) { + return nil, fmt.Errorf("Fatal: %w: unable to open config file: %v\nIs there a repository at the following location?\n%v", ErrNoRepository, err, location.StripPassword(gopts.backends, s)) + } if err != nil { - return nil, errors.Fatal(err.Error()) + return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(gopts.backends, s)) } - factory := gopts.backends.Lookup(loc.Scheme) - if factory == nil { - return nil, errors.Fatalf("invalid backend: %q", loc.Scheme) + if fi.Size == 0 { + return nil, errors.New("config file has zero size, invalid repository?") } - be, err := factory.Create(ctx, cfg, rt, nil) - if err != nil { - return nil, err - } + return be, nil +} - return logger.New(sema.NewBackend(be)), nil +// Create the backend specified by URI. +func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) { + return innerOpen(ctx, s, gopts, opts, true) } diff --git a/mover-restic/restic/cmd/restic/global_debug.go b/mover-restic/restic/cmd/restic/global_debug.go index b798074d1..502b2cf6e 100644 --- a/mover-restic/restic/cmd/restic/global_debug.go +++ b/mover-restic/restic/cmd/restic/global_debug.go @@ -15,23 +15,28 @@ import ( "github.com/pkg/profile" ) -var ( - listenProfile string - memProfilePath string - cpuProfilePath string - traceProfilePath string - blockProfilePath string - insecure bool -) +type ProfileOptions struct { + listen string + memPath string + cpuPath string + tracePath string + blockPath string + insecure bool +} + +var profileOpts ProfileOptions +var prof interface { + Stop() +} func init() { f := cmdRoot.PersistentFlags() - f.StringVar(&listenProfile, "listen-profile", "", "listen on this `address:port` for memory profiling") - f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`") - f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`") - f.StringVar(&traceProfilePath, "trace-profile", "", "write trace to `dir`") - f.StringVar(&blockProfilePath, "block-profile", "", "write block profile to `dir`") - f.BoolVar(&insecure, "insecure-kdf", false, "use insecure KDF settings") + f.StringVar(&profileOpts.listen, "listen-profile", "", "listen on this `address:port` for memory profiling") + f.StringVar(&profileOpts.memPath, "mem-profile", "", "write memory profile to `dir`") + f.StringVar(&profileOpts.cpuPath, "cpu-profile", "", "write cpu profile to `dir`") + f.StringVar(&profileOpts.tracePath, "trace-profile", "", "write trace to `dir`") + f.StringVar(&profileOpts.blockPath, "block-profile", "", "write block profile to `dir`") + f.BoolVar(&profileOpts.insecure, "insecure-kdf", false, "use insecure KDF settings") } type fakeTestingTB struct{} @@ -41,10 +46,10 @@ func (fakeTestingTB) Logf(msg string, args ...interface{}) { } func runDebug() error { - if listenProfile != "" { - fmt.Fprintf(os.Stderr, "running profile HTTP server on %v\n", listenProfile) + if profileOpts.listen != "" { + fmt.Fprintf(os.Stderr, "running profile HTTP server on %v\n", profileOpts.listen) go func() { - err := http.ListenAndServe(listenProfile, nil) + err := http.ListenAndServe(profileOpts.listen, nil) if err != nil { fmt.Fprintf(os.Stderr, "profile HTTP server listen failed: %v\n", err) } @@ -52,16 +57,16 @@ func runDebug() error { } profilesEnabled := 0 - if memProfilePath != "" { + if profileOpts.memPath != "" { profilesEnabled++ } - if cpuProfilePath != "" { + if profileOpts.cpuPath != "" { profilesEnabled++ } - if traceProfilePath != "" { + if profileOpts.tracePath != "" { profilesEnabled++ } - if blockProfilePath != "" { + if profileOpts.blockPath != "" { profilesEnabled++ } @@ -69,30 +74,25 @@ func runDebug() error { return errors.Fatal("only one profile (memory, CPU, trace, or block) may be activated at the same time") } - var prof interface { - Stop() + if profileOpts.memPath != "" { + prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(profileOpts.memPath)) + } else if profileOpts.cpuPath != "" { + prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(profileOpts.cpuPath)) + } else if profileOpts.tracePath != "" { + prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(profileOpts.tracePath)) + } else if profileOpts.blockPath != "" { + prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(profileOpts.blockPath)) } - if memProfilePath != "" { - prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(memProfilePath)) - } else if cpuProfilePath != "" { - prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(cpuProfilePath)) - } else if traceProfilePath != "" { - prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(traceProfilePath)) - } else if blockProfilePath != "" { - prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(blockProfilePath)) - } - - if prof != nil { - AddCleanupHandler(func(code int) (int, error) { - prof.Stop() - return code, nil - }) - } - - if insecure { + if profileOpts.insecure { repository.TestUseLowSecurityKDFParameters(fakeTestingTB{}) } return nil } + +func stopDebug() { + if prof != nil { + prof.Stop() + } +} diff --git a/mover-restic/restic/cmd/restic/global_release.go b/mover-restic/restic/cmd/restic/global_release.go index 7cb2e6caf..1dab5a293 100644 --- a/mover-restic/restic/cmd/restic/global_release.go +++ b/mover-restic/restic/cmd/restic/global_release.go @@ -5,3 +5,6 @@ package main // runDebug is a noop without the debug tag. func runDebug() error { return nil } + +// stopDebug is a noop without the debug tag. +func stopDebug() {} diff --git a/mover-restic/restic/cmd/restic/global_test.go b/mover-restic/restic/cmd/restic/global_test.go index 4f5c29e9a..8e97ece29 100644 --- a/mover-restic/restic/cmd/restic/global_test.go +++ b/mover-restic/restic/cmd/restic/global_test.go @@ -1,10 +1,13 @@ package main import ( + "context" "os" "path/filepath" + "strings" "testing" + "github.com/restic/restic/internal/errors" rtest "github.com/restic/restic/internal/test" ) @@ -22,6 +25,16 @@ func Test_PrintFunctionsRespectsGlobalStdout(t *testing.T) { } } +type errorReader struct{ err error } + +func (r *errorReader) Read([]byte) (int, error) { return 0, r.err } + +func TestReadPassword(t *testing.T) { + want := errors.New("foo") + _, err := readPassword(&errorReader{want}) + rtest.Assert(t, errors.Is(err, want), "wrong error %v", err) +} + func TestReadRepo(t *testing.T) { tempDir := rtest.TempDir(t) @@ -50,3 +63,14 @@ func TestReadRepo(t *testing.T) { t.Fatal("must not read repository path from invalid file path") } } + +func TestReadEmptyPassword(t *testing.T) { + opts := GlobalOptions{InsecureNoPassword: true} + password, err := ReadPassword(context.TODO(), opts, "test") + rtest.OK(t, err) + rtest.Equals(t, "", password, "got unexpected password") + + opts.password = "invalid" + _, err = ReadPassword(context.TODO(), opts, "test") + rtest.Assert(t, strings.Contains(err.Error(), "must not be specified together with providing a password via a cli option or environment variable"), "unexpected error message, got %v", err) +} diff --git a/mover-restic/restic/cmd/restic/include.go b/mover-restic/restic/cmd/restic/include.go new file mode 100644 index 000000000..dcc4c7f37 --- /dev/null +++ b/mover-restic/restic/cmd/restic/include.go @@ -0,0 +1,100 @@ +package main + +import ( + "strings" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" + "github.com/spf13/pflag" +) + +// IncludeByNameFunc is a function that takes a filename that should be included +// in the restore process and returns whether it should be included. +type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool) + +type includePatternOptions struct { + Includes []string + InsensitiveIncludes []string + IncludeFiles []string + InsensitiveIncludeFiles []string +} + +func initIncludePatternOptions(f *pflag.FlagSet, opts *includePatternOptions) { + f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames") + f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)") + f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns") +} + +func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) { + var fs []IncludeByNameFunc + if len(opts.IncludeFiles) > 0 { + includePatterns, err := readPatternsFromFiles(opts.IncludeFiles) + if err != nil { + return nil, err + } + + if err := filter.ValidatePatterns(includePatterns); err != nil { + return nil, errors.Fatalf("--include-file: %s", err) + } + + opts.Includes = append(opts.Includes, includePatterns...) + } + + if len(opts.InsensitiveIncludeFiles) > 0 { + includePatterns, err := readPatternsFromFiles(opts.InsensitiveIncludeFiles) + if err != nil { + return nil, err + } + + if err := filter.ValidatePatterns(includePatterns); err != nil { + return nil, errors.Fatalf("--iinclude-file: %s", err) + } + + opts.InsensitiveIncludes = append(opts.InsensitiveIncludes, includePatterns...) + } + + if len(opts.InsensitiveIncludes) > 0 { + if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil { + return nil, errors.Fatalf("--iinclude: %s", err) + } + + fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes)) + } + + if len(opts.Includes) > 0 { + if err := filter.ValidatePatterns(opts.Includes); err != nil { + return nil, errors.Fatalf("--include: %s", err) + } + + fs = append(fs, includeByPattern(opts.Includes)) + } + return fs, nil +} + +// includeByPattern returns a IncludeByNameFunc which includes files that match +// one of the patterns. +func includeByPattern(patterns []string) IncludeByNameFunc { + parsedPatterns := filter.ParsePatterns(patterns) + return func(item string) (matched bool, childMayMatch bool) { + matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item) + if err != nil { + Warnf("error for include pattern: %v", err) + } + + return matched, childMayMatch + } +} + +// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match +// one of the patterns, ignoring the casing of the filenames. +func includeByInsensitivePattern(patterns []string) IncludeByNameFunc { + for index, path := range patterns { + patterns[index] = strings.ToLower(path) + } + + includeFunc := includeByPattern(patterns) + return func(item string) (matched bool, childMayMatch bool) { + return includeFunc(strings.ToLower(item)) + } +} diff --git a/mover-restic/restic/cmd/restic/include_test.go b/mover-restic/restic/cmd/restic/include_test.go new file mode 100644 index 000000000..751bfbb76 --- /dev/null +++ b/mover-restic/restic/cmd/restic/include_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "testing" +) + +func TestIncludeByPattern(t *testing.T) { + var tests = []struct { + filename string + include bool + }{ + {filename: "/home/user/foo.go", include: true}, + {filename: "/home/user/foo.c", include: false}, + {filename: "/home/user/foobar", include: false}, + {filename: "/home/user/foobar/x", include: false}, + {filename: "/home/user/README", include: false}, + {filename: "/home/user/README.md", include: true}, + } + + patterns := []string{"*.go", "README.md"} + + for _, tc := range tests { + t.Run(tc.filename, func(t *testing.T) { + includeFunc := includeByPattern(patterns) + matched, _ := includeFunc(tc.filename) + if matched != tc.include { + t.Fatalf("wrong result for filename %v: want %v, got %v", + tc.filename, tc.include, matched) + } + }) + } +} + +func TestIncludeByInsensitivePattern(t *testing.T) { + var tests = []struct { + filename string + include bool + }{ + {filename: "/home/user/foo.GO", include: true}, + {filename: "/home/user/foo.c", include: false}, + {filename: "/home/user/foobar", include: false}, + {filename: "/home/user/FOObar/x", include: false}, + {filename: "/home/user/README", include: false}, + {filename: "/home/user/readme.MD", include: true}, + } + + patterns := []string{"*.go", "README.md"} + + for _, tc := range tests { + t.Run(tc.filename, func(t *testing.T) { + includeFunc := includeByInsensitivePattern(patterns) + matched, _ := includeFunc(tc.filename) + if matched != tc.include { + t.Fatalf("wrong result for filename %v: want %v, got %v", + tc.filename, tc.include, matched) + } + }) + } +} diff --git a/mover-restic/restic/cmd/restic/integration_filter_pattern_test.go b/mover-restic/restic/cmd/restic/integration_filter_pattern_test.go index 2eacdeea9..dccbcc0a0 100644 --- a/mover-restic/restic/cmd/restic/integration_filter_pattern_test.go +++ b/mover-restic/restic/cmd/restic/integration_filter_pattern_test.go @@ -70,30 +70,64 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) { var err error // Test --exclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{Exclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iexclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{InsensitiveExclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --include - err = testRunRestoreAssumeFailure("latest", RestoreOptions{Include: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) // Test --iinclude - err = testRunRestoreAssumeFailure("latest", RestoreOptions{InsensitiveInclude: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts) + err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts) rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided: *[._]log[.-][0-9] !*[._]log[.-][0-9]`, err.Error()) } + +func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + + // Create an include file with some invalid patterns + patternsFile := env.base + "/patternsFile" + fileErr := os.WriteFile(patternsFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644) + if fileErr != nil { + t.Fatalf("Could not write include file: %v", fileErr) + } + + err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts) + rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts) + rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts) + rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) + + err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts) + rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided: +*[._]log[.-][0-9] +!*[._]log[.-][0-9]`, err.Error()) +} diff --git a/mover-restic/restic/cmd/restic/integration_helpers_test.go b/mover-restic/restic/cmd/restic/integration_helpers_test.go index d97589e80..978deab3d 100644 --- a/mover-restic/restic/cmd/restic/integration_helpers_test.go +++ b/mover-restic/restic/cmd/restic/integration_helpers_test.go @@ -12,6 +12,7 @@ import ( "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/retry" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" @@ -204,7 +205,7 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) { extended: make(options.Options), // replace this hook with "nil" if listing a filetype more than once is necessary - backendTestHook: func(r restic.Backend) (restic.Backend, error) { return newOrderedListOnceBackend(r), nil }, + backendTestHook: func(r backend.Backend) (backend.Backend, error) { return newOrderedListOnceBackend(r), nil }, // start with default set of backends backends: globalOptions.backends, } @@ -231,47 +232,66 @@ func testSetupBackupData(t testing.TB, env *testEnvironment) string { } func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet { - r, err := OpenRepository(context.TODO(), gopts) + ctx, r, unlock, err := openWithReadLock(context.TODO(), gopts, false) rtest.OK(t, err) + defer unlock() packs := restic.NewIDSet() - rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { + rtest.OK(t, r.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { packs.Insert(id) return nil })) return packs } +func listTreePacks(gopts GlobalOptions, t *testing.T) restic.IDSet { + ctx, r, unlock, err := openWithReadLock(context.TODO(), gopts, false) + rtest.OK(t, err) + defer unlock() + + rtest.OK(t, r.LoadIndex(ctx, nil)) + treePacks := restic.NewIDSet() + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { + if pb.Type == restic.TreeBlob { + treePacks.Insert(pb.PackID) + } + })) + + return treePacks +} + func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) { - r, err := OpenRepository(context.TODO(), gopts) + ctx, r, unlock, err := openWithExclusiveLock(context.TODO(), gopts, false) rtest.OK(t, err) + defer unlock() for id := range remove { - rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})) + rtest.OK(t, r.RemoveUnpacked(ctx, restic.PackFile, id)) } } func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, removeTreePacks bool) { - r, err := OpenRepository(context.TODO(), gopts) + ctx, r, unlock, err := openWithExclusiveLock(context.TODO(), gopts, false) rtest.OK(t, err) + defer unlock() // Get all tree packs - rtest.OK(t, r.LoadIndex(context.TODO(), nil)) + rtest.OK(t, r.LoadIndex(ctx, nil)) treePacks := restic.NewIDSet() - r.Index().Each(context.TODO(), func(pb restic.PackedBlob) { + rtest.OK(t, r.ListBlobs(ctx, func(pb restic.PackedBlob) { if pb.Type == restic.TreeBlob { treePacks.Insert(pb.PackID) } - }) + })) // remove all packs containing data blobs - rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { + rtest.OK(t, r.List(ctx, restic.PackFile, func(id restic.ID, size int64) error { if treePacks.Has(id) != removeTreePacks || keep.Has(id) { return nil } - return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) + return r.RemoveUnpacked(ctx, restic.PackFile, id) })) } diff --git a/mover-restic/restic/cmd/restic/integration_test.go b/mover-restic/restic/cmd/restic/integration_test.go index 8ea4d17d9..4cecec6bc 100644 --- a/mover-restic/restic/cmd/restic/integration_test.go +++ b/mover-restic/restic/cmd/restic/integration_test.go @@ -8,9 +8,11 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/termstatus" ) func TestCheckRestoreNoLock(t *testing.T) { @@ -41,12 +43,12 @@ func TestCheckRestoreNoLock(t *testing.T) { // backends (like e.g. Amazon S3) as the second listing may be inconsistent to what // is expected by the first listing + some operations. type listOnceBackend struct { - restic.Backend + backend.Backend listedFileType map[restic.FileType]bool strictOrder bool } -func newListOnceBackend(be restic.Backend) *listOnceBackend { +func newListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), @@ -54,7 +56,7 @@ func newListOnceBackend(be restic.Backend) *listOnceBackend { } } -func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { +func newOrderedListOnceBackend(be backend.Backend) *listOnceBackend { return &listOnceBackend{ Backend: be, listedFileType: make(map[restic.FileType]bool), @@ -62,7 +64,7 @@ func newOrderedListOnceBackend(be restic.Backend) *listOnceBackend { } } -func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *listOnceBackend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) error { if t != restic.LockFile && be.listedFileType[t] { return errors.Errorf("tried listing type %v the second time", t) } @@ -77,7 +79,7 @@ func TestListOnce(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } pruneOpts := PruneOptions{MaxUnused: "0"} @@ -85,10 +87,15 @@ func TestListOnce(t *testing.T) { createPrunableRepo(t, env) testRunPrune(t, env.gopts, pruneOpts) - rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil)) - - rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)) - rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts)) + rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runCheck(context.TODO(), checkOpts, env.gopts, nil, term) + })) + rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term) + })) + rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + return runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts, term) + })) } type writeToOnly struct { @@ -104,10 +111,10 @@ func (r *writeToOnly) WriteTo(w io.Writer) (int64, error) { } type onlyLoadWithWriteToBackend struct { - restic.Backend + backend.Backend } -func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h restic.Handle, +func (be *onlyLoadWithWriteToBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return be.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { @@ -120,7 +127,7 @@ func TestBackendLoadWriteTo(t *testing.T) { defer cleanup() // setup backend which only works if it's WriteTo method is correctly propagated upwards - env.gopts.backendInnerTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendInnerTestHook = func(r backend.Backend) (backend.Backend, error) { return &onlyLoadWithWriteToBackend{Backend: r}, nil } @@ -140,7 +147,7 @@ func TestFindListOnce(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil } @@ -153,12 +160,13 @@ func TestFindListOnce(t *testing.T) { testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts) thirdSnapshot := restic.NewIDSet(testListSnapshots(t, env.gopts, 3)...) - repo, err := OpenRepository(context.TODO(), env.gopts) + ctx, repo, unlock, err := openWithReadLock(context.TODO(), env.gopts, false) rtest.OK(t, err) + defer unlock() snapshotIDs := restic.NewIDSet() // specify the two oldest snapshots explicitly and use "latest" to reference the newest one - for sn := range FindFilteredSnapshots(context.TODO(), repo.Backend(), repo, &restic.SnapshotFilter{}, []string{ + for sn := range FindFilteredSnapshots(ctx, repo, repo, &restic.SnapshotFilter{}, []string{ secondSnapshot[0].String(), secondSnapshot[1].String()[:8], "latest", diff --git a/mover-restic/restic/cmd/restic/lock.go b/mover-restic/restic/cmd/restic/lock.go index 11c1ed8f5..0e3dea6d5 100644 --- a/mover-restic/restic/cmd/restic/lock.go +++ b/mover-restic/restic/cmd/restic/lock.go @@ -2,315 +2,47 @@ package main import ( "context" - "fmt" - "sync" - "time" - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/repository" ) -type lockContext struct { - lock *restic.Lock - cancel context.CancelFunc - refreshWG sync.WaitGroup -} - -var globalLocks struct { - locks map[*restic.Lock]*lockContext - sync.Mutex - sync.Once -} - -func lockRepo(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { - return lockRepository(ctx, repo, false, retryLock, json) -} - -func lockRepoExclusive(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { - return lockRepository(ctx, repo, true, retryLock, json) -} - -var ( - retrySleepStart = 5 * time.Second - retrySleepMax = 60 * time.Second -) - -func minDuration(a, b time.Duration) time.Duration { - if a <= b { - return a - } - return b -} - -// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked -// cancelling the original context also stops the lock refresh -func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) { - // make sure that a repository is unlocked properly and after cancel() was - // called by the cleanup handler in global.go - globalLocks.Do(func() { - AddCleanupHandler(unlockAll) - }) - - lockFn := restic.NewLock - if exclusive { - lockFn = restic.NewExclusiveLock - } - - var lock *restic.Lock - var err error - - retrySleep := minDuration(retrySleepStart, retryLock) - retryMessagePrinted := false - retryTimeout := time.After(retryLock) - -retryLoop: - for { - lock, err = lockFn(ctx, repo) - if err != nil && restic.IsAlreadyLocked(err) { - - if !retryMessagePrinted { - if !json { - Verbosef("repo already locked, waiting up to %s for the lock\n", retryLock) - } - retryMessagePrinted = true - } - - debug.Log("repo already locked, retrying in %v", retrySleep) - retrySleepCh := time.After(retrySleep) - - select { - case <-ctx.Done(): - return nil, ctx, ctx.Err() - case <-retryTimeout: - debug.Log("repo already locked, timeout expired") - // Last lock attempt - lock, err = lockFn(ctx, repo) - break retryLoop - case <-retrySleepCh: - retrySleep = minDuration(retrySleep*2, retrySleepMax) - } - } else { - // anything else, either a successful lock or another error - break retryLoop - } - } - if restic.IsInvalidLock(err) { - return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err) - } +func internalOpenWithLocked(ctx context.Context, gopts GlobalOptions, dryRun bool, exclusive bool) (context.Context, *repository.Repository, func(), error) { + repo, err := OpenRepository(ctx, gopts) if err != nil { - return nil, ctx, fmt.Errorf("unable to create lock in backend: %w", err) - } - debug.Log("create lock %p (exclusive %v)", lock, exclusive) - - ctx, cancel := context.WithCancel(ctx) - lockInfo := &lockContext{ - lock: lock, - cancel: cancel, - } - lockInfo.refreshWG.Add(2) - refreshChan := make(chan struct{}) - forceRefreshChan := make(chan refreshLockRequest) - - globalLocks.Lock() - globalLocks.locks[lock] = lockInfo - go refreshLocks(ctx, repo.Backend(), lockInfo, refreshChan, forceRefreshChan) - go monitorLockRefresh(ctx, lockInfo, refreshChan, forceRefreshChan) - globalLocks.Unlock() - - return lock, ctx, err -} - -var refreshInterval = 5 * time.Minute - -// consider a lock refresh failed a bit before the lock actually becomes stale -// the difference allows to compensate for a small time drift between clients. -var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2 - -type refreshLockRequest struct { - result chan bool -} - -func refreshLocks(ctx context.Context, backend restic.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) { - debug.Log("start") - lock := lockInfo.lock - ticker := time.NewTicker(refreshInterval) - lastRefresh := lock.Time - - defer func() { - ticker.Stop() - // ensure that the context was cancelled before removing the lock - lockInfo.cancel() - - // remove the lock from the repo - debug.Log("unlocking repository with lock %v", lock) - if err := lock.Unlock(); err != nil { - debug.Log("error while unlocking: %v", err) - Warnf("error while unlocking: %v", err) - } - - lockInfo.refreshWG.Done() - }() - - for { - select { - case <-ctx.Done(): - debug.Log("terminate") - return - - case req := <-forceRefresh: - debug.Log("trying to refresh stale lock") - // keep on going if our current lock still exists - success := tryRefreshStaleLock(ctx, backend, lock, lockInfo.cancel) - // inform refresh goroutine about forced refresh - select { - case <-ctx.Done(): - case req.result <- success: - } - - if success { - // update lock refresh time - lastRefresh = lock.Time - } - - case <-ticker.C: - if time.Since(lastRefresh) > refreshabilityTimeout { - // the lock is too old, wait until the expiry monitor cancels the context - continue - } - - debug.Log("refreshing locks") - err := lock.Refresh(context.TODO()) - if err != nil { - Warnf("unable to refresh lock: %v\n", err) - } else { - lastRefresh = lock.Time - // inform monitor goroutine about successful refresh - select { - case <-ctx.Done(): - case refreshed <- struct{}{}: - } - } - } + return nil, nil, nil, err } -} -func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-chan struct{}, forceRefresh chan<- refreshLockRequest) { - // time.Now() might use a monotonic timer which is paused during standby - // convert to unix time to ensure we compare real time values - lastRefresh := time.Now().UnixNano() - pollDuration := 1 * time.Second - if refreshInterval < pollDuration { - // require for TestLockFailedRefresh - pollDuration = refreshInterval / 5 - } - // timers are paused during standby, which is a problem as the refresh timeout - // _must_ expire if the host was too long in standby. Thus fall back to periodic checks - // https://github.com/golang/go/issues/35012 - ticker := time.NewTicker(pollDuration) - defer func() { - ticker.Stop() - lockInfo.cancel() - lockInfo.refreshWG.Done() - }() - - var refreshStaleLockResult chan bool + unlock := func() {} + if !dryRun { + var lock *repository.Unlocker - for { - select { - case <-ctx.Done(): - debug.Log("terminate expiry monitoring") - return - case <-refreshed: - if refreshStaleLockResult != nil { - // ignore delayed refresh notifications while the stale lock is refreshed - continue + lock, ctx, err = repository.Lock(ctx, repo, exclusive, gopts.RetryLock, func(msg string) { + if !gopts.JSON { + Verbosef("%s", msg) } - lastRefresh = time.Now().UnixNano() - case <-ticker.C: - if time.Now().UnixNano()-lastRefresh < refreshabilityTimeout.Nanoseconds() || refreshStaleLockResult != nil { - continue - } - - debug.Log("trying to refreshStaleLock") - // keep on going if our current lock still exists - refreshReq := refreshLockRequest{ - result: make(chan bool), - } - refreshStaleLockResult = refreshReq.result - - // inform refresh goroutine about forced refresh - select { - case <-ctx.Done(): - case forceRefresh <- refreshReq: - } - case success := <-refreshStaleLockResult: - if success { - lastRefresh = time.Now().UnixNano() - refreshStaleLockResult = nil - continue - } - - Warnf("Fatal: failed to refresh lock in time\n") - return + }, Warnf) + if err != nil { + return nil, nil, nil, err } - } -} - -func tryRefreshStaleLock(ctx context.Context, backend restic.Backend, lock *restic.Lock, cancel context.CancelFunc) bool { - freeze := restic.AsBackend[restic.FreezeBackend](backend) - if freeze != nil { - debug.Log("freezing backend") - freeze.Freeze() - defer freeze.Unfreeze() - } - err := lock.RefreshStaleLock(ctx) - if err != nil { - Warnf("failed to refresh stale lock: %v\n", err) - // cancel context while the backend is still frozen to prevent accidental modifications - cancel() - return false + unlock = lock.Unlock + } else { + repo.SetDryRun() } - return true + return ctx, repo, unlock, nil } -func unlockRepo(lock *restic.Lock) { - if lock == nil { - return - } - - globalLocks.Lock() - lockInfo, exists := globalLocks.locks[lock] - delete(globalLocks.locks, lock) - globalLocks.Unlock() - - if !exists { - debug.Log("unable to find lock %v in the global list of locks, ignoring", lock) - return - } - lockInfo.cancel() - lockInfo.refreshWG.Wait() +func openWithReadLock(ctx context.Context, gopts GlobalOptions, noLock bool) (context.Context, *repository.Repository, func(), error) { + // TODO enforce read-only operations once the locking code has moved to the repository + return internalOpenWithLocked(ctx, gopts, noLock, false) } -func unlockAll(code int) (int, error) { - globalLocks.Lock() - locks := globalLocks.locks - debug.Log("unlocking %d locks", len(globalLocks.locks)) - for _, lockInfo := range globalLocks.locks { - lockInfo.cancel() - } - globalLocks.locks = make(map[*restic.Lock]*lockContext) - globalLocks.Unlock() - - for _, lockInfo := range locks { - lockInfo.refreshWG.Wait() - } - - return code, nil +func openWithAppendLock(ctx context.Context, gopts GlobalOptions, dryRun bool) (context.Context, *repository.Repository, func(), error) { + // TODO enforce non-exclusive operations once the locking code has moved to the repository + return internalOpenWithLocked(ctx, gopts, dryRun, false) } -func init() { - globalLocks.locks = make(map[*restic.Lock]*lockContext) +func openWithExclusiveLock(ctx context.Context, gopts GlobalOptions, dryRun bool) (context.Context, *repository.Repository, func(), error) { + return internalOpenWithLocked(ctx, gopts, dryRun, true) } diff --git a/mover-restic/restic/cmd/restic/main.go b/mover-restic/restic/cmd/restic/main.go index 4595e8161..5818221a5 100644 --- a/mover-restic/restic/cmd/restic/main.go +++ b/mover-restic/restic/cmd/restic/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "bytes" + "context" "fmt" "log" "os" @@ -14,6 +15,7 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/options" "github.com/restic/restic/internal/restic" ) @@ -23,6 +25,8 @@ func init() { _, _ = maxprocs.Set() } +var ErrOK = errors.New("ok") + // cmdRoot is the base command when no other command has been specified. var cmdRoot = &cobra.Command{ Use: "restic", @@ -37,7 +41,7 @@ The full documentation can be found at https://restic.readthedocs.io/ . SilenceUsage: true, DisableAutoGenTag: true, - PersistentPreRunE: func(c *cobra.Command, args []string) error { + PersistentPreRunE: func(c *cobra.Command, _ []string) error { // set verbosity, default is one globalOptions.verbosity = 1 if globalOptions.Quiet && globalOptions.Verbose > 0 { @@ -73,6 +77,9 @@ The full documentation can be found at https://restic.readthedocs.io/ . // enabled) return runDebug() }, + PersistentPostRun: func(_ *cobra.Command, _ []string) { + stopDebug() + }, } // Distinguish commands that need the password from those that work without, @@ -87,8 +94,6 @@ func needsPassword(cmd string) bool { } } -var logBuffer = bytes.NewBuffer(nil) - func tweakGoGC() { // lower GOGC from 100 to 50, unless it was manually overwritten by the user oldValue := godebug.SetGCPercent(50) @@ -101,12 +106,30 @@ func main() { tweakGoGC() // install custom global logger into a buffer, if an error occurs // we can show the logs + logBuffer := bytes.NewBuffer(nil) log.SetOutput(logBuffer) + err := feature.Flag.Apply(os.Getenv("RESTIC_FEATURES"), func(s string) { + fmt.Fprintln(os.Stderr, s) + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + Exit(1) + } + debug.Log("main %#v", os.Args) debug.Log("restic %s compiled with %v on %v/%v", version, runtime.Version(), runtime.GOOS, runtime.GOARCH) - err := cmdRoot.ExecuteContext(internalGlobalCtx) + + ctx := createGlobalContext() + err = cmdRoot.ExecuteContext(ctx) + + if err == nil { + err = ctx.Err() + } else if err == ErrOK { + // ErrOK overwrites context cancelation errors + err = nil + } switch { case restic.IsAlreadyLocked(err): @@ -128,11 +151,17 @@ func main() { } var exitCode int - switch err { - case nil: + switch { + case err == nil: exitCode = 0 - case ErrInvalidSourceData: + case err == ErrInvalidSourceData: exitCode = 3 + case errors.Is(err, ErrNoRepository): + exitCode = 10 + case restic.IsAlreadyLocked(err): + exitCode = 11 + case errors.Is(err, context.Canceled): + exitCode = 130 default: exitCode = 1 } diff --git a/mover-restic/restic/cmd/restic/progress.go b/mover-restic/restic/cmd/restic/progress.go index 8b33f94c9..d9ff634ce 100644 --- a/mover-restic/restic/cmd/restic/progress.go +++ b/mover-restic/restic/cmd/restic/progress.go @@ -30,7 +30,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration { } // newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided. -func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter { +func newGenericProgressMax(show bool, max uint64, description string, print func(status string, final bool)) *progress.Counter { if !show { return nil } @@ -46,16 +46,18 @@ func newGenericProgressMax(show bool, max uint64, description string, print func ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description) } - print(status) - if final { - fmt.Print("\n") - } + print(status, final) }) } func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter { - return newGenericProgressMax(show, max, description, func(status string) { - term.SetStatus([]string{status}) + return newGenericProgressMax(show, max, description, func(status string, final bool) { + if final { + term.SetStatus(nil) + term.Print(status) + } else { + term.SetStatus([]string{status}) + } }) } @@ -64,7 +66,7 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter return newGenericProgressMax(show, max, description, printProgress) } -func printProgress(status string) { +func printProgress(status string, final bool) { canUpdateStatus := stdoutCanUpdateStatus() @@ -95,6 +97,9 @@ func printProgress(status string) { } _, _ = os.Stdout.Write([]byte(clear + status + carriageControl)) + if final { + _, _ = os.Stdout.Write([]byte("\n")) + } } func newIndexProgress(quiet bool, json bool) *progress.Counter { @@ -104,3 +109,21 @@ func newIndexProgress(quiet bool, json bool) *progress.Counter { func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter { return newTerminalProgressMax(!quiet && !json && stdoutIsTerminal(), 0, "index files loaded", term) } + +type terminalProgressPrinter struct { + term *termstatus.Terminal + ui.Message + show bool +} + +func (t *terminalProgressPrinter) NewCounter(description string) *progress.Counter { + return newTerminalProgressMax(t.show, 0, description, t.term) +} + +func newTerminalProgressPrinter(verbosity uint, term *termstatus.Terminal) progress.Printer { + return &terminalProgressPrinter{ + term: term, + Message: *ui.NewMessage(term, verbosity), + show: verbosity > 0, + } +} diff --git a/mover-restic/restic/cmd/restic/secondary_repo.go b/mover-restic/restic/cmd/restic/secondary_repo.go index 4c46b60df..9a3eb5fe2 100644 --- a/mover-restic/restic/cmd/restic/secondary_repo.go +++ b/mover-restic/restic/cmd/restic/secondary_repo.go @@ -1,6 +1,7 @@ package main import ( + "context" "os" "github.com/restic/restic/internal/errors" @@ -10,11 +11,12 @@ import ( type secondaryRepoOptions struct { password string // from-repo options - Repo string - RepositoryFile string - PasswordFile string - PasswordCommand string - KeyHint string + Repo string + RepositoryFile string + PasswordFile string + PasswordCommand string + KeyHint string + InsecureNoPassword bool // repo2 options LegacyRepo string LegacyRepositoryFile string @@ -48,6 +50,7 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)") f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)") f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)") + f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository, must be passed to every restic command (insecure)") opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY") opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE") @@ -56,13 +59,13 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo opts.PasswordCommand = os.Getenv("RESTIC_FROM_PASSWORD_COMMAND") } -func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) { +func fillSecondaryGlobalOpts(ctx context.Context, opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) { if opts.Repo == "" && opts.RepositoryFile == "" && opts.LegacyRepo == "" && opts.LegacyRepositoryFile == "" { return GlobalOptions{}, false, errors.Fatal("Please specify a source repository location (--from-repo or --from-repository-file)") } hasFromRepo := opts.Repo != "" || opts.RepositoryFile != "" || opts.PasswordFile != "" || - opts.KeyHint != "" || opts.PasswordCommand != "" + opts.KeyHint != "" || opts.PasswordCommand != "" || opts.InsecureNoPassword hasRepo2 := opts.LegacyRepo != "" || opts.LegacyRepositoryFile != "" || opts.LegacyPasswordFile != "" || opts.LegacyKeyHint != "" || opts.LegacyPasswordCommand != "" @@ -84,6 +87,7 @@ func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, rep dstGopts.PasswordFile = opts.PasswordFile dstGopts.PasswordCommand = opts.PasswordCommand dstGopts.KeyHint = opts.KeyHint + dstGopts.InsecureNoPassword = opts.InsecureNoPassword pwdEnv = "RESTIC_FROM_PASSWORD" repoPrefix = "source" @@ -97,6 +101,8 @@ func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, rep dstGopts.PasswordFile = opts.LegacyPasswordFile dstGopts.PasswordCommand = opts.LegacyPasswordCommand dstGopts.KeyHint = opts.LegacyKeyHint + // keep existing bevhaior for legacy options + dstGopts.InsecureNoPassword = false pwdEnv = "RESTIC_PASSWORD2" } @@ -109,7 +115,7 @@ func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, rep return GlobalOptions{}, false, err } } - dstGopts.password, err = ReadPassword(dstGopts, "enter password for "+repoPrefix+" repository: ") + dstGopts.password, err = ReadPassword(ctx, dstGopts, "enter password for "+repoPrefix+" repository: ") if err != nil { return GlobalOptions{}, false, err } diff --git a/mover-restic/restic/cmd/restic/secondary_repo_test.go b/mover-restic/restic/cmd/restic/secondary_repo_test.go index ff1a10b03..aa511ca99 100644 --- a/mover-restic/restic/cmd/restic/secondary_repo_test.go +++ b/mover-restic/restic/cmd/restic/secondary_repo_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "os" "path/filepath" "testing" @@ -170,7 +171,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { // Test all valid cases for _, testCase := range validSecondaryRepoTestCases { - DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination") + DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(context.TODO(), testCase.Opts, gOpts, "destination") rtest.OK(t, err) rtest.Equals(t, DstGOpts, testCase.DstGOpts) rtest.Equals(t, isFromRepo, testCase.FromRepo) @@ -178,7 +179,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) { // Test all invalid cases for _, testCase := range invalidSecondaryRepoTestCases { - _, _, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination") + _, _, err := fillSecondaryGlobalOpts(context.TODO(), testCase.Opts, gOpts, "destination") rtest.Assert(t, err != nil, "Expected error, but function did not return an error") } } diff --git a/mover-restic/restic/cmd/restic/termstatus.go b/mover-restic/restic/cmd/restic/termstatus.go new file mode 100644 index 000000000..c0e9a045b --- /dev/null +++ b/mover-restic/restic/cmd/restic/termstatus.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "sync" + + "github.com/restic/restic/internal/ui/termstatus" +) + +// setupTermstatus creates a new termstatus and reroutes globalOptions.{stdout,stderr} to it +// The returned function must be called to shut down the termstatus, +// +// Expected usage: +// ``` +// term, cancel := setupTermstatus() +// defer cancel() +// // do stuff +// ``` +func setupTermstatus() (*termstatus.Terminal, func()) { + var wg sync.WaitGroup + // only shutdown once cancel is called to ensure that no output is lost + cancelCtx, cancel := context.WithCancel(context.Background()) + + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() + + // use the termstatus for stdout/stderr + prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr + globalOptions.stdout, globalOptions.stderr = termstatus.WrapStdio(term) + + return term, func() { + // shutdown termstatus + globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr + cancel() + wg.Wait() + } +} diff --git a/mover-restic/restic/cmd/restic/testdata/repo-restore-permissions-test.tar.gz b/mover-restic/restic/cmd/restic/testdata/repo-restore-permissions-test.tar.gz index 36aa62dbfb0ad52d5c0cafab528cfee393d388a2..dc8e9bc80e992fca9a59632cbb89344dcd870869 100644 GIT binary patch literal 4256 zcmV;R5MS>fiwFP!000001MS**I92<<2XI0~DaUEfl*r6p(;k${tZkmh6MI2cBcS8f1-|!^(ALS1s2#f$20REgm03kR|{!8JP z`~f^nfXD!i0FkH!3>izNpg5EYk!U0W1Q8I@;x`zSfCp%clM_ZL7zjlO2%fU|J&2+Z zME?CTJ_ii{i~O-T+^_QoK?nx9B>r_E|407j`3Ff-7%0gXrAn$wf*1f%!r+uZ*a8Gp zK~NO{lnHnQgW&N51q`T)!AL4fem)MKBoB0P`^7`HxPtD*^!?`wHxkpu!-cW9(v3=6 z{LG{<0=#_}KN?A@Vi5$wDoQdWRq=|F-jb?-q9l{#?z^}Zq-$ei3X&+g-V8%c3#N;O zjW5|#JG=~ddTdIdD-0W31(29pb~ff@Ka`}fkLhX2 zHgcw#niH%j8rBq+mjSBHG+q3KA^&#%U*Ud!g%HoxxHY6R^LA1>y+@-~$yf|j zIfwh6*RSgiKJAqKct*^zq)>`)ZoGh}FtT#Xe#|nH+^FG0y;EY8H4|kS;XNW1$qi@d zawF@Qt#|6yUR@Aaxz_Yz8B?8GGkMT`w$ej7RF#jIbazdkvS?adqRmU4E413s39IbP zO|lkZqYv26CW7f0itBYYW;~77PW#|2!vC=H{DvN1;+l;9^`L4gY($ zfM4nV004r&>Hi>%133MEDY%X74zboTV(JV^UF!1x`50-VNo?vfiUh0W!Qpr}l?5J= z_FWFqo&<%r=V z%Ur5Sze4YO-u1+(s4{oGQ`UX}qX8&icT_M)Yu7@dFbUrQ6i-9v4}@ z!80v$S;+7$r={l488j}Q~Rhk0I_?LKW~ockBObDlks?Zd{6lHVQuFhbf&}e zA|?;%PPdEs3~p3<>7lzb$nS=_H1TSK-w97_%%~uGL}fzroS*ditI8`xBj3&Fw=zrn zryq`ri1^)um7w`#`OP27DJna=Y8gL14Vj4Ok*s_MKkvaYN zF7LIS5=!hsHdf+7ukMsmYnNJGdd;ZGfby$(d0wec@~1b}9YLgi=Xl3<3gV049+M zSQ>&ws1ObZ0w^A!fm8^@V{pGS##j132m-&o|6jcR!*crnQb?RDl|yFsYbrVtl3PtY z6aCkFL%eL0zOdX6sQGeh?nYaw`Q}o+W%2uqLX`Ji z_1R}akIG7qe|I3#w)TSJnpl!pfuetjt#RSuG7KBV7H!er>Av~w_wmGRBDWf3=Gn;u zJX2s_dQC8ijo7k|xb6Aw;rQ4C!|KU`;sG0<0NIRN`85zS5V51c>zH)CLdnL7fUb&G zk#@5WFKW8HYWW`;ay-WQc$i9~&YnjdLnd{l zA@$$$xm`=otst#_xW%(MBmT@|C_QG^u`=bMWa;*Ymb&H#7*yL~tCgMZ!Gt|M0j!{g z(CL(0yJv1m+b_%u2MFEfRy`+`F&chISoG@KdZFHvBaF0gI6ULY;-YZ`9*=Hn6 zde-%s^2Pe;X1*m^bhzK^ByGSF1P$^HlAXm}$A|%R#^~9(lpo?69#5aE?@i7e6Np-FD1tW>Y>z1;JU2^DwC7G;k2AQcZryi9&CV%vueI96?LS_Ry6Sa{ zKFkQXeAOl?a|eyMocGA_Q|sB-Xh*TmX0ap609lkU4W zwi)V6t?RBZB$GprAM-z}l%C^Dd(j$lBy~4yMm{pJRh+OCIriq>O2{qNB${MP#@blXSkpC8!} z%rzZu`$Xd=1LVUbL@3r_sU2*Qh z&aC1xk?V%G^`>jR#O<6f8eG`nFr_vXGcQnTx-;d;)U@RA^o8QKnWVbAQOXH+&i8@a0U3V>CP>3d_jUQgsJTARSI?H6WgyCRDTjrQR zT1ks-i^jd|C$q6cr;~cUg!Q%9~Ag+pNxd)ca{hR~r%bqckfOWXzka}x^5CLg9(Txv^I48MgpbN@F&UwPmbG8`ZMiqaF~%D-VeWB|MwHY-l~Z!}FM<<;i=m^?l|qzmIs5+RTo| z4_&8zVx2RUeg5&oe3Z_@^188x==2B5tgLvbsJ*n^E-xK2*#(K;A`Du5fG<@(%Qx$e{7UNce%oQgzyi1Q4Dcn@YJfRt!jFs$L!553x{K`gWuzn@f@Ke6ph&Z=SMafj?;UdzdkckHZXBBSo_wKCUr zD0+xB8U3@5&ELV7`S0#Uar=gU0Qw{U|3VPp%lzj&|6dwk@^|s1qW*ue{5kjkOX9!e zkH?eoG#o-EU|=kcN}-@2L;w%~qJcP=hEf(=Gl<0ys3-+R5rjm;k||Ug8Bf4NI1GS- zG}`Zu@wMmw2#EW2{vh}ze@_4Z_70!$8tHzCz86ao(x;aRxddlka!)^0q|B^+FJzG^ zpL*ZIRrr&+;!}LtXhfrdb5{v!(BGtK-oY~0*0Hy2b6Ivi_tZO6=R|hw275^j-xReQ zcdcIz9TdoT)%qrgUy}FeJC+V_LGHltT!Jq<9iKQ^=$N}TY5QG14?EKqk>DTHyU`p` z)zYnd_TL`-Iscf#3m>7t7}xgZK;b6GO3T^oS7&SV2RzQE6w)ddQtd+fyLbt+lK34} zXJ?;}T1=mvT~%oIzNei{v^(<5O}tx~{CKm-T^0M=N#Bof74+s{YiCRyxK?HuxomPe z3OYEPJh(4{-q_h>zUkJ2+ZxK*7bS8>TqlY&Zs1b2pAJ;xROeN51sr9o)LCx)m=5rP z^S1RKlREd7C4_t@?6s%5U%AaKi5i_O&((YI%{VDyta|7={sKzPN;^#l z^u{-7>fCSQ=D)l*YCi2?Z=h0*;YvBeka~q4uoNeBz*4qd7g((Np^?DDg>OHsKCHqQLQp=gE zL08(iz7w6yeYl`qY-dogp~9Azb@BcNS>qHR8zrjn3qAcQ%l_1Y+{gefqu9Os-gK?K zhqvAJpX!U zCT^8qe;gX}cJEOFCb)=dlqgl?RZz83%~D`V?s05^tO z^QSKjwRT6wMz@bJtLd8a`LER+i>ys{81?M0emxV)6M9jH-b^H?w>R`5f_z=;B4hXa z$aIQkDN0R;xE~lbUoOg>-+R$V(Qd;gW$&vi1dHQ{Qx)aX1$%ad{q%ZzMZ%Q)L zt?Rz8@BRCIuitgC=)OLheBT!o7YqvZOABC-g%$zY^0D{Kw z$^COd{+IX<*FQp}<)UPlrHMe02;*V2o*!0yywY(ianw$Xy);YDw=Dg#>TO+AAH)1~ ztbEVPAmj43wLI$g<$*e`O3lFMYvr@K5W-BEw-0^e5~sH*ye)V<^39?&Sy%5Y~O#q?`W?a z75#XZosF^c<)>(?!M}QL{u5Y;e}8Y^@96=5BL4$0un_+s3gYGe#b7~yPkJ!>JN^KF z%Kza7{UHS5>Ax8KLw^7Z;~)yaz(Hgh4n;vzsB{dS29X(L90cJIGWQOOj>7^B6gNf@ zDhi?_I0Q@O-UsP)2%`M)7~k@LGzRlq{Xy=E0LAnF#UNCk%A%9~=`?vQc@PC4>L`pl z2opg-3k0MVD3rXK{4evsBzx1j&vU1Y+u`QJ_W!lRlgxJac4u)r zJ!uT?HJi!`_VwppTF7gm5d=c3$+P6Ouxj$Y@>+nJJe%z0&wUCqc5onoWU8?*3$IUP zyAvJ!DNNlEOE(`oFC;|M(%9WXUzLsv@pRP#OfA?9j-e)-f+3PZSgMA$h9;^3bX^0Y zwSfc6pWyCc;O6hcfHmp5+S=T4X=HygcLW<(4whg_rdnt+Frd4c5sP6T5&&UB^aysY z4%UFBnS%#`7-Z`~a`RzOV75M8)r`e(*E2MtQ0-s?S4Rj<;;@3PnITMHstMv~v0D}N zHFdD01_oRBTH9K2Ld?xAC_so?2uQWG)b#WuP|Wp-s^)+y9%S2HUOrUM?`Q!3g8w4`nwS3$W(eZ&6&2m}3Af3E*y0G|IZ2AS#vgA42?0f6A7@d4PS z>v2%gC=@%70xqOB5Si+RmAi9$8s-Tm9eHze_ykmV)z|ez-0FuXEpsNW%7`XIOVN`2 z{nkH|_eCBRZWr%=oWV&%ACzNR06vYr(|uW%=_H55e0)xA`G9=LhL=(w*Aw@~C096q z)v#9CGHK58i4fVN-Ef{W>o9I79X_DBHWa}>UOrRh9=z&~JoGL@<&a+Tb5epFzOm6J zW5xX%-AVKXrb6iem0_utceXiCM=E4y`))(;%h9&SKjHkiCCTAz5I9jD+PIh*X+Z4!EG z*x1c*fZ(>{+47}_3dPthGUEPwStqA#P8L;KZf(+imb^S%`jawaS7*Z6riie$kKTOl z%w5(`SwTKG`Qv=7p@!96)FTCNKO$xK(%HgL`=9CEahnBuaXl@E%WopBi1x1GZRtDsgC9RB6+VQIbfi`f@w*)ol8S=@(1p zkE`sBt)pWK9+|Hij!}(zWHE(_07ZxUn?F6L@Ri>6q_dKVxT$>g`v*_&_moho+8j}B zqbnCP(x3thq$d23RVJRS+YvjI5|S2fw}uw^@%SD$W5;;-!|1gx(&eEzNtc1Mhwp9E zVD?ky%3lk;t(Sp_MRV*=!lbZX%EX4QZPreUcY`(``~5u zvJlgXp~5tQ8H2`;`!mgs;+;Cx#UkuPk1d&yDisQFuNLAf8TbjXn-nRMtKNq%yBoNE zxsyQJ=j_Ccd?ytmMk(#KhM};fP*O@8H)yR06>Eo3^YK7xt&z5j37*=V`&T=0CATEFpNS| z>2wT@Lc?IOR66>99OGN{KZO3B{vd>cK%W0E26aBm5>mV=)aQD$7T2Y>zdh}cOJTs!b(Dd)^dG(FESE61)+&l3Xz7sF>S* zGJ3f(vG+1MFp&YXVN|L+&f|Ib*6|6jIA{wDjszt(@;dV-h#7lVK3PlLbLFYqtukA?uA z{)@qH^ryizI*m>R$p{rkCBqmx4hG2}h^C++kP6ZeItai383&>P7{pLv3=IM>R2q#8 zq9F)mVE^a{;9K=Szzz1_)&DT>{C{!q9MB1~H?m+Gy;8i=8T2TPEM*m&GE9{VYXnV&r;O6|f!RW?HME4RHjk(eAUc41ftgNl*ktnZ-bDM?bG@XJ^Nl*K<^uVFocvOu z)#VPz2cPz>%DYK2Exotr3G411pB3wkSr$DW{Buq=aPz?4ks~H9OZ~PMTsz&kF`!3$ zTG=+Nxm-~3)qt|;b)`66X*Y+tLMAeB%uW>F;HE+qjI3PZ9 zm1rSqP@#OK=2W#?&zsIZm&;Xok}lcKgX2jJF4sn)=T05AI&rW@p;5Ra?qqbw*_V;I z0lQw9rnvAcT5kz&J`>9{Rd?+KZ_j}8(e$~f8hdtDj5+!$wnXiSQJw6)}kRUoyC zfN$obY-?bqW(tJ$-*H*C>SpP(A6=o(Ungsy)dnMuedp(F5VH}PqB>>oq4b>*3jKzz zdyFO`@+03LF`j6X@p~nu{?6NYduYHS5P&#ch<6w4~0#}jK+lq=*#u{$(s{r3gTPc#x|^^ z=w(hke<*ZwyPP_wn1hzP*lj$n-qNXEcdBO4s#oK~+%_S_*==W~YVw3;-d_?zImI*j zCM2fq>H|Hh_>?cI%#UKGIIrfLyvi#*UyD7Ny8}u}s15{vWt=dgg8>0L7GQuh2*jc=e=x>3{2v5?-`D@#^&iah|HUA2rfe%RxmRCp zA1=T^s?o-yUu zhOC(#g21pU=gLQ%Rcz#jHKv>2==8_O9_rUk5|a&<>I2TO?wqfIklx5Gg+6IY^{S;( zW5J!3Ez)h)({F1!eQJfB&+P16^`HWUr?hr7m>%Gh6rR4x`Q&%}cH56;QVrK?weId~ z5Bj-{zxPyqg}tI$)QXRtC|P^0UeZY2DM3LQiQ*uy7f)#Ga>a7|s;Wj~T+ZJcmxj>_{uuc%7uw_SDwc<`9cU>bG0g;k6&P z^F?rAXhZl!@*SPYJ4#OTa}vR;?(%CDDP{~t9Ff>`W2$~t_vryvY6S8)Z)1t&iEPLRVWr?cx zUHYNNDI-aqS=U1ljtwx*oFWt3y_!47Ytc9{^Yi9OuCg8@ronEk!E-aod*T|NPZZUP zWNlpkys1yYUNo`!{bZVG^b))@7BAKoQ;vIMos{UrpCXAdzoBd2Q?2djlDW%X`;x-( zThTXs?zr`{g0J3iIF`AEVY)=@bt8xbLyvZ{TXjt!cL^<1PB((-6{hl{ZzZ0 z4w_vUSi7HZB7)SX_m1C9#;;)VhqE&sp(}c?DOTrxZVqM_UNJCi#yge>vc_abOHDq? z&exUX&2P^xDVM&5C)E?gePkV7FPUH5;ykWB9y2FeM%bR*H$EZXKXI|*ilOvQzGE|K z`*WH^L&Nl`ts?O9#!;npO4(K`#IIQMehz>P<9b}jna82;bt{+$0WI4atNorZCcCztG?TaPn zA)f=(>s~Ku2EO8ABB%mQ&QNRMgF4xn#2v&<#oMgkcR8%ce%3uZsiNOg;~j3}GS;vH zI}uSgm>(5rzf}WwJArDvB&fc3AN9#4lYIWLix;x@wT5UDKgDRZ&qeNuub|7wZDPN9 zIZ+Y*lA_yWoKqIFexx;HY3iT>X{T;*sL(b@HuUiD#8tfgmo3b+eYU5YKbrZ?UHv(- zFQt)l5c~QTZ8oropz`MH^jx&j{F1tnhJ&Y`Xar`*Lzmmi+8jgjyIL=ptvOgzJlk+@ z!~UM3mK>ug8HwFp99yf=HyMnigGGfNe{~Y%!Gi}69z1yP;K73j4<0;t@ZiCN2M-=R Yc<|uCg9i^DJpPaKUr7c%#{f_O02$#w+W-In diff --git a/mover-restic/restic/doc/020_installation.rst b/mover-restic/restic/doc/020_installation.rst index 0f1cd6c04..17b581a87 100644 --- a/mover-restic/restic/doc/020_installation.rst +++ b/mover-restic/restic/doc/020_installation.rst @@ -77,8 +77,7 @@ avoid any conflicts: macOS ===== -If you are using macOS, you can install restic using the -`homebrew `__ package manager: +If you are using macOS, you can install restic using `Homebrew `__: .. code-block:: console @@ -363,3 +362,18 @@ Example for using sudo to write a zsh completion script directly to the system-w the operating system used, e.g. ``/usr/share/bash-completion/completions/restic`` in Debian and derivatives. Please look up the correct path in the appropriate documentation. + +Example for setting up a powershell completion script for the local user's profile: + +.. code-block:: pwsh-session + + # Create profile if one does not exist + PS> If (!(Test-Path $PROFILE.CurrentUserAllHosts)) {New-Item -Path $PROFILE.CurrentUserAllHosts -Force} + + PS> $ProfileDir = (Get-Item $PROFILE.CurrentUserAllHosts).Directory + + # Generate Restic completions in the same directory as the profile + PS> restic generate --powershell-completion "$ProfileDir\restic-completion.ps1" + + # Append to the profile file the command to load Restic completions + PS> Add-Content -Path $PROFILE.CurrentUserAllHosts -Value "`r`nImport-Module $ProfileDir\restic-completion.ps1" diff --git a/mover-restic/restic/doc/030_preparing_a_new_repo.rst b/mover-restic/restic/doc/030_preparing_a_new_repo.rst index 62499a1d6..87975f9fa 100644 --- a/mover-restic/restic/doc/030_preparing_a_new_repo.rst +++ b/mover-restic/restic/doc/030_preparing_a_new_repo.rst @@ -35,15 +35,15 @@ environment variable ``RESTIC_REPOSITORY_FILE``. For automating the supply of the repository password to restic, several options exist: - * Setting the environment variable ``RESTIC_PASSWORD`` +* Setting the environment variable ``RESTIC_PASSWORD`` - * Specifying the path to a file with the password via the option - ``--password-file`` or the environment variable ``RESTIC_PASSWORD_FILE`` +* Specifying the path to a file with the password via the option + ``--password-file`` or the environment variable ``RESTIC_PASSWORD_FILE`` + +* Configuring a program to be called when the password is needed via the + option ``--password-command`` or the environment variable + ``RESTIC_PASSWORD_COMMAND`` - * Configuring a program to be called when the password is needed via the - option ``--password-command`` or the environment variable - ``RESTIC_PASSWORD_COMMAND`` - The ``init`` command has an option called ``--repository-version`` which can be used to explicitly set the version of the new repository. By default, the current stable version is used (see table below). The alias ``latest`` will @@ -201,15 +201,16 @@ scheme like this: $ restic -r rest:http://host:8000/ init Depending on your REST server setup, you can use HTTPS protocol, -password protection, multiple repositories or any combination of -those features. The TCP/IP port is also configurable. Here -are some more examples: +unix socket, password protection, multiple repositories or any +combination of those features. The TCP/IP port is also configurable. +Here are some more examples: .. code-block:: console $ restic -r rest:https://host:8000/ init $ restic -r rest:https://user:pass@host:8000/ init $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init + $ restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ init The server username and password can be specified using environment variables as well: @@ -487,7 +488,8 @@ Backblaze B2 Different from the B2 backend, restic's S3 backend will only hide no longer necessary files. Thus, make sure to setup lifecycle rules to eventually - delete hidden files. + delete hidden files. The lifecycle setting "Keep only the last version of the file" + will keep only the most current version of a file. Read the [Backblaze documentation](https://www.backblaze.com/docs/cloud-storage-lifecycle-rules). Restic can backup data to any Backblaze B2 bucket. You need to first setup the following environment variables with the credentials you can find in the @@ -722,9 +724,9 @@ For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``. The rclone backend has three additional options: - * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone`` - * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete`` - * ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m`` +* ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone`` +* ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete`` +* ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m`` The reason for the ``--b2-hard-delete`` parameters can be found in the corresponding GitHub `issue #1657`_. @@ -850,3 +852,26 @@ and then grants read/write permissions for group access. .. note:: To manage who has access to the repository you can use ``usermod`` on Linux systems, to change which group controls repository access ``chgrp -R`` is your friend. + + +Repositories with empty password +******************************** + +Restic by default refuses to create or operate on repositories that use an +empty password. Since restic 0.17.0, the option ``--insecure-no-password`` allows +disabling this check. Restic will not prompt for a password when using this option. +Specifying ``--insecure-no-password`` while also passing a password to restic +via a CLI option or via environment variable results in an error. + +For security reasons, the option must always be specified when operating on +repositories with an empty password. For example to create a new repository +with an empty password, use the following command. + +.. code-block:: console + + restic init --insecure-no-password + + +The ``init`` and ``copy`` command also support the option ``--from-insecure-no-password`` +which applies to the source repository. The ``key add`` and ``key passwd`` commands +include the ``--new-insecure-no-password`` option to add or set and empty password. diff --git a/mover-restic/restic/doc/040_backup.rst b/mover-restic/restic/doc/040_backup.rst index 621b07e2e..81d99e071 100644 --- a/mover-restic/restic/doc/040_backup.rst +++ b/mover-restic/restic/doc/040_backup.rst @@ -24,16 +24,17 @@ again: $ restic -r /srv/restic-repo --verbose backup ~/work open repository enter password for repository: - password is correct - lock repository + repository a14e5863 opened (version 2, compression level auto) load index files - start scan - start backup - scan finished in 1.837s - processed 1.720 GiB in 0:12 + start scan on [/home/user/work] + start backup on [/home/user/work] + scan finished in 1.837s: 5307 files, 1.720 GiB + Files: 5307 new, 0 changed, 0 unmodified Dirs: 1867 new, 0 changed, 0 unmodified - Added: 1.200 GiB + Added to the repository: 1.200 GiB (1.103 GiB stored) + + processed 5307 files, 1.720 GiB in 0:12 snapshot 40dc1520 saved As you can see, restic created a backup of the directory and was pretty @@ -44,6 +45,7 @@ You can see that restic tells us it processed 1.720 GiB of data, this is the size of the files and directories in ``~/work`` on the local file system. It also tells us that only 1.200 GiB was added to the repository. This means that some of the data was duplicate and restic was able to efficiently reduce it. +The data compression also managed to compress the data down to 1.103 GiB. If you don't pass the ``--verbose`` option, restic will print less data. You'll still get a nice live status display. Be aware that the live status shows the @@ -56,6 +58,39 @@ snapshot for each volume that contains files to backup. Files are read from the VSS snapshot instead of the regular filesystem. This allows to backup files that are exclusively locked by another process during the backup. +You can use the following extended options to change the VSS behavior: + + * ``-o vss.timeout`` specifies timeout for VSS snapshot creation, default value being 120 seconds + * ``-o vss.exclude-all-mount-points`` disable auto snapshotting of all volume mount points + * ``-o vss.exclude-volumes`` allows excluding specific volumes or volume mount points from snapshotting + * ``-o vss.provider`` specifies VSS provider used for snapshotting + +For example a 2.5 minutes timeout with snapshotting of mount points disabled can be specified as: + +.. code-block:: console + + -o vss.timeout=2m30s -o vss.exclude-all-mount-points=true + +and excluding drive ``d:\``, mount point ``c:\mnt`` and volume ``\\?\Volume{04ce0545-3391-11e0-ba2f-806e6f6e6963}\`` as: + +.. code-block:: console + + -o vss.exclude-volumes="d:;c:\mnt\;\\?\volume{04ce0545-3391-11e0-ba2f-806e6f6e6963}" + +VSS provider can be specified by GUID: + +.. code-block:: console + + -o vss.provider={3f900f90-00e9-440e-873a-96ca5eb079e5} + +or by name: + +.. code-block:: console + + -o vss.provider="Hyper-V IC Software Shadow Copy Provider" + +Also, ``MS`` can be used as alias for ``Microsoft Software Shadow Copy provider 1.0``. + By default VSS ignores Outlook OST files. This is not a restriction of restic but the default Windows VSS configuration. The files not to snapshot are configured in the Windows registry under the following key: @@ -76,17 +111,18 @@ repository (since all data is already there). This is de-duplication at work! $ restic -r /srv/restic-repo --verbose backup ~/work open repository enter password for repository: - password is correct - lock repository + repository a14e5863 opened (version 2, compression level auto) load index files - using parent snapshot d875ae93 - start scan - start backup - scan finished in 1.881s - processed 1.720 GiB in 0:03 + using parent snapshot 40dc1520 + start scan on [/home/user/work] + start backup on [/home/user/work] + scan finished in 1.881s: 5307 files, 1.720 GiB + Files: 0 new, 0 changed, 5307 unmodified Dirs: 0 new, 0 changed, 1867 unmodified - Added: 0 B + Added to the repository: 0 B (0 B stored) + + processed 5307 files, 1.720 GiB in 0:03 snapshot 79766175 saved You can even backup individual files in the same repository (not passing @@ -96,7 +132,6 @@ You can even backup individual files in the same repository (not passing $ restic -r /srv/restic-repo backup ~/work.txt enter password for repository: - password is correct snapshot 249d0210 saved If you're interested in what restic does, pass ``--verbose`` twice (or @@ -110,7 +145,6 @@ restic encounters: $ restic -r /srv/restic-repo --verbose --verbose backup ~/work.txt open repository enter password for repository: - password is correct lock repository load index files using parent snapshot f3f8d56b @@ -170,10 +204,10 @@ On **Unix** (including Linux and Mac), given that a file lives at the same location as a file in a previous backup, the following file metadata attributes have to match for its contents to be presumed unchanged: - * Modification timestamp (mtime). - * Metadata change timestamp (ctime). - * File size. - * Inode number (internal number used to reference a file in a filesystem). +* Modification timestamp (mtime). +* Metadata change timestamp (ctime). +* File size. +* Inode number (internal number used to reference a file in a filesystem). The reason for requiring both mtime and ctime to match is that Unix programs can freely change mtime (and some do). In such cases, a ctime change may be @@ -182,9 +216,9 @@ the only hint that a file did change. The following ``restic backup`` command line flags modify the change detection rules: - * ``--force``: turn off change detection and rescan all files. - * ``--ignore-ctime``: require mtime to match, but allow ctime to differ. - * ``--ignore-inode``: require mtime to match, but allow inode number +* ``--force``: turn off change detection and rescan all files. +* ``--ignore-ctime``: require mtime to match, but allow ctime to differ. +* ``--ignore-inode``: require mtime to match, but allow inode number and ctime to differ. The option ``--ignore-inode`` exists to support FUSE-based filesystems and @@ -198,6 +232,40 @@ On **Windows**, a file is considered unchanged when its path, size and modification time match, and only ``--force`` has any effect. The other options are recognized but ignored. +Skip creating snapshots if unchanged +************************************ + +By default, restic always creates a new snapshot even if nothing has changed +compared to the parent snapshot. To omit the creation of a new snapshot in this +case, specify the ``--skip-if-unchanged`` option. + +Note that when using absolute paths to specify the backup source, then also +changes to the parent folders result in a changed snapshot. For example, a backup +of ``/home/user/work`` will create a new snapshot if the metadata of either +``/``, ``/home`` or ``/home/user`` change. To avoid this problem run restic from +the corresponding folder and use relative paths. + +.. code-block:: console + + $ cd /home/user/work && restic -r /srv/restic-repo backup . --skip-if-unchanged + + open repository + enter password for repository: + repository a14e5863 opened (version 2, compression level auto) + load index files + using parent snapshot 40dc1520 + start scan on [.] + start backup on [.] + scan finished in 1.814s: 5307 files, 1.720 GiB + + Files: 0 new, 0 changed, 5307 unmodified + Dirs: 0 new, 0 changed, 1867 unmodified + Added to the repository: 0 B (0 B stored) + + processed 5307 files, 1.720 GiB in 0:03 + skipped creating snapshot + + Dry Runs ******** @@ -250,9 +318,9 @@ It can be used like this: This instructs restic to exclude files matching the following criteria: - * All files matching ``*.c`` (parameter ``--exclude``) - * All files matching ``*.go`` (second line in ``excludes.txt``) - * All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) +* All files matching ``*.c`` (parameter ``--exclude``) +* All files matching ``*.go`` (second line in ``excludes.txt``) +* All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) Patterns use the syntax of the Go function `filepath.Match `__ @@ -270,8 +338,8 @@ environment variable (depending on your operating system). Patterns need to match on complete path components. For example, the pattern ``foo``: - * matches ``/dir1/foo/dir2/file`` and ``/dir/foo`` - * does not match ``/dir/foobar`` or ``barfoo`` +* matches ``/dir1/foo/dir2/file`` and ``/dir/foo`` +* does not match ``/dir/foobar`` or ``barfoo`` A trailing ``/`` is ignored, a leading ``/`` anchors the pattern at the root directory. This means, ``/bin`` matches ``/bin/bash`` but does not match ``/usr/bin/restic``. @@ -281,9 +349,9 @@ e.g. ``b*ash`` matches ``/bin/bash`` but does not match ``/bin/ash``. For this, the special wildcard ``**`` can be used to match arbitrary sub-directories: The pattern ``foo/**/bar`` matches: - * ``/dir1/foo/dir2/bar/file`` - * ``/foo/bar/file`` - * ``/tmp/foo/bar`` +* ``/dir1/foo/dir2/bar/file`` +* ``/foo/bar/file`` +* ``/tmp/foo/bar`` Spaces in patterns listed in an exclude file can be specified verbatim. That is, in order to exclude a file named ``foo bar star.txt``, put that just as it reads @@ -298,9 +366,9 @@ some escaping in order to pass the name/pattern as a single argument to restic. On most Unixy shells, you can either quote or use backslashes. For example: - * ``--exclude='foo bar star/foo.txt'`` - * ``--exclude="foo bar star/foo.txt"`` - * ``--exclude=foo\ bar\ star/foo.txt`` +* ``--exclude='foo bar star/foo.txt'`` +* ``--exclude="foo bar star/foo.txt"`` +* ``--exclude=foo\ bar\ star/foo.txt`` If a pattern starts with exclamation mark and matches a file that was previously matched by a regular pattern, the match is cancelled. @@ -381,8 +449,8 @@ contains one *pattern* per line. The file must be encoded as UTF-8, or UTF-16 with a byte-order mark. Leading and trailing whitespace is removed from the patterns. Empty lines and lines starting with a ``#`` are ignored and each pattern is expanded when read, such that special characters in it are expanded -using the Go function `filepath.Glob `__ -- please see its documentation for the syntax you can use in the patterns. +according to the syntax described in the documentation of the Go function +`filepath.Match `__. The argument passed to ``--files-from-verbatim`` must be the name of a text file that contains one *path* per line, e.g. as generated by GNU ``find`` with the @@ -430,18 +498,17 @@ You can combine all three options with each other and with the normal file argum Comparing Snapshots ******************* -Restic has a `diff` command which shows the difference between two snapshots +Restic has a ``diff`` command which shows the difference between two snapshots and displays a small statistic, just pass the command two snapshot IDs: .. code-block:: console $ restic -r /srv/restic-repo diff 5845b002 2ab627a6 - password is correct comparing snapshot ea657ce5 to 2ab627a6: - C /restic/cmd_diff.go + M /restic/cmd_diff.go + /restic/foo - C /restic/restic + M /restic/restic Files: 0 new, 0 removed, 2 changed Dirs: 1 new, 0 removed @@ -460,6 +527,24 @@ folder, you could use the following command: $ restic -r /srv/restic-repo diff 5845b002:/restic 2ab627a6:/restic +By default, the ``diff`` command only lists differences in file contents. +The flag ``--metadata`` shows changes to file metadata, too. + +The characters left of the file path show what has changed for this file: + ++-------+-----------------------+ +| ``+`` | added | ++-------+-----------------------+ +| ``-`` | removed | ++-------+-----------------------+ +| ``T`` | entry type changed | ++-------+-----------------------+ +| ``M`` | file content changed | ++-------+-----------------------+ +| ``U`` | metadata changed | ++-------+-----------------------+ +| ``?`` | bitrot detected | ++-------+-----------------------+ Backing up special items and metadata ************************************* @@ -481,44 +566,81 @@ written, and the next backup needs to write new metadata again. If you really want to save the access time for files and directories, you can pass the ``--with-atime`` option to the ``backup`` command. +Backing up full security descriptors on Windows is only possible when the user +has ``SeBackupPrivilege`` privilege or is running as admin. This is a restriction +of Windows not restic. +If either of these conditions are not met, only the owner, group and DACL will +be backed up. + Note that ``restic`` does not back up some metadata associated with files. Of -particular note are:: +particular note are: + +* File creation date on Unix platforms +* Inode flags on Unix platforms + +Reading data from a command +*************************** - - file creation date on Unix platforms - - inode flags on Unix platforms - - file ownership and ACLs on Windows - - the "hidden" flag on Windows +Sometimes, it can be useful to directly save the output of a program, for example, +``mysqldump`` so that the SQL can later be restored. Restic supports this mode +of operation; just supply the option ``--stdin-from-command`` when using the +``backup`` action, and write the command in place of the files/directories: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup --stdin-from-command mysqldump [...] + +This command creates a new snapshot based on the standard output of ``mysqldump``. +By default, the command's standard output is saved in a file named ``stdin``. +A different name can be specified with ``--stdin-filename``: + +.. code-block:: console + + $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command mysqldump [...] + +Restic uses the command exit code to determine whether the command succeeded. A +non-zero exit code from the command causes restic to cancel the backup. This causes +restic to fail with exit code 1. No snapshot will be created in this case. Reading data from stdin *********************** -Sometimes it can be nice to directly save the output of a program, e.g. -``mysqldump`` so that the SQL can later be restored. Restic supports -this mode of operation, just supply the option ``--stdin`` to the -``backup`` command like this: +.. warning:: + + Restic cannot detect if data read from stdin is complete or not. As explained + below, this can cause incomplete backup unless additional checks (outside of + restic) are configured. If possible, use ``--stdin-from-command`` instead. + +Alternatively, restic supports reading arbitrary data directly from the standard +input. Use the option ``--stdin`` of the ``backup`` command as follows: .. code-block:: console - $ set -o pipefail - $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin + # Will not notice failures, see the warning below + $ gzip bigfile.dat | restic -r /srv/restic-repo backup --stdin -This creates a new snapshot of the output of ``mysqldump``. You can then -use e.g. the fuse mounting option (see below) to mount the repository -and read the file. +This creates a new snapshot of the content of ``bigfile.dat``. +As for ``--stdin-from-command``, the default file name is ``stdin``; a +different name can be specified with ``--stdin-filename``. -By default, the file name ``stdin`` is used, a different name can be -specified with ``--stdin-filename``, e.g. like this: +**Important**: while it is possible to pipe a command output to restic using +``--stdin``, doing so is discouraged as it will mask errors from the +command, leading to corrupted backups. For example, in the following code +block, if ``mysqldump`` fails to connect to the MySQL database, the restic +backup will nevertheless succeed in creating an _empty_ backup: .. code-block:: console - $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin --stdin-filename production.sql - -The option ``pipefail`` is highly recommended so that a non-zero exit code from -one of the programs in the pipe (e.g. ``mysqldump`` here) makes the whole chain -return a non-zero exit code. Refer to the `Use the Unofficial Bash Strict Mode -`__ for more -details on this. + # Will not notice failures, read the warning above + $ mysqldump [...] | restic -r /srv/restic-repo backup --stdin +A simple solution is to use ``--stdin-from-command`` (see above). If you +still need to use the ``--stdin`` flag, you must use the shell option ``set -o pipefail`` +(so that a non-zero exit code from one of the programs in the pipe makes the +whole chain return a non-zero exit code) and you must check the exit code of +the pipe and act accordingly (e.g., remove the last backup). Refer to the +`Use the Unofficial Bash Strict Mode `__ +for more details on this. Tags for backup *************** @@ -584,7 +706,8 @@ environment variables. The following lists these environment variables: RESTIC_PACK_SIZE Target size for pack files RESTIC_READ_CONCURRENCY Concurrency for file reads - TMPDIR Location for temporary files + TMPDIR Location for temporary files (except Windows) + TMP Location for temporary files (only Windows) AWS_ACCESS_KEY_ID Amazon S3 access key ID AWS_SECRET_ACCESS_KEY Amazon S3 secret access key @@ -592,6 +715,12 @@ environment variables. The following lists these environment variables: AWS_DEFAULT_REGION Amazon S3 default region AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) + RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials + RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy + RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1) + RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only. AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure @@ -643,15 +772,14 @@ The external programs that restic may execute include ``rclone`` (for rclone backends) and ``ssh`` (for the SFTP backend). These may respond to further environment variables and configuration files; see their respective manuals. - Exit status codes ***************** Restic returns one of the following exit status codes after the backup command is run: - * 0 when the backup was successful (snapshot with all source files created) - * 1 when there was a fatal error (no snapshot created) - * 3 when some source files could not be read (incomplete snapshot with remaining files created) +* 0 when the backup was successful (snapshot with all source files created) +* 1 when there was a fatal error (no snapshot created) +* 3 when some source files could not be read (incomplete snapshot with remaining files created) Fatal errors occur for example when restic is unable to write to the backup destination, when there are network connectivity issues preventing successful communication, or when an invalid diff --git a/mover-restic/restic/doc/045_working_with_repos.rst b/mover-restic/restic/doc/045_working_with_repos.rst index 77c7a15b5..8dba8439f 100644 --- a/mover-restic/restic/doc/045_working_with_repos.rst +++ b/mover-restic/restic/doc/045_working_with_repos.rst @@ -18,19 +18,21 @@ Working with repositories Listing all snapshots ===================== -Now, you can list all the snapshots stored in the repository: +Now, you can list all the snapshots stored in the repository. The size column +only exists for snapshots created using restic 0.17.0 or later. It reflects the +size of the contained files at the time when the snapshot was created. .. code-block:: console $ restic -r /srv/restic-repo snapshots enter password for repository: - ID Date Host Tags Directory - ---------------------------------------------------------------------- - 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work - 79766175 2015-05-08 21:40:19 kasimir /home/user/work - bdbd3439 2015-05-08 21:45:17 luigi /home/art - 590c8fc8 2015-05-08 21:47:38 kazik /srv - 9f0bc19e 2015-05-08 21:46:11 luigi /srv + ID Date Host Tags Directory Size + ------------------------------------------------------------------------- + 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB + 79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB + bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB + 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB + 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB You can filter the listing by directory path: @@ -38,10 +40,10 @@ You can filter the listing by directory path: $ restic -r /srv/restic-repo snapshots --path="/srv" enter password for repository: - ID Date Host Tags Directory - ---------------------------------------------------------------------- - 590c8fc8 2015-05-08 21:47:38 kazik /srv - 9f0bc19e 2015-05-08 21:46:11 luigi /srv + ID Date Host Tags Directory Size + ------------------------------------------------------------------- + 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB + 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB Or filter by host: @@ -49,10 +51,10 @@ Or filter by host: $ restic -r /srv/restic-repo snapshots --host luigi enter password for repository: - ID Date Host Tags Directory - ---------------------------------------------------------------------- - bdbd3439 2015-05-08 21:45:17 luigi /home/art - 9f0bc19e 2015-05-08 21:46:11 luigi /srv + ID Date Host Tags Directory Size + ------------------------------------------------------------------- + bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB + 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB Combining filters is also possible. @@ -64,24 +66,94 @@ Furthermore you can group the output by the same filters (host, paths, tags): enter password for repository: snapshots for (host [kasimir]) - ID Date Host Tags Directory - ---------------------------------------------------------------------- - 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work - 79766175 2015-05-08 21:40:19 kasimir /home/user/work + ID Date Host Tags Directory Size + ------------------------------------------------------------------------ + 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB + 79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB 2 snapshots snapshots for (host [luigi]) - ID Date Host Tags Directory - ---------------------------------------------------------------------- - bdbd3439 2015-05-08 21:45:17 luigi /home/art - 9f0bc19e 2015-05-08 21:46:11 luigi /srv + ID Date Host Tags Directory Size + ------------------------------------------------------------------- + bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB + 9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB 2 snapshots snapshots for (host [kazik]) - ID Date Host Tags Directory - ---------------------------------------------------------------------- - 590c8fc8 2015-05-08 21:47:38 kazik /srv + ID Date Host Tags Directory Size + ------------------------------------------------------------------- + 590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB 1 snapshots +Listing files in a snapshot +=========================== + +To get a list of the files in a specific snapshot you can use the ``ls`` command: + +.. code-block:: console + + $ restic ls 073a90db + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +The special snapshot ID ``latest`` can be used to list files and directories of the latest snapshot in the repository. +The ``--host`` flag can be used in conjunction to select the latest snapshot originating from a certain host only. + +.. code-block:: console + + $ restic ls --host kasimir latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +By default, ``ls`` prints all files in a snapshot. + +File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted +as absolute directory paths, and only files inside those directories will be listed. Files in subdirectories are not +listed when filtering by directories. If the ``--recursive`` flag is used, then subdirectories are also included. +Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/' +as separator. + +.. code-block:: console + + $ restic ls latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + +.. code-block:: console + + $ restic ls --recursive latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +To show more details about the files in a snapshot, you can use the ``--long`` option. The columns include +file permissions, UID, GID, file size, modification time and file path. For scripting usage, the +``ls`` command supports the ``--json`` flag; the JSON output format is described at :ref:`ls json`. + +.. code-block:: console + + $ restic ls --long latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + drwxr-xr-x 0 0 0 2024-01-21 16:50:52 /home + drwxr-xr-x 0 0 0 2024-01-21 16:51:03 /home/user + -rw-r--r-- 0 0 18 2024-01-21 16:51:03 /home/user/work.txt + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. The ``ls`` command supports +outputting information about a snapshot in the NCDU format using the ``--ncdu`` option. + +You can use it as follows: ``restic ls latest --ncdu | ncdu -f -`` + + Copying snapshots between repositories ====================================== @@ -91,14 +163,14 @@ example from a local to a remote repository, you can use the ``copy`` command: .. code-block:: console $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo - repository d6504c63 opened successfully, password is correct - repository 3dd0878c opened successfully, password is correct + repository d6504c63 opened successfully + repository 3dd0878c opened successfully - snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST) + snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir copy started, this may take a while... snapshot 7a746a07 saved - snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST) + snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST by user@kasimir skipping snapshot 4e5d5487, was already copied to snapshot 50eb62b7 The example command copies all snapshots from the source repository @@ -191,20 +263,20 @@ the unwanted files from affected snapshots by rewriting them using the .. code-block:: console $ restic -r /srv/restic-repo rewrite --exclude secret-file - repository c881945a opened (repository version 2) successfully, password is correct + repository c881945a opened (repository version 2) successfully - snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST) + snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir excluding /home/user/work/secret-file saved new snapshot b6aee1ff - snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST) + snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST by user@kasimir modified 1 snapshots $ restic -r /srv/restic-repo rewrite --exclude secret-file 6160ddb2 - repository c881945a opened (repository version 2) successfully, password is correct + repository c881945a opened (repository version 2) successfully - snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST) + snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir excluding /home/user/work/secret-file new snapshot saved as b6aee1ff @@ -234,6 +306,28 @@ modifying the repository. Instead restic will only print the actions it would perform. +Modifying metadata of snapshots +=============================== + +Sometimes it may be desirable to change the metadata of an existing snapshot. +Currently, rewriting the hostname and the time of the backup is supported. +This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. + +.. code-block:: console + + $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11" + + repository b7dbade3 opened (version 2, compression level auto) + [0:00] 100.00% 1 / 1 index files loaded + + snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET by user@kasimir + setting time to 1999-01-01 11:11:11 +0100 CET + setting host to newhost + saved new snapshot c05da643 + + modified 1 snapshots + + .. _checking-integrity: Checking integrity and consistency @@ -274,10 +368,22 @@ detect this and yield the same error as when you tried to restore: $ restic -r /srv/restic-repo check ... load indexes - error: error loading index de30f323: load : invalid data returned - Fatal: LoadIndex returned errors + error: error loading index de30f3231ca2e6a59af4aa84216dfe2ef7339c549dc11b09b84000997b139628: LoadRaw(): invalid data returned + + The repository index is damaged and must be repaired. You must run `restic repair index' to correct this. + + Fatal: repository contains errors + +.. warning:: + + If ``check`` reports an error in the repository, then you must repair the repository. + As long as a repository is damaged, restoring some files or directories will fail. New + snapshots are not guaranteed to be restorable either. + + For instructions how to repair a damaged repository, see the :ref:`troubleshooting` + section or follow the instructions provided by the ``check`` command. -If the repository structure is intact, restic will show that no errors were found: +If the repository structure is intact, restic will show that ``no errors were found``: .. code-block:: console diff --git a/mover-restic/restic/doc/047_tuning_backup_parameters.rst b/mover-restic/restic/doc/047_tuning_backup_parameters.rst index d8fb2c9b6..650f111be 100644 --- a/mover-restic/restic/doc/047_tuning_backup_parameters.rst +++ b/mover-restic/restic/doc/047_tuning_backup_parameters.rst @@ -26,7 +26,8 @@ When you start a backup, restic will concurrently count the number of files and their total size, which is used to estimate how long it will take. This will cause some extra I/O, which can slow down backups of network file systems or FUSE mounts. To avoid this overhead at the cost of not seeing a progress -estimate, use the ``--no-scan`` option which disables this file scanning. +estimate, use the ``--no-scan`` option of the ``backup`` command which disables +this file scanning. Backend Connections =================== @@ -98,7 +99,8 @@ to a 16 MiB pack size. The side effect of increasing the pack size is requiring more disk space for temporary pack files created before uploading. The space must be available in the system default temp -directory, unless overwritten by setting the ``$TMPDIR`` environment variable. In addition, +directory, unless overwritten by setting the ``$TMPDIR`` (except Windows) environment +variable (on Windows use ``$TMP`` or ``$TEMP``). In addition, depending on the backend the memory usage can also increase by a similar amount. Restic requires temporary space according to the pack size, multiplied by the number of backend connections plus one. For example, if the backend uses 5 connections (the default @@ -111,3 +113,28 @@ to disk. An operating system usually caches file write operations in memory and them to disk after a short delay. As larger pack files take longer to upload, this increases the chance of these files being written to disk. This can increase disk wear for SSDs. + + +Feature Flags +============= + +Feature flags allow disabling or enabling certain experimental restic features. The flags +can be specified via the ``RESTIC_FEATURES`` environment variable. The variable expects a +comma-separated list of ``key[=value],key2[=value2]`` pairs. The key is the name of a feature +flag. The value is optional and can contain either the value ``true`` (default if omitted) +or ``false``. The list of currently available feature flags is shown by the ``features`` +command. + +Restic will return an error if an invalid feature flag is specified. No longer relevant +feature flags may be removed in a future restic release. Thus, make sure to no longer +specify these flags. + +A feature can either be in alpha, beta, stable or deprecated state. + +- An _alpha_ feature is disabled by default and may change in arbitrary ways between restic + versions or be removed. +- A _beta_ feature is enabled by default, but still can change in minor ways or be removed. +- A _stable_ feature is always enabled and cannot be disabled. This allows for a transition + period after which the flag will be removed in a future restic version. +- A _deprecated_ feature is always disabled and cannot be enabled. The flag will be removed + in a future restic version. diff --git a/mover-restic/restic/doc/050_restore.rst b/mover-restic/restic/doc/050_restore.rst index 56f6458ed..1a920fad4 100644 --- a/mover-restic/restic/doc/050_restore.rst +++ b/mover-restic/restic/doc/050_restore.rst @@ -68,10 +68,18 @@ There are case insensitive variants of ``--exclude`` and ``--include`` called ``--iexclude`` and ``--iinclude``. These options will behave the same way but ignore the casing of paths. +There are also ``--include-file``, ``--exclude-file``, ``--iinclude-file`` and +``--iexclude-file`` flags that read the include and exclude patterns from a file. + Restoring symbolic links on windows is only possible when the user has ``SeCreateSymbolicLinkPrivilege`` privilege or is running as admin. This is a restriction of windows not restic. +Restoring full security descriptors on Windows is only possible when the user has +``SeRestorePrivilege``, ``SeSecurityPrivilege`` and ``SeTakeOwnershipPrivilege`` +privilege or is running as admin. This is a restriction of Windows not restic. +If either of these conditions are not met, only the DACL will be restored. + By default, restic does not restore files as sparse. Use ``restore --sparse`` to enable the creation of sparse files if supported by the filesystem. Then restic will restore long runs of zero bytes as holes in the corresponding files. @@ -80,6 +88,77 @@ disk space. Note that the exact location of the holes can differ from those in the original file, as their location is determined while restoring and is not stored explicitly. +Restoring in-place +------------------ + +.. note:: + + Restoring data in-place can leave files in a partially restored state if the ``restore`` + operation is interrupted. To ensure you can revert back to the previous state, create + a current ``backup`` before restoring a different snapshot. + +By default, the ``restore`` command overwrites already existing files at the target +directory. This behavior can be configured via the ``--overwrite`` option. The following +values are supported: + +* ``--overwrite always`` (default): always overwrites already existing files. ``restore`` + will verify the existing file content and only restore mismatching parts to minimize + downloads. Updates the metadata of all files. +* ``--overwrite if-changed``: like the previous case, but speeds up the file content check + by assuming that files with matching size and modification time (mtime) are already up to date. + In case of a mismatch, the full file content is verified. Updates the metadata of all files. +* ``--overwrite if-newer``: only overwrite existing files if the file in the snapshot has a + newer modification time (mtime). +* ``--overwrite never``: never overwrite existing files. + +Delete files not in snapshot +---------------------------- + +When restoring into a directory that already contains files, it can be useful to remove all +files that do not exist in the snapshot. For this, pass the ``--delete`` option to the ``restore`` +command. The command will then **delete all files** from the target directory that do not +exist in the snapshot. + +The ``--delete`` option also allows overwriting a non-empty directory if the snapshot contains a +file with the same name. + +.. warning:: + + Always use the ``--dry-run -vv`` option to verify what would be deleted before running the actual + command. + +When specifying ``--include`` or ``--exclude`` options, only files or directories matched by those +options will be deleted. For example, the command +``restic -r /srv/restic-repo restore 79766175:/work --target /tmp/restore-work --include /foo --delete`` +would only delete files within ``/tmp/restore-work/foo``. + +Dry run +------- + +As restore operations can take a long time, it can be useful to perform a dry-run to +see what would be restored without having to run the full restore operation. The +restore command supports the ``--dry-run`` option and prints information about the +restored files when specifying ``--verbose=2``. + +.. code-block:: console + + $ restic restore --target /tmp/restore-work --dry-run --verbose=2 latest + + unchanged /restic/internal/walker/walker.go with size 2.812 KiB + updated /restic/internal/walker/walker_test.go with size 11.143 KiB + restored /restic/restic with size 35.318 MiB + restored /restic + [...] + Summary: Restored 9072 files/dirs (153.597 MiB) in 0:00 + +Files with already up to date content are reported as ``unchanged``. Files whose content +was modified are ``updated`` and files that are new are shown as ``restored``. Directories +and other file types like symlinks are always reported as ``restored``. + +To reliably determine which files would be updated, a dry-run also verifies the content of +already existing files according to the specified overwrite behavior. To skip these checks +either specify ``--overwrite never`` or specify a non-existing ``--target`` directory. + Restore using mount =================== @@ -98,9 +177,9 @@ command to serve the repository with FUSE: Mounting repositories via FUSE is only possible on Linux, macOS and FreeBSD. On Linux, the ``fuse`` kernel module needs to be loaded and the ``fusermount`` -command needs to be in the ``PATH``. On macOS, you need `FUSE for macOS -`__. On FreeBSD, you may need to install FUSE -and load the kernel module (``kldload fuse``). +command needs to be in the ``PATH``. On macOS, you need `FUSE-T +`__ or `FUSE for macOS `__. +On FreeBSD, you may need to install FUSE and load the kernel module (``kldload fuse``). Restic supports storage and preservation of hard links. However, since hard links exist in the scope of a filesystem by definition, restoring @@ -174,3 +253,10 @@ To include the folder content at the root of the archive, you can use the `` restore.tar + +It is also possible to ``dump`` the contents of a selected snapshot and folder +structure to a file using the ``--target`` flag. + +.. code-block:: console + + $ restic -r /srv/restic-repo dump latest / --target /home/linux.user/output.tar -a tar diff --git a/mover-restic/restic/doc/060_forget.rst b/mover-restic/restic/doc/060_forget.rst index caeb6313a..fe0236f12 100644 --- a/mover-restic/restic/doc/060_forget.rst +++ b/mover-restic/restic/doc/060_forget.rst @@ -80,7 +80,7 @@ command must be run: $ restic -r /srv/restic-repo prune enter password for repository: - repository 33002c5e opened successfully, password is correct + repository 33002c5e opened successfully loading all snapshots... loading indexes... finding data that is still in use for 4 snapshots @@ -182,7 +182,9 @@ The ``forget`` command accepts the following policy options: - ``--keep-yearly n`` for the last ``n`` years which have one or more snapshots, keep only the most recent one for each year. - ``--keep-tag`` keep all snapshots which have all tags specified by - this option (can be specified multiple times). + this option (can be specified multiple times). The ``forget`` command will + exit with an error if all snapshots in a snapshot group would be removed + as none of them have the specified tags. - ``--keep-within duration`` keep all snapshots having a timestamp within the specified duration of the latest snapshot, where ``duration`` is a number of years, months, days, and hours. E.g. ``2y5m7d3h`` will keep all @@ -205,7 +207,7 @@ The ``forget`` command accepts the following policy options: natural time boundaries and *not* relative to when you run ``forget``. Weeks are Monday 00:00 to Sunday 23:59, days 00:00 to 23:59, hours :00 to :59, etc. They also only count hours/days/weeks/etc which have one or more snapshots. - A value of ``-1`` will be interpreted as "forever", i.e. "keep all". + A value of ``unlimited`` will be interpreted as "forever", i.e. "keep all". .. note:: All duration related options (``--keep-{within-,}*``) ignore snapshots with a timestamp in the future (relative to when the ``forget`` command is @@ -263,7 +265,7 @@ Sunday for 12 weeks: .. code-block:: console $ restic snapshots - repository f00c6e2a opened successfully, password is correct + repository f00c6e2a opened successfully ID Time Host Tags Paths --------------------------------------------------------------- 0a1f9759 2019-09-01 11:00:00 mopped /home/user/work @@ -287,7 +289,7 @@ four Sundays, and remove the other snapshots: .. code-block:: console $ restic forget --keep-daily 4 --dry-run - repository f00c6e2a opened successfully, password is correct + repository f00c6e2a opened successfully Applying Policy: keep the last 4 daily snapshots keep 4 snapshots: ID Time Host Tags Reasons Paths @@ -336,12 +338,23 @@ year and yearly for the last 75 years, you can instead specify ``forget --keep-within-yearly 75y`` (note that `1w` is not a recognized duration, so you will have to specify `7d` instead). + +Removing all snapshots +====================== + For safety reasons, restic refuses to act on an "empty" policy. For example, if one were to specify ``--keep-last 0`` to forget *all* snapshots in the repository, restic will respond that no snapshots will be removed. To delete all snapshots, use ``--keep-last 1`` and then finally remove the last snapshot manually (by passing the ID to ``forget``). +Since restic 0.17.0, it is possible to delete all snapshots for a specific +host, tag or path using the ``--unsafe-allow-remove-all`` option. The option +must always be combined with a snapshot filter (by host, path or tag). +For example the command ``forget --tag example --unsafe-allow-remove-all`` +removes all snapshots with tag ``example``. + + Security considerations in append-only mode =========================================== diff --git a/mover-restic/restic/doc/075_scripting.rst b/mover-restic/restic/doc/075_scripting.rst index fe41ac870..87ae4fcf4 100644 --- a/mover-restic/restic/doc/075_scripting.rst +++ b/mover-restic/restic/doc/075_scripting.rst @@ -21,23 +21,51 @@ Check if a repository is already initialized ******************************************** You may find a need to check if a repository is already initialized, -perhaps to prevent your script from initializing a repository multiple -times. The command ``cat config`` may be used for this purpose: +perhaps to prevent your script from trying to initialize a repository multiple +times (the ``init`` command contains a check to prevent overwriting existing +repositories). The command ``cat config`` may be used for this purpose: .. code-block:: console $ restic -r /srv/restic-repo cat config - Fatal: unable to open config file: stat /srv/restic-repo/config: no such file or directory + Fatal: repository does not exist: unable to open config file: stat /srv/restic-repo/config: no such file or directory Is there a repository at the following location? /srv/restic-repo -If a repository does not exist, restic will return a non-zero exit code -and print an error message. Note that restic will also return a non-zero -exit code if a different error is encountered (e.g.: incorrect password -to ``cat config``) and it may print a different error message. If there -are no errors, restic will return a zero exit code and print the repository +If a repository does not exist, restic (since 0.17.0) will return exit code ``10`` +and print a corresponding error message. Older versions return exit code ``1``. +Note that restic will also return exit code ``1`` if a different error is encountered +(e.g.: incorrect password to ``cat config``) and it may print a different error message. +If there are no errors, restic will return a zero exit code and print the repository metadata. +Exit codes +********** + +Restic commands return an exit code that signals whether the command was successful. +The following table provides a general description, see the help of each command for +a more specific description. + +.. warning:: + New exit codes will be added over time. If an unknown exit code is returned, then it + MUST be treated as a command failure. + ++-----+----------------------------------------------------+ +| 0 | Command was successful | ++-----+----------------------------------------------------+ +| 1 | Command failed, see command help for more details | ++-----+----------------------------------------------------+ +| 2 | Go runtime error | ++-----+----------------------------------------------------+ +| 3 | ``backup`` command could not read some source data | ++-----+----------------------------------------------------+ +| 10 | Repository does not exist (since restic 0.17.0) | ++-----+----------------------------------------------------+ +| 11 | Failed to lock repository (since restic 0.17.0) | ++-----+----------------------------------------------------+ +| 130 | Restic was interrupted using SIGINT or SIGSTOP | ++-----+----------------------------------------------------+ + JSON output *********** @@ -75,9 +103,6 @@ Several commands, in particular long running ones or those that generate a large use a format also known as JSON lines. It consists of a stream of new-line separated JSON messages. You can determine the nature of the message using the ``message_type`` field. -As an exception, the ``ls`` command uses the field ``struct_type`` instead. - - backup ------ @@ -166,7 +191,9 @@ Summary is the last output line in a successful backup. +---------------------------+---------------------------------------------------------+ | ``tree_blobs`` | Number of tree blobs | +---------------------------+---------------------------------------------------------+ -| ``data_added`` | Amount of data added, in bytes | +| ``data_added`` | Amount of (uncompressed) data added, in bytes | ++---------------------------+---------------------------------------------------------+ +| ``data_added_packed`` | Amount of data added (after compression), in bytes | +---------------------------+---------------------------------------------------------+ | ``total_files_processed`` | Total number of files processed | +---------------------------+---------------------------------------------------------+ @@ -174,7 +201,8 @@ Summary is the last output line in a successful backup. +---------------------------+---------------------------------------------------------+ | ``total_duration`` | Total time it took for the operation to complete | +---------------------------+---------------------------------------------------------+ -| ``snapshot_id`` | ID of the new snapshot | +| ``snapshot_id`` | ID of the new snapshot. Field is omitted if snapshot | +| | creation was skipped | +---------------------------+---------------------------------------------------------+ @@ -201,7 +229,8 @@ change +------------------+--------------------------------------------------------------+ | ``modifier`` | Type of change, a concatenation of the following characters: | | | "+" = added, "-" = removed, "T" = entry type changed, | -| | "M" = file content changed, "U" = metadata changed | +| | "M" = file content changed, "U" = metadata changed, | +| | "?" = bitrot detected | +------------------+--------------------------------------------------------------+ statistics @@ -367,13 +396,13 @@ Snapshot object Reason object -+----------------+---------------------------------------------------------+ -| ``snapshot`` | Snapshot object, without ``id`` and ``short_id`` fields | -+----------------+---------------------------------------------------------+ -| ``matches`` | Array containing descriptions of the matching criteria | -+----------------+---------------------------------------------------------+ -| ``counters`` | Object containing counters used by the policies | -+----------------+---------------------------------------------------------+ ++----------------+-----------------------------------------------------------+ +| ``snapshot`` | Snapshot object, including ``id`` and ``short_id`` fields | ++----------------+-----------------------------------------------------------+ +| ``matches`` | Array containing descriptions of the matching criteria | ++----------------+-----------------------------------------------------------+ +| ``counters`` | Object containing counters used by the policies | ++----------------+-----------------------------------------------------------+ init @@ -408,6 +437,8 @@ The ``key list`` command returns an array of objects with the following structur +--------------+------------------------------------+ +.. _ls json: + ls -- @@ -417,63 +448,67 @@ As an exception, the ``struct_type`` field is used to determine the message type snapshot ^^^^^^^^ -+----------------+--------------------------------------------------+ -| ``struct_type``| Always "snapshot" | -+----------------+--------------------------------------------------+ -| ``time`` | Timestamp of when the backup was started | -+----------------+--------------------------------------------------+ -| ``parent`` | ID of the parent snapshot | -+----------------+--------------------------------------------------+ -| ``tree`` | ID of the root tree blob | -+----------------+--------------------------------------------------+ -| ``paths`` | List of paths included in the backup | -+----------------+--------------------------------------------------+ -| ``hostname`` | Hostname of the backed up machine | -+----------------+--------------------------------------------------+ -| ``username`` | Username the backup command was run as | -+----------------+--------------------------------------------------+ -| ``uid`` | ID of owner | -+----------------+--------------------------------------------------+ -| ``gid`` | ID of group | -+----------------+--------------------------------------------------+ -| ``excludes`` | List of paths and globs excluded from the backup | -+----------------+--------------------------------------------------+ -| ``tags`` | List of tags for the snapshot in question | -+----------------+--------------------------------------------------+ -| ``id`` | Snapshot ID | -+----------------+--------------------------------------------------+ -| ``short_id`` | Snapshot ID, short form | -+----------------+--------------------------------------------------+ ++------------------+--------------------------------------------------+ +| ``message_type`` | Always "snapshot" | ++------------------+--------------------------------------------------+ +| ``struct_type`` | Always "snapshot" (deprecated) | ++------------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++------------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++------------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++------------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++------------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++------------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++------------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++------------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++------------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++------------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++------------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++------------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++------------------+--------------------------------------------------+ node ^^^^ -+-----------------+--------------------------+ -| ``struct_type`` | Always "node" | -+-----------------+--------------------------+ -| ``name`` | Node name | -+-----------------+--------------------------+ -| ``type`` | Node type | -+-----------------+--------------------------+ -| ``path`` | Node path | -+-----------------+--------------------------+ -| ``uid`` | UID of node | -+-----------------+--------------------------+ -| ``gid`` | GID of node | -+-----------------+--------------------------+ -| ``size`` | Size in bytes | -+-----------------+--------------------------+ -| ``mode`` | Node mode | -+-----------------+--------------------------+ -| ``atime`` | Node access time | -+-----------------+--------------------------+ -| ``mtime`` | Node modification time | -+-----------------+--------------------------+ -| ``ctime`` | Node creation time | -+-----------------+--------------------------+ -| ``inode`` | Inode number of node | -+-----------------+--------------------------+ ++------------------+----------------------------+ +| ``message_type`` | Always "node" | ++------------------+----------------------------+ +| ``struct_type`` | Always "node" (deprecated) | ++------------------+----------------------------+ +| ``name`` | Node name | ++------------------+----------------------------+ +| ``type`` | Node type | ++------------------+----------------------------+ +| ``path`` | Node path | ++------------------+----------------------------+ +| ``uid`` | UID of node | ++------------------+----------------------------+ +| ``gid`` | GID of node | ++------------------+----------------------------+ +| ``size`` | Size in bytes | ++------------------+----------------------------+ +| ``mode`` | Node mode | ++------------------+----------------------------+ +| ``atime`` | Node access time | ++------------------+----------------------------+ +| ``mtime`` | Node modification time | ++------------------+----------------------------+ +| ``ctime`` | Node creation time | ++------------------+----------------------------+ +| ``inode`` | Inode number of node | ++------------------+----------------------------+ restore @@ -495,11 +530,30 @@ Status +----------------------+------------------------------------------------------------+ |``files_restored`` | Files restored | +----------------------+------------------------------------------------------------+ +|``files_skipped`` | Files skipped due to overwrite setting | ++----------------------+------------------------------------------------------------+ |``total_bytes`` | Total number of bytes in restore set | +----------------------+------------------------------------------------------------+ |``bytes_restored`` | Number of bytes restored | +----------------------+------------------------------------------------------------+ +|``bytes_skipped`` | Total size of skipped files | ++----------------------+------------------------------------------------------------+ + +Verbose Status +^^^^^^^^^^^^^^ + +Verbose status provides details about the progress, including details about restored files. +Only printed if `--verbose=2` is specified. ++----------------------+-----------------------------------------------------------+ +| ``message_type`` | Always "verbose_status" | ++----------------------+-----------------------------------------------------------+ +| ``action`` | Either "restored", "updated", "unchanged" or "deleted" | ++----------------------+-----------------------------------------------------------+ +| ``item`` | The item in question | ++----------------------+-----------------------------------------------------------+ +| ``size`` | Size of the item in bytes | ++----------------------+-----------------------------------------------------------+ Summary ^^^^^^^ @@ -513,10 +567,14 @@ Summary +----------------------+------------------------------------------------------------+ |``files_restored`` | Files restored | +----------------------+------------------------------------------------------------+ +|``files_skipped`` | Files skipped due to overwrite setting | ++----------------------+------------------------------------------------------------+ |``total_bytes`` | Total number of bytes in restore set | +----------------------+------------------------------------------------------------+ |``bytes_restored`` | Number of bytes restored | +----------------------+------------------------------------------------------------+ +|``bytes_skipped`` | Total size of skipped files | ++----------------------+------------------------------------------------------------+ snapshots @@ -547,11 +605,48 @@ The snapshots command returns a single JSON object, an array with objects of the +---------------------+--------------------------------------------------+ | ``program_version`` | restic version used to create snapshot | +---------------------+--------------------------------------------------+ +| ``summary`` | Snapshot statistics, see "Summary object" | ++---------------------+--------------------------------------------------+ | ``id`` | Snapshot ID | +---------------------+--------------------------------------------------+ | ``short_id`` | Snapshot ID, short form | +---------------------+--------------------------------------------------+ +Summary object + +The contained statistics reflect the information at the point in time when the snapshot +was created. + ++---------------------------+---------------------------------------------------------+ +| ``backup_start`` | Time at which the backup was started | ++---------------------------+---------------------------------------------------------+ +| ``backup_end`` | Time at which the backup was completed | ++---------------------------+---------------------------------------------------------+ +| ``files_new`` | Number of new files | ++---------------------------+---------------------------------------------------------+ +| ``files_changed`` | Number of files that changed | ++---------------------------+---------------------------------------------------------+ +| ``files_unmodified`` | Number of files that did not change | ++---------------------------+---------------------------------------------------------+ +| ``dirs_new`` | Number of new directories | ++---------------------------+---------------------------------------------------------+ +| ``dirs_changed`` | Number of directories that changed | ++---------------------------+---------------------------------------------------------+ +| ``dirs_unmodified`` | Number of directories that did not change | ++---------------------------+---------------------------------------------------------+ +| ``data_blobs`` | Number of data blobs | ++---------------------------+---------------------------------------------------------+ +| ``tree_blobs`` | Number of tree blobs | ++---------------------------+---------------------------------------------------------+ +| ``data_added`` | Amount of (uncompressed) data added, in bytes | ++---------------------------+---------------------------------------------------------+ +| ``data_added_packed`` | Amount of data added (after compression), in bytes | ++---------------------------+---------------------------------------------------------+ +| ``total_files_processed`` | Total number of files processed | ++---------------------------+---------------------------------------------------------+ +| ``total_bytes_processed`` | Total number of bytes processed | ++---------------------------+---------------------------------------------------------+ + stats ----- @@ -576,3 +671,19 @@ The stats command returns a single JSON object. +------------------------------+-----------------------------------------------------+ | ``compression_space_saving`` | Overall space saving due to compression | +------------------------------+-----------------------------------------------------+ + + +version +------- + +The version command returns a single JSON object. + ++----------------+--------------------+ +| ``version`` | restic version | ++----------------+--------------------+ +| ``go_version`` | Go compile version | ++----------------+--------------------+ +| ``go_os`` | Go OS | ++----------------+--------------------+ +| ``go_arch`` | Go architecture | ++----------------+--------------------+ diff --git a/mover-restic/restic/doc/077_troubleshooting.rst b/mover-restic/restic/doc/077_troubleshooting.rst index fe317acfc..36c9d63ec 100644 --- a/mover-restic/restic/doc/077_troubleshooting.rst +++ b/mover-restic/restic/doc/077_troubleshooting.rst @@ -10,6 +10,8 @@ ^ for subsubsections " for paragraphs +.. _troubleshooting: + ######################### Troubleshooting ######################### @@ -71,11 +73,15 @@ some blobs in the repository, then please ask for help in the forum or our IRC channel. These errors are often caused by hardware problems which **must** be investigated and fixed. Otherwise, the backup will be damaged again and again. -Similarly, if a repository is repeatedly damaged, please open an `issue on Github +Similarly, if a repository is repeatedly damaged, please open an `issue on GitHub `_ as this could indicate a bug somewhere. Please include the check output and additional information that might help locate the problem. +If ``check`` detects damaged pack files, it will show instructions on how to repair +them using the ``repair pack`` command. Use that command instead of the "Repair the +index" section in this guide. + 2. Backup the repository ************************ @@ -98,12 +104,17 @@ remove data unexpectedly. Please take the time to understand what the commands described in the following do. If you are unsure, then ask for help in the forum or our IRC channel. Search whether your issue is already known and solved. Please take a look at the -`forum`_ and `Github issues `_. +`forum`_ and `GitHub issues `_. 3. Repair the index ******************* +.. note:: + + If the `check` command tells you to run `restic repair pack`, then use that + command instead. It will repair the damaged pack files and also update the index. + Restic relies on its index to contain correct information about what data is stored in the repository. Thus, the first step to repair a repository is to repair the index: @@ -153,7 +164,7 @@ command will automatically remove the original, damaged snapshots. $ restic repair snapshots --forget - snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET) + snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET by user@host file "/restic/internal/fuse/snapshots_dir.go": removed missing content file "/restic/internal/restorer/restorer_unix_test.go": removed missing content file "/restic/internal/walker/walker.go": removed missing content diff --git a/mover-restic/restic/doc/REST_backend.rst b/mover-restic/restic/doc/REST_backend.rst index f9d72cf06..9e85187f9 100644 --- a/mover-restic/restic/doc/REST_backend.rst +++ b/mover-restic/restic/doc/REST_backend.rst @@ -7,18 +7,18 @@ API. The following values are valid for ``{type}``: - * ``data`` - * ``keys`` - * ``locks`` - * ``snapshots`` - * ``index`` - * ``config`` +* ``data`` +* ``keys`` +* ``locks`` +* ``snapshots`` +* ``index`` +* ``config`` The API version is selected via the ``Accept`` HTTP header in the request. The following values are defined: - * ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1 - * ``application/vnd.x.restic.rest.v2``: Select API version 2 +* ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1 +* ``application/vnd.x.restic.rest.v2``: Select API version 2 The server will respond with the value of the highest version it supports in the ``Content-Type`` HTTP response header for the HTTP requests which should diff --git a/mover-restic/restic/doc/bash-completion.sh b/mover-restic/restic/doc/bash-completion.sh index cae37a6ca..9d64871ca 100644 --- a/mover-restic/restic/doc/bash-completion.sh +++ b/mover-restic/restic/doc/bash-completion.sh @@ -49,7 +49,7 @@ __restic_handle_go_custom_completion() local out requestComp lastParam lastChar comp directive args # Prepare the command to request completions for the program. - # Calling ${words[0]} instead of directly restic allows to handle aliases + # Calling ${words[0]} instead of directly restic allows handling aliases args=("${words[@]:1}") # Disable ActiveHelp which is not supported for bash completion v1 requestComp="RESTIC_ACTIVE_HELP=0 ${words[0]} __completeNoDesc ${args[*]}" @@ -456,12 +456,16 @@ _restic_backup() two_word_flags+=("--read-concurrency") local_nonpersistent_flags+=("--read-concurrency") local_nonpersistent_flags+=("--read-concurrency=") + flags+=("--skip-if-unchanged") + local_nonpersistent_flags+=("--skip-if-unchanged") flags+=("--stdin") local_nonpersistent_flags+=("--stdin") flags+=("--stdin-filename=") two_word_flags+=("--stdin-filename") local_nonpersistent_flags+=("--stdin-filename") local_nonpersistent_flags+=("--stdin-filename=") + flags+=("--stdin-from-command") + local_nonpersistent_flags+=("--stdin-from-command") flags+=("--tag=") two_word_flags+=("--tag") local_nonpersistent_flags+=("--tag") @@ -479,6 +483,9 @@ _restic_backup() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -552,6 +559,9 @@ _restic_cache() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -617,6 +627,9 @@ _restic_cat() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -690,6 +703,9 @@ _restic_check() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -744,6 +760,8 @@ _restic_copy() flags_with_completion=() flags_completion=() + flags+=("--from-insecure-no-password") + local_nonpersistent_flags+=("--from-insecure-no-password") flags+=("--from-key-hint=") two_word_flags+=("--from-key-hint") local_nonpersistent_flags+=("--from-key-hint") @@ -789,6 +807,9 @@ _restic_copy() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -856,6 +877,9 @@ _restic_diff() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -934,6 +958,12 @@ _restic_dump() two_word_flags+=("--tag") local_nonpersistent_flags+=("--tag") local_nonpersistent_flags+=("--tag=") + flags+=("--target=") + two_word_flags+=("--target") + two_word_flags+=("-t") + local_nonpersistent_flags+=("--target") + local_nonpersistent_flags+=("--target=") + local_nonpersistent_flags+=("-t") flags+=("--cacert=") two_word_flags+=("--cacert") flags+=("--cache-dir=") @@ -941,6 +971,9 @@ _restic_dump() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1056,6 +1089,9 @@ _restic_find() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1174,6 +1210,8 @@ _restic_forget() two_word_flags+=("--keep-tag") local_nonpersistent_flags+=("--keep-tag") local_nonpersistent_flags+=("--keep-tag=") + flags+=("--unsafe-allow-remove-all") + local_nonpersistent_flags+=("--unsafe-allow-remove-all") flags+=("--host=") two_word_flags+=("--host") local_nonpersistent_flags+=("--host") @@ -1227,6 +1265,9 @@ _restic_forget() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1312,6 +1353,9 @@ _restic_generate() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1373,6 +1417,9 @@ _restic_help() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1430,6 +1477,8 @@ _restic_init() flags+=("--copy-chunker-params") local_nonpersistent_flags+=("--copy-chunker-params") + flags+=("--from-insecure-no-password") + local_nonpersistent_flags+=("--from-insecure-no-password") flags+=("--from-key-hint=") two_word_flags+=("--from-key-hint") local_nonpersistent_flags+=("--from-key-hint") @@ -1465,6 +1514,9 @@ _restic_init() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1505,9 +1557,224 @@ _restic_init() noun_aliases=() } -_restic_key() +_restic_key_add() { - last_command="restic_key" + last_command="restic_key_add" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--host=") + two_word_flags+=("--host") + local_nonpersistent_flags+=("--host") + local_nonpersistent_flags+=("--host=") + flags+=("--new-insecure-no-password") + local_nonpersistent_flags+=("--new-insecure-no-password") + flags+=("--new-password-file=") + two_word_flags+=("--new-password-file") + local_nonpersistent_flags+=("--new-password-file") + local_nonpersistent_flags+=("--new-password-file=") + flags+=("--user=") + two_word_flags+=("--user") + local_nonpersistent_flags+=("--user") + local_nonpersistent_flags+=("--user=") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_key_help() +{ + last_command="restic_key_help" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + has_completion_function=1 + noun_aliases=() +} + +_restic_key_list() +{ + last_command="restic_key_list" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_key_passwd() +{ + last_command="restic_key_passwd" command_aliases=() @@ -1527,6 +1794,8 @@ _restic_key() two_word_flags+=("--host") local_nonpersistent_flags+=("--host") local_nonpersistent_flags+=("--host=") + flags+=("--new-insecure-no-password") + local_nonpersistent_flags+=("--new-insecure-no-password") flags+=("--new-password-file=") two_word_flags+=("--new-password-file") local_nonpersistent_flags+=("--new-password-file") @@ -1542,6 +1811,150 @@ _restic_key() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_key_remove() +{ + last_command="restic_key_remove" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_key() +{ + last_command="restic_key" + + command_aliases=() + + commands=() + commands+=("add") + commands+=("help") + commands+=("list") + commands+=("passwd") + commands+=("remove") + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1607,6 +2020,9 @@ _restic_list() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1677,6 +2093,8 @@ _restic_ls() flags+=("-l") local_nonpersistent_flags+=("--long") local_nonpersistent_flags+=("-l") + flags+=("--ncdu") + local_nonpersistent_flags+=("--ncdu") flags+=("--path=") two_word_flags+=("--path") local_nonpersistent_flags+=("--path") @@ -1694,6 +2112,9 @@ _restic_ls() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1763,6 +2184,9 @@ _restic_migrate() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1856,6 +2280,9 @@ _restic_mount() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -1943,6 +2370,9 @@ _restic_prune() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2008,6 +2438,9 @@ _restic_recover() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2069,6 +2502,9 @@ _restic_repair_help() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2137,6 +2573,9 @@ _restic_repair_index() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2202,6 +2641,9 @@ _restic_repair_packs() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2287,6 +2729,9 @@ _restic_repair_snapshots() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2356,6 +2801,9 @@ _restic_repair() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2410,12 +2858,20 @@ _restic_restore() flags_with_completion=() flags_completion=() + flags+=("--delete") + local_nonpersistent_flags+=("--delete") + flags+=("--dry-run") + local_nonpersistent_flags+=("--dry-run") flags+=("--exclude=") two_word_flags+=("--exclude") two_word_flags+=("-e") local_nonpersistent_flags+=("--exclude") local_nonpersistent_flags+=("--exclude=") local_nonpersistent_flags+=("-e") + flags+=("--exclude-file=") + two_word_flags+=("--exclude-file") + local_nonpersistent_flags+=("--exclude-file") + local_nonpersistent_flags+=("--exclude-file=") flags+=("--help") flags+=("-h") local_nonpersistent_flags+=("--help") @@ -2430,16 +2886,32 @@ _restic_restore() two_word_flags+=("--iexclude") local_nonpersistent_flags+=("--iexclude") local_nonpersistent_flags+=("--iexclude=") + flags+=("--iexclude-file=") + two_word_flags+=("--iexclude-file") + local_nonpersistent_flags+=("--iexclude-file") + local_nonpersistent_flags+=("--iexclude-file=") flags+=("--iinclude=") two_word_flags+=("--iinclude") local_nonpersistent_flags+=("--iinclude") local_nonpersistent_flags+=("--iinclude=") + flags+=("--iinclude-file=") + two_word_flags+=("--iinclude-file") + local_nonpersistent_flags+=("--iinclude-file") + local_nonpersistent_flags+=("--iinclude-file=") flags+=("--include=") two_word_flags+=("--include") two_word_flags+=("-i") local_nonpersistent_flags+=("--include") local_nonpersistent_flags+=("--include=") local_nonpersistent_flags+=("-i") + flags+=("--include-file=") + two_word_flags+=("--include-file") + local_nonpersistent_flags+=("--include-file") + local_nonpersistent_flags+=("--include-file=") + flags+=("--overwrite=") + two_word_flags+=("--overwrite") + local_nonpersistent_flags+=("--overwrite") + local_nonpersistent_flags+=("--overwrite=") flags+=("--path=") two_word_flags+=("--path") local_nonpersistent_flags+=("--path") @@ -2465,6 +2937,9 @@ _restic_restore() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2553,6 +3028,14 @@ _restic_rewrite() two_word_flags+=("--iexclude-file") local_nonpersistent_flags+=("--iexclude-file") local_nonpersistent_flags+=("--iexclude-file=") + flags+=("--new-host=") + two_word_flags+=("--new-host") + local_nonpersistent_flags+=("--new-host") + local_nonpersistent_flags+=("--new-host=") + flags+=("--new-time=") + two_word_flags+=("--new-time") + local_nonpersistent_flags+=("--new-time") + local_nonpersistent_flags+=("--new-time=") flags+=("--path=") two_word_flags+=("--path") local_nonpersistent_flags+=("--path") @@ -2568,6 +3051,9 @@ _restic_rewrite() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2637,6 +3123,9 @@ _restic_self-update() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2730,6 +3219,9 @@ _restic_snapshots() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2813,6 +3305,9 @@ _restic_stats() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2904,6 +3399,9 @@ _restic_tag() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -2971,6 +3469,9 @@ _restic_unlock() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -3036,6 +3537,9 @@ _restic_version() flags+=("--cleanup-cache") flags+=("--compression=") two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") @@ -3129,6 +3633,9 @@ _restic_root_command() flags+=("-h") local_nonpersistent_flags+=("--help") local_nonpersistent_flags+=("-h") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") flags+=("--insecure-tls") flags+=("--json") flags+=("--key-hint=") diff --git a/mover-restic/restic/doc/design.rst b/mover-restic/restic/doc/design.rst index a58f803ea..7fb8b71b2 100644 --- a/mover-restic/restic/doc/design.rst +++ b/mover-restic/restic/doc/design.rst @@ -48,7 +48,7 @@ be used instead of the complete filename. Apart from the files stored within the ``keys`` and ``data`` directories, all files are encrypted with AES-256 in counter mode (CTR). The integrity of the encrypted data is secured by a Poly1305-AES message authentication -code (sometimes also referred to as a "signature"). +code (MAC). Files in the ``data`` directory ("pack files") consist of multiple parts which are all independently encrypted and authenticated, see below. @@ -296,8 +296,8 @@ of a JSON document like the following: } This JSON document lists Packs and the blobs contained therein. In this -example, the Pack ``73d04e61`` contains two data Blobs and one Tree -blob, the plaintext hashes are listed afterwards. The ``length`` field +example, the Pack ``73d04e61`` contains three data Blobs, +the plaintext hashes are listed afterwards. The ``length`` field corresponds to ``Length(encrypted_blob)`` in the pack file header. Field ``uncompressed_length`` is only present for compressed blobs and therefore is never present in version 1 of the repository format. It is @@ -824,4 +824,4 @@ Changes Repository Version 2 -------------------- - * Support compression for blobs (data/tree) and index / lock / snapshot files +* Support compression for blobs (data/tree) and index / lock / snapshot files diff --git a/mover-restic/restic/doc/developer_information.rst b/mover-restic/restic/doc/developer_information.rst index 9de517901..f0fe28c32 100644 --- a/mover-restic/restic/doc/developer_information.rst +++ b/mover-restic/restic/doc/developer_information.rst @@ -9,14 +9,14 @@ restic for version 0.10.0 and later. For restic versions down to 0.9.3 please refer to the documentation for the respective version. The binary produced depends on the following things: - * The source code for the release - * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) - * The architecture and operating system the Go compiler runs on (Linux, ``amd64``) - * The build tags (for official binaries, it's the tag ``selfupdate``) - * The path where the source code is extracted to (``/restic``) - * The path to the Go compiler (``/usr/local/go``) - * The path to the Go workspace (``GOPATH=/home/build/go``) - * Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``) +* The source code for the release +* The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) +* The architecture and operating system the Go compiler runs on (Linux, ``amd64``) +* The build tags (for official binaries, it's the tag ``selfupdate``) +* The path where the source code is extracted to (``/restic``) +* The path to the Go compiler (``/usr/local/go``) +* The path to the Go workspace (``GOPATH=/home/build/go``) +* Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``) In addition, The compressed ZIP files for Windows depends on the modification timestamp and filename of the binary contained in it. In order to reproduce the @@ -69,9 +69,9 @@ container can be found in the `GitHub repository `__ The container serves the following goals: - * Have a very controlled environment which is independent from the local system - * Make it easy to have the correct version of the Go compiler at the right path - * Make it easy to pass in the source code to build at a well-defined path +* Have a very controlled environment which is independent from the local system +* Make it easy to have the correct version of the Go compiler at the right path +* Make it easy to pass in the source code to build at a well-defined path The following steps are necessary to build the binaries: @@ -113,6 +113,26 @@ The following steps are necessary to build the binaries: restic/builder \ go run helpers/build-release-binaries/main.go --version 0.14.0 --verbose +Verifying the Official Binaries +******************************* + +To verify the official binaries, you can either build them yourself using the above +instructions or use the ``helpers/verify-release-binaries.sh`` script from the restic +repository. Run it as ``helpers/verify-release-binaries.sh restic_version go_version``. +The specified go compiler version must match the one used to build the official +binaries. For example, for restic 0.16.2 the command would be +``helpers/verify-release-binaries.sh 0.16.2 1.21.3``. + +The script requires bash, curl, docker (version >= 25.0), git, gpg, shasum and tar. + +The script first downloads all release binaries, checks the SHASUM256 file and its +signature. Afterwards it checks that the tarball matches the restic git repository +contents, before first reproducing the builder docker container and finally the +restic binaries. As final step, the restic binary in both the docker hub images +and the GitHub container registry is verified. If any step fails, then the script +will issue a warning. + + Prepare a New Release ********************* diff --git a/mover-restic/restic/doc/faq.rst b/mover-restic/restic/doc/faq.rst index 8e56b5d9e..19879d817 100644 --- a/mover-restic/restic/doc/faq.rst +++ b/mover-restic/restic/doc/faq.rst @@ -100,7 +100,7 @@ Restic handles globbing and expansion in the following ways: - Globbing is only expanded for lines read via ``--files-from`` - Environment variables are not expanded in the file read via ``--files-from`` - ``*`` is expanded for paths read via ``--files-from`` -- e.g. For backup targets given to restic as arguments on the shell, neither glob expansion nor shell variable replacement is done. If restic is called as ``restic backup '*' '$HOME'``, it will try to backup the literal file(s)/dir(s) ``*`` and ``$HOME`` +- e.g. For backup sources given to restic as arguments on the shell, neither glob expansion nor shell variable replacement is done. If restic is called as ``restic backup '*' '$HOME'``, it will try to backup the literal file(s)/dir(s) ``*`` and ``$HOME`` - Double-asterisk ``**`` only works in exclude patterns as this is a custom extension built into restic; the shell must not expand it diff --git a/mover-restic/restic/doc/fish-completion.fish b/mover-restic/restic/doc/fish-completion.fish index f9d7801e1..7db10cb20 100644 --- a/mover-restic/restic/doc/fish-completion.fish +++ b/mover-restic/restic/doc/fish-completion.fish @@ -79,7 +79,7 @@ function __restic_clear_perform_completion_once_result __restic_debug "" __restic_debug "========= clearing previously set __restic_perform_completion_once_result variable ==========" set --erase __restic_perform_completion_once_result - __restic_debug "Succesfully erased the variable __restic_perform_completion_once_result" + __restic_debug "Successfully erased the variable __restic_perform_completion_once_result" end function __restic_requires_order_preservation diff --git a/mover-restic/restic/doc/man/restic-backup.1 b/mover-restic/restic/doc/man/restic-backup.1 index 730685271..cda4aadff 100644 --- a/mover-restic/restic/doc/man/restic-backup.1 +++ b/mover-restic/restic/doc/man/restic-backup.1 @@ -22,6 +22,8 @@ given as the arguments. Exit status is 0 if the command was successful. Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -63,7 +65,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-f\fP, \fB--force\fP[=false] - force re-reading the target files/directories (overrides the "parent" flag) + force re-reading the source files/directories (overrides the "parent" flag) .PP \fB-g\fP, \fB--group-by\fP=host,paths @@ -75,7 +77,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB-H\fP, \fB--host\fP="" - set the \fBhostname\fR for the snapshot manually. To prevent an expensive rescan use the "parent" flag + set the \fBhostname\fR for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the "parent" flag .PP \fB--iexclude\fP=[] @@ -91,7 +93,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea .PP \fB--ignore-inode\fP[=false] - ignore inode number changes when checking for modified files + ignore inode number and ctime changes when checking for modified files .PP \fB--no-scan\fP[=false] @@ -109,6 +111,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--read-concurrency\fP=0 read \fBn\fR files concurrently (default: $RESTIC_READ_CONCURRENCY or 2) +.PP +\fB--skip-if-unchanged\fP[=false] + skip snapshot creation if identical to parent snapshot + .PP \fB--stdin\fP[=false] read backup from stdin @@ -117,6 +123,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--stdin-filename\fP="stdin" \fBfilename\fR to use when reading from stdin +.PP +\fB--stdin-from-command\fP[=false] + interpret arguments as command to execute and store its stdout + .PP \fB--tag\fP=[] add \fBtags\fR for the new snapshot in the format \fBtag[,tag,...]\fR (can be specified multiple times) @@ -147,6 +157,14 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-cache.1 b/mover-restic/restic/doc/man/restic-cache.1 index c170c1624..f868b8a6b 100644 --- a/mover-restic/restic/doc/man/restic-cache.1 +++ b/mover-restic/restic/doc/man/restic-cache.1 @@ -18,7 +18,8 @@ The "cache" command allows listing and cleaning local cache directories. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. .SH OPTIONS @@ -56,6 +57,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-cat.1 b/mover-restic/restic/doc/man/restic-cat.1 index b42a58e14..2298c58cf 100644 --- a/mover-restic/restic/doc/man/restic-cat.1 +++ b/mover-restic/restic/doc/man/restic-cat.1 @@ -18,7 +18,10 @@ The "cat" command is used to print internal objects to stdout. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -44,6 +47,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-check.1 b/mover-restic/restic/doc/man/restic-check.1 index 9c1dc77e5..c0d1b07a8 100644 --- a/mover-restic/restic/doc/man/restic-check.1 +++ b/mover-restic/restic/doc/man/restic-check.1 @@ -23,7 +23,10 @@ repository and not use a local cache. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -61,6 +64,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-copy.1 b/mover-restic/restic/doc/man/restic-copy.1 index bd9795f44..63b67e5e7 100644 --- a/mover-restic/restic/doc/man/restic-copy.1 +++ b/mover-restic/restic/doc/man/restic-copy.1 @@ -30,7 +30,19 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a new destination repository using the "init" command. +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + + .SH OPTIONS +.PP +\fB--from-insecure-no-password\fP[=false] + use an empty password for the source repository, must be passed to every restic command (insecure) + .PP \fB--from-key-hint\fP="" key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT) @@ -57,11 +69,11 @@ new destination repository using the "init" command. .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--tag\fP=[] @@ -85,6 +97,14 @@ new destination repository using the "init" command. \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-diff.1 b/mover-restic/restic/doc/man/restic-diff.1 index 28f3a4838..f4ffa2737 100644 --- a/mover-restic/restic/doc/man/restic-diff.1 +++ b/mover-restic/restic/doc/man/restic-diff.1 @@ -28,18 +28,27 @@ U The metadata (access mode, timestamps, ...) for the item was updated M The file's content was modified .IP \(bu 2 T The type was changed, e.g. a file was made a symlink +.IP \(bu 2 +? Bitrot detected: The file's content has changed but all metadata is the same .RE +.PP +Metadata comparison will likely not work if a backup was created using the +\&'--ignore-inode' or '--ignore-ctime' option. + .PP To only compare files in specific subfolders, you can use the -":" syntax, where "subfolder" is a path within the +"snapshotID:subfolder" syntax, where "subfolder" is a path within the snapshot. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -69,6 +78,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-dump.1 b/mover-restic/restic/doc/man/restic-dump.1 index 7fa3f777d..00cb3c8b6 100644 --- a/mover-restic/restic/doc/man/restic-dump.1 +++ b/mover-restic/restic/doc/man/restic-dump.1 @@ -24,13 +24,16 @@ repository. .PP To include the folder content at the root of the archive, you can use the -":" syntax, where "subfolder" is a path within the +"snapshotID:subfolder" syntax, where "subfolder" is a path within the snapshot. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -44,16 +47,20 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times, snapshots must include all specified paths) .PP \fB--tag\fP=[] only consider snapshots including \fBtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times) +.PP +\fB-t\fP, \fB--target\fP="" + write the output to target \fBpath\fR + .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP @@ -72,6 +79,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-find.1 b/mover-restic/restic/doc/man/restic-find.1 index c3297c43f..2d81decd3 100644 --- a/mover-restic/restic/doc/man/restic-find.1 +++ b/mover-restic/restic/doc/man/restic-find.1 @@ -29,7 +29,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--human-readable\fP[=false] @@ -57,7 +57,7 @@ It can also be used to search for restic blobs or trees for troubleshooting. .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--show-pack-id\fP[=false] @@ -93,6 +93,14 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) @@ -178,8 +186,10 @@ restic find --pack 025c1d06 EXIT STATUS =========== -Exit status is 0 if the command was successful, and non-zero if there was any error. - +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .EE diff --git a/mover-restic/restic/doc/man/restic-forget.1 b/mover-restic/restic/doc/man/restic-forget.1 index d0c4cfc74..55705288f 100644 --- a/mover-restic/restic/doc/man/restic-forget.1 +++ b/mover-restic/restic/doc/man/restic-forget.1 @@ -15,7 +15,10 @@ restic-forget - Remove snapshots from the repository .PP The "forget" command removes snapshots according to a policy. All snapshots are first divided into groups according to "--group-by", and after that the policy -specified by the "--keep-*" options is applied to each group individually. +specified by the "--keep-\fI" options is applied to each group individually. +If there are not enough snapshots to keep one for each duration related +"--keep-{within-,}\fP" option, the oldest snapshot in the group is kept +additionally. .PP Please note that this command really only deletes the snapshot object in the @@ -29,7 +32,10 @@ security considerations. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -85,9 +91,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--keep-tag\fP=[] keep snapshots with this \fBtaglist\fR (can be specified multiple times) +.PP +\fB--unsafe-allow-remove-all\fP[=false] + allow deleting all snapshots of a snapshot group + .PP \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--tag\fP=[] @@ -95,7 +105,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB-c\fP, \fB--compact\fP[=false] @@ -155,6 +165,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-generate.1 b/mover-restic/restic/doc/man/restic-generate.1 index 84f659ef2..f2db39bac 100644 --- a/mover-restic/restic/doc/man/restic-generate.1 +++ b/mover-restic/restic/doc/man/restic-generate.1 @@ -19,7 +19,8 @@ and the auto-completion files for bash, fish and zsh). .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. .SH OPTIONS @@ -65,6 +66,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-init.1 b/mover-restic/restic/doc/man/restic-init.1 index 5f19c8f8c..de439add5 100644 --- a/mover-restic/restic/doc/man/restic-init.1 +++ b/mover-restic/restic/doc/man/restic-init.1 @@ -18,7 +18,8 @@ The "init" command initializes a new repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. .SH OPTIONS @@ -26,6 +27,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--copy-chunker-params\fP[=false] copy chunker parameters from the secondary repository (useful with the copy command) +.PP +\fB--from-insecure-no-password\fP[=false] + use an empty password for the source repository, must be passed to every restic command (insecure) + .PP \fB--from-key-hint\fP="" key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT) @@ -72,6 +77,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-key-add.1 b/mover-restic/restic/doc/man/restic-key-add.1 new file mode 100644 index 000000000..6a24e1e67 --- /dev/null +++ b/mover-restic/restic/doc/man/restic-key-add.1 @@ -0,0 +1,149 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-key-add - Add a new key (password) to the repository; returns the new key ID + + +.SH SYNOPSIS +.PP +\fBrestic key add [flags]\fP + + +.SH DESCRIPTION +.PP +The "add" sub-command creates a new key and validates the key. Returns the new key ID. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for add + +.PP +\fB--host\fP="" + the hostname for new key + +.PP +\fB--new-insecure-no-password\fP[=false] + add an empty password for the repository (insecure) + +.PP +\fB--new-password-file\fP="" + \fBfile\fR from which to read the new password + +.PP +\fB--user\fP="" + the username for new key + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-key(1)\fP diff --git a/mover-restic/restic/doc/man/restic-key-list.1 b/mover-restic/restic/doc/man/restic-key-list.1 new file mode 100644 index 000000000..a00b116b9 --- /dev/null +++ b/mover-restic/restic/doc/man/restic-key-list.1 @@ -0,0 +1,135 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-key-list - List keys (passwords) + + +.SH SYNOPSIS +.PP +\fBrestic key list [flags]\fP + + +.SH DESCRIPTION +.PP +The "list" sub-command lists all the keys (passwords) associated with the repository. +Returns the key ID, username, hostname, created time and if it's the current key being +used to access the repository. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for list + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-key(1)\fP diff --git a/mover-restic/restic/doc/man/restic-key-passwd.1 b/mover-restic/restic/doc/man/restic-key-passwd.1 new file mode 100644 index 000000000..42315d72a --- /dev/null +++ b/mover-restic/restic/doc/man/restic-key-passwd.1 @@ -0,0 +1,150 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-key-passwd - Change key (password); creates a new key ID and removes the old key ID, returns new key ID + + +.SH SYNOPSIS +.PP +\fBrestic key passwd [flags]\fP + + +.SH DESCRIPTION +.PP +The "passwd" sub-command creates a new key, validates the key and remove the old key ID. +Returns the new key ID. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for passwd + +.PP +\fB--host\fP="" + the hostname for new key + +.PP +\fB--new-insecure-no-password\fP[=false] + add an empty password for the repository (insecure) + +.PP +\fB--new-password-file\fP="" + \fBfile\fR from which to read the new password + +.PP +\fB--user\fP="" + the username for new key + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-key(1)\fP diff --git a/mover-restic/restic/doc/man/restic-key-remove.1 b/mover-restic/restic/doc/man/restic-key-remove.1 new file mode 100644 index 000000000..6ee826059 --- /dev/null +++ b/mover-restic/restic/doc/man/restic-key-remove.1 @@ -0,0 +1,134 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-key-remove - Remove key ID (password) from the repository. + + +.SH SYNOPSIS +.PP +\fBrestic key remove [ID] [flags]\fP + + +.SH DESCRIPTION +.PP +The "remove" sub-command removes the selected key ID. The "remove" command does not allow +removing the current key being used to access the repository. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for remove + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic-key(1)\fP diff --git a/mover-restic/restic/doc/man/restic-key.1 b/mover-restic/restic/doc/man/restic-key.1 index 8d1813188..43da808cc 100644 --- a/mover-restic/restic/doc/man/restic-key.1 +++ b/mover-restic/restic/doc/man/restic-key.1 @@ -8,17 +8,13 @@ restic-key - Manage keys (passwords) .SH SYNOPSIS .PP -\fBrestic key [flags] [list|add|remove|passwd] [ID]\fP +\fBrestic key [flags]\fP .SH DESCRIPTION .PP -The "key" command manages keys (passwords) for accessing the repository. - - -.SH EXIT STATUS -.PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +The "key" command allows you to set multiple access keys or passwords +per repository. .SH OPTIONS @@ -26,18 +22,6 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB-h\fP, \fB--help\fP[=false] help for key -.PP -\fB--host\fP="" - the hostname for new keys - -.PP -\fB--new-password-file\fP="" - \fBfile\fR from which to read the new password - -.PP -\fB--user\fP="" - the username for new keys - .SH OPTIONS INHERITED FROM PARENT COMMANDS .PP @@ -56,6 +40,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) @@ -131,4 +123,4 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .SH SEE ALSO .PP -\fBrestic(1)\fP +\fBrestic(1)\fP, \fBrestic-key-add(1)\fP, \fBrestic-key-list(1)\fP, \fBrestic-key-passwd(1)\fP, \fBrestic-key-remove(1)\fP diff --git a/mover-restic/restic/doc/man/restic-list.1 b/mover-restic/restic/doc/man/restic-list.1 index e399038a2..f8a1db005 100644 --- a/mover-restic/restic/doc/man/restic-list.1 +++ b/mover-restic/restic/doc/man/restic-list.1 @@ -18,7 +18,10 @@ The "list" command allows listing objects in the repository based on type. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -44,6 +47,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-ls.1 b/mover-restic/restic/doc/man/restic-ls.1 index 10b0657a3..6cc662583 100644 --- a/mover-restic/restic/doc/man/restic-ls.1 +++ b/mover-restic/restic/doc/man/restic-ls.1 @@ -33,7 +33,10 @@ a path separator); paths use the forward slash '/' as separator. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -43,7 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--human-readable\fP[=false] @@ -53,9 +56,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB-l\fP, \fB--long\fP[=false] use a long listing format showing size and mode +.PP +\fB--ncdu\fP[=false] + output NCDU export format (pipe into 'ncdu -f -') + .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times, snapshots must include all specified paths) .PP \fB--recursive\fP[=false] @@ -83,6 +90,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-migrate.1 b/mover-restic/restic/doc/man/restic-migrate.1 index 7e48f726c..2272294bf 100644 --- a/mover-restic/restic/doc/man/restic-migrate.1 +++ b/mover-restic/restic/doc/man/restic-migrate.1 @@ -20,7 +20,10 @@ names are specified, these migrations are applied. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -50,6 +53,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-mount.1 b/mover-restic/restic/doc/man/restic-mount.1 index aab607fcf..a256d2a5f 100644 --- a/mover-restic/restic/doc/man/restic-mount.1 +++ b/mover-restic/restic/doc/man/restic-mount.1 @@ -28,7 +28,6 @@ Example time template without colons: .EX --time-template "2006-01-02_15-04-05" - .EE .PP @@ -36,7 +35,6 @@ You need to specify a sample format for exactly the following timestamp: .EX Mon Jan 2 15:04:05 -0700 MST 2006 - .EE .PP @@ -62,7 +60,10 @@ The default path templates are: .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -76,7 +77,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--no-default-permissions\fP[=false] @@ -88,7 +89,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--path-template\fP=[] @@ -120,6 +121,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-prune.1 b/mover-restic/restic/doc/man/restic-prune.1 index c54d5d7ff..7e16748ab 100644 --- a/mover-restic/restic/doc/man/restic-prune.1 +++ b/mover-restic/restic/doc/man/restic-prune.1 @@ -19,7 +19,10 @@ referenced and therefore not needed any more. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -73,6 +76,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-recover.1 b/mover-restic/restic/doc/man/restic-recover.1 index 010fbafd7..0529360ae 100644 --- a/mover-restic/restic/doc/man/restic-recover.1 +++ b/mover-restic/restic/doc/man/restic-recover.1 @@ -20,7 +20,10 @@ It can be used if, for example, a snapshot has been removed by accident with "fo .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -46,6 +49,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-repair-index.1 b/mover-restic/restic/doc/man/restic-repair-index.1 index f06be64c0..60327a916 100644 --- a/mover-restic/restic/doc/man/restic-repair-index.1 +++ b/mover-restic/restic/doc/man/restic-repair-index.1 @@ -19,7 +19,10 @@ repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -49,6 +52,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-repair-packs.1 b/mover-restic/restic/doc/man/restic-repair-packs.1 index f3671fe18..01a2f6540 100644 --- a/mover-restic/restic/doc/man/restic-repair-packs.1 +++ b/mover-restic/restic/doc/man/restic-repair-packs.1 @@ -12,9 +12,6 @@ restic-repair-packs - Salvage damaged pack files .SH DESCRIPTION -.PP -WARNING: The CLI for this command is experimental and will likely change in the future! - .PP The "repair packs" command extracts intact blobs from the specified pack files, rebuilds the index to remove the damaged pack files and removes the pack files from the repository. @@ -22,7 +19,10 @@ the index to remove the damaged pack files and removes the pack files from the r .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -48,6 +48,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-repair-snapshots.1 b/mover-restic/restic/doc/man/restic-repair-snapshots.1 index 9369f25f2..c4439f131 100644 --- a/mover-restic/restic/doc/man/restic-repair-snapshots.1 +++ b/mover-restic/restic/doc/man/restic-repair-snapshots.1 @@ -37,7 +37,10 @@ snapshot! .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -55,11 +58,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--tag\fP=[] @@ -83,6 +86,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-repair.1 b/mover-restic/restic/doc/man/restic-repair.1 index 77aecc173..7fa313aab 100644 --- a/mover-restic/restic/doc/man/restic-repair.1 +++ b/mover-restic/restic/doc/man/restic-repair.1 @@ -39,6 +39,14 @@ Repair the repository \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-restore.1 b/mover-restic/restic/doc/man/restic-restore.1 index 4635b1e43..876b18bf8 100644 --- a/mover-restic/restic/doc/man/restic-restore.1 +++ b/mover-restic/restic/doc/man/restic-restore.1 @@ -21,43 +21,74 @@ The special snapshotID "latest" can be used to restore the latest snapshot in th repository. .PP -To only restore a specific subfolder, you can use the ":" +To only restore a specific subfolder, you can use the "snapshotID:subfolder" syntax, where "subfolder" is a path within the snapshot. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS +.PP +\fB--delete\fP[=false] + delete files from target directory if they do not exist in snapshot. Use '--dry-run -vv' to check what would be deleted + +.PP +\fB--dry-run\fP[=false] + do not write any data, just show what would be done + .PP \fB-e\fP, \fB--exclude\fP=[] exclude a \fBpattern\fR (can be specified multiple times) +.PP +\fB--exclude-file\fP=[] + read exclude patterns from a \fBfile\fR (can be specified multiple times) + .PP \fB-h\fP, \fB--help\fP[=false] help for restore .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots for this \fBhost\fR, when snapshot ID "latest" is given (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--iexclude\fP=[] - same as --exclude but ignores the casing of \fBpattern\fR + same as --exclude \fBpattern\fR but ignores the casing of filenames + +.PP +\fB--iexclude-file\fP=[] + same as --exclude-file but ignores casing of \fBfile\fRnames in patterns .PP \fB--iinclude\fP=[] - same as --include but ignores the casing of \fBpattern\fR + same as --include \fBpattern\fR but ignores the casing of filenames + +.PP +\fB--iinclude-file\fP=[] + same as --include-file but ignores casing of \fBfile\fRnames in patterns .PP \fB-i\fP, \fB--include\fP=[] - include a \fBpattern\fR, exclude everything else (can be specified multiple times) + include a \fBpattern\fR (can be specified multiple times) + +.PP +\fB--include-file\fP=[] + read include patterns from a \fBfile\fR (can be specified multiple times) + +.PP +\fB--overwrite\fP=always + overwrite behavior, one of (always|if-changed|if-newer|never) (default: always) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR, when snapshot ID "latest" is given (can be specified multiple times, snapshots must include all specified paths) .PP \fB--sparse\fP[=false] @@ -93,6 +124,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-rewrite.1 b/mover-restic/restic/doc/man/restic-rewrite.1 index d63c653e6..d3dd92436 100644 --- a/mover-restic/restic/doc/man/restic-rewrite.1 +++ b/mover-restic/restic/doc/man/restic-rewrite.1 @@ -35,7 +35,10 @@ use the "prune" command. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -61,7 +64,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--iexclude\fP=[] @@ -71,9 +74,17 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--iexclude-file\fP=[] same as --exclude-file but ignores casing of \fBfile\fRnames in patterns +.PP +\fB--new-host\fP="" + replace hostname + +.PP +\fB--new-time\fP="" + replace time of the backup + .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--tag\fP=[] @@ -97,6 +108,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-self-update.1 b/mover-restic/restic/doc/man/restic-self-update.1 index 92ab5add3..e6dd4faf2 100644 --- a/mover-restic/restic/doc/man/restic-self-update.1 +++ b/mover-restic/restic/doc/man/restic-self-update.1 @@ -21,7 +21,10 @@ files. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -51,6 +54,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-snapshots.1 b/mover-restic/restic/doc/man/restic-snapshots.1 index 6203bbf2b..25d5274e3 100644 --- a/mover-restic/restic/doc/man/restic-snapshots.1 +++ b/mover-restic/restic/doc/man/restic-snapshots.1 @@ -18,7 +18,10 @@ The "snapshots" command lists all snapshots stored in the repository. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -36,7 +39,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--latest\fP=0 @@ -44,7 +47,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--tag\fP=[] @@ -68,6 +71,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-stats.1 b/mover-restic/restic/doc/man/restic-stats.1 index 9d37163de..fe4074ca5 100644 --- a/mover-restic/restic/doc/man/restic-stats.1 +++ b/mover-restic/restic/doc/man/restic-stats.1 @@ -32,7 +32,7 @@ The modes are: .IP \(bu 2 restore-size: (default) Counts the size of the restored files. .IP \(bu 2 -files-by-contents: Counts total size of files, where a file is +files-by-contents: Counts total size of unique files, where a file is considered unique if it has unique contents. .IP \(bu 2 raw-data: Counts the size of blobs in the repository, regardless of @@ -48,7 +48,10 @@ Refer to the online manual for more details about each mode. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -58,7 +61,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--mode\fP="restore-size" @@ -66,7 +69,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--tag\fP=[] @@ -90,6 +93,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-tag.1 b/mover-restic/restic/doc/man/restic-tag.1 index b1468c74d..7ab1911e5 100644 --- a/mover-restic/restic/doc/man/restic-tag.1 +++ b/mover-restic/restic/doc/man/restic-tag.1 @@ -25,7 +25,10 @@ When no snapshotID is given, all snapshots matching the host, tag and path filte .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. +Exit status is 10 if the repository does not exist. +Exit status is 11 if the repository is already locked. .SH OPTIONS @@ -39,11 +42,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er .PP \fB-H\fP, \fB--host\fP=[] - only consider snapshots for this \fBhost\fR (can be specified multiple times) + only consider snapshots for this \fBhost\fR (can be specified multiple times) (default: $RESTIC_HOST) .PP \fB--path\fP=[] - only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times) + only consider snapshots including this (absolute) \fBpath\fR (can be specified multiple times, snapshots must include all specified paths) .PP \fB--remove\fP=[] @@ -75,6 +78,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-unlock.1 b/mover-restic/restic/doc/man/restic-unlock.1 index 0b3b43f2a..a24a4f815 100644 --- a/mover-restic/restic/doc/man/restic-unlock.1 +++ b/mover-restic/restic/doc/man/restic-unlock.1 @@ -18,7 +18,8 @@ The "unlock" command removes stale locks that have been created by other restic .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. .SH OPTIONS @@ -48,6 +49,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic-version.1 b/mover-restic/restic/doc/man/restic-version.1 index ccc23038f..e9df439ed 100644 --- a/mover-restic/restic/doc/man/restic-version.1 +++ b/mover-restic/restic/doc/man/restic-version.1 @@ -19,7 +19,8 @@ and the version of this software. .SH EXIT STATUS .PP -Exit status is 0 if the command was successful, and non-zero if there was any error. +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. .SH OPTIONS @@ -45,6 +46,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--compression\fP=auto compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/man/restic.1 b/mover-restic/restic/doc/man/restic.1 index 333eab76a..ee423c6ad 100644 --- a/mover-restic/restic/doc/man/restic.1 +++ b/mover-restic/restic/doc/man/restic.1 @@ -41,6 +41,14 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB-h\fP, \fB--help\fP[=false] help for restic +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + .PP \fB--insecure-tls\fP[=false] skip TLS certificate verification when connecting to the repository (insecure) diff --git a/mover-restic/restic/doc/manual_rest.rst b/mover-restic/restic/doc/manual_rest.rst index d1c64ba6e..a7a0f96e0 100644 --- a/mover-restic/restic/doc/manual_rest.rst +++ b/mover-restic/restic/doc/manual_rest.rst @@ -13,6 +13,8 @@ Usage help is available: restic is a backup program which allows saving multiple revisions of files and directories in an encrypted repository stored on different backends. + The full documentation can be found at https://restic.readthedocs.io/ . + Usage: restic [command] @@ -47,17 +49,20 @@ Usage help is available: version Print version information Flags: - --cacert file file to load root certificates from (default: use system certificates) + --cacert file file to load root certificates from (default: use system certificates or $RESTIC_CACERT) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) (default auto) -h, --help help for restic + --http-user-agent string set a http user agent for outgoing http requests + --insecure-no-password use an empty password for the repository, must be passed to every restic command (insecure) --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --limit-download rate limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-upload rate limits uploads to a maximum rate in KiB/s. (default: unlimited) --no-cache do not use a local cache + --no-extra-verify skip additional verification of data before upload (see documentation) --no-lock do not lock the repository, this allows some operations on read-only repositories -o, --option key=value set extended option (key=value, can be specified multiple times) --pack-size size set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) @@ -67,7 +72,7 @@ Usage help is available: -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) --retry-lock duration retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) - --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key + --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Use "restic [command] --help" for more information about a command. @@ -105,36 +110,41 @@ command: --files-from file read the files to backup from file (can be combined with file args; can be specified multiple times) --files-from-raw file read the files to backup from file (can be combined with file args; can be specified multiple times) --files-from-verbatim file read the files to backup from file (can be combined with file args; can be specified multiple times) - -f, --force force re-reading the target files/directories (overrides the "parent" flag) + -f, --force force re-reading the source files/directories (overrides the "parent" flag) -g, --group-by group group snapshots by host, paths and/or tags, separated by comma (disable grouping with '') (default host,paths) -h, --help help for backup - -H, --host hostname set the hostname for the snapshot manually. To prevent an expensive rescan use the "parent" flag + -H, --host hostname set the hostname for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the "parent" flag --iexclude pattern same as --exclude pattern but ignores the casing of filenames --iexclude-file file same as --exclude-file but ignores casing of filenames in patterns --ignore-ctime ignore ctime changes when checking for modified files - --ignore-inode ignore inode number changes when checking for modified files + --ignore-inode ignore inode number and ctime changes when checking for modified files --no-scan do not run scanner to estimate size of backup -x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes --parent snapshot use this parent snapshot (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time) --read-concurrency n read n files concurrently (default: $RESTIC_READ_CONCURRENCY or 2) + --skip-if-unchanged skip snapshot creation if identical to parent snapshot --stdin read backup from stdin --stdin-filename filename filename to use when reading from stdin (default "stdin") + --stdin-from-command interpret arguments as command to execute and store its stdout --tag tags add tags for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times) (default []) --time time time of the backup (ex. '2012-11-01 22:08:41') (default: now) --use-fs-snapshot use filesystem snapshot where possible (currently only Windows VSS) --with-atime store the atime for all files and directories Global Flags: - --cacert file file to load root certificates from (default: use system certificates) + --cacert file file to load root certificates from (default: use system certificates or $RESTIC_CACERT) --cache-dir directory set the cache directory. (default: use system default cache directory) --cleanup-cache auto remove old cache directories --compression mode compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) (default auto) + --http-user-agent string set a http user agent for outgoing http requests + --insecure-no-password use an empty password for the repository, must be passed to every restic command (insecure) --insecure-tls skip TLS certificate verification when connecting to the repository (insecure) --json set output mode to JSON for commands that support it --key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT) --limit-download rate limits downloads to a maximum rate in KiB/s. (default: unlimited) --limit-upload rate limits uploads to a maximum rate in KiB/s. (default: unlimited) --no-cache do not use a local cache + --no-extra-verify skip additional verification of data before upload (see documentation) --no-lock do not lock the repository, this allows some operations on read-only repositories -o, --option key=value set extended option (key=value, can be specified multiple times) --pack-size size set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) @@ -144,7 +154,7 @@ command: -r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY) --repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE) --retry-lock duration retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) - --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key + --tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Subcommands that support showing progress information such as ``backup``, @@ -322,7 +332,6 @@ required to restore the latest snapshot (from any host that made it): .. code-block:: console $ restic stats latest - password is correct Total File Count: 10538 Total Size: 37.824 GiB @@ -333,7 +342,6 @@ host by using the ``--host`` flag: .. code-block:: console $ restic stats --host myserver latest - password is correct Total File Count: 21766 Total Size: 481.783 GiB @@ -350,7 +358,6 @@ has restic's deduplication helped? We can check: .. code-block:: console $ restic stats --host myserver --mode raw-data latest - password is correct Total Blob Count: 340847 Total Size: 458.663 GiB @@ -408,9 +415,12 @@ Temporary files During some operations (e.g. ``backup`` and ``prune``) restic uses temporary files to store data. These files will, by default, be saved to the system's temporary directory, on Linux this is usually located in -``/tmp/``. The environment variable ``TMPDIR`` can be used to specify a -different directory, e.g. to use the directory ``/var/tmp/restic-tmp`` -instead of the default, set the environment variable like this: +``/tmp/``. To specify a different directory for temporary files, set +the appropriate environment variable. On non-Windows operating systems, +use the ``TMPDIR`` environment variable. On Windows, use either the +``TMP`` or ``TEMP`` environment variable. For example, to use the +directory ``/var/tmp/restic-tmp`` instead of the default, set the +environment variable as follows: .. code-block:: console @@ -428,10 +438,10 @@ This allows faster operations, since meta data does not need to be loaded from a remote repository. The cache is automatically created, usually in an OS-specific cache folder: - * Linux/other: ``$XDG_CACHE_HOME/restic``, or ``~/.cache/restic`` if - ``XDG_CACHE_HOME`` is not set - * macOS: ``~/Library/Caches/restic`` - * Windows: ``%LOCALAPPDATA%/restic`` +* Linux/other: ``$XDG_CACHE_HOME/restic``, or ``~/.cache/restic`` if + ``XDG_CACHE_HOME`` is not set +* macOS: ``~/Library/Caches/restic`` +* Windows: ``%LOCALAPPDATA%/restic`` If the relevant environment variables are not set, restic exits with an error message. diff --git a/mover-restic/restic/doc/powershell-completion.ps1 b/mover-restic/restic/doc/powershell-completion.ps1 index d8aa5a1af..033477e7b 100644 --- a/mover-restic/restic/doc/powershell-completion.ps1 +++ b/mover-restic/restic/doc/powershell-completion.ps1 @@ -10,7 +10,7 @@ filter __restic_escapeStringWithSpecialChars { $_ -replace '\s|#|@|\$|;|,|''|\{|\}|\(|\)|"|`|\||<|>|&','`$&' } -[scriptblock]$__resticCompleterBlock = { +[scriptblock]${__resticCompleterBlock} = { param( $WordToComplete, $CommandAst, @@ -85,7 +85,7 @@ filter __restic_escapeStringWithSpecialChars { __restic_debug "Calling $RequestComp" # First disable ActiveHelp which is not supported for Powershell - $env:RESTIC_ACTIVE_HELP=0 + ${env:RESTIC_ACTIVE_HELP}=0 #call the command store the output in $out and redirect stderr and stdout to null # $Out is an array contains each line per element @@ -242,4 +242,4 @@ filter __restic_escapeStringWithSpecialChars { } } -Register-ArgumentCompleter -CommandName 'restic' -ScriptBlock $__resticCompleterBlock +Register-ArgumentCompleter -CommandName 'restic' -ScriptBlock ${__resticCompleterBlock} diff --git a/mover-restic/restic/docker/Dockerfile b/mover-restic/restic/docker/Dockerfile index 978da7960..02b53261f 100644 --- a/mover-restic/restic/docker/Dockerfile +++ b/mover-restic/restic/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20-alpine AS builder +FROM golang:1.22-alpine AS builder WORKDIR /go/src/github.com/restic/restic diff --git a/mover-restic/restic/go.mod b/mover-restic/restic/go.mod index fbfd56364..bdef600b9 100644 --- a/mover-restic/restic/go.mod +++ b/mover-restic/restic/go.mod @@ -1,51 +1,50 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.41.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 + cloud.google.com/go/storage v1.43.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 github.com/Backblaze/blazer v0.6.1 - github.com/anacrolix/fuse v0.2.0 - github.com/cenkalti/backoff/v4 v4.2.1 - github.com/cespare/xxhash/v2 v2.2.0 + github.com/anacrolix/fuse v0.3.1 + github.com/cenkalti/backoff/v4 v4.3.0 + github.com/cespare/xxhash/v2 v2.3.0 github.com/elithrar/simple-scrypt v1.3.0 github.com/go-ole/go-ole v1.3.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.4 + github.com/klauspost/compress v1.17.9 github.com/minio/minio-go/v7 v7.0.66 github.com/ncw/swift/v2 v2.0.2 + github.com/peterbourgon/unixtransport v0.0.4 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.7.0 github.com/pkg/sftp v1.13.6 - github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013 + github.com/pkg/xattr v0.4.10 github.com/restic/chunker v0.4.0 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 go.uber.org/automaxprocs v1.5.3 golang.org/x/crypto v0.24.0 golang.org/x/net v0.26.0 - golang.org/x/oauth2 v0.20.0 + golang.org/x/oauth2 v0.21.0 golang.org/x/sync v0.7.0 - golang.org/x/sys v0.21.0 - golang.org/x/term v0.21.0 + golang.org/x/sys v0.22.0 + golang.org/x/term v0.22.0 golang.org/x/text v0.16.0 golang.org/x/time v0.5.0 - google.golang.org/api v0.182.0 + google.golang.org/api v0.187.0 ) -replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2 - require ( - cloud.google.com/go v0.114.0 // indirect - cloud.google.com/go/auth v0.4.2 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.8 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -58,7 +57,7 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.4 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect @@ -77,11 +76,11 @@ require ( go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect - google.golang.org/grpc v1.64.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d // indirect + google.golang.org/grpc v1.64.1 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/mover-restic/restic/go.sum b/mover-restic/restic/go.sum index a4b0a9d3f..12151ade3 100644 --- a/mover-restic/restic/go.sum +++ b/mover-restic/restic/go.sum @@ -1,52 +1,60 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.114.0 h1:OIPFAdfrFDFO2ve2U7r/H5SwSbBzEdrBdE7xkgwc+kY= -cloud.google.com/go v0.114.0/go.mod h1:ZV9La5YYxctro1HTPug5lXH/GefROyW8PPD4T8n9J8E= -cloud.google.com/go/auth v0.4.2 h1:sb0eyLkhRtpq5jA+a8KWw0W70YcdVca7KJ8TM0AFYDg= -cloud.google.com/go/auth v0.4.2/go.mod h1:Kqvlz1cf1sNA0D+sYJnkPQOP+JMHkuHeIgVmCRtZOLc= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= -cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= -cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= +cloud.google.com/go/longrunning v0.5.7 h1:WLbHekDbjK1fVFD3ibpFFVoyizlLRl73I7YKuAKilhU= +cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= +cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 h1:H+U3Gk9zY56G3u872L82bk4thcsy2Gghb9ExT4Zvm1o= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0/go.mod h1:mgrmMSgaLp9hmax62XQTd0N4aAqSE5E0DulSpVYK7vc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2 h1:YUUxeiOWgdAQE3pXt2H7QXzZs0q8UBjgRbl56qo8GYM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.2/go.mod h1:dmXQgZuiSubAecswZE+Sm8jkvEa7kQgTPVRvwL/nd0E= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/Backblaze/blazer v0.6.1 h1:xC9HyC7OcxRzzmtfRiikIEvq4HZYWjU6caFwX2EXw1s= github.com/Backblaze/blazer v0.6.1/go.mod h1:7/jrGx4O6OKOto6av+hLwelPR8rwZ+PLxQ5ZOiYAjwY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74= -github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do= -github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tcJDk= +github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= +github.com/anacrolix/fuse v0.3.1 h1:oT8s3B5HFkBdLe/WKJO5MNo9iIyEtc+BhvTZYp4jhDM= +github.com/anacrolix/fuse v0.3.1/go.mod h1:vN3X/6E+uHNjg5F8Oy9FD9I+pYxeDWeB8mNjIoxL5ds= +github.com/anacrolix/generics v0.0.0-20230113004304-d6428d516633 h1:TO3pytMIJ98CO1nYtqbFx/iuTHi4OgIUoE2wNfDdKxw= +github.com/anacrolix/generics v0.0.0-20230113004304-d6428d516633/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/log v0.13.1/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= +github.com/anacrolix/log v0.14.1 h1:j2FcIpYZ5FbANetUcm5JNu+zUBGADSp/VbjhUPrAY0k= +github.com/anacrolix/log v0.14.1/go.mod h1:1OmJESOtxQGNMlUO5rcv96Vpp9mfMqXXbe2RdinFLdY= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/dvyukov/go-fuzz v0.0.0-20220726122315-1d375ef9f9f6/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg= github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1qPwrl/EJwEqnZo= @@ -58,6 +66,7 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -89,6 +98,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -103,8 +113,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= -github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= @@ -112,17 +122,24 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -132,16 +149,22 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ= +github.com/peterbourgon/unixtransport v0.0.4 h1:UTF0FxXCAglvoZz9jaGPYjEg52DjBLDYGMJvJni6Tfw= +github.com/peterbourgon/unixtransport v0.0.4/go.mod h1:o8aUkOCa8W/BIXpi15uKvbSabjtBh0JhSOJGSfoOhAU= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA= github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= -github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013 h1:aqByeeNnF7NiEbXCi7nBxZ272+6f6FUBmj/dUzWCdvc= -github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= +github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= @@ -149,6 +172,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw= github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw= github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -156,8 +182,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc= @@ -172,7 +198,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= @@ -197,57 +223,68 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171 h1:TfdoLivD44QwvssI9Sv1xwa5DcL5XQr4au4sZ2F2NV4= +golang.org/x/exp v0.0.0-20220428152302-39d4317da171/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= @@ -260,32 +297,33 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -google.golang.org/api v0.182.0 h1:if5fPvudRQ78GeRx3RayIoiuV7modtErPIZC/T2bIvE= -google.golang.org/api v0.182.0/go.mod h1:cGhjy4caqA5yXRzEhkHI8Y9mfyC2VLTlER2l08xaqtM= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= -google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8 h1:W5Xj/70xIA4x60O/IFyXivR5MGqblAb8R3w26pnD6No= -google.golang.org/genproto/googleapis/api v0.0.0-20240513163218-0867130af1f8/go.mod h1:vPrPUTsDCYxXWjP7clS81mZ6/803D8K4iM9Ma27VKas= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e h1:Elxv5MwEkCI9f5SkoL6afed6NTdxaGoAo39eANBwHL8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls= +google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc= +google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d h1:k3zyW3BYYR30e8v3x0bTDdE9vpYFjZHK+HcyqkrppWk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -295,12 +333,16 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/mover-restic/restic/helpers/build-release-binaries/main.go b/mover-restic/restic/helpers/build-release-binaries/main.go index 169e1e001..81d126b00 100644 --- a/mover-restic/restic/helpers/build-release-binaries/main.go +++ b/mover-restic/restic/helpers/build-release-binaries/main.go @@ -126,7 +126,8 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) { "GOARCH="+goarch, ) if goarch == "arm" { - c.Env = append(c.Env, "GOARM=5") + // the raspberry pi 1 only supports the ARMv6 instruction set + c.Env = append(c.Env, "GOARM=6") } verbose("run %v %v in %v", "go", c.Args, c.Dir) diff --git a/mover-restic/restic/helpers/prepare-release/main.go b/mover-restic/restic/helpers/prepare-release/main.go index a6c7bd4f4..ba3de38a5 100644 --- a/mover-restic/restic/helpers/prepare-release/main.go +++ b/mover-restic/restic/helpers/prepare-release/main.go @@ -323,6 +323,11 @@ func updateVersion() { } func updateVersionDev() { + err := os.WriteFile("VERSION", []byte(opts.Version+"-dev\n"), 0644) + if err != nil { + die("unable to write version to file: %v", err) + } + newVersion := fmt.Sprintf(`var version = "%s-dev (compiled manually)"`, opts.Version) replace(versionCodeFile, versionPattern, newVersion) @@ -379,7 +384,7 @@ func readdir(dir string) []string { } func sha256sums(inputDir, outputFile string) { - msg("runnnig sha256sum in %v", inputDir) + msg("running sha256sum in %v", inputDir) filenames := readdir(inputDir) diff --git a/mover-restic/restic/internal/archiver/archiver.go b/mover-restic/restic/internal/archiver/archiver.go index d2ddbf00a..d9f089e81 100644 --- a/mover-restic/restic/internal/archiver/archiver.go +++ b/mover-restic/restic/internal/archiver/archiver.go @@ -8,10 +8,12 @@ import ( "runtime" "sort" "strings" + "sync" "time" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" @@ -40,6 +42,18 @@ type ItemStats struct { TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead) } +type ChangeStats struct { + New uint + Changed uint + Unchanged uint +} + +type Summary struct { + Files, Dirs ChangeStats + ProcessedBytes uint64 + ItemStats +} + // Add adds other to the current ItemStats. func (s *ItemStats) Add(other ItemStats) { s.DataBlobs += other.DataBlobs @@ -50,9 +64,19 @@ func (s *ItemStats) Add(other ItemStats) { s.TreeSizeInRepo += other.TreeSizeInRepo } +type archiverRepo interface { + restic.Loader + restic.BlobSaver + restic.SaverUnpacked + + Config() restic.Config + StartPackUploader(ctx context.Context, wg *errgroup.Group) + Flush(ctx context.Context) error +} + // Archiver saves a directory structure to the repo. type Archiver struct { - Repo restic.Repository + Repo archiverRepo SelectByName SelectByNameFunc Select SelectFunc FS fs.FS @@ -61,6 +85,8 @@ type Archiver struct { blobSaver *BlobSaver fileSaver *FileSaver treeSaver *TreeSaver + mu sync.Mutex + summary *Summary // Error is called for all errors that occur during backup. Error ErrorFunc @@ -144,11 +170,11 @@ func (o Options) ApplyDefaults() Options { } // New initializes a new archiver. -func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver { +func New(repo archiverRepo, fs fs.FS, opts Options) *Archiver { arch := &Archiver{ Repo: repo, - SelectByName: func(item string) bool { return true }, - Select: func(item string, fi os.FileInfo) bool { return true }, + SelectByName: func(_ string) bool { return true }, + Select: func(_ string, _ os.FileInfo) bool { return true }, FS: fs, Options: opts.ApplyDefaults(), @@ -182,12 +208,58 @@ func (arch *Archiver) error(item string, err error) error { return errf } +func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) { + arch.CompleteItem(item, previous, current, s, d) + + arch.mu.Lock() + defer arch.mu.Unlock() + + arch.summary.ItemStats.Add(s) + + if current != nil { + arch.summary.ProcessedBytes += current.Size + } else { + // last item or an error occurred + return + } + + switch current.Type { + case "dir": + switch { + case previous == nil: + arch.summary.Dirs.New++ + case previous.Equals(*current): + arch.summary.Dirs.Unchanged++ + default: + arch.summary.Dirs.Changed++ + } + + case "file": + switch { + case previous == nil: + arch.summary.Files.New++ + case previous.Equals(*current): + arch.summary.Files.Unchanged++ + default: + arch.summary.Files.Changed++ + } + } +} + // nodeFromFileInfo returns the restic node from an os.FileInfo. -func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) { - node, err := restic.NodeFromFileInfo(filename, fi) +func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { + node, err := restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) if !arch.WithAtime { node.AccessTime = node.ModTime } + if feature.Flag.Enabled(feature.DeviceIDForHardlinks) { + if node.Links == 1 || node.Type == "dir" { + // the DeviceID is only necessary for hardlinked files + // when using subvolumes or snapshots their deviceIDs tend to change which causes + // restic to upload new tree blobs + node.DeviceID = 0 + } + } // overwrite name to match that within the snapshot node.Name = path.Base(snPath) if err != nil { @@ -214,7 +286,7 @@ func (arch *Archiver) loadSubtree(ctx context.Context, node *restic.Node) (*rest } func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { - if arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.TreeBlob}) { + if _, ok := arch.Repo.LookupBlobSize(restic.TreeBlob, id); ok { err = errors.Errorf("tree %v could not be loaded; the repository could be damaged: %v", id, err) } else { err = errors.Errorf("tree %v is not known; the repository could be damaged, run `repair index` to try to repair it", id) @@ -222,17 +294,17 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error { return err } -// SaveDir stores a directory in the repo and returns the node. snPath is the +// saveDir stores a directory in the repo and returns the node. snPath is the // path within the current snapshot. -func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) { +func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) { debug.Log("%v %v", snPath, dir) - treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi) + treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false) if err != nil { return FutureNode{}, err } - names, err := readdirnames(arch.FS, dir, fs.O_NOFOLLOW) + names, err := fs.Readdirnames(arch.FS, dir, fs.O_NOFOLLOW) if err != nil { return FutureNode{}, err } @@ -250,7 +322,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi pathname := arch.FS.Join(dir, name) oldNode := previous.Find(name) snItem := join(snPath, name) - fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode) + fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode) // return error early if possible if err != nil { @@ -277,7 +349,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi // FutureNode holds a reference to a channel that returns a FutureNodeResult // or a reference to an already existing result. If the result is available -// immediatelly, then storing a reference directly requires less memory than +// immediately, then storing a reference directly requires less memory than // using the indirection via a channel. type FutureNode struct { ch <-chan futureNodeResult @@ -318,6 +390,7 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult { return res } case <-ctx.Done(): + return futureNodeResult{err: ctx.Err()} } return futureNodeResult{err: errors.Errorf("no result")} } @@ -327,21 +400,21 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult { func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool { // check if all blobs are contained in index for _, id := range previous.Content { - if !arch.Repo.Index().Has(restic.BlobHandle{ID: id, Type: restic.DataBlob}) { + if _, ok := arch.Repo.LookupBlobSize(restic.DataBlob, id); !ok { return false } } return true } -// Save saves a target (file or directory) to the repo. If the item is +// save saves a target (file or directory) to the repo. If the item is // excluded, this function returns a nil node and error, with excluded set to // true. // // Errors and completion needs to be handled by the caller. // // snPath is the path within the current snapshot. -func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) { +func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) { start := time.Now() debug.Log("%v target %q, previous %v", snPath, target, previous) @@ -380,9 +453,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) { if arch.allBlobsPresent(previous) { debug.Log("%v hasn't changed, using old list of blobs", target) - arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start)) + arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start)) arch.CompleteBlob(previous.Size) - node, err := arch.nodeFromFileInfo(snPath, target, fi) + node, err := arch.nodeFromFileInfo(snPath, target, fi, false) if err != nil { return FutureNode{}, false, err } @@ -445,9 +518,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() { arch.StartFile(snPath) }, func() { - arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0) + arch.trackItem(snPath, nil, nil, ItemStats{}, 0) }, func(node *restic.Node, stats ItemStats) { - arch.CompleteItem(snPath, previous, node, stats, time.Since(start)) + arch.trackItem(snPath, previous, node, stats, time.Since(start)) }) case fi.IsDir(): @@ -462,9 +535,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous return FutureNode{}, false, err } - fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree, + fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree, func(node *restic.Node, stats ItemStats) { - arch.CompleteItem(snItem, previous, node, stats, time.Since(start)) + arch.trackItem(snItem, previous, node, stats, time.Since(start)) }) if err != nil { debug.Log("SaveDir for %v returned error: %v", snPath, err) @@ -478,7 +551,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous default: debug.Log(" %v other", target) - node, err := arch.nodeFromFileInfo(snPath, target, fi) + node, err := arch.nodeFromFileInfo(snPath, target, fi, false) if err != nil { return FutureNode{}, false, err } @@ -545,9 +618,9 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) { return fi, nil } -// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path +// saveTree stores a Tree in the repo, returned is the tree. snPath is the path // within the current snapshot. -func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) { +func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) { var node *restic.Node if snPath != "/" { @@ -561,7 +634,9 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, } debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath) - node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi) + // in some cases reading xattrs for directories above the backup source is not allowed + // thus ignore errors for such folders. + node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi, true) if err != nil { return FutureNode{}, 0, err } @@ -585,7 +660,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, // this is a leaf node if subatree.Leaf() { - fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name)) + fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name)) if err != nil { err = arch.error(subatree.Path, err) @@ -619,8 +694,8 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, } // not a leaf node, archive subtree - fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { - arch.CompleteItem(snItem, oldNode, n, is, time.Since(start)) + fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) { + arch.trackItem(snItem, oldNode, n, is, time.Since(start)) }) if err != nil { return FutureNode{}, 0, err @@ -632,27 +707,6 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, return fn, len(nodes), nil } -// flags are passed to fs.OpenFile. O_RDONLY is implied. -func readdirnames(filesystem fs.FS, dir string, flags int) ([]string, error) { - f, err := filesystem.OpenFile(dir, fs.O_RDONLY|flags, 0) - if err != nil { - return nil, errors.WithStack(err) - } - - entries, err := f.Readdirnames(-1) - if err != nil { - _ = f.Close() - return nil, errors.Wrapf(err, "Readdirnames %v failed", dir) - } - - err = f.Close() - if err != nil { - return nil, err - } - - return entries, nil -} - // resolveRelativeTargets replaces targets that only contain relative // directories ("." or "../../") with the contents of the directory. Each // element of target is processed with fs.Clean(). @@ -668,7 +722,7 @@ func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) { } debug.Log("replacing %q with readdir(%q)", target, target) - entries, err := readdirnames(filesys, target, fs.O_NOFOLLOW) + entries, err := fs.Readdirnames(filesys, target, fs.O_NOFOLLOW) if err != nil { return nil, err } @@ -688,9 +742,12 @@ type SnapshotOptions struct { Tags restic.TagList Hostname string Excludes []string + BackupStart time.Time Time time.Time ParentSnapshot *restic.Snapshot ProgramVersion string + // SkipIfUnchanged omits the snapshot creation if it is identical to the parent snapshot. + SkipIfUnchanged bool } // loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned. @@ -738,15 +795,17 @@ func (arch *Archiver) stopWorkers() { } // Snapshot saves several targets and returns a snapshot. -func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) { +func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) { + arch.summary = &Summary{} + cleanTargets, err := resolveRelativeTargets(arch.FS, targets) if err != nil { - return nil, restic.ID{}, err + return nil, restic.ID{}, nil, err } atree, err := NewTree(arch.FS, cleanTargets) if err != nil { - return nil, restic.ID{}, err + return nil, restic.ID{}, nil, err } var rootTreeID restic.ID @@ -762,8 +821,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps arch.runWorkers(wgCtx, wg) debug.Log("starting snapshot") - fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(n *restic.Node, is ItemStats) { - arch.CompleteItem("/", nil, nil, is, time.Since(start)) + fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) { + arch.trackItem("/", nil, nil, is, time.Since(start)) }) if err != nil { return err @@ -799,12 +858,19 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps }) err = wgUp.Wait() if err != nil { - return nil, restic.ID{}, err + return nil, restic.ID{}, nil, err + } + + if opts.ParentSnapshot != nil && opts.SkipIfUnchanged { + ps := opts.ParentSnapshot + if ps.Tree != nil && rootTreeID.Equal(*ps.Tree) { + return nil, restic.ID{}, arch.summary, nil + } } sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time) if err != nil { - return nil, restic.ID{}, err + return nil, restic.ID{}, nil, err } sn.ProgramVersion = opts.ProgramVersion @@ -813,11 +879,28 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps sn.Parent = opts.ParentSnapshot.ID() } sn.Tree = &rootTreeID + sn.Summary = &restic.SnapshotSummary{ + BackupStart: opts.BackupStart, + BackupEnd: time.Now(), + + FilesNew: arch.summary.Files.New, + FilesChanged: arch.summary.Files.Changed, + FilesUnmodified: arch.summary.Files.Unchanged, + DirsNew: arch.summary.Dirs.New, + DirsChanged: arch.summary.Dirs.Changed, + DirsUnmodified: arch.summary.Dirs.Unchanged, + DataBlobs: arch.summary.ItemStats.DataBlobs, + TreeBlobs: arch.summary.ItemStats.TreeBlobs, + DataAdded: arch.summary.ItemStats.DataSize + arch.summary.ItemStats.TreeSize, + DataAddedPacked: arch.summary.ItemStats.DataSizeInRepo + arch.summary.ItemStats.TreeSizeInRepo, + TotalFilesProcessed: arch.summary.Files.New + arch.summary.Files.Changed + arch.summary.Files.Unchanged, + TotalBytesProcessed: arch.summary.ProcessedBytes, + } id, err := restic.SaveSnapshot(ctx, arch.Repo, sn) if err != nil { - return nil, restic.ID{}, err + return nil, restic.ID{}, nil, err } - return sn, id, nil + return sn, id, arch.summary, nil } diff --git a/mover-restic/restic/internal/archiver/archiver_test.go b/mover-restic/restic/internal/archiver/archiver_test.go index 5a9896a48..f38d5b0de 100644 --- a/mover-restic/restic/internal/archiver/archiver_test.go +++ b/mover-restic/restic/internal/archiver/archiver_test.go @@ -15,18 +15,20 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" - restictest "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) repo := repository.TestRepository(t) TestCreateFiles(t, tempdir, src) @@ -34,7 +36,7 @@ func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository return tempdir, repo } -func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem fs.FS) (*restic.Node, ItemStats) { +func saveFile(t testing.TB, repo archiverRepo, filename string, filesystem fs.FS) (*restic.Node, ItemStats) { wg, ctx := errgroup.WithContext(context.TODO()) repo.StartPackUploader(ctx, wg) @@ -131,7 +133,7 @@ func TestArchiverSaveFile(t *testing.T) { var tests = []TestFile{ {Content: ""}, {Content: "foo"}, - {Content: string(restictest.Random(23, 12*1024*1024+1287898))}, + {Content: string(rtest.Random(23, 12*1024*1024+1287898))}, } for _, testfile := range tests { @@ -164,7 +166,7 @@ func TestArchiverSaveFileReaderFS(t *testing.T) { Data string }{ {Data: "foo"}, - {Data: string(restictest.Random(23, 12*1024*1024+1287898))}, + {Data: string(rtest.Random(23, 12*1024*1024+1287898))}, } for _, test := range tests { @@ -206,7 +208,7 @@ func TestArchiverSave(t *testing.T) { var tests = []TestFile{ {Content: ""}, {Content: "foo"}, - {Content: string(restictest.Random(23, 12*1024*1024+1287898))}, + {Content: string(rtest.Random(23, 12*1024*1024+1287898))}, } for _, testfile := range tests { @@ -225,8 +227,9 @@ func TestArchiverSave(t *testing.T) { return err } arch.runWorkers(ctx, wg) + arch.summary = &Summary{} - node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil) + node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil) if err != nil { t.Fatal(err) } @@ -274,7 +277,7 @@ func TestArchiverSaveReaderFS(t *testing.T) { Data string }{ {Data: "foo"}, - {Data: string(restictest.Random(23, 12*1024*1024+1287898))}, + {Data: string(rtest.Random(23, 12*1024*1024+1287898))}, } for _, test := range tests { @@ -302,8 +305,9 @@ func TestArchiverSaveReaderFS(t *testing.T) { return err } arch.runWorkers(ctx, wg) + arch.summary = &Summary{} - node, excluded, err := arch.Save(ctx, "/", filename, nil) + node, excluded, err := arch.save(ctx, "/", filename, nil) t.Logf("Save returned %v %v", node, err) if err != nil { t.Fatal(err) @@ -350,7 +354,7 @@ func TestArchiverSaveReaderFS(t *testing.T) { func BenchmarkArchiverSaveFileSmall(b *testing.B) { const fileSize = 4 * 1024 d := TestDir{"file": TestFile{ - Content: string(restictest.Random(23, fileSize)), + Content: string(rtest.Random(23, fileSize)), }} b.SetBytes(fileSize) @@ -382,7 +386,7 @@ func BenchmarkArchiverSaveFileSmall(b *testing.B) { func BenchmarkArchiverSaveFileLarge(b *testing.B) { const fileSize = 40*1024*1024 + 1287898 d := TestDir{"file": TestFile{ - Content: string(restictest.Random(23, fileSize)), + Content: string(rtest.Random(23, fileSize)), }} b.SetBytes(fileSize) @@ -412,14 +416,14 @@ func BenchmarkArchiverSaveFileLarge(b *testing.B) { } type blobCountingRepo struct { - restic.Repository + archiverRepo m sync.Mutex saved map[restic.BlobHandle]uint } func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) { - id, exists, size, err := repo.Repository.SaveBlob(ctx, t, buf, id, storeDuplicate) + id, exists, size, err := repo.archiverRepo.SaveBlob(ctx, t, buf, id, storeDuplicate) if exists { return id, exists, size, err } @@ -431,7 +435,7 @@ func (repo *blobCountingRepo) SaveBlob(ctx context.Context, t restic.BlobType, b } func (repo *blobCountingRepo) SaveTree(ctx context.Context, t *restic.Tree) (restic.ID, error) { - id, err := restic.SaveTree(ctx, repo.Repository, t) + id, err := restic.SaveTree(ctx, repo.archiverRepo, t) h := restic.BlobHandle{ID: id, Type: restic.TreeBlob} repo.m.Lock() repo.saved[h]++ @@ -458,14 +462,14 @@ func appendToFile(t testing.TB, filename string, data []byte) { } func TestArchiverSaveFileIncremental(t *testing.T) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) repo := &blobCountingRepo{ - Repository: repository.TestRepository(t), - saved: make(map[restic.BlobHandle]uint), + archiverRepo: repository.TestRepository(t), + saved: make(map[restic.BlobHandle]uint), } - data := restictest.Random(23, 512*1024+887898) + data := rtest.Random(23, 512*1024+887898) testfile := filepath.Join(tempdir, "testfile") for i := 0; i < 3; i++ { @@ -508,12 +512,12 @@ func chmodTwice(t testing.TB, name string) { // POSIX says that ctime is updated "even if the file status does not // change", but let's make sure it does change, just in case. err := os.Chmod(name, 0700) - restictest.OK(t, err) + rtest.OK(t, err) sleep() err = os.Chmod(name, 0600) - restictest.OK(t, err) + rtest.OK(t, err) } func lstat(t testing.TB, name string) os.FileInfo { @@ -552,7 +556,7 @@ func rename(t testing.TB, oldname, newname string) { } func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node { - node, err := restic.NodeFromFileInfo(filename, fi) + node, err := restic.NodeFromFileInfo(filename, fi, false) if err != nil { t.Fatal(err) } @@ -672,7 +676,7 @@ func TestFileChanged(t *testing.T) { t.Skip("don't run test on Windows") } - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) filename := filepath.Join(tempdir, "file") content := defaultContent @@ -708,7 +712,7 @@ func TestFileChanged(t *testing.T) { } func TestFilChangedSpecialCases(t *testing.T) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) filename := filepath.Join(tempdir, "file") content := []byte("foobar") @@ -742,12 +746,12 @@ func TestArchiverSaveDir(t *testing.T) { }{ { src: TestDir{ - "targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))}, + "targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, }, target: ".", want: TestDir{ "targetdir": TestDir{ - "targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))}, + "targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, }, }, }, @@ -757,8 +761,8 @@ func TestArchiverSaveDir(t *testing.T) { "foo": TestFile{Content: "foo"}, "emptyfile": TestFile{Content: ""}, "bar": TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"}, - "largefile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))}, - "largerfile": TestFile{Content: string(restictest.Random(234, 5*1024*1024+5000))}, + "largefile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, + "largerfile": TestFile{Content: string(rtest.Random(234, 5*1024*1024+5000))}, }, }, target: "targetdir", @@ -830,13 +834,14 @@ func TestArchiverSaveDir(t *testing.T) { arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch.runWorkers(ctx, wg) + arch.summary = &Summary{} chdir := tempdir if test.chdir != "" { chdir = filepath.Join(chdir, test.chdir) } - back := restictest.Chdir(t, chdir) + back := rtest.Chdir(t, chdir) defer back() fi, err := fs.Lstat(test.target) @@ -844,7 +849,7 @@ func TestArchiverSaveDir(t *testing.T) { t.Fatal(err) } - ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil) + ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil) if err != nil { t.Fatal(err) } @@ -894,11 +899,11 @@ func TestArchiverSaveDir(t *testing.T) { } func TestArchiverSaveDirIncremental(t *testing.T) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) repo := &blobCountingRepo{ - Repository: repository.TestRepository(t), - saved: make(map[restic.BlobHandle]uint), + archiverRepo: repository.TestRepository(t), + saved: make(map[restic.BlobHandle]uint), } appendToFile(t, filepath.Join(tempdir, "testfile"), []byte("foobar")) @@ -911,13 +916,14 @@ func TestArchiverSaveDirIncremental(t *testing.T) { arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch.runWorkers(ctx, wg) + arch.summary = &Summary{} fi, err := fs.Lstat(tempdir) if err != nil { t.Fatal(err) } - ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil) + ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil) if err != nil { t.Fatal(err) } @@ -981,9 +987,9 @@ func TestArchiverSaveDirIncremental(t *testing.T) { // bothZeroOrNeither fails the test if only one of exp, act is zero. func bothZeroOrNeither(tb testing.TB, exp, act uint64) { + tb.Helper() if (exp == 0 && act != 0) || (exp != 0 && act == 0) { - _, file, line, _ := runtime.Caller(1) - tb.Fatalf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) + rtest.Equals(tb, exp, act) } } @@ -1003,7 +1009,7 @@ func TestArchiverSaveTree(t *testing.T) { prepare func(t testing.TB) targets []string want TestDir - stat ItemStats + stat Summary }{ { src: TestDir{ @@ -1013,7 +1019,12 @@ func TestArchiverSaveTree(t *testing.T) { want: TestDir{ "targetfile": TestFile{Content: string("foobar")}, }, - stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, + stat: Summary{ + ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0}, + ProcessedBytes: 6, + Files: ChangeStats{1, 0, 0}, + Dirs: ChangeStats{0, 0, 0}, + }, }, { src: TestDir{ @@ -1025,7 +1036,12 @@ func TestArchiverSaveTree(t *testing.T) { "targetfile": TestFile{Content: string("foobar")}, "filesymlink": TestSymlink{Target: "targetfile"}, }, - stat: ItemStats{1, 6, 32 + 6, 0, 0, 0}, + stat: Summary{ + ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0}, + ProcessedBytes: 6, + Files: ChangeStats{1, 0, 0}, + Dirs: ChangeStats{0, 0, 0}, + }, }, { src: TestDir{ @@ -1045,7 +1061,12 @@ func TestArchiverSaveTree(t *testing.T) { "symlink": TestSymlink{Target: "subdir"}, }, }, - stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a}, + stat: Summary{ + ItemStats: ItemStats{0, 0, 0, 1, 0x154, 0x16a}, + ProcessedBytes: 0, + Files: ChangeStats{0, 0, 0}, + Dirs: ChangeStats{1, 0, 0}, + }, }, { src: TestDir{ @@ -1069,7 +1090,12 @@ func TestArchiverSaveTree(t *testing.T) { }, }, }, - stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1}, + stat: Summary{ + ItemStats: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1}, + ProcessedBytes: 6, + Files: ChangeStats{1, 0, 0}, + Dirs: ChangeStats{3, 0, 0}, + }, }, } @@ -1081,20 +1107,13 @@ func TestArchiverSaveTree(t *testing.T) { arch := New(repo, testFS, Options{}) - var stat ItemStats - lock := &sync.Mutex{} - arch.CompleteItem = func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) { - lock.Lock() - defer lock.Unlock() - stat.Add(s) - } - wg, ctx := errgroup.WithContext(context.TODO()) repo.StartPackUploader(ctx, wg) arch.runWorkers(ctx, wg) + arch.summary = &Summary{} - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() if test.prepare != nil { @@ -1106,7 +1125,7 @@ func TestArchiverSaveTree(t *testing.T) { t.Fatal(err) } - fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil) + fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil) if err != nil { t.Fatal(err) } @@ -1133,11 +1152,15 @@ func TestArchiverSaveTree(t *testing.T) { want = test.src } TestEnsureTree(context.TODO(), t, "/", repo, treeID, want) + stat := arch.summary bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs)) bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs)) bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize) bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo) bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo) + rtest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes) + rtest.Equals(t, test.stat.Files, stat.Files) + rtest.Equals(t, test.stat.Dirs, stat.Dirs) }) } } @@ -1385,7 +1408,7 @@ func TestArchiverSnapshot(t *testing.T) { chdir = filepath.Join(chdir, filepath.FromSlash(test.chdir)) } - back := restictest.Chdir(t, chdir) + back := rtest.Chdir(t, chdir) defer back() var targets []string @@ -1394,7 +1417,7 @@ func TestArchiverSnapshot(t *testing.T) { } t.Logf("targets: %v", targets) - sn, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()}) + sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -1407,7 +1430,7 @@ func TestArchiverSnapshot(t *testing.T) { } TestEnsureSnapshot(t, repo, snapshotID, want) - checker.TestCheckRepo(t, repo) + checker.TestCheckRepo(t, repo, false) // check that the snapshot contains the targets with absolute paths for i, target := range sn.Paths { @@ -1538,11 +1561,11 @@ func TestArchiverSnapshotSelect(t *testing.T) { arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch.Select = test.selFn - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() targets := []string{"."} - _, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()}) + _, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()}) if test.err != "" { if err == nil { t.Fatalf("expected error not found, got %v, wanted %q", err, test.err) @@ -1567,7 +1590,7 @@ func TestArchiverSnapshotSelect(t *testing.T) { } TestEnsureSnapshot(t, repo, snapshotID, want) - checker.TestCheckRepo(t, repo) + checker.TestCheckRepo(t, repo, false) }) } } @@ -1615,17 +1638,85 @@ func (f MockFile) Read(p []byte) (int, error) { return n, err } +func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) { + rtest.Equals(t, stat.Files.New, sn.Summary.FilesNew) + rtest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged) + rtest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified) + rtest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew) + rtest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged) + rtest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified) + rtest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed) + rtest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed) + bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs)) + bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs)) + bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded)) + bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked)) +} + func TestArchiverParent(t *testing.T) { var tests = []struct { - src TestDir - read map[string]int // tracks number of times a file must have been read + src TestDir + modify func(path string) + statInitial Summary + statSecond Summary }{ { src: TestDir{ - "targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))}, + "targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))}, + }, + statInitial: Summary{ + Files: ChangeStats{1, 0, 0}, + Dirs: ChangeStats{0, 0, 0}, + ProcessedBytes: 2102152, + ItemStats: ItemStats{3, 0x201593, 0x201632, 1, 0, 0}, + }, + statSecond: Summary{ + Files: ChangeStats{0, 0, 1}, + Dirs: ChangeStats{0, 0, 0}, + ProcessedBytes: 2102152, + }, + }, + { + src: TestDir{ + "targetDir": TestDir{ + "targetfile": TestFile{Content: string(rtest.Random(888, 1234))}, + "targetfile2": TestFile{Content: string(rtest.Random(888, 1235))}, + }, + }, + statInitial: Summary{ + Files: ChangeStats{2, 0, 0}, + Dirs: ChangeStats{1, 0, 0}, + ProcessedBytes: 2469, + ItemStats: ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0}, + }, + statSecond: Summary{ + Files: ChangeStats{0, 0, 2}, + Dirs: ChangeStats{0, 0, 1}, + ProcessedBytes: 2469, + }, + }, + { + src: TestDir{ + "targetDir": TestDir{ + "targetfile": TestFile{Content: string(rtest.Random(888, 1234))}, + }, + "targetfile2": TestFile{Content: string(rtest.Random(888, 1235))}, }, - read: map[string]int{ - "targetfile": 1, + modify: func(path string) { + remove(t, filepath.Join(path, "targetDir", "targetfile")) + save(t, filepath.Join(path, "targetfile2"), []byte("foobar")) + }, + statInitial: Summary{ + Files: ChangeStats{2, 0, 0}, + Dirs: ChangeStats{1, 0, 0}, + ProcessedBytes: 2469, + ItemStats: ItemStats{2, 0xe13, 0xcf8, 2, 0, 0}, + }, + statSecond: Summary{ + Files: ChangeStats{0, 1, 0}, + Dirs: ChangeStats{0, 1, 0}, + ProcessedBytes: 6, + ItemStats: ItemStats{1, 0x305, 0x233, 2, 0, 0}, }, }, } @@ -1644,10 +1735,10 @@ func TestArchiverParent(t *testing.T) { arch := New(repo, testFS, Options{}) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() - firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) + firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if err != nil { t.Fatal(err) } @@ -1672,38 +1763,38 @@ func TestArchiverParent(t *testing.T) { } return nil }) + rtest.Equals(t, test.statInitial.Files, summary.Files) + rtest.Equals(t, test.statInitial.Dirs, summary.Dirs) + rtest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes) + checkSnapshotStats(t, firstSnapshot, test.statInitial) + + if test.modify != nil { + test.modify(tempdir) + } opts := SnapshotOptions{ Time: time.Now(), ParentSnapshot: firstSnapshot, } - _, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts) + testFS.bytesRead = map[string]int{} + secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts) if err != nil { t.Fatal(err) } - // check that all files still been read exactly once - TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error { - file, ok := item.(TestFile) - if !ok { - return nil - } - - n, ok := testFS.bytesRead[filename] - if !ok { - t.Fatalf("file %v was not read at all", filename) - } - - if n != len(file.Content) { - t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content)) - } - return nil - }) + if test.modify == nil { + // check that no files were read this time + rtest.Equals(t, map[string]int{}, testFS.bytesRead) + } + rtest.Equals(t, test.statSecond.Files, summary.Files) + rtest.Equals(t, test.statSecond.Dirs, summary.Dirs) + rtest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes) + checkSnapshotStats(t, secondSnapshot, test.statSecond) t.Logf("second backup saved as %v", secondSnapshotID.Str()) t.Logf("testfs: %v", testFS) - checker.TestCheckRepo(t, repo) + checker.TestCheckRepo(t, repo, false) }) } } @@ -1803,7 +1894,7 @@ func TestArchiverErrorReporting(t *testing.T) { tempdir, repo := prepareTempdirRepoSrc(t, test.src) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() if test.prepare != nil { @@ -1813,7 +1904,7 @@ func TestArchiverErrorReporting(t *testing.T) { arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) arch.Error = test.errFn - _, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) + _, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if test.mustError { if err != nil { t.Logf("found expected error (%v), skipping further checks", err) @@ -1836,32 +1927,32 @@ func TestArchiverErrorReporting(t *testing.T) { } TestEnsureSnapshot(t, repo, snapshotID, want) - checker.TestCheckRepo(t, repo) + checker.TestCheckRepo(t, repo, false) }) } } type noCancelBackend struct { - restic.Backend + backend.Backend } -func (c *noCancelBackend) Remove(_ context.Context, h restic.Handle) error { +func (c *noCancelBackend) Remove(_ context.Context, h backend.Handle) error { return c.Backend.Remove(context.Background(), h) } -func (c *noCancelBackend) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { +func (c *noCancelBackend) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error { return c.Backend.Save(context.Background(), h, rd) } -func (c *noCancelBackend) Load(_ context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (c *noCancelBackend) Load(_ context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { return c.Backend.Load(context.Background(), h, length, offset, fn) } -func (c *noCancelBackend) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (c *noCancelBackend) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { return c.Backend.Stat(context.Background(), h) } -func (c *noCancelBackend) List(_ context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (c *noCancelBackend) List(_ context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return c.Backend.List(context.Background(), t, fn) } @@ -1873,20 +1964,20 @@ func TestArchiverContextCanceled(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) TestCreateFiles(t, tempdir, TestDir{ "targetfile": TestFile{Content: "foobar"}, }) // Ensure that the archiver itself reports the canceled context and not just the backend - repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) + repo, _ := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() arch := New(repo, fs.Track{FS: fs.Local{}}, Options{}) - _, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) + _, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if err != nil { t.Logf("found expected error (%v)", err) @@ -1926,7 +2017,7 @@ func (m *TrackFS) OpenFile(name string, flag int, perm os.FileMode) (fs.File, er } type failSaveRepo struct { - restic.Repository + archiverRepo failAfter int32 cnt int32 err error @@ -1938,7 +2029,7 @@ func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []by return restic.Hash(buf), false, 0, f.err } - return f.Repository.SaveBlob(ctx, t, buf, id, storeDuplicate) + return f.archiverRepo.SaveBlob(ctx, t, buf, id, storeDuplicate) } func TestArchiverAbortEarlyOnError(t *testing.T) { @@ -1967,16 +2058,16 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { { src: TestDir{ "dir": TestDir{ - "file0": TestFile{Content: string(restictest.Random(0, 1024))}, - "file1": TestFile{Content: string(restictest.Random(1, 1024))}, - "file2": TestFile{Content: string(restictest.Random(2, 1024))}, - "file3": TestFile{Content: string(restictest.Random(3, 1024))}, - "file4": TestFile{Content: string(restictest.Random(4, 1024))}, - "file5": TestFile{Content: string(restictest.Random(5, 1024))}, - "file6": TestFile{Content: string(restictest.Random(6, 1024))}, - "file7": TestFile{Content: string(restictest.Random(7, 1024))}, - "file8": TestFile{Content: string(restictest.Random(8, 1024))}, - "file9": TestFile{Content: string(restictest.Random(9, 1024))}, + "file0": TestFile{Content: string(rtest.Random(0, 1024))}, + "file1": TestFile{Content: string(rtest.Random(1, 1024))}, + "file2": TestFile{Content: string(rtest.Random(2, 1024))}, + "file3": TestFile{Content: string(rtest.Random(3, 1024))}, + "file4": TestFile{Content: string(rtest.Random(4, 1024))}, + "file5": TestFile{Content: string(rtest.Random(5, 1024))}, + "file6": TestFile{Content: string(rtest.Random(6, 1024))}, + "file7": TestFile{Content: string(rtest.Random(7, 1024))}, + "file8": TestFile{Content: string(rtest.Random(8, 1024))}, + "file9": TestFile{Content: string(rtest.Random(9, 1024))}, }, }, wantOpen: map[string]uint{ @@ -2001,7 +2092,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { tempdir, repo := prepareTempdirRepoSrc(t, test.src) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() testFS := &TrackFS{ @@ -2014,9 +2105,9 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { } testRepo := &failSaveRepo{ - Repository: repo, - failAfter: int32(test.failAfter), - err: test.err, + archiverRepo: repo, + failAfter: int32(test.failAfter), + err: test.err, } // at most two files may be queued @@ -2025,7 +2116,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { SaveBlobConcurrency: 1, }) - _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) + _, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()}) if !errors.Is(err, test.err) { t.Errorf("expected error (%v) not found, got %v", test.err, err) } @@ -2043,7 +2134,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) { } } -func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Snapshot, filename string) (*restic.Snapshot, *restic.Node) { +func snapshot(t testing.TB, repo archiverRepo, fs fs.FS, parent *restic.Snapshot, filename string) (*restic.Snapshot, *restic.Node) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2053,7 +2144,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Sna Time: time.Now(), ParentSnapshot: parent, } - snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts) + snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts) if err != nil { t.Fatal(err) } @@ -2124,6 +2215,8 @@ const ( ) func TestMetadataChanged(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)() + files := TestDir{ "testfile": TestFile{ Content: "foo bar test file", @@ -2132,12 +2225,12 @@ func TestMetadataChanged(t *testing.T) { tempdir, repo := prepareTempdirRepoSrc(t, files) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() // get metadata fi := lstat(t, "testfile") - want, err := restic.NodeFromFileInfo("testfile", fi) + want, err := restic.NodeFromFileInfo("testfile", fi, false) if err != nil { t.Fatal(err) } @@ -2152,6 +2245,7 @@ func TestMetadataChanged(t *testing.T) { sn, node2 := snapshot(t, repo, fs, nil, "testfile") // set some values so we can then compare the nodes + want.DeviceID = 0 want.Content = node2.Content want.Path = "" if len(want.ExtendedAttributes) == 0 { @@ -2194,7 +2288,7 @@ func TestMetadataChanged(t *testing.T) { // make sure the content matches TestEnsureFileContent(context.Background(), t, repo, "testfile", node3, files["testfile"].(TestFile)) - checker.TestCheckRepo(t, repo) + checker.TestCheckRepo(t, repo, false) } func TestRacyFileSwap(t *testing.T) { @@ -2206,7 +2300,7 @@ func TestRacyFileSwap(t *testing.T) { tempdir, repo := prepareTempdirRepoSrc(t, files) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() // get metadata of current folder @@ -2235,7 +2329,7 @@ func TestRacyFileSwap(t *testing.T) { arch.runWorkers(ctx, wg) // fs.Track will panic if the file was not closed - _, excluded, err := arch.Save(ctx, "/", tempfile, nil) + _, excluded, err := arch.save(ctx, "/", tempfile, nil) if err == nil { t.Errorf("Save() should have failed") } diff --git a/mover-restic/restic/internal/archiver/archiver_unix_test.go b/mover-restic/restic/internal/archiver/archiver_unix_test.go index 7523f0749..4a380dff8 100644 --- a/mover-restic/restic/internal/archiver/archiver_unix_test.go +++ b/mover-restic/restic/internal/archiver/archiver_unix_test.go @@ -6,6 +6,12 @@ package archiver import ( "os" "syscall" + "testing" + + "github.com/restic/restic/internal/feature" + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" ) type wrappedFileInfo struct { @@ -39,3 +45,45 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo { return res } + +func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) { + fi := lstat(t, name) + want, err := restic.NodeFromFileInfo(name, fi, false) + rtest.OK(t, err) + + _, node := snapshot(t, repo, fs.Local{}, nil, name) + return want, node +} + +func TestHardlinkMetadata(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)() + + files := TestDir{ + "testfile": TestFile{ + Content: "foo bar test file", + }, + "linktarget": TestFile{ + Content: "test file", + }, + "testlink": TestHardlink{ + Target: "./linktarget", + }, + "testdir": TestDir{}, + } + + tempdir, repo := prepareTempdirRepoSrc(t, files) + + back := rtest.Chdir(t, tempdir) + defer back() + + want, node := statAndSnapshot(t, repo, "testlink") + rtest.Assert(t, node.DeviceID == want.DeviceID, "device id mismatch expected %v got %v", want.DeviceID, node.DeviceID) + rtest.Assert(t, node.Links == want.Links, "link count mismatch expected %v got %v", want.Links, node.Links) + rtest.Assert(t, node.Inode == want.Inode, "inode mismatch expected %v got %v", want.Inode, node.Inode) + + _, node = statAndSnapshot(t, repo, "testfile") + rtest.Assert(t, node.DeviceID == 0, "device id mismatch for testfile expected %v got %v", 0, node.DeviceID) + + _, node = statAndSnapshot(t, repo, "testdir") + rtest.Assert(t, node.DeviceID == 0, "device id mismatch for testdir expected %v got %v", 0, node.DeviceID) +} diff --git a/mover-restic/restic/internal/archiver/blob_saver.go b/mover-restic/restic/internal/archiver/blob_saver.go index ae4879ff4..d4347a169 100644 --- a/mover-restic/restic/internal/archiver/blob_saver.go +++ b/mover-restic/restic/internal/archiver/blob_saver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -43,9 +44,9 @@ func (s *BlobSaver) TriggerShutdown() { // Save stores a blob in the repo. It checks the index and the known blobs // before saving anything. It takes ownership of the buffer passed in. -func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) { +func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, filename string, cb func(res SaveBlobResponse)) { select { - case s.ch <- saveBlobJob{BlobType: t, buf: buf, cb: cb}: + case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}: case <-ctx.Done(): debug.Log("not sending job, context is cancelled") } @@ -54,6 +55,7 @@ func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb type saveBlobJob struct { restic.BlobType buf *Buffer + fn string cb func(res SaveBlobResponse) } @@ -95,7 +97,7 @@ func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) if err != nil { debug.Log("saveBlob returned error, exiting: %v", err) - return err + return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err) } job.cb(res) job.buf.Release() diff --git a/mover-restic/restic/internal/archiver/blob_saver_test.go b/mover-restic/restic/internal/archiver/blob_saver_test.go index 1996c35b8..f7ef2f47d 100644 --- a/mover-restic/restic/internal/archiver/blob_saver_test.go +++ b/mover-restic/restic/internal/archiver/blob_saver_test.go @@ -4,20 +4,20 @@ import ( "context" "fmt" "runtime" + "strings" "sync" "sync/atomic" "testing" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) var errTest = errors.New("test error") type saveFail struct { - idx restic.MasterIndex cnt int32 failAt int32 } @@ -31,18 +31,12 @@ func (b *saveFail) SaveBlob(_ context.Context, _ restic.BlobType, _ []byte, id r return id, false, 0, nil } -func (b *saveFail) Index() restic.MasterIndex { - return b.idx -} - func TestBlobSaver(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wg, ctx := errgroup.WithContext(ctx) - saver := &saveFail{ - idx: index.NewMasterIndex(), - } + saver := &saveFail{} b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU())) @@ -57,7 +51,7 @@ func TestBlobSaver(t *testing.T) { lock.Lock() results = append(results, SaveBlobResponse{}) lock.Unlock() - b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) { + b.Save(ctx, restic.DataBlob, buf, "file", func(res SaveBlobResponse) { lock.Lock() results[idx] = res lock.Unlock() @@ -98,7 +92,6 @@ func TestBlobSaverError(t *testing.T) { wg, ctx := errgroup.WithContext(ctx) saver := &saveFail{ - idx: index.NewMasterIndex(), failAt: int32(test.failAt), } @@ -106,7 +99,7 @@ func TestBlobSaverError(t *testing.T) { for i := 0; i < test.blobs; i++ { buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))} - b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) {}) + b.Save(ctx, restic.DataBlob, buf, "errfile", func(res SaveBlobResponse) {}) } b.TriggerShutdown() @@ -116,9 +109,8 @@ func TestBlobSaverError(t *testing.T) { t.Errorf("expected error not found") } - if err != errTest { - t.Fatalf("unexpected error found: %v", err) - } + rtest.Assert(t, errors.Is(err, errTest), "unexpected error %v", err) + rtest.Assert(t, strings.Contains(err.Error(), "errfile"), "expected error to contain 'errfile' got: %v", err) }) } } diff --git a/mover-restic/restic/internal/archiver/file_saver.go b/mover-restic/restic/internal/archiver/file_saver.go index 0742c8b57..d10334301 100644 --- a/mover-restic/restic/internal/archiver/file_saver.go +++ b/mover-restic/restic/internal/archiver/file_saver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "io" "os" "sync" @@ -15,7 +16,7 @@ import ( ) // SaveBlobFn saves a blob to a repo. -type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, func(res SaveBlobResponse)) +type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, string, func(res SaveBlobResponse)) // FileSaver concurrently saves incoming files to the repo. type FileSaver struct { @@ -28,7 +29,7 @@ type FileSaver struct { CompleteBlob func(bytes uint64) - NodeFromFileInfo func(snPath, filename string, fi os.FileInfo) (*restic.Node, error) + NodeFromFileInfo func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) } // NewFileSaver returns a new file saver. A worker pool with fileWorkers is @@ -146,7 +147,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat panic("completed twice") } isCompleted = true - fnr.err = err + fnr.err = fmt.Errorf("failed to save %v: %w", target, err) fnr.node = nil fnr.stats = ItemStats{} finish(fnr) @@ -155,7 +156,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat debug.Log("%v", snPath) - node, err := s.NodeFromFileInfo(snPath, f.Name(), fi) + node, err := s.NodeFromFileInfo(snPath, f.Name(), fi, false) if err != nil { _ = f.Close() completeError(err) @@ -204,7 +205,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat node.Content = append(node.Content, restic.ID{}) lock.Unlock() - s.saveBlob(ctx, restic.DataBlob, buf, func(sbr SaveBlobResponse) { + s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr SaveBlobResponse) { lock.Lock() if !sbr.known { fnr.stats.DataBlobs++ diff --git a/mover-restic/restic/internal/archiver/file_saver_test.go b/mover-restic/restic/internal/archiver/file_saver_test.go index b088eeeed..409bdedd0 100644 --- a/mover-restic/restic/internal/archiver/file_saver_test.go +++ b/mover-restic/restic/internal/archiver/file_saver_test.go @@ -33,7 +33,7 @@ func createTestFiles(t testing.TB, num int) (files []string) { func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) { wg, ctx := errgroup.WithContext(ctx) - saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, cb func(SaveBlobResponse)) { + saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, _ string, cb func(SaveBlobResponse)) { cb(SaveBlobResponse{ id: restic.Hash(buf.Data), length: len(buf.Data), @@ -49,8 +49,8 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont } s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers) - s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo) (*restic.Node, error) { - return restic.NodeFromFileInfo(filename, fi) + s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) { + return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError) } return s, ctx, wg diff --git a/mover-restic/restic/internal/archiver/scanner.go b/mover-restic/restic/internal/archiver/scanner.go index 6ce2a4700..d61e5ce47 100644 --- a/mover-restic/restic/internal/archiver/scanner.go +++ b/mover-restic/restic/internal/archiver/scanner.go @@ -25,10 +25,10 @@ type Scanner struct { func NewScanner(fs fs.FS) *Scanner { return &Scanner{ FS: fs, - SelectByName: func(item string) bool { return true }, - Select: func(item string, fi os.FileInfo) bool { return true }, - Error: func(item string, err error) error { return err }, - Result: func(item string, s ScanStats) {}, + SelectByName: func(_ string) bool { return true }, + Select: func(_ string, _ os.FileInfo) bool { return true }, + Error: func(_ string, err error) error { return err }, + Result: func(_ string, _ ScanStats) {}, } } @@ -124,7 +124,7 @@ func (s *Scanner) scan(ctx context.Context, stats ScanStats, target string) (Sca stats.Files++ stats.Bytes += uint64(fi.Size()) case fi.Mode().IsDir(): - names, err := readdirnames(s.FS, target, fs.O_NOFOLLOW) + names, err := fs.Readdirnames(s.FS, target, fs.O_NOFOLLOW) if err != nil { return stats, s.Error(target, err) } diff --git a/mover-restic/restic/internal/archiver/scanner_test.go b/mover-restic/restic/internal/archiver/scanner_test.go index 1b4cd1f7f..b5b7057b8 100644 --- a/mover-restic/restic/internal/archiver/scanner_test.go +++ b/mover-restic/restic/internal/archiver/scanner_test.go @@ -9,7 +9,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/fs" - restictest "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" ) func TestScanner(t *testing.T) { @@ -81,10 +81,10 @@ func TestScanner(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) TestCreateFiles(t, tempdir, test.src) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() cur, err := os.Getwd() @@ -216,10 +216,10 @@ func TestScannerError(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) TestCreateFiles(t, tempdir, test.src) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() cur, err := os.Getwd() @@ -288,10 +288,10 @@ func TestScannerCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) TestCreateFiles(t, tempdir, src) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() cur, err := os.Getwd() diff --git a/mover-restic/restic/internal/archiver/testing.go b/mover-restic/restic/internal/archiver/testing.go index c7482d160..106e68445 100644 --- a/mover-restic/restic/internal/archiver/testing.go +++ b/mover-restic/restic/internal/archiver/testing.go @@ -6,11 +6,11 @@ import ( "path" "path/filepath" "runtime" + "sort" "strings" "testing" "time" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -25,13 +25,13 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res Tags: []string{"test"}, } if parent != nil { - sn, err := restic.LoadSnapshot(context.TODO(), arch.Repo, *parent) + sn, err := restic.LoadSnapshot(context.TODO(), repo, *parent) if err != nil { t.Fatal(err) } opts.ParentSnapshot = sn } - sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts) + sn, _, _, err := arch.Snapshot(context.TODO(), []string{path}, opts) if err != nil { t.Fatal(err) } @@ -63,11 +63,29 @@ func (s TestSymlink) String() string { return "" } +// TestHardlink describes a hardlink created for a test. +type TestHardlink struct { + Target string +} + +func (s TestHardlink) String() string { + return "" +} + // TestCreateFiles creates a directory structure described by dir at target, // which must already exist. On Windows, symlinks aren't created. func TestCreateFiles(t testing.TB, target string, dir TestDir) { t.Helper() - for name, item := range dir { + + // ensure a stable order such that it can be guaranteed that a hardlink target already exists + var names []string + for name := range dir { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + item := dir[name] targetPath := filepath.Join(target, name) switch it := item.(type) { @@ -81,6 +99,11 @@ func TestCreateFiles(t testing.TB, target string, dir TestDir) { if err != nil { t.Fatal(err) } + case TestHardlink: + err := fs.Link(filepath.Join(target, filepath.FromSlash(it.Target)), targetPath) + if err != nil { + t.Fatal(err) + } case TestDir: err := fs.Mkdir(targetPath, 0755) if err != nil { @@ -209,13 +232,13 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { } // TestEnsureFileContent checks if the file in the repo is the same as file. -func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Repository, filename string, node *restic.Node, file TestFile) { +func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLoader, filename string, node *restic.Node, file TestFile) { if int(node.Size) != len(file.Content) { t.Fatalf("%v: wrong node size: want %d, got %d", filename, node.Size, len(file.Content)) return } - content := make([]byte, crypto.CiphertextLength(len(file.Content))) + content := make([]byte, len(file.Content)) pos := 0 for _, id := range node.Content { part, err := repo.LoadBlob(ctx, restic.DataBlob, id, content[pos:]) @@ -237,7 +260,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi // TestEnsureTree checks that the tree ID in the repo matches dir. On Windows, // Symlinks are ignored. -func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.Repository, treeID restic.ID, dir TestDir) { +func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.BlobLoader, treeID restic.ID, dir TestDir) { t.Helper() tree, err := restic.LoadTree(ctx, repo, treeID) diff --git a/mover-restic/restic/internal/archiver/testing_test.go b/mover-restic/restic/internal/archiver/testing_test.go index ada7261f1..ff3bd3668 100644 --- a/mover-restic/restic/internal/archiver/testing_test.go +++ b/mover-restic/restic/internal/archiver/testing_test.go @@ -11,7 +11,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" - restictest "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" ) // MockT passes through all logging functions from T, but catches Fail(), @@ -101,7 +101,7 @@ func TestTestCreateFiles(t *testing.T) { } for i, test := range tests { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) t.Run("", func(t *testing.T) { tempdir := filepath.Join(tempdir, fmt.Sprintf("test-%d", i)) @@ -191,7 +191,7 @@ func TestTestWalkFiles(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) got := make(map[string]string) @@ -321,7 +321,7 @@ func TestTestEnsureFiles(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) createFilesAt(t, tempdir, test.files) subtestT := testing.TB(t) @@ -452,7 +452,7 @@ func TestTestEnsureSnapshot(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) targetDir := filepath.Join(tempdir, "target") err := fs.Mkdir(targetDir, 0700) @@ -462,7 +462,7 @@ func TestTestEnsureSnapshot(t *testing.T) { createFilesAt(t, targetDir, test.files) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() repo := repository.TestRepository(t) @@ -473,7 +473,7 @@ func TestTestEnsureSnapshot(t *testing.T) { Hostname: "localhost", Tags: []string{"test"}, } - _, id, err := arch.Snapshot(ctx, []string{"."}, opts) + _, id, _, err := arch.Snapshot(ctx, []string{"."}, opts) if err != nil { t.Fatal(err) } diff --git a/mover-restic/restic/internal/archiver/tree.go b/mover-restic/restic/internal/archiver/tree.go index 16a78ee70..cd03ba521 100644 --- a/mover-restic/restic/internal/archiver/tree.go +++ b/mover-restic/restic/internal/archiver/tree.go @@ -233,7 +233,7 @@ func unrollTree(f fs.FS, t *Tree) error { // nodes, add the contents of Path to the nodes. if t.Path != "" && len(t.Nodes) > 0 { debug.Log("resolve path %v", t.Path) - entries, err := readdirnames(f, t.Path, 0) + entries, err := fs.Readdirnames(f, t.Path, 0) if err != nil { return err } diff --git a/mover-restic/restic/internal/archiver/tree_saver.go b/mover-restic/restic/internal/archiver/tree_saver.go index a7dae3873..9c11b48f0 100644 --- a/mover-restic/restic/internal/archiver/tree_saver.go +++ b/mover-restic/restic/internal/archiver/tree_saver.go @@ -11,7 +11,7 @@ import ( // TreeSaver concurrently saves incoming trees to the repo. type TreeSaver struct { - saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) + saveBlob SaveBlobFn errFn ErrorFunc ch chan<- saveTreeJob @@ -19,7 +19,7 @@ type TreeSaver struct { // NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is // started, it is stopped when ctx is cancelled. -func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)), errFn ErrorFunc) *TreeSaver { +func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob SaveBlobFn, errFn ErrorFunc) *TreeSaver { ch := make(chan saveTreeJob) s := &TreeSaver{ @@ -90,6 +90,10 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I // return the error if it wasn't ignored if fnr.err != nil { debug.Log("err for %v: %v", fnr.snPath, fnr.err) + if fnr.err == context.Canceled { + return nil, stats, fnr.err + } + fnr.err = s.errFn(fnr.target, fnr.err) if fnr.err == nil { // ignore error @@ -126,7 +130,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I b := &Buffer{Data: buf} ch := make(chan SaveBlobResponse, 1) - s.saveBlob(ctx, restic.TreeBlob, b, func(res SaveBlobResponse) { + s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res SaveBlobResponse) { ch <- res }) diff --git a/mover-restic/restic/internal/archiver/tree_saver_test.go b/mover-restic/restic/internal/archiver/tree_saver_test.go index 5de4375d6..47a3f3842 100644 --- a/mover-restic/restic/internal/archiver/tree_saver_test.go +++ b/mover-restic/restic/internal/archiver/tree_saver_test.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sync/errgroup" ) -func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) { +func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, _ string, cb func(res SaveBlobResponse)) { cb(SaveBlobResponse{ id: restic.NewRandomID(), known: false, diff --git a/mover-restic/restic/internal/archiver/tree_test.go b/mover-restic/restic/internal/archiver/tree_test.go index 7852a4c2e..a9d2d97ff 100644 --- a/mover-restic/restic/internal/archiver/tree_test.go +++ b/mover-restic/restic/internal/archiver/tree_test.go @@ -8,7 +8,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/fs" - restictest "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" ) // debug.Log requires Tree.String. @@ -439,10 +439,10 @@ func TestTree(t *testing.T) { t.Skip("skip test on unix") } - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) TestCreateFiles(t, tempdir, test.src) - back := restictest.Chdir(t, tempdir) + back := rtest.Chdir(t, tempdir) defer back() tree, err := NewTree(fs.Local{}, test.targets) diff --git a/mover-restic/restic/internal/backend/azure/azure.go b/mover-restic/restic/internal/backend/azure/azure.go index 4ccfb9664..737cf0e14 100644 --- a/mover-restic/restic/internal/backend/azure/azure.go +++ b/mover-restic/restic/internal/backend/azure/azure.go @@ -15,9 +15,9 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" @@ -43,7 +43,7 @@ const saveLargeSize = 256 * 1024 * 1024 const defaultListMaxItems = 5000 // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("azure", ParseConfig, location.NoPassword, Create, Open) @@ -161,7 +161,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, errors.Wrap(err, "container.Create") } } else if err != nil { - return be, err + return be, errors.Wrap(err, "container.GetProperties") } return be, nil @@ -177,6 +177,20 @@ func (be *Backend) IsNotExist(err error) bool { return bloberror.HasCode(err, bloberror.BlobNotFound) } +func (be *Backend) IsPermanentError(err error) bool { + if be.IsNotExist(err) { + return true + } + + var aerr *azcore.ResponseError + if errors.As(err, &aerr) { + if aerr.StatusCode == http.StatusRequestedRangeNotSatisfiable || aerr.StatusCode == http.StatusUnauthorized || aerr.StatusCode == http.StatusForbidden { + return true + } + } + return false +} + // Join combines path components with slashes. func (be *Backend) Join(p ...string) string { return path.Join(p...) @@ -186,11 +200,6 @@ func (be *Backend) Connections() uint { return be.connections } -// Location returns this backend's location (the container name). -func (be *Backend) Location() string { - return be.Join(be.cfg.AccountName, be.cfg.Prefix) -} - // Hasher may return a hash function for calculating a content hash for the backend func (be *Backend) Hasher() hash.Hash { return md5.New() @@ -207,7 +216,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName) @@ -224,7 +233,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe return err } -func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error { +func (be *Backend) saveSmall(ctx context.Context, objName string, rd backend.RewindReader) error { blockBlobClient := be.container.NewBlockBlobClient(objName) // upload it as a new "block", use the base64 hash for the ID @@ -249,7 +258,7 @@ func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.Rewi return errors.Wrap(err, "CommitBlockList") } -func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error { +func (be *Backend) saveLarge(ctx context.Context, objName string, rd backend.RewindReader) error { blockBlobClient := be.container.NewBlockBlobClient(objName) buf := make([]byte, 100*1024*1024) @@ -304,11 +313,11 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) blockBlobClient := be.container.NewBlobClient(objName) @@ -323,21 +332,26 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, return nil, err } + if length > 0 && (resp.ContentLength == nil || *resp.ContentLength != int64(length)) { + _ = resp.Body.Close() + return nil, &azcore.ResponseError{ErrorCode: "restic-file-too-short", StatusCode: http.StatusRequestedRangeNotSatisfiable} + } + return resp.Body, err } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { objName := be.Filename(h) blobClient := be.container.NewBlobClient(objName) props, err := blobClient.GetProperties(ctx, nil) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties") + return backend.FileInfo{}, errors.Wrap(err, "blob.GetProperties") } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Size: *props.ContentLength, Name: h.Name, } @@ -345,7 +359,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) blob := be.container.NewBlobClient(objName) @@ -360,7 +374,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -391,7 +405,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: *item.Properties.ContentLength, } @@ -417,7 +431,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing diff --git a/mover-restic/restic/internal/backend/azure/azure_test.go b/mover-restic/restic/internal/backend/azure/azure_test.go index 33f65bd52..7df27d325 100644 --- a/mover-restic/restic/internal/backend/azure/azure_test.go +++ b/mover-restic/restic/internal/backend/azure/azure_test.go @@ -122,11 +122,11 @@ func TestUploadLargeFile(t *testing.T) { data := rtest.Random(23, 300*1024*1024) id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} t.Logf("hash of %d bytes: %v", len(data), id) - err = be.Save(ctx, h, restic.NewByteReader(data, be.Hasher())) + err = be.Save(ctx, h, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } diff --git a/mover-restic/restic/internal/backend/azure/config.go b/mover-restic/restic/internal/backend/azure/config.go index 61c413efa..7d69719ef 100644 --- a/mover-restic/restic/internal/backend/azure/config.go +++ b/mover-restic/restic/internal/backend/azure/config.go @@ -6,9 +6,9 @@ import ( "strconv" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an azure compatible @@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/mover-restic/restic/internal/backend/b2/b2.go b/mover-restic/restic/internal/backend/b2/b2.go index 2351d21c7..9717cdd0e 100644 --- a/mover-restic/restic/internal/backend/b2/b2.go +++ b/mover-restic/restic/internal/backend/b2/b2.go @@ -2,6 +2,7 @@ package b2 import ( "context" + "fmt" "hash" "io" "net/http" @@ -12,9 +13,9 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/Backblaze/blazer/b2" "github.com/Backblaze/blazer/base" @@ -31,11 +32,13 @@ type b2Backend struct { canDelete bool } -// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips +var errTooShort = fmt.Errorf("file is too short") + +// Billing happens in 1000 item granularity, but we are more interested in reducing the number of network round trips const defaultListMaxItems = 10 * 1000 -// ensure statically that *b2Backend implements restic.Backend. -var _ restic.Backend = &b2Backend{} +// ensure statically that *b2Backend implements backend.Backend. +var _ backend.Backend = &b2Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("b2", ParseConfig, location.NoPassword, Create, Open) @@ -85,7 +88,7 @@ func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Clien } // Open opens a connection to the B2 service. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(ctx) @@ -97,7 +100,9 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend } bucket, err := client.Bucket(ctx, cfg.Bucket) - if err != nil { + if b2.IsNotExist(err) { + return nil, backend.ErrNoRepository + } else if err != nil { return nil, errors.Wrap(err, "Bucket") } @@ -118,7 +123,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend // Create opens a connection to the B2 service. If the bucket does not exist yet, // it is created. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("cfg %#v", cfg) ctx, cancel := context.WithCancel(ctx) @@ -159,11 +164,6 @@ func (be *b2Backend) Connections() uint { return be.cfg.Connections } -// Location returns the location for the backend. -func (be *b2Backend) Location() string { - return be.cfg.Bucket -} - // Hasher may return a hash function for calculating a content hash for the backend func (be *b2Backend) Hasher() hash.Hash { return nil @@ -186,16 +186,39 @@ func (be *b2Backend) IsNotExist(err error) bool { return false } +func (be *b2Backend) IsPermanentError(err error) bool { + // the library unfortunately endlessly retries authentication errors + return be.IsNotExist(err) || errors.Is(err, errTooShort) +} + // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *b2Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *b2Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) + return util.DefaultLoad(ctx, h, length, offset, be.openReader, func(rd io.Reader) error { + if length == 0 { + return fn(rd) + } + + // there is no direct way to efficiently check whether the file is too short + // use a LimitedReader to track the number of bytes read + limrd := &io.LimitedReader{R: rd, N: int64(length)} + err := fn(limrd) + + // check the underlying reader to be agnostic to however fn() handles the returned error + _, rderr := rd.Read([]byte{0}) + if rderr == io.EOF && limrd.N != 0 { + // file is too short + return fmt.Errorf("%w: %v", errTooShort, err) + } + + return err + }) } -func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *b2Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { name := be.Layout.Filename(h) obj := be.bucket.Object(name) @@ -213,7 +236,7 @@ func (be *b2Backend) openReader(ctx context.Context, h restic.Handle, length int } // Save stores data in the backend at the handle. -func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *b2Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -237,18 +260,18 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd restic.Rewind } // Stat returns information about a blob. -func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *b2Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { name := be.Filename(h) obj := be.bucket.Object(name) info, err := obj.Attrs(ctx) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Stat") + return backend.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: info.Size, Name: h.Name}, nil + return backend.FileInfo{Size: info.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *b2Backend) Remove(ctx context.Context, h backend.Handle) error { // the retry backend will also repeat the remove method up to 10 times for i := 0; i < 3; i++ { obj := be.bucket.Object(be.Filename(h)) @@ -284,7 +307,7 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error { } // List returns a channel that yields all names of blobs of type t. -func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *b2Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -299,7 +322,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic return err } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(obj.Name()), Size: attrs.Size, } @@ -313,7 +336,7 @@ func (be *b2Backend) List(ctx context.Context, t restic.FileType, fn func(restic // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *b2Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing diff --git a/mover-restic/restic/internal/backend/b2/config.go b/mover-restic/restic/internal/backend/b2/config.go index 94614e44f..8d947fc1b 100644 --- a/mover-restic/restic/internal/backend/b2/config.go +++ b/mover-restic/restic/internal/backend/b2/config.go @@ -6,9 +6,9 @@ import ( "regexp" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an b2 compatible @@ -82,7 +82,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/mover-restic/restic/internal/restic/backend.go b/mover-restic/restic/internal/backend/backend.go similarity index 77% rename from mover-restic/restic/internal/restic/backend.go rename to mover-restic/restic/internal/backend/backend.go index df3281641..f606e1123 100644 --- a/mover-restic/restic/internal/restic/backend.go +++ b/mover-restic/restic/internal/backend/backend.go @@ -1,11 +1,14 @@ -package restic +package backend import ( "context" + "fmt" "hash" "io" ) +var ErrNoRepository = fmt.Errorf("repository does not exist") + // Backend is used to store and access data. // // Backend operations that return an error will be retried when a Backend is @@ -14,11 +17,7 @@ import ( // the context package need not be wrapped, as context cancellation is checked // separately by the retrying logic. type Backend interface { - // Location returns a string that describes the type and location of the - // repository. - Location() string - - // Connections returns the maxmimum number of concurrent backend operations. + // Connections returns the maximum number of concurrent backend operations. Connections() uint // Hasher may return a hash function for calculating a content hash for the backend @@ -27,7 +26,7 @@ type Backend interface { // HasAtomicReplace returns whether Save() can atomically replace files HasAtomicReplace() bool - // Remove removes a File described by h. + // Remove removes a File described by h. Remove(ctx context.Context, h Handle) error // Close the backend @@ -38,12 +37,14 @@ type Backend interface { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. If length is larger than zero, only a portion of the file - // is read. + // is read. If the length is larger than zero and the file is too short to return + // the requested length bytes, then an error MUST be returned that is recognized + // by IsPermanentError(). // // The function fn may be called multiple times during the same Load invocation // and therefore must be idempotent. // - // Implementations are encouraged to use backend.DefaultLoad + // Implementations are encouraged to use util.DefaultLoad Load(ctx context.Context, h Handle, length int, offset int64, fn func(rd io.Reader) error) error // Stat returns information about the File identified by h. @@ -66,11 +67,17 @@ type Backend interface { // for unwrapping it. IsNotExist(err error) bool + // IsPermanentError returns true if the error can very likely not be resolved + // by retrying the operation. Backends should return true if the file is missing, + // the requested range does not (completely) exist in the file or the user is + // not authorized to perform the requested operation. + IsPermanentError(err error) bool + // Delete removes all data in the backend. Delete(ctx context.Context) error } -type BackendUnwrapper interface { +type Unwrapper interface { // Unwrap returns the underlying backend or nil if there is none. Unwrap() Backend } @@ -81,7 +88,7 @@ func AsBackend[B Backend](b Backend) B { return be } - if be, ok := b.(BackendUnwrapper); ok { + if be, ok := b.(Unwrapper); ok { b = be.Unwrap() } else { // not the backend we're looking for diff --git a/mover-restic/restic/internal/backend/backend_test.go b/mover-restic/restic/internal/backend/backend_test.go new file mode 100644 index 000000000..28ece55df --- /dev/null +++ b/mover-restic/restic/internal/backend/backend_test.go @@ -0,0 +1,38 @@ +package backend_test + +import ( + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/test" +) + +type testBackend struct { + backend.Backend +} + +func (t *testBackend) Unwrap() backend.Backend { + return nil +} + +type otherTestBackend struct { + backend.Backend +} + +func (t *otherTestBackend) Unwrap() backend.Backend { + return t.Backend +} + +func TestAsBackend(t *testing.T) { + other := otherTestBackend{} + test.Assert(t, backend.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend") + + testBe := &testBackend{} + test.Assert(t, backend.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned") + + wrapper := &otherTestBackend{Backend: testBe} + test.Assert(t, backend.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend") + + wrapper.Backend = other + test.Assert(t, backend.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend") +} diff --git a/mover-restic/restic/internal/cache/backend.go b/mover-restic/restic/internal/backend/cache/backend.go similarity index 70% rename from mover-restic/restic/internal/cache/backend.go rename to mover-restic/restic/internal/backend/cache/backend.go index 311b099ee..63bb6f85f 100644 --- a/mover-restic/restic/internal/cache/backend.go +++ b/mover-restic/restic/internal/backend/cache/backend.go @@ -5,56 +5,57 @@ import ( "io" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend wraps a restic.Backend and adds a cache. type Backend struct { - restic.Backend + backend.Backend *Cache // inProgress contains the handle for all files that are currently // downloaded. The channel in the value is closed as soon as the download // is finished. inProgressMutex sync.Mutex - inProgress map[restic.Handle]chan struct{} + inProgress map[backend.Handle]chan struct{} } -// ensure Backend implements restic.Backend -var _ restic.Backend = &Backend{} +// ensure Backend implements backend.Backend +var _ backend.Backend = &Backend{} -func newBackend(be restic.Backend, c *Cache) *Backend { +func newBackend(be backend.Backend, c *Cache) *Backend { return &Backend{ Backend: be, Cache: c, - inProgress: make(map[restic.Handle]chan struct{}), + inProgress: make(map[backend.Handle]chan struct{}), } } // Remove deletes a file from the backend and the cache if it has been cached. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("cache Remove(%v)", h) err := b.Backend.Remove(ctx, h) if err != nil { return err } - return b.Cache.remove(h) + _, err = b.Cache.remove(h) + return err } -func autoCacheTypes(h restic.Handle) bool { +func autoCacheTypes(h backend.Handle) bool { switch h.Type { - case restic.IndexFile, restic.SnapshotFile: + case backend.IndexFile, backend.SnapshotFile: return true - case restic.PackFile: - return h.ContainedBlobType == restic.TreeBlob + case backend.PackFile: + return h.IsMetadata } return false } // Save stores a new file in the backend and the cache. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if !autoCacheTypes(h) { return b.Backend.Save(ctx, h, rd) } @@ -79,17 +80,16 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea return err } - err = b.Cache.Save(h, rd) + err = b.Cache.save(h, rd) if err != nil { debug.Log("unable to save %v to cache: %v", h, err) - _ = b.Cache.remove(h) return err } return nil } -func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { +func (b *Backend) cacheFile(ctx context.Context, h backend.Handle) error { finish := make(chan struct{}) b.inProgressMutex.Lock() @@ -120,11 +120,11 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { if !b.Cache.Has(h) { // nope, it's still not in the cache, pull it from the repo and save it err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error { - return b.Cache.Save(h, rd) + return b.Cache.save(h, rd) }) if err != nil { // try to remove from the cache, ignore errors - _ = b.Cache.remove(h) + _, _ = b.Cache.remove(h) } return err } @@ -133,10 +133,10 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error { } // loadFromCache will try to load the file from the cache. -func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { - rd, err := b.Cache.load(h, length, offset) +func (b *Backend) loadFromCache(h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) { + rd, inCache, err := b.Cache.load(h, length, offset) if err != nil { - return false, err + return inCache, err } err = consumer(rd) @@ -148,7 +148,7 @@ func (b *Backend) loadFromCache(h restic.Handle, length int, offset int64, consu } // Load loads a file from the cache or the backend. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { b.inProgressMutex.Lock() waitForFinish, inProgress := b.inProgress[h] b.inProgressMutex.Unlock() @@ -162,14 +162,10 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset // try loading from cache without checking that the handle is actually cached inCache, err := b.loadFromCache(h, length, offset, consumer) if inCache { - if err == nil { - return nil - } - - // drop from cache and retry once - _ = b.Cache.remove(h) + debug.Log("error loading %v from cache: %v", h, err) + // the caller must explicitly use cache.Forget() to remove the cache entry + return err } - debug.Log("error loading %v from cache: %v", h, err) // if we don't automatically cache this file type, fall back to the backend if !autoCacheTypes(h) { @@ -185,6 +181,9 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset inCache, err = b.loadFromCache(h, length, offset, consumer) if inCache { + if err != nil { + debug.Log("error loading %v from cache: %v", h, err) + } return err } @@ -194,17 +193,13 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset // Stat tests whether the backend has a file. If it does not exist but still // exists in the cache, it is removed from the cache. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("cache Stat(%v)", h) fi, err := b.Backend.Stat(ctx, h) - if err != nil { - if b.Backend.IsNotExist(err) { - // try to remove from the cache, ignore errors - _ = b.Cache.remove(h) - } - - return fi, err + if err != nil && b.Backend.IsNotExist(err) { + // try to remove from the cache, ignore errors + _, _ = b.Cache.remove(h) } return fi, err @@ -215,6 +210,6 @@ func (b *Backend) IsNotExist(err error) bool { return b.Backend.IsNotExist(err) } -func (b *Backend) Unwrap() restic.Backend { +func (b *Backend) Unwrap() backend.Backend { return b.Backend } diff --git a/mover-restic/restic/internal/backend/cache/backend_test.go b/mover-restic/restic/internal/backend/cache/backend_test.go new file mode 100644 index 000000000..7addc275d --- /dev/null +++ b/mover-restic/restic/internal/backend/cache/backend_test.go @@ -0,0 +1,240 @@ +package cache + +import ( + "bytes" + "context" + "io" + "math/rand" + "strings" + "sync" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/mem" + backendtest "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) { + buf, err := backendtest.LoadAll(context.TODO(), be, h) + if err != nil { + t.Fatal(err) + } + + if len(buf) != len(data) { + t.Fatalf("wrong number of bytes read, want %v, got %v", len(data), len(buf)) + } + + if !bytes.Equal(buf, data) { + t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[:16], buf[:16]) + } +} + +func save(t testing.TB, be backend.Backend, h backend.Handle, data []byte) { + err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher())) + if err != nil { + t.Fatal(err) + } +} + +func remove(t testing.TB, be backend.Backend, h backend.Handle) { + err := be.Remove(context.TODO(), h) + if err != nil { + t.Fatal(err) + } +} + +func randomData(n int) (backend.Handle, []byte) { + data := test.Random(rand.Int(), n) + id := restic.Hash(data) + h := backend.Handle{ + Type: backend.IndexFile, + Name: id.String(), + } + return h, data +} + +func TestBackend(t *testing.T) { + be := mem.New() + c := TestNewCache(t) + wbe := c.Wrap(be) + + h, data := randomData(5234142) + + // save directly in backend + save(t, be, h, data) + if c.Has(h) { + t.Errorf("cache has file too early") + } + + // load data via cache + loadAndCompare(t, wbe, h, data) + if !c.Has(h) { + t.Errorf("cache doesn't have file after load") + } + + // remove via cache + remove(t, wbe, h) + if c.Has(h) { + t.Errorf("cache has file after remove") + } + + // save via cache + save(t, wbe, h, data) + if !c.Has(h) { + t.Errorf("cache doesn't have file after load") + } + + // load data directly from backend + loadAndCompare(t, be, h, data) + + // load data via cache + loadAndCompare(t, wbe, h, data) + + // remove directly + remove(t, be, h) + if !c.Has(h) { + t.Errorf("file not in cache any more") + } + + // run stat + _, err := wbe.Stat(context.TODO(), h) + if err == nil { + t.Errorf("expected error for removed file not found, got nil") + } + + if !wbe.IsNotExist(err) { + t.Errorf("Stat() returned error that does not match IsNotExist(): %v", err) + } + + if c.Has(h) { + t.Errorf("removed file still in cache after stat") + } +} + +type loadCountingBackend struct { + backend.Backend + ctr int +} + +func (l *loadCountingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + l.ctr++ + return l.Backend.Load(ctx, h, length, offset, fn) +} + +func TestOutOfBoundsAccess(t *testing.T) { + be := &loadCountingBackend{Backend: mem.New()} + c := TestNewCache(t) + wbe := c.Wrap(be) + + h, data := randomData(50) + save(t, be, h, data) + + // load out of bounds + err := wbe.Load(context.TODO(), h, 100, 100, func(rd io.Reader) error { + t.Error("cache returned non-existent file section") + return errors.New("broken") + }) + test.Assert(t, strings.Contains(err.Error(), " is too short"), "expected too short error, got %v", err) + test.Equals(t, 1, be.ctr, "expected file to be loaded only once") + // file must nevertheless get cached + if !c.Has(h) { + t.Errorf("cache doesn't have file after load") + } + + // start within bounds, but request too large chunk + err = wbe.Load(context.TODO(), h, 100, 0, func(rd io.Reader) error { + t.Error("cache returned non-existent file section") + return errors.New("broken") + }) + test.Assert(t, strings.Contains(err.Error(), " is too short"), "expected too short error, got %v", err) + test.Equals(t, 1, be.ctr, "expected file to be loaded only once") +} + +func TestForget(t *testing.T) { + be := &loadCountingBackend{Backend: mem.New()} + c := TestNewCache(t) + wbe := c.Wrap(be) + + h, data := randomData(50) + save(t, be, h, data) + + loadAndCompare(t, wbe, h, data) + test.Equals(t, 1, be.ctr, "expected file to be loaded once") + + // must still exist even if load returns an error + exp := errors.New("error") + err := wbe.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error { + return exp + }) + test.Equals(t, exp, err, "wrong error") + test.Assert(t, c.Has(h), "missing cache entry") + + test.OK(t, c.Forget(h)) + test.Assert(t, !c.Has(h), "cache entry should have been removed") + + // cache it again + loadAndCompare(t, wbe, h, data) + test.Assert(t, c.Has(h), "missing cache entry") + + // forget must delete file only once + err = c.Forget(h) + test.Assert(t, strings.Contains(err.Error(), "circuit breaker prevents repeated deletion of cached file"), "wrong error message %q", err) + test.Assert(t, c.Has(h), "cache entry should still exist") +} + +type loadErrorBackend struct { + backend.Backend + loadError error +} + +func (be loadErrorBackend) Load(_ context.Context, _ backend.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { + time.Sleep(10 * time.Millisecond) + return be.loadError +} + +func TestErrorBackend(t *testing.T) { + be := mem.New() + c := TestNewCache(t) + h, data := randomData(5234142) + + // save directly in backend + save(t, be, h, data) + + testErr := errors.New("test error") + errBackend := loadErrorBackend{ + Backend: be, + loadError: testErr, + } + + loadTest := func(wg *sync.WaitGroup, be backend.Backend) { + defer wg.Done() + + buf, err := backendtest.LoadAll(context.TODO(), be, h) + if err == testErr { + return + } + + if err != nil { + t.Error(err) + return + } + + if !bytes.Equal(buf, data) { + t.Errorf("data does not match") + } + time.Sleep(time.Millisecond) + } + + wrappedBE := c.Wrap(errBackend) + var wg sync.WaitGroup + for i := 0; i < 5; i++ { + wg.Add(1) + go loadTest(&wg, wrappedBE) + } + + wg.Wait() +} diff --git a/mover-restic/restic/internal/cache/cache.go b/mover-restic/restic/internal/backend/cache/cache.go similarity index 97% rename from mover-restic/restic/internal/cache/cache.go rename to mover-restic/restic/internal/backend/cache/cache.go index 5b3601741..a55b51c70 100644 --- a/mover-restic/restic/internal/cache/cache.go +++ b/mover-restic/restic/internal/backend/cache/cache.go @@ -6,9 +6,11 @@ import ( "path/filepath" "regexp" "strconv" + "sync" "time" "github.com/pkg/errors" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" @@ -19,6 +21,8 @@ type Cache struct { path string Base string Created bool + + forgotten sync.Map } const dirMode = 0700 @@ -234,7 +238,7 @@ func IsOld(t time.Time, maxAge time.Duration) bool { } // Wrap returns a backend with a cache. -func (c *Cache) Wrap(be restic.Backend) restic.Backend { +func (c *Cache) Wrap(be backend.Backend) backend.Backend { return newBackend(be, c) } diff --git a/mover-restic/restic/internal/cache/cache_test.go b/mover-restic/restic/internal/backend/cache/cache_test.go similarity index 100% rename from mover-restic/restic/internal/cache/cache_test.go rename to mover-restic/restic/internal/backend/cache/cache_test.go diff --git a/mover-restic/restic/internal/cache/dir.go b/mover-restic/restic/internal/backend/cache/dir.go similarity index 100% rename from mover-restic/restic/internal/cache/dir.go rename to mover-restic/restic/internal/backend/cache/dir.go diff --git a/mover-restic/restic/internal/cache/dir_test.go b/mover-restic/restic/internal/backend/cache/dir_test.go similarity index 100% rename from mover-restic/restic/internal/cache/dir_test.go rename to mover-restic/restic/internal/backend/cache/dir_test.go diff --git a/mover-restic/restic/internal/cache/file.go b/mover-restic/restic/internal/backend/cache/file.go similarity index 65% rename from mover-restic/restic/internal/cache/file.go rename to mover-restic/restic/internal/backend/cache/file.go index c315be19f..12f5f23c5 100644 --- a/mover-restic/restic/internal/cache/file.go +++ b/mover-restic/restic/internal/backend/cache/file.go @@ -1,6 +1,7 @@ package cache import ( + "fmt" "io" "os" "path/filepath" @@ -8,13 +9,14 @@ import ( "github.com/pkg/errors" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" ) -func (c *Cache) filename(h restic.Handle) string { +func (c *Cache) filename(h backend.Handle) string { if len(h.Name) < 2 { panic("Name is empty or too short") } @@ -22,7 +24,7 @@ func (c *Cache) filename(h restic.Handle) string { return filepath.Join(c.path, cacheLayoutPaths[h.Type], subdir, h.Name) } -func (c *Cache) canBeCached(t restic.FileType) bool { +func (c *Cache) canBeCached(t backend.FileType) bool { if c == nil { return false } @@ -31,54 +33,54 @@ func (c *Cache) canBeCached(t restic.FileType) bool { return ok } -// Load returns a reader that yields the contents of the file with the +// load returns a reader that yields the contents of the file with the // given handle. rd must be closed after use. If an error is returned, the -// ReadCloser is nil. -func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +// ReadCloser is nil. The bool return value indicates whether the requested +// file exists in the cache. It can be true even when no reader is returned +// because length or offset are out of bounds +func (c *Cache) load(h backend.Handle, length int, offset int64) (io.ReadCloser, bool, error) { debug.Log("Load(%v, %v, %v) from cache", h, length, offset) if !c.canBeCached(h.Type) { - return nil, errors.New("cannot be cached") + return nil, false, errors.New("cannot be cached") } f, err := fs.Open(c.filename(h)) if err != nil { - return nil, errors.WithStack(err) + return nil, false, errors.WithStack(err) } fi, err := f.Stat() if err != nil { _ = f.Close() - return nil, errors.WithStack(err) + return nil, true, errors.WithStack(err) } size := fi.Size() if size <= int64(crypto.CiphertextLength(0)) { _ = f.Close() - _ = c.remove(h) - return nil, errors.Errorf("cached file %v is truncated, removing", h) + return nil, true, errors.Errorf("cached file %v is truncated", h) } if size < offset+int64(length) { _ = f.Close() - _ = c.remove(h) - return nil, errors.Errorf("cached file %v is too small, removing", h) + return nil, true, errors.Errorf("cached file %v is too short", h) } if offset > 0 { if _, err = f.Seek(offset, io.SeekStart); err != nil { _ = f.Close() - return nil, err + return nil, true, err } } if length <= 0 { - return f, nil + return f, true, nil } - return backend.LimitReadCloser(f, int64(length)), nil + return util.LimitReadCloser(f, int64(length)), true, nil } -// Save saves a file in the cache. -func (c *Cache) Save(h restic.Handle, rd io.Reader) error { +// save saves a file in the cache. +func (c *Cache) save(h backend.Handle, rd io.Reader) error { debug.Log("Save to cache: %v", h) if rd == nil { return errors.New("Save() called with nil reader") @@ -138,13 +140,34 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error { return errors.WithStack(err) } -// Remove deletes a file. When the file is not cache, no error is returned. -func (c *Cache) remove(h restic.Handle) error { - if !c.Has(h) { - return nil +func (c *Cache) Forget(h backend.Handle) error { + h.IsMetadata = false + + if _, ok := c.forgotten.Load(h); ok { + // Delete a file at most once while restic runs. + // This prevents repeatedly caching and forgetting broken files + return fmt.Errorf("circuit breaker prevents repeated deletion of cached file %v", h) } - return fs.Remove(c.filename(h)) + removed, err := c.remove(h) + if removed { + c.forgotten.Store(h, struct{}{}) + } + return err +} + +// remove deletes a file. When the file is not cached, no error is returned. +func (c *Cache) remove(h backend.Handle) (bool, error) { + if !c.canBeCached(h.Type) { + return false, nil + } + + err := fs.Remove(c.filename(h)) + removed := err == nil + if errors.Is(err, os.ErrNotExist) { + err = nil + } + return removed, err } // Clear removes all files of type t from the cache that are not contained in @@ -165,7 +188,8 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error { continue } - if err = fs.Remove(c.filename(restic.Handle{Type: t, Name: id.String()})); err != nil { + // ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently + if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) { return err } } @@ -207,7 +231,7 @@ func (c *Cache) list(t restic.FileType) (restic.IDSet, error) { } // Has returns true if the file is cached. -func (c *Cache) Has(h restic.Handle) bool { +func (c *Cache) Has(h backend.Handle) bool { if !c.canBeCached(h.Type) { return false } diff --git a/mover-restic/restic/internal/cache/file_test.go b/mover-restic/restic/internal/backend/cache/file_test.go similarity index 82% rename from mover-restic/restic/internal/cache/file_test.go rename to mover-restic/restic/internal/backend/cache/file_test.go index e72133cd7..331e3251d 100644 --- a/mover-restic/restic/internal/cache/file_test.go +++ b/mover-restic/restic/internal/backend/cache/file_test.go @@ -10,26 +10,27 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) -func generateRandomFiles(t testing.TB, tpe restic.FileType, c *Cache) restic.IDSet { +func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet { ids := restic.NewIDSet() for i := 0; i < rand.Intn(15)+10; i++ { - buf := test.Random(rand.Int(), 1<<19) + buf := rtest.Random(rand.Int(), 1<<19) id := restic.Hash(buf) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} if c.Has(h) { t.Errorf("index %v present before save", id) } - err := c.Save(h, bytes.NewReader(buf)) + err := c.save(h, bytes.NewReader(buf)) if err != nil { t.Fatal(err) } @@ -46,11 +47,12 @@ func randomID(s restic.IDSet) restic.ID { panic("set is empty") } -func load(t testing.TB, c *Cache, h restic.Handle) []byte { - rd, err := c.load(h, 0, 0) +func load(t testing.TB, c *Cache, h backend.Handle) []byte { + rd, inCache, err := c.load(h, 0, 0) if err != nil { t.Fatal(err) } + rtest.Equals(t, true, inCache, "expected inCache flag to be true") if rd == nil { t.Fatalf("load() returned nil reader") @@ -101,7 +103,7 @@ func TestFiles(t *testing.T) { ids := generateRandomFiles(t, tpe, c) id := randomID(ids) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} id2 := restic.Hash(load(t, c, h)) if !id.Equal(id2) { @@ -143,14 +145,14 @@ func TestFileLoad(t *testing.T) { c := TestNewCache(t) // save about 5 MiB of data in the cache - data := test.Random(rand.Int(), 5234142) + data := rtest.Random(rand.Int(), 5234142) id := restic.ID{} copy(id[:], data) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } - if err := c.Save(h, bytes.NewReader(data)); err != nil { + if err := c.save(h, bytes.NewReader(data)); err != nil { t.Fatalf("Save() returned error: %v", err) } @@ -168,10 +170,11 @@ func TestFileLoad(t *testing.T) { for _, test := range tests { t.Run(fmt.Sprintf("%v/%v", test.length, test.offset), func(t *testing.T) { - rd, err := c.load(h, test.length, test.offset) + rd, inCache, err := c.load(h, test.length, test.offset) if err != nil { t.Fatal(err) } + rtest.Equals(t, true, inCache, "expected inCache flag to be true") buf, err := io.ReadAll(rd) if err != nil { @@ -224,19 +227,19 @@ func TestFileSaveConcurrent(t *testing.T) { var ( c = TestNewCache(t) - data = test.Random(1, 10000) + data = rtest.Random(1, 10000) g errgroup.Group id restic.ID ) rand.Read(id[:]) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } for i := 0; i < nproc/2; i++ { - g.Go(func() error { return c.Save(h, bytes.NewReader(data)) }) + g.Go(func() error { return c.save(h, bytes.NewReader(data)) }) // Can't use load because only the main goroutine may call t.Fatal. g.Go(func() error { @@ -244,7 +247,7 @@ func TestFileSaveConcurrent(t *testing.T) { // ensure is ENOENT or nil error. time.Sleep(time.Duration(100+rand.Intn(200)) * time.Millisecond) - f, err := c.load(h, 0, 0) + f, _, err := c.load(h, 0, 0) t.Logf("Load error: %v", err) switch { case err == nil: @@ -263,23 +266,23 @@ func TestFileSaveConcurrent(t *testing.T) { }) } - test.OK(t, g.Wait()) + rtest.OK(t, g.Wait()) saved := load(t, c, h) - test.Equals(t, data, saved) + rtest.Equals(t, data, saved) } func TestFileSaveAfterDamage(t *testing.T) { c := TestNewCache(t) - test.OK(t, fs.RemoveAll(c.path)) + rtest.OK(t, fs.RemoveAll(c.path)) // save a few bytes of data in the cache - data := test.Random(123456789, 42) + data := rtest.Random(123456789, 42) id := restic.Hash(data) - h := restic.Handle{ + h := backend.Handle{ Type: restic.PackFile, Name: id.String(), } - if err := c.Save(h, bytes.NewReader(data)); err == nil { + if err := c.save(h, bytes.NewReader(data)); err == nil { t.Fatal("Missing error when saving to deleted cache directory") } } diff --git a/mover-restic/restic/internal/cache/testing.go b/mover-restic/restic/internal/backend/cache/testing.go similarity index 100% rename from mover-restic/restic/internal/cache/testing.go rename to mover-restic/restic/internal/backend/cache/testing.go diff --git a/mover-restic/restic/internal/backend/dryrun/dry_backend.go b/mover-restic/restic/internal/backend/dryrun/dry_backend.go index f7acb10dd..8af0ce9ad 100644 --- a/mover-restic/restic/internal/backend/dryrun/dry_backend.go +++ b/mover-restic/restic/internal/backend/dryrun/dry_backend.go @@ -5,8 +5,8 @@ import ( "hash" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) // Backend passes reads through to an underlying layer and accepts writes, but @@ -15,20 +15,20 @@ import ( // the repo and does normal operations else. // This is used for `backup --dry-run`. type Backend struct { - b restic.Backend + b backend.Backend } -// statically ensure that Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that Backend implements backend.Backend. +var _ backend.Backend = &Backend{} -func New(be restic.Backend) *Backend { +func New(be backend.Backend) *Backend { b := &Backend{b: be} debug.Log("created new dry backend") return b } // Save adds new Data to the backend. -func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReader) error { +func (be *Backend) Save(_ context.Context, h backend.Handle, _ backend.RewindReader) error { if err := h.Valid(); err != nil { return err } @@ -38,7 +38,7 @@ func (be *Backend) Save(_ context.Context, h restic.Handle, _ restic.RewindReade } // Remove deletes a file from the backend. -func (be *Backend) Remove(_ context.Context, _ restic.Handle) error { +func (be *Backend) Remove(_ context.Context, _ backend.Handle) error { return nil } @@ -46,11 +46,6 @@ func (be *Backend) Connections() uint { return be.b.Connections() } -// Location returns the location of the backend. -func (be *Backend) Location() string { - return "DRY:" + be.b.Location() -} - // Delete removes all data in the backend. func (be *Backend) Delete(_ context.Context) error { return nil @@ -72,14 +67,18 @@ func (be *Backend) IsNotExist(err error) bool { return be.b.IsNotExist(err) } -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) IsPermanentError(err error) bool { + return be.b.IsPermanentError(err) +} + +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { return be.b.List(ctx, t, fn) } -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error { return be.b.Load(ctx, h, length, offset, fn) } -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { return be.b.Stat(ctx, h) } diff --git a/mover-restic/restic/internal/backend/dryrun/dry_backend_test.go b/mover-restic/restic/internal/backend/dryrun/dry_backend_test.go index 69716c340..be98f5310 100644 --- a/mover-restic/restic/internal/backend/dryrun/dry_backend_test.go +++ b/mover-restic/restic/internal/backend/dryrun/dry_backend_test.go @@ -8,16 +8,16 @@ import ( "strings" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/dryrun" "github.com/restic/restic/internal/backend/mem" ) // make sure that Backend implements backend.Backend -var _ restic.Backend = &dryrun.Backend{} +var _ backend.Backend = &dryrun.Backend{} -func newBackends() (*dryrun.Backend, restic.Backend) { +func newBackends() (*dryrun.Backend, backend.Backend) { m := mem.New() return dryrun.New(m), m } @@ -30,13 +30,12 @@ func TestDry(t *testing.T) { // won't pass. Instead, perform a series of operations over the backend, testing the state // at each step. steps := []struct { - be restic.Backend + be backend.Backend op string fname string content string wantErr string }{ - {d, "loc", "", "DRY:RAM", ""}, {d, "delete", "", "", ""}, {d, "stat", "a", "", "not found"}, {d, "list", "", "", ""}, @@ -61,13 +60,13 @@ func TestDry(t *testing.T) { for i, step := range steps { var err error - handle := restic.Handle{Type: restic.PackFile, Name: step.fname} + handle := backend.Handle{Type: backend.PackFile, Name: step.fname} switch step.op { case "save": - err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher())) + err = step.be.Save(ctx, handle, backend.NewByteReader([]byte(step.content), step.be.Hasher())) case "list": fileList := []string{} - err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err = step.be.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { fileList = append(fileList, fi.Name) return nil }) @@ -76,17 +75,12 @@ func TestDry(t *testing.T) { if files != step.content { t.Errorf("%d. List = %q, want %q", i, files, step.content) } - case "loc": - loc := step.be.Location() - if loc != step.content { - t.Errorf("%d. Location = %q, want %q", i, loc, step.content) - } case "delete": err = step.be.Delete(ctx) case "remove": err = step.be.Remove(ctx, handle) case "stat": - var fi restic.FileInfo + var fi backend.FileInfo fi, err = step.be.Stat(ctx, handle) if err == nil { fis := fmt.Sprintf("%s %d", fi.Name, fi.Size) @@ -96,7 +90,7 @@ func TestDry(t *testing.T) { } case "load": data := "" - err = step.be.Load(ctx, handle, 100, 0, func(rd io.Reader) error { + err = step.be.Load(ctx, handle, 0, 0, func(rd io.Reader) error { buf, err := io.ReadAll(rd) data = string(buf) return err diff --git a/mover-restic/restic/internal/restic/file.go b/mover-restic/restic/internal/backend/file.go similarity index 92% rename from mover-restic/restic/internal/restic/file.go rename to mover-restic/restic/internal/backend/file.go index 0e9f046ae..990175f9c 100644 --- a/mover-restic/restic/internal/restic/file.go +++ b/mover-restic/restic/internal/backend/file.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "fmt" @@ -41,9 +41,9 @@ func (t FileType) String() string { // Handle is used to store and access data in a backend. type Handle struct { - Type FileType - ContainedBlobType BlobType - Name string + Type FileType + IsMetadata bool + Name string } func (h Handle) String() string { diff --git a/mover-restic/restic/internal/restic/file_test.go b/mover-restic/restic/internal/backend/file_test.go similarity index 98% rename from mover-restic/restic/internal/restic/file_test.go rename to mover-restic/restic/internal/backend/file_test.go index cc54c2924..45f1c2ee7 100644 --- a/mover-restic/restic/internal/restic/file_test.go +++ b/mover-restic/restic/internal/backend/file_test.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "testing" diff --git a/mover-restic/restic/internal/backend/gs/config.go b/mover-restic/restic/internal/backend/gs/config.go index 61a31113f..7dc181ce9 100644 --- a/mover-restic/restic/internal/backend/gs/config.go +++ b/mover-restic/restic/internal/backend/gs/config.go @@ -5,9 +5,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to a Google Cloud Storage @@ -59,7 +59,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/mover-restic/restic/internal/backend/gs/gs.go b/mover-restic/restic/internal/backend/gs/gs.go index 5c12654d6..0af226f5d 100644 --- a/mover-restic/restic/internal/backend/gs/gs.go +++ b/mover-restic/restic/internal/backend/gs/gs.go @@ -12,12 +12,13 @@ import ( "strings" "cloud.google.com/go/storage" - "github.com/pkg/errors" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/errors" "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -45,8 +46,8 @@ type Backend struct { layout.Layout } -// Ensure that *Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// Ensure that *Backend implements backend.Backend. +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("gs", ParseConfig, location.NoPassword, Create, Open) @@ -122,7 +123,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) { } // Open opens the gs backend at the specified bucket. -func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(_ context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { return open(cfg, rt) } @@ -131,10 +132,10 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, // // The service account must have the "storage.buckets.create" permission to // create a bucket the does not yet exist. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { be, err := open(cfg, rt) if err != nil { - return nil, errors.Wrap(err, "open") + return nil, err } // Try to determine if the bucket exists. If it does not, try to create it. @@ -145,7 +146,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe // however, the client doesn't have storage.bucket.get permission return be, nil } - return nil, errors.Wrap(err, "service.Buckets.Get") + return nil, errors.WithStack(err) } if !exists { @@ -155,7 +156,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe // Bucket doesn't exist, try to create it. if err := be.bucket.Create(ctx, be.projectID, bucketAttrs); err != nil { // Always an error, as the bucket definitely doesn't exist. - return nil, errors.Wrap(err, "service.Buckets.Insert") + return nil, errors.WithStack(err) } } @@ -173,6 +174,21 @@ func (be *Backend) IsNotExist(err error) bool { return errors.Is(err, storage.ErrObjectNotExist) } +func (be *Backend) IsPermanentError(err error) bool { + if be.IsNotExist(err) { + return true + } + + var gerr *googleapi.Error + if errors.As(err, &gerr) { + if gerr.Code == http.StatusRequestedRangeNotSatisfiable || gerr.Code == http.StatusUnauthorized || gerr.Code == http.StatusForbidden { + return true + } + } + + return false +} + // Join combines path components with slashes. func (be *Backend) Join(p ...string) string { return path.Join(p...) @@ -182,11 +198,6 @@ func (be *Backend) Connections() uint { return be.connections } -// Location returns this backend's location (the bucket name). -func (be *Backend) Location() string { - return be.Join(be.bucketName, be.prefix) -} - // Hasher may return a hash function for calculating a content hash for the backend func (be *Backend) Hasher() hash.Hash { return md5.New() @@ -203,7 +214,7 @@ func (be *Backend) Path() string { } // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) // Set chunk size to zero to disable resumable uploads. @@ -241,7 +252,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } if err != nil { - return errors.Wrap(err, "service.Objects.Insert") + return errors.WithStack(err) } // sanity check @@ -253,14 +264,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length == 0 { // negative length indicates read till end to GCS lib length = -1 @@ -273,24 +284,29 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, return nil, err } + if length > 0 && r.Attrs.Size < offset+int64(length) { + _ = r.Close() + return nil, &googleapi.Error{Code: http.StatusRequestedRangeNotSatisfiable, Message: "restic-file-too-short"} + } + return r, err } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) attr, err := be.bucket.Object(objName).Attrs(ctx) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "service.Objects.Get") + return backend.FileInfo{}, errors.WithStack(err) } - return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil + return backend.FileInfo{Size: attr.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.bucket.Object(objName).Delete(ctx) @@ -299,12 +315,12 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { err = nil } - return errors.Wrap(err, "client.RemoveObject") + return errors.WithStack(err) } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) // make sure prefix ends with a slash @@ -330,7 +346,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: int64(attrs.Size), } @@ -350,7 +366,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing. diff --git a/mover-restic/restic/internal/backend/http_transport.go b/mover-restic/restic/internal/backend/http_transport.go index 9ee1c91f1..5162d3571 100644 --- a/mover-restic/restic/internal/backend/http_transport.go +++ b/mover-restic/restic/internal/backend/http_transport.go @@ -10,8 +10,11 @@ import ( "strings" "time" + "github.com/peterbourgon/unixtransport" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" + "golang.org/x/net/http2" ) // TransportOptions collects various options which can be set for an HTTP based @@ -25,6 +28,9 @@ type TransportOptions struct { // Skip TLS certificate verification InsecureTLS bool + + // Specify Custom User-Agent for the http Client + HTTPUserAgent string } // readPEMCertKey reads a file and returns the PEM encoded certificate and key @@ -73,7 +79,6 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) { KeepAlive: 30 * time.Second, DualStack: true, }).DialContext, - ForceAttemptHTTP2: true, MaxIdleConns: 100, MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, @@ -82,6 +87,19 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) { TLSClientConfig: &tls.Config{}, } + // ensure that http2 connections are closed if they are broken + h2, err := http2.ConfigureTransports(tr) + if err != nil { + panic(err) + } + if feature.Flag.Enabled(feature.BackendErrorRedesign) { + h2.WriteByteTimeout = 120 * time.Second + h2.ReadIdleTimeout = 60 * time.Second + h2.PingTimeout = 60 * time.Second + } + + unixtransport.Register(tr) + if opts.InsecureTLS { tr.TLSClientConfig.InsecureSkipVerify = true } @@ -116,6 +134,18 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) { tr.TLSClientConfig.RootCAs = pool } + rt := http.RoundTripper(tr) + + // if the userAgent is set in the Transport Options, wrap the + // http.RoundTripper + if opts.HTTPUserAgent != "" { + rt = newCustomUserAgentRoundTripper(rt, opts.HTTPUserAgent) + } + + if feature.Flag.Enabled(feature.BackendErrorRedesign) { + rt = newWatchdogRoundtripper(rt, 5*time.Minute, 128*1024) + } + // wrap in the debug round tripper (if active) - return debug.RoundTripper(tr), nil + return debug.RoundTripper(rt), nil } diff --git a/mover-restic/restic/internal/backend/httpuseragent_roundtripper.go b/mover-restic/restic/internal/backend/httpuseragent_roundtripper.go new file mode 100644 index 000000000..6272aa41a --- /dev/null +++ b/mover-restic/restic/internal/backend/httpuseragent_roundtripper.go @@ -0,0 +1,25 @@ +package backend + +import "net/http" + +// httpUserAgentRoundTripper is a custom http.RoundTripper that modifies the User-Agent header +// of outgoing HTTP requests. +type httpUserAgentRoundTripper struct { + userAgent string + rt http.RoundTripper +} + +func newCustomUserAgentRoundTripper(rt http.RoundTripper, userAgent string) *httpUserAgentRoundTripper { + return &httpUserAgentRoundTripper{ + rt: rt, + userAgent: userAgent, + } +} + +// RoundTrip modifies the User-Agent header of the request and then delegates the request +// to the underlying RoundTripper. +func (c *httpUserAgentRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + req.Header.Set("User-Agent", c.userAgent) + return c.rt.RoundTrip(req) +} diff --git a/mover-restic/restic/internal/backend/httpuseragent_roundtripper_test.go b/mover-restic/restic/internal/backend/httpuseragent_roundtripper_test.go new file mode 100644 index 000000000..0a81c418a --- /dev/null +++ b/mover-restic/restic/internal/backend/httpuseragent_roundtripper_test.go @@ -0,0 +1,50 @@ +package backend + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestCustomUserAgentTransport(t *testing.T) { + // Create a mock HTTP handler that checks the User-Agent header + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent := r.Header.Get("User-Agent") + if userAgent != "TestUserAgent" { + t.Errorf("Expected User-Agent: TestUserAgent, got: %s", userAgent) + } + w.WriteHeader(http.StatusOK) + }) + + // Create a test server with the mock handler + server := httptest.NewServer(handler) + defer server.Close() + + // Create a custom user agent transport + customUserAgent := "TestUserAgent" + transport := &httpUserAgentRoundTripper{ + userAgent: customUserAgent, + rt: http.DefaultTransport, + } + + // Create an HTTP client with the custom transport + client := &http.Client{ + Transport: transport, + } + + // Make a request to the test server + resp, err := client.Get(server.URL) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + defer func() { + if err := resp.Body.Close(); err != nil { + t.Log("failed to close response body") + } + }() + + // Check the response status code + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status code: %d, got: %d", http.StatusOK, resp.StatusCode) + } +} diff --git a/mover-restic/restic/internal/backend/layout/layout.go b/mover-restic/restic/internal/backend/layout/layout.go index b83f4c05b..052fd66ca 100644 --- a/mover-restic/restic/internal/backend/layout/layout.go +++ b/mover-restic/restic/internal/backend/layout/layout.go @@ -7,17 +7,19 @@ import ( "path/filepath" "regexp" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" ) // Layout computes paths for file name storage. type Layout interface { - Filename(restic.Handle) string - Dirname(restic.Handle) string - Basedir(restic.FileType) (dir string, subdirs bool) + Filename(backend.Handle) string + Dirname(backend.Handle) string + Basedir(backend.FileType) (dir string, subdirs bool) Paths() []string Name() string } @@ -92,6 +94,8 @@ func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error // cannot be detected automatically. var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed") +var ErrLegacyLayoutFound = errors.New("detected legacy S3 layout. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your repository") + // DetectLayout tries to find out which layout is used in a local (or sftp) // filesystem at the given path. If repo is nil, an instance of LocalFilesystem // is used. @@ -102,13 +106,13 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err } // key file in the "keys" dir (DefaultLayout) - foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[restic.KeyFile])) + foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile])) if err != nil { return nil, err } // key file in the "key" dir (S3LegacyLayout) - foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[restic.KeyFile])) + foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile])) if err != nil { return nil, err } @@ -122,6 +126,10 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err } if foundKeyFile && !foundKeysFile { + if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) { + return nil, ErrLegacyLayoutFound + } + debug.Log("found s3 layout at %v", dir) return &S3LegacyLayout{ Path: dir, @@ -144,6 +152,10 @@ func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, pa Join: repo.Join, } case "s3legacy": + if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) { + return nil, ErrLegacyLayoutFound + } + l = &S3LegacyLayout{ Path: path, Join: repo.Join, diff --git a/mover-restic/restic/internal/backend/layout/layout_default.go b/mover-restic/restic/internal/backend/layout/layout_default.go index 17c250e8f..9a8419f10 100644 --- a/mover-restic/restic/internal/backend/layout/layout_default.go +++ b/mover-restic/restic/internal/backend/layout/layout_default.go @@ -3,7 +3,7 @@ package layout import ( "encoding/hex" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) // DefaultLayout implements the default layout for local and sftp backends, as @@ -15,12 +15,12 @@ type DefaultLayout struct { Join func(...string) string } -var defaultLayoutPaths = map[restic.FileType]string{ - restic.PackFile: "data", - restic.SnapshotFile: "snapshots", - restic.IndexFile: "index", - restic.LockFile: "locks", - restic.KeyFile: "keys", +var defaultLayoutPaths = map[backend.FileType]string{ + backend.PackFile: "data", + backend.SnapshotFile: "snapshots", + backend.IndexFile: "index", + backend.LockFile: "locks", + backend.KeyFile: "keys", } func (l *DefaultLayout) String() string { @@ -33,10 +33,10 @@ func (l *DefaultLayout) Name() string { } // Dirname returns the directory path for a given file type and name. -func (l *DefaultLayout) Dirname(h restic.Handle) string { +func (l *DefaultLayout) Dirname(h backend.Handle) string { p := defaultLayoutPaths[h.Type] - if h.Type == restic.PackFile && len(h.Name) > 2 { + if h.Type == backend.PackFile && len(h.Name) > 2 { p = l.Join(p, h.Name[:2]) + "/" } @@ -44,9 +44,9 @@ func (l *DefaultLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *DefaultLayout) Filename(h restic.Handle) string { +func (l *DefaultLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { return l.Join(l.Path, "config") } @@ -62,15 +62,15 @@ func (l *DefaultLayout) Paths() (dirs []string) { // also add subdirs for i := 0; i < 256; i++ { subdir := hex.EncodeToString([]byte{byte(i)}) - dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[restic.PackFile], subdir)) + dirs = append(dirs, l.Join(l.Path, defaultLayoutPaths[backend.PackFile], subdir)) } return dirs } // Basedir returns the base dir name for type t. -func (l *DefaultLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { - if t == restic.PackFile { +func (l *DefaultLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { + if t == backend.PackFile { subdirs = true } diff --git a/mover-restic/restic/internal/backend/layout/layout_rest.go b/mover-restic/restic/internal/backend/layout/layout_rest.go index 2aa869995..822dd4a7e 100644 --- a/mover-restic/restic/internal/backend/layout/layout_rest.go +++ b/mover-restic/restic/internal/backend/layout/layout_rest.go @@ -1,6 +1,8 @@ package layout -import "github.com/restic/restic/internal/restic" +import ( + "github.com/restic/restic/internal/backend" +) // RESTLayout implements the default layout for the REST protocol. type RESTLayout struct { @@ -21,8 +23,8 @@ func (l *RESTLayout) Name() string { } // Dirname returns the directory path for a given file type and name. -func (l *RESTLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { +func (l *RESTLayout) Dirname(h backend.Handle) string { + if h.Type == backend.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -30,10 +32,10 @@ func (l *RESTLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *RESTLayout) Filename(h restic.Handle) string { +func (l *RESTLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { name = "config" } @@ -49,6 +51,6 @@ func (l *RESTLayout) Paths() (dirs []string) { } // Basedir returns the base dir name for files of type t. -func (l *RESTLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { +func (l *RESTLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { return l.URL + l.Join(l.Path, restLayoutPaths[t]), false } diff --git a/mover-restic/restic/internal/backend/layout/layout_s3legacy.go b/mover-restic/restic/internal/backend/layout/layout_s3legacy.go index ac88e77ad..8b90789d8 100644 --- a/mover-restic/restic/internal/backend/layout/layout_s3legacy.go +++ b/mover-restic/restic/internal/backend/layout/layout_s3legacy.go @@ -1,6 +1,8 @@ package layout -import "github.com/restic/restic/internal/restic" +import ( + "github.com/restic/restic/internal/backend" +) // S3LegacyLayout implements the old layout used for s3 cloud storage backends, as // described in the Design document. @@ -10,12 +12,12 @@ type S3LegacyLayout struct { Join func(...string) string } -var s3LayoutPaths = map[restic.FileType]string{ - restic.PackFile: "data", - restic.SnapshotFile: "snapshot", - restic.IndexFile: "index", - restic.LockFile: "lock", - restic.KeyFile: "key", +var s3LayoutPaths = map[backend.FileType]string{ + backend.PackFile: "data", + backend.SnapshotFile: "snapshot", + backend.IndexFile: "index", + backend.LockFile: "lock", + backend.KeyFile: "key", } func (l *S3LegacyLayout) String() string { @@ -44,8 +46,8 @@ func (l *S3LegacyLayout) join(url string, items ...string) string { } // Dirname returns the directory path for a given file type and name. -func (l *S3LegacyLayout) Dirname(h restic.Handle) string { - if h.Type == restic.ConfigFile { +func (l *S3LegacyLayout) Dirname(h backend.Handle) string { + if h.Type == backend.ConfigFile { return l.URL + l.Join(l.Path, "/") } @@ -53,10 +55,10 @@ func (l *S3LegacyLayout) Dirname(h restic.Handle) string { } // Filename returns a path to a file, including its name. -func (l *S3LegacyLayout) Filename(h restic.Handle) string { +func (l *S3LegacyLayout) Filename(h backend.Handle) string { name := h.Name - if h.Type == restic.ConfigFile { + if h.Type == backend.ConfigFile { name = "config" } @@ -72,6 +74,6 @@ func (l *S3LegacyLayout) Paths() (dirs []string) { } // Basedir returns the base dir name for type t. -func (l *S3LegacyLayout) Basedir(t restic.FileType) (dirname string, subdirs bool) { +func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { return l.Join(l.Path, s3LayoutPaths[t]), false } diff --git a/mover-restic/restic/internal/backend/layout/layout_test.go b/mover-restic/restic/internal/backend/layout/layout_test.go index fc9c6e214..55a0749c9 100644 --- a/mover-restic/restic/internal/backend/layout/layout_test.go +++ b/mover-restic/restic/internal/backend/layout/layout_test.go @@ -9,7 +9,8 @@ import ( "sort" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/feature" rtest "github.com/restic/restic/internal/test" ) @@ -19,79 +20,79 @@ func TestDefaultLayout(t *testing.T) { var tests = []struct { path string join func(...string) string - restic.Handle + backend.Handle filename string }{ { tempdir, filepath.Join, - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(tempdir, "data", "01", "0123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(tempdir, "config"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(tempdir, "snapshots", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(tempdir, "index", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(tempdir, "locks", "123456"), }, { tempdir, filepath.Join, - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(tempdir, "keys", "123456"), }, { "", path.Join, - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, "data/01/0123456", }, { "", path.Join, - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, "config", }, { "", path.Join, - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, "snapshots/123456", }, { "", path.Join, - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, "index/123456", }, { "", path.Join, - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, "locks/123456", }, { "", path.Join, - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, "keys/123456", }, } @@ -143,31 +144,31 @@ func TestRESTLayout(t *testing.T) { path := rtest.TempDir(t) var tests = []struct { - restic.Handle + backend.Handle filename string }{ { - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(path, "data", "0123456"), }, { - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(path, "config"), }, { - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(path, "snapshots", "123456"), }, { - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(path, "index", "123456"), }, { - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(path, "locks", "123456"), }, { - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(path, "keys", "123456"), }, } @@ -209,61 +210,61 @@ func TestRESTLayout(t *testing.T) { func TestRESTLayoutURLs(t *testing.T) { var tests = []struct { l Layout - h restic.Handle + h backend.Handle fn string dir string }{ { &RESTLayout{URL: "https://hostname.foo", Path: "", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/locks/foobar", "https://hostname.foo:1234/prefix/repo/locks/", }, { &RESTLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { &S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "https://hostname.foo/data/foobar", "https://hostname.foo/data/", }, { &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/lock/foobar", "https://hostname.foo:1234/prefix/repo/lock/", }, { &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, { &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - restic.Handle{Type: restic.PackFile, Name: "foobar"}, + backend.Handle{Type: backend.PackFile, Name: "foobar"}, "data/foobar", "data/", }, { &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - restic.Handle{Type: restic.LockFile, Name: "foobar"}, + backend.Handle{Type: backend.LockFile, Name: "foobar"}, "lock/foobar", "lock/", }, { &S3LegacyLayout{URL: "", Path: "/", Join: path.Join}, - restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, "/config", "/", }, @@ -288,31 +289,31 @@ func TestS3LegacyLayout(t *testing.T) { path := rtest.TempDir(t) var tests = []struct { - restic.Handle + backend.Handle filename string }{ { - restic.Handle{Type: restic.PackFile, Name: "0123456"}, + backend.Handle{Type: backend.PackFile, Name: "0123456"}, filepath.Join(path, "data", "0123456"), }, { - restic.Handle{Type: restic.ConfigFile, Name: "CFG"}, + backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, filepath.Join(path, "config"), }, { - restic.Handle{Type: restic.SnapshotFile, Name: "123456"}, + backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, filepath.Join(path, "snapshot", "123456"), }, { - restic.Handle{Type: restic.IndexFile, Name: "123456"}, + backend.Handle{Type: backend.IndexFile, Name: "123456"}, filepath.Join(path, "index", "123456"), }, { - restic.Handle{Type: restic.LockFile, Name: "123456"}, + backend.Handle{Type: backend.LockFile, Name: "123456"}, filepath.Join(path, "lock", "123456"), }, { - restic.Handle{Type: restic.KeyFile, Name: "123456"}, + backend.Handle{Type: backend.KeyFile, Name: "123456"}, filepath.Join(path, "key", "123456"), }, } @@ -352,6 +353,7 @@ func TestS3LegacyLayout(t *testing.T) { } func TestDetectLayout(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() path := rtest.TempDir(t) var tests = []struct { @@ -389,6 +391,7 @@ func TestDetectLayout(t *testing.T) { } func TestParseLayout(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() path := rtest.TempDir(t) var tests = []struct { @@ -415,8 +418,8 @@ func TestParseLayout(t *testing.T) { } // test that the functions work (and don't panic) - _ = layout.Dirname(restic.Handle{Type: restic.PackFile}) - _ = layout.Filename(restic.Handle{Type: restic.PackFile, Name: "1234"}) + _ = layout.Dirname(backend.Handle{Type: backend.PackFile}) + _ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"}) _ = layout.Paths() layoutName := fmt.Sprintf("%T", layout) diff --git a/mover-restic/restic/internal/backend/limiter/limiter.go b/mover-restic/restic/internal/backend/limiter/limiter.go index 8cbe297fe..7ba5ad02b 100644 --- a/mover-restic/restic/internal/backend/limiter/limiter.go +++ b/mover-restic/restic/internal/backend/limiter/limiter.go @@ -5,8 +5,8 @@ import ( "net/http" ) -// Limiter defines an interface that implementors can use to rate limit I/O -// according to some policy defined and configured by the implementor. +// Limiter defines an interface that implementers can use to rate limit I/O +// according to some policy defined and configured by the implementer. type Limiter interface { // Upstream returns a rate limited reader that is intended to be used in // uploads. diff --git a/mover-restic/restic/internal/backend/limiter/limiter_backend.go b/mover-restic/restic/internal/backend/limiter/limiter_backend.go index a91794037..ac1a4188a 100644 --- a/mover-restic/restic/internal/backend/limiter/limiter_backend.go +++ b/mover-restic/restic/internal/backend/limiter/limiter_backend.go @@ -4,12 +4,12 @@ import ( "context" "io" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" ) -func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { - return func(ctx context.Context, cfg C, lim Limiter) (restic.Backend, error) { - var be restic.Backend +func WrapBackendConstructor[B backend.Backend, C any](constructor func(ctx context.Context, cfg C) (B, error)) func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) { + return func(ctx context.Context, cfg C, lim Limiter) (backend.Backend, error) { + var be backend.Backend be, err := constructor(ctx, cfg) if err != nil { return nil, err @@ -24,7 +24,7 @@ func WrapBackendConstructor[B restic.Backend, C any](constructor func(ctx contex // LimitBackend wraps a Backend and applies rate limiting to Load() and Save() // calls on the backend. -func LimitBackend(be restic.Backend, l Limiter) restic.Backend { +func LimitBackend(be backend.Backend, l Limiter) backend.Backend { return rateLimitedBackend{ Backend: be, limiter: l, @@ -32,11 +32,11 @@ func LimitBackend(be restic.Backend, l Limiter) restic.Backend { } type rateLimitedBackend struct { - restic.Backend + backend.Backend limiter Limiter } -func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (r rateLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { limited := limitedRewindReader{ RewindReader: rd, limited: r.limiter.Upstream(rd), @@ -46,7 +46,7 @@ func (r rateLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic } type limitedRewindReader struct { - restic.RewindReader + backend.RewindReader limited io.Reader } @@ -55,13 +55,13 @@ func (l limitedRewindReader) Read(b []byte) (int, error) { return l.limited.Read(b) } -func (r rateLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (r rateLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return r.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { return consumer(newDownstreamLimitedReader(rd, r.limiter)) }) } -func (r rateLimitedBackend) Unwrap() restic.Backend { return r.Backend } +func (r rateLimitedBackend) Unwrap() backend.Backend { return r.Backend } type limitedReader struct { io.Reader @@ -85,4 +85,4 @@ func (l *limitedReader) WriteTo(w io.Writer) (int64, error) { return l.writerTo.WriteTo(l.limiter.DownstreamWriter(w)) } -var _ restic.Backend = (*rateLimitedBackend)(nil) +var _ backend.Backend = (*rateLimitedBackend)(nil) diff --git a/mover-restic/restic/internal/backend/limiter/limiter_backend_test.go b/mover-restic/restic/internal/backend/limiter/limiter_backend_test.go index 1014dbed1..491d2ef69 100644 --- a/mover-restic/restic/internal/backend/limiter/limiter_backend_test.go +++ b/mover-restic/restic/internal/backend/limiter/limiter_backend_test.go @@ -8,8 +8,8 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -21,11 +21,11 @@ func randomBytes(t *testing.T, size int) []byte { } func TestLimitBackendSave(t *testing.T) { - testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} + testHandle := backend.Handle{Type: backend.PackFile, Name: "test"} data := randomBytes(t, 1234) be := mock.NewBackend() - be.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + be.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { buf := new(bytes.Buffer) _, err := io.Copy(buf, rd) if err != nil { @@ -39,7 +39,7 @@ func TestLimitBackendSave(t *testing.T) { limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) limbe := LimitBackend(be, limiter) - rd := restic.NewByteReader(data, nil) + rd := backend.NewByteReader(data, nil) err := limbe.Save(context.TODO(), testHandle, rd) rtest.OK(t, err) } @@ -64,7 +64,7 @@ func (r *tracedReadWriteToCloser) Close() error { } func TestLimitBackendLoad(t *testing.T) { - testHandle := restic.Handle{Type: restic.PackFile, Name: "test"} + testHandle := backend.Handle{Type: backend.PackFile, Name: "test"} data := randomBytes(t, 1234) for _, test := range []struct { @@ -72,7 +72,7 @@ func TestLimitBackendLoad(t *testing.T) { }{{false, false}, {false, true}, {true, false}, {true, true}} { be := mock.NewBackend() src := newTracedReadWriteToCloser(bytes.NewReader(data)) - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if length != 0 || offset != 0 { return nil, fmt.Errorf("Not supported") } diff --git a/mover-restic/restic/internal/backend/limiter/static_limiter_test.go b/mover-restic/restic/internal/backend/limiter/static_limiter_test.go index 8a839518f..79a1d02f3 100644 --- a/mover-restic/restic/internal/backend/limiter/static_limiter_test.go +++ b/mover-restic/restic/internal/backend/limiter/static_limiter_test.go @@ -118,6 +118,7 @@ func TestRoundTripperReader(t *testing.T) { test.Assert(t, bytes.Equal(data, out.Bytes()), "data ping-pong failed") } +// nolint:bodyclose // the http response is just a mock func TestRoundTripperCornerCases(t *testing.T) { limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) diff --git a/mover-restic/restic/internal/backend/local/config.go b/mover-restic/restic/internal/backend/local/config.go index dc5e7948c..e08f05550 100644 --- a/mover-restic/restic/internal/backend/local/config.go +++ b/mover-restic/restic/internal/backend/local/config.go @@ -10,7 +10,7 @@ import ( // Config holds all information needed to open a local repository. type Config struct { Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"` Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` } diff --git a/mover-restic/restic/internal/backend/local/layout_test.go b/mover-restic/restic/internal/backend/local/layout_test.go index a4fccd2cb..00c91376a 100644 --- a/mover-restic/restic/internal/backend/local/layout_test.go +++ b/mover-restic/restic/internal/backend/local/layout_test.go @@ -5,11 +5,13 @@ import ( "path/filepath" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/feature" rtest "github.com/restic/restic/internal/test" ) func TestLayout(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() path := rtest.TempDir(t) var tests = []struct { @@ -49,7 +51,7 @@ func TestLayout(t *testing.T) { } packs := make(map[string]bool) - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { packs[fi.Name] = false return nil }) diff --git a/mover-restic/restic/internal/backend/local/local.go b/mover-restic/restic/internal/backend/local/local.go index 4198102c2..f041d608a 100644 --- a/mover-restic/restic/internal/backend/local/local.go +++ b/mover-restic/restic/internal/backend/local/local.go @@ -2,6 +2,7 @@ package local import ( "context" + "fmt" "hash" "io" "os" @@ -12,10 +13,10 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/restic" "github.com/cenkalti/backoff/v4" ) @@ -24,11 +25,13 @@ import ( type Local struct { Config layout.Layout - backend.Modes + util.Modes } -// ensure statically that *Local implements restic.Backend. -var _ restic.Backend = &Local{} +// ensure statically that *Local implements backend.Backend. +var _ backend.Backend = &Local{} + +var errTooShort = fmt.Errorf("file is too short") func NewFactory() location.Factory { return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -42,8 +45,8 @@ func open(ctx context.Context, cfg Config) (*Local, error) { return nil, err } - fi, err := fs.Stat(l.Filename(restic.Handle{Type: restic.ConfigFile})) - m := backend.DeriveModesFromFileInfo(fi, err) + fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile})) + m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) return &Local{ @@ -70,7 +73,7 @@ func Create(ctx context.Context, cfg Config) (*Local, error) { } // test if config file already exists - _, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = fs.Lstat(be.Filename(backend.Handle{Type: backend.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -90,11 +93,6 @@ func (b *Local) Connections() uint { return b.Config.Connections } -// Location returns this backend's location (the directory name). -func (b *Local) Location() string { - return b.Path -} - // Hasher may return a hash function for calculating a content hash for the backend func (b *Local) Hasher() hash.Hash { return nil @@ -110,8 +108,12 @@ func (b *Local) IsNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) } +func (b *Local) IsPermanentError(err error) bool { + return b.IsNotExist(err) || errors.Is(err, errTooShort) || errors.Is(err, os.ErrPermission) +} + // Save stores data in the backend at the handle. -func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) (err error) { +func (b *Local) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) (err error) { finalname := b.Filename(h) dir := filepath.Dir(finalname) @@ -194,7 +196,7 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) } } - // try to mark file as read-only to avoid accidential modifications + // try to mark file as read-only to avoid accidental modifications // ignore if the operation fails as some filesystems don't allow the chmod call // e.g. exfat and network file systems with certain mount options err = setFileReadonly(finalname, b.Modes.File) @@ -209,16 +211,28 @@ var tempFile = os.CreateTemp // Overridden by test. // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn) +func (b *Local) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn) } -func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Local) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { f, err := fs.Open(b.Filename(h)) if err != nil { return nil, err } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + + size := fi.Size() + if size < offset+int64(length) { + _ = f.Close() + return nil, errTooShort + } + if offset > 0 { _, err = f.Seek(offset, 0) if err != nil { @@ -228,24 +242,24 @@ func (b *Local) openReader(_ context.Context, h restic.Handle, length int, offse } if length > 0 { - return backend.LimitReadCloser(f, int64(length)), nil + return util.LimitReadCloser(f, int64(length)), nil } return f, nil } // Stat returns information about a blob. -func (b *Local) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Local) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { fi, err := fs.Stat(b.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (b *Local) Remove(_ context.Context, h restic.Handle) error { +func (b *Local) Remove(_ context.Context, h backend.Handle) error { fn := b.Filename(h) // reset read-only flag @@ -259,7 +273,7 @@ func (b *Local) Remove(_ context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { +func (b *Local) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) (err error) { basedir, subdirs := b.Basedir(t) if subdirs { err = visitDirs(ctx, basedir, fn) @@ -279,7 +293,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File // two levels of directory structure (including dir itself as the first level). // Also, visitDirs assumes it sees a directory full of directories, while // visitFiles wants a directory full or regular files. -func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error { +func visitDirs(ctx context.Context, dir string, fn func(backend.FileInfo) error) error { d, err := fs.Open(dir) if err != nil { return err @@ -306,7 +320,7 @@ func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) return ctx.Err() } -func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, ignoreNotADirectory bool) error { +func visitFiles(ctx context.Context, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error { d, err := fs.Open(dir) if err != nil { return err @@ -340,7 +354,7 @@ func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error, default: } - err := fn(restic.FileInfo{ + err := fn(backend.FileInfo{ Name: fi.Name(), Size: fi.Size(), }) diff --git a/mover-restic/restic/internal/backend/local/local_internal_test.go b/mover-restic/restic/internal/backend/local/local_internal_test.go index 1e80e72ed..6cad26d0a 100644 --- a/mover-restic/restic/internal/backend/local/local_internal_test.go +++ b/mover-restic/restic/internal/backend/local/local_internal_test.go @@ -8,7 +8,7 @@ import ( "syscall" "testing" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/backend" rtest "github.com/restic/restic/internal/test" "github.com/cenkalti/backoff/v4" @@ -32,7 +32,7 @@ func TestNoSpacePermanent(t *testing.T) { rtest.OK(t, be.Close()) }() - h := restic.Handle{Type: restic.ConfigFile} + h := backend.Handle{Type: backend.ConfigFile} err = be.Save(context.Background(), h, nil) _, ok := err.(*backoff.PermanentError) rtest.Assert(t, ok, diff --git a/mover-restic/restic/internal/backend/location/display_location_test.go b/mover-restic/restic/internal/backend/location/display_location_test.go index 19502d85b..4011abbf0 100644 --- a/mover-restic/restic/internal/backend/location/display_location_test.go +++ b/mover-restic/restic/internal/backend/location/display_location_test.go @@ -3,15 +3,15 @@ package location_test import ( "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) func TestStripPassword(t *testing.T) { registry := location.NewRegistry() registry.Register( - location.NewHTTPBackendFactory[any, restic.Backend]("test", nil, + location.NewHTTPBackendFactory[any, backend.Backend]("test", nil, func(s string) string { return "cleaned" }, nil, nil, diff --git a/mover-restic/restic/internal/backend/location/location_test.go b/mover-restic/restic/internal/backend/location/location_test.go index b2623032e..fe550a586 100644 --- a/mover-restic/restic/internal/backend/location/location_test.go +++ b/mover-restic/restic/internal/backend/location/location_test.go @@ -3,8 +3,8 @@ package location_test import ( "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -13,7 +13,7 @@ type testConfig struct { } func testFactory() location.Factory { - return location.NewHTTPBackendFactory[testConfig, restic.Backend]( + return location.NewHTTPBackendFactory[testConfig, backend.Backend]( "local", func(s string) (*testConfig, error) { return &testConfig{loc: s}, nil diff --git a/mover-restic/restic/internal/backend/location/registry.go b/mover-restic/restic/internal/backend/location/registry.go index a8818bd73..b50371add 100644 --- a/mover-restic/restic/internal/backend/location/registry.go +++ b/mover-restic/restic/internal/backend/location/registry.go @@ -4,8 +4,8 @@ import ( "context" "net/http" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/limiter" - "github.com/restic/restic/internal/restic" ) type Registry struct { @@ -33,11 +33,11 @@ type Factory interface { Scheme() string ParseConfig(s string) (interface{}, error) StripPassword(s string) string - Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) - Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) + Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) + Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) } -type genericBackendFactory[C any, T restic.Backend] struct { +type genericBackendFactory[C any, T backend.Backend] struct { scheme string parseConfigFn func(s string) (*C, error) stripPasswordFn func(s string) string @@ -58,14 +58,14 @@ func (f *genericBackendFactory[C, T]) StripPassword(s string) string { } return s } -func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Create(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) { return f.createFn(ctx, *cfg.(*C), rt, lim) } -func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (restic.Backend, error) { +func (f *genericBackendFactory[C, T]) Open(ctx context.Context, cfg interface{}, rt http.RoundTripper, lim limiter.Limiter) (backend.Backend, error) { return f.openFn(ctx, *cfg.(*C), rt, lim) } -func NewHTTPBackendFactory[C any, T restic.Backend]( +func NewHTTPBackendFactory[C any, T backend.Backend]( scheme string, parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, @@ -85,7 +85,7 @@ func NewHTTPBackendFactory[C any, T restic.Backend]( } } -func NewLimitedBackendFactory[C any, T restic.Backend]( +func NewLimitedBackendFactory[C any, T backend.Backend]( scheme string, parseConfigFn func(s string) (*C, error), stripPasswordFn func(s string) string, diff --git a/mover-restic/restic/internal/backend/logger/log.go b/mover-restic/restic/internal/backend/logger/log.go index 6c860cfae..6fdf92295 100644 --- a/mover-restic/restic/internal/backend/logger/log.go +++ b/mover-restic/restic/internal/backend/logger/log.go @@ -4,18 +4,18 @@ import ( "context" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" ) type Backend struct { - restic.Backend + backend.Backend } -// statically ensure that Backend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that Backend implements backend.Backend. +var _ backend.Backend = &Backend{} -func New(be restic.Backend) *Backend { +func New(be backend.Backend) *Backend { return &Backend{Backend: be} } @@ -26,7 +26,7 @@ func (be *Backend) IsNotExist(err error) bool { } // Save adds new Data to the backend. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { debug.Log("Save(%v, %v)", h, rd.Length()) err := be.Backend.Save(ctx, h, rd) debug.Log(" save err %v", err) @@ -34,28 +34,28 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } // Remove deletes a file from the backend. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { debug.Log("Remove(%v)", h) err := be.Backend.Remove(ctx, h) debug.Log(" remove err %v", err) return err } -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(io.Reader) error) error { debug.Log("Load(%v, length %v, offset %v)", h, length, offset) err := be.Backend.Load(ctx, h, length, offset, fn) debug.Log(" load err %v", err) return err } -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { debug.Log("Stat(%v)", h) fi, err := be.Backend.Stat(ctx, h) debug.Log(" stat err %v", err) return fi, err } -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { debug.Log("List(%v)", t) err := be.Backend.List(ctx, t, fn) debug.Log(" list err %v", err) @@ -76,4 +76,4 @@ func (be *Backend) Close() error { return err } -func (be *Backend) Unwrap() restic.Backend { return be.Backend } +func (be *Backend) Unwrap() backend.Backend { return be.Backend } diff --git a/mover-restic/restic/internal/backend/mem/mem_backend.go b/mover-restic/restic/internal/backend/mem/mem_backend.go index 86ec48756..981c0a182 100644 --- a/mover-restic/restic/internal/backend/mem/mem_backend.go +++ b/mover-restic/restic/internal/backend/mem/mem_backend.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/base64" + "fmt" "hash" "io" "net/http" @@ -12,15 +13,15 @@ import ( "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -type memMap map[restic.Handle][]byte +type memMap map[backend.Handle][]byte // make sure that MemoryBackend implements backend.Backend -var _ restic.Backend = &MemoryBackend{} +var _ backend.Backend = &MemoryBackend{} // NewFactory creates a persistent mem backend func NewFactory() location.Factory { @@ -28,7 +29,7 @@ func NewFactory() location.Factory { return location.NewHTTPBackendFactory[struct{}, *MemoryBackend]( "mem", - func(s string) (*struct{}, error) { + func(_ string) (*struct{}, error) { return &struct{}{}, nil }, location.NoPassword, @@ -41,7 +42,8 @@ func NewFactory() location.Factory { ) } -var errNotFound = errors.New("not found") +var errNotFound = fmt.Errorf("not found") +var errTooSmall = errors.New("access beyond end of file") const connectionCount = 2 @@ -68,13 +70,17 @@ func (be *MemoryBackend) IsNotExist(err error) bool { return errors.Is(err, errNotFound) } +func (be *MemoryBackend) IsPermanentError(err error) bool { + return be.IsNotExist(err) || errors.Is(err, errTooSmall) +} + // Save adds new Data to the backend. -func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *MemoryBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob - if h.Type == restic.ConfigFile { + h.IsMetadata = false + if h.Type == backend.ConfigFile { h.Name = "" } @@ -112,16 +118,16 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *MemoryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) +func (be *MemoryBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *MemoryBackend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob - if h.Type == restic.ConfigFile { + h.IsMetadata = false + if h.Type == backend.ConfigFile { h.Name = "" } @@ -130,12 +136,12 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length } buf := be.data[h] - if offset > int64(len(buf)) { - return nil, errors.New("offset beyond end of file") + if offset+int64(length) > int64(len(buf)) { + return nil, errTooSmall } buf = buf[offset:] - if length > 0 && len(buf) > length { + if length > 0 { buf = buf[:length] } @@ -143,29 +149,29 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length } // Stat returns information about a file in the backend. -func (be *MemoryBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *MemoryBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob - if h.Type == restic.ConfigFile { + h.IsMetadata = false + if h.Type == backend.ConfigFile { h.Name = "" } e, ok := be.data[h] if !ok { - return restic.FileInfo{}, errNotFound + return backend.FileInfo{}, errNotFound } - return restic.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() + return backend.FileInfo{Size: int64(len(e)), Name: h.Name}, ctx.Err() } // Remove deletes a file from the backend. -func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *MemoryBackend) Remove(ctx context.Context, h backend.Handle) error { be.m.Lock() defer be.m.Unlock() - h.ContainedBlobType = restic.InvalidBlob + h.IsMetadata = false if _, ok := be.data[h]; !ok { return errNotFound } @@ -176,7 +182,7 @@ func (be *MemoryBackend) Remove(ctx context.Context, h restic.Handle) error { } // List returns a channel which yields entries from the backend. -func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *MemoryBackend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { entries := make(map[string]int64) be.m.Lock() @@ -190,7 +196,7 @@ func (be *MemoryBackend) List(ctx context.Context, t restic.FileType, fn func(re be.m.Unlock() for name, size := range entries { - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: name, Size: size, } @@ -216,11 +222,6 @@ func (be *MemoryBackend) Connections() uint { return connectionCount } -// Location returns the location of the backend (RAM). -func (be *MemoryBackend) Location() string { - return "RAM" -} - // Hasher may return a hash function for calculating a content hash for the backend func (be *MemoryBackend) Hasher() hash.Hash { return xxhash.New() diff --git a/mover-restic/restic/internal/backend/mock/backend.go b/mover-restic/restic/internal/backend/mock/backend.go index 875e55e71..a03198443 100644 --- a/mover-restic/restic/internal/backend/mock/backend.go +++ b/mover-restic/restic/internal/backend/mock/backend.go @@ -5,22 +5,22 @@ import ( "hash" "io" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) // Backend implements a mock backend. type Backend struct { CloseFn func() error IsNotExistFn func(err error) bool - SaveFn func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error - OpenReaderFn func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) - StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) - ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error - RemoveFn func(ctx context.Context, h restic.Handle) error + IsPermanentErrorFn func(err error) bool + SaveFn func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error + OpenReaderFn func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) + StatFn func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) + ListFn func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error + RemoveFn func(ctx context.Context, h backend.Handle) error DeleteFn func(ctx context.Context) error ConnectionsFn func() uint - LocationFn func() string HasherFn func() hash.Hash HasAtomicReplaceFn func() bool } @@ -48,15 +48,6 @@ func (m *Backend) Connections() uint { return m.ConnectionsFn() } -// Location returns a location string. -func (m *Backend) Location() string { - if m.LocationFn == nil { - return "" - } - - return m.LocationFn() -} - // Hasher may return a hash function for calculating a content hash for the backend func (m *Backend) Hasher() hash.Hash { if m.HasherFn == nil { @@ -83,8 +74,16 @@ func (m *Backend) IsNotExist(err error) bool { return m.IsNotExistFn(err) } +func (m *Backend) IsPermanentError(err error) bool { + if m.IsPermanentErrorFn == nil { + return false + } + + return m.IsPermanentErrorFn(err) +} + // Save data in the backend. -func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (m *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if m.SaveFn == nil { return errors.New("not implemented") } @@ -94,7 +93,7 @@ func (m *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (m *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { rd, err := m.openReader(ctx, h, length, offset) if err != nil { return err @@ -107,7 +106,7 @@ func (m *Backend) Load(ctx context.Context, h restic.Handle, length int, offset return rd.Close() } -func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (m *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { if m.OpenReaderFn == nil { return nil, errors.New("not implemented") } @@ -116,16 +115,16 @@ func (m *Backend) openReader(ctx context.Context, h restic.Handle, length int, o } // Stat an object in the backend. -func (m *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (m *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { if m.StatFn == nil { - return restic.FileInfo{}, errors.New("not implemented") + return backend.FileInfo{}, errors.New("not implemented") } return m.StatFn(ctx, h) } // List items of type t. -func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (m *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { if m.ListFn == nil { return nil } @@ -134,7 +133,7 @@ func (m *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi } // Remove data from the backend. -func (m *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (m *Backend) Remove(ctx context.Context, h backend.Handle) error { if m.RemoveFn == nil { return errors.New("not implemented") } @@ -152,4 +151,4 @@ func (m *Backend) Delete(ctx context.Context) error { } // Make sure that Backend implements the backend interface. -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} diff --git a/mover-restic/restic/internal/backend/rclone/backend.go b/mover-restic/restic/internal/backend/rclone/backend.go index fd6f5b262..25082598f 100644 --- a/mover-restic/restic/internal/backend/rclone/backend.go +++ b/mover-restic/restic/internal/backend/rclone/backend.go @@ -21,6 +21,7 @@ import ( "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/rest" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "golang.org/x/net/http2" @@ -81,7 +82,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru cmd.Stdin = r cmd.Stdout = w - bg, err := backend.StartForeground(cmd) + bg, err := util.StartForeground(cmd) // close rclone side of pipes errR := r.Close() errW := w.Close() @@ -93,7 +94,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan stru err = errW } if err != nil { - if backend.IsErrDot(err) { + if util.IsErrDot(err) { return nil, nil, nil, nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o rclone.program=./ to override", cmd.Path) } return nil, nil, nil, nil, err @@ -182,7 +183,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, dialCount := 0 tr := &http2.Transport{ AllowHTTP: true, // this is not really HTTP, just stdin/stdout - DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) { + DialTLS: func(network, address string, _ *tls.Config) (net.Conn, error) { debug.Log("new connection requested, %v %v", network, address) if dialCount > 0 { // the connection to the child process is already closed @@ -251,6 +252,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, return nil, fmt.Errorf("error talking HTTP to rclone: %w", err) } + _ = res.Body.Close() debug.Log("HTTP status %q returned, moving instance to background", res.Status) err = bg() if err != nil { diff --git a/mover-restic/restic/internal/backend/rclone/internal_test.go b/mover-restic/restic/internal/backend/rclone/internal_test.go index 32fe850a0..34d52885e 100644 --- a/mover-restic/restic/internal/backend/rclone/internal_test.go +++ b/mover-restic/restic/internal/backend/rclone/internal_test.go @@ -5,8 +5,8 @@ import ( "os/exec" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -32,9 +32,9 @@ func TestRcloneExit(t *testing.T) { t.Log("killed rclone") for i := 0; i < 10; i++ { - _, err = be.Stat(context.TODO(), restic.Handle{ + _, err = be.Stat(context.TODO(), backend.Handle{ Name: "foo", - Type: restic.PackFile, + Type: backend.PackFile, }) rtest.Assert(t, err != nil, "expected an error") } diff --git a/mover-restic/restic/internal/backend/readerat.go b/mover-restic/restic/internal/backend/readerat.go index ff2e40393..f4164cc6e 100644 --- a/mover-restic/restic/internal/backend/readerat.go +++ b/mover-restic/restic/internal/backend/readerat.go @@ -6,13 +6,12 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) type backendReaderAt struct { ctx context.Context - be restic.Backend - h restic.Handle + be Backend + h Handle } func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { @@ -22,12 +21,12 @@ func (brd backendReaderAt) ReadAt(p []byte, offset int64) (n int, err error) { // ReaderAt returns an io.ReaderAt for a file in the backend. The returned reader // should not escape the caller function to avoid unexpected interactions with the // embedded context -func ReaderAt(ctx context.Context, be restic.Backend, h restic.Handle) io.ReaderAt { +func ReaderAt(ctx context.Context, be Backend, h Handle) io.ReaderAt { return backendReaderAt{ctx: ctx, be: be, h: h} } // ReadAt reads from the backend handle h at the given position. -func ReadAt(ctx context.Context, be restic.Backend, h restic.Handle, offset int64, p []byte) (n int, err error) { +func ReadAt(ctx context.Context, be Backend, h Handle, offset int64, p []byte) (n int, err error) { debug.Log("ReadAt(%v) at %v, len %v", h, offset, len(p)) err = be.Load(ctx, h, len(p), offset, func(rd io.Reader) (ierr error) { diff --git a/mover-restic/restic/internal/backend/rest/config.go b/mover-restic/restic/internal/backend/rest/config.go index 8458b0df2..8f17d444a 100644 --- a/mover-restic/restic/internal/backend/rest/config.go +++ b/mover-restic/restic/internal/backend/rest/config.go @@ -5,9 +5,9 @@ import ( "os" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to a REST server. @@ -73,7 +73,7 @@ func prepareURL(s string) string { return s } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/mover-restic/restic/internal/backend/rest/config_test.go b/mover-restic/restic/internal/backend/rest/config_test.go index 23ea9095b..13a1ebb13 100644 --- a/mover-restic/restic/internal/backend/rest/config_test.go +++ b/mover-restic/restic/internal/backend/rest/config_test.go @@ -31,6 +31,13 @@ var configTests = []test.ConfigTestData[Config]{ Connections: 5, }, }, + { + S: "rest:http+unix:///tmp/rest.socket:/my_backup_repo/", + Cfg: Config{ + URL: parseURL("http+unix:///tmp/rest.socket:/my_backup_repo/"), + Connections: 5, + }, + }, } func TestParseConfig(t *testing.T) { diff --git a/mover-restic/restic/internal/backend/rest/rest.go b/mover-restic/restic/internal/backend/rest/rest.go index de730e21f..1af88ec3f 100644 --- a/mover-restic/restic/internal/backend/rest/rest.go +++ b/mover-restic/restic/internal/backend/rest/rest.go @@ -14,13 +14,14 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/feature" ) -// make sure the rest backend implements restic.Backend -var _ restic.Backend = &Backend{} +// make sure the rest backend implements backend.Backend +var _ backend.Backend = &Backend{} // Backend uses the REST protocol to access data stored on a server. type Backend struct { @@ -30,6 +31,20 @@ type Backend struct { layout.Layout } +// restError is returned whenever the server returns a non-successful HTTP status. +type restError struct { + backend.Handle + StatusCode int + Status string +} + +func (e *restError) Error() string { + if e.StatusCode == http.StatusNotFound && e.Handle.Type.String() != "invalid" { + return fmt.Sprintf("%v does not exist", e.Handle) + } + return fmt.Sprintf("unexpected HTTP response (%v): %v", e.StatusCode, e.Status) +} + func NewFactory() location.Factory { return location.NewHTTPBackendFactory("rest", ParseConfig, StripPassword, Create, Open) } @@ -58,6 +73,17 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) return be, nil } +func drainAndClose(resp *http.Response) error { + _, err := io.Copy(io.Discard, resp.Body) + cerr := resp.Body.Close() + + // return first error + if err != nil { + return errors.Errorf("drain: %w", err) + } + return cerr +} + // Create creates a new REST on server configured in config. func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { be, err := Open(ctx, cfg, rt) @@ -65,7 +91,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, err } - _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(ctx, backend.Handle{Type: backend.ConfigFile}) if err == nil { return nil, errors.New("config file already exists") } @@ -80,18 +106,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, err } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) - } - - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { + if err := drainAndClose(resp); err != nil { return nil, err } - err = resp.Body.Close() - if err != nil { - return nil, err + if resp.StatusCode != http.StatusOK { + return nil, &restError{backend.Handle{}, resp.StatusCode, resp.Status} } return be, nil @@ -101,11 +121,6 @@ func (b *Backend) Connections() uint { return b.connections } -// Location returns this backend's location (the server's URL). -func (b *Backend) Location() string { - return b.url.String() -} - // Hasher may return a hash function for calculating a content hash for the backend func (b *Backend) Hasher() hash.Hash { return nil @@ -118,7 +133,7 @@ func (b *Backend) HasAtomicReplace() bool { } // Save stores data in the backend at the handle. -func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { ctx, cancel := context.WithCancel(ctx) defer cancel() @@ -136,43 +151,45 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea req.ContentLength = rd.Length() resp, err := b.client.Do(req) - - var cerr error - if resp != nil { - _, _ = io.Copy(io.Discard, resp.Body) - cerr = resp.Body.Close() - } - if err != nil { return errors.WithStack(err) } + if err := drainAndClose(resp); err != nil { + return err + } + if resp.StatusCode != http.StatusOK { - return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) + return &restError{h, resp.StatusCode, resp.Status} } - return errors.Wrap(cerr, "Close") + return nil } -// notExistError is returned whenever the requested file does not exist on the -// server. -type notExistError struct { - restic.Handle +// IsNotExist returns true if the error was caused by a non-existing file. +func (b *Backend) IsNotExist(err error) bool { + var e *restError + return errors.As(err, &e) && e.StatusCode == http.StatusNotFound } -func (e *notExistError) Error() string { - return fmt.Sprintf("%v does not exist", e.Handle) -} +func (b *Backend) IsPermanentError(err error) bool { + if b.IsNotExist(err) { + return true + } -// IsNotExist returns true if the error was caused by a non-existing file. -func (b *Backend) IsNotExist(err error) bool { - var e *notExistError - return errors.As(err, &e) + var rerr *restError + if errors.As(err, &rerr) { + if rerr.StatusCode == http.StatusRequestedRangeNotSatisfiable || rerr.StatusCode == http.StatusUnauthorized || rerr.StatusCode == http.StatusForbidden { + return true + } + } + + return false } // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { r, err := b.openReader(ctx, h, length, offset) if err != nil { return err @@ -201,7 +218,7 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset return err } -func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil) if err != nil { return nil, errors.WithStack(err) @@ -215,60 +232,48 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { - if resp != nil { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - } return nil, errors.Wrap(err, "client.Do") } - if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() - return nil, ¬ExistError{h} + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + _ = drainAndClose(resp) + return nil, &restError{h, resp.StatusCode, resp.Status} } - if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - _ = resp.Body.Close() - return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) + if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 && resp.ContentLength != int64(length) { + return nil, &restError{h, http.StatusRequestedRangeNotSatisfiable, "partial out of bounds read"} } return resp.Body, nil } // Stat returns information about a blob. -func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) if err != nil { - return restic.FileInfo{}, errors.WithStack(err) + return backend.FileInfo{}, errors.WithStack(err) } - _, _ = io.Copy(io.Discard, resp.Body) - if err = resp.Body.Close(); err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Close") - } - - if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() - return restic.FileInfo{}, ¬ExistError{h} + if err = drainAndClose(resp); err != nil { + return backend.FileInfo{}, err } if resp.StatusCode != http.StatusOK { - return restic.FileInfo{}, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) + return backend.FileInfo{}, &restError{h, resp.StatusCode, resp.Status} } if resp.ContentLength < 0 { - return restic.FileInfo{}, errors.New("negative content length") + return backend.FileInfo{}, errors.New("negative content length") } - bi := restic.FileInfo{ + bi := backend.FileInfo{ Size: resp.ContentLength, Name: h.Name, } @@ -277,7 +282,7 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e } // Remove removes the blob with the given name and type. -func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil) if err != nil { return errors.WithStack(err) @@ -285,32 +290,25 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error { req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { return errors.Wrap(err, "client.Do") } - if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() - return ¬ExistError{h} + if err = drainAndClose(resp); err != nil { + return err } if resp.StatusCode != http.StatusOK { - return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode) + return &restError{h, resp.StatusCode, resp.Status} } - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - return errors.Wrap(err, "Copy") - } - - return errors.Wrap(resp.Body.Close(), "Close") + return nil } // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - url := b.Dirname(restic.Handle{Type: t}) +func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { + url := b.Dirname(backend.Handle{Type: t}) if !strings.HasSuffix(url, "/") { url += "/" } @@ -322,7 +320,6 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { return errors.Wrap(err, "List") } @@ -333,25 +330,31 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi // already ignores missing directories, but misuses "not found" to // report certain internal errors, see // https://github.com/rclone/rclone/pull/7550 for details. - return nil + return drainAndClose(resp) } } if resp.StatusCode != http.StatusOK { - return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode) + _ = drainAndClose(resp) + return &restError{backend.Handle{Type: t}, resp.StatusCode, resp.Status} } if resp.Header.Get("Content-Type") == ContentTypeV2 { - return b.listv2(ctx, resp, fn) + err = b.listv2(ctx, resp, fn) + } else { + err = b.listv1(ctx, t, resp, fn) } - return b.listv1(ctx, t, resp, fn) + if cerr := drainAndClose(resp); cerr != nil && err == nil { + err = cerr + } + return err } // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET // /data/`) only returns the names of the files, so we need to issue an HTTP // HEAD request for each file. -func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Response, fn func(restic.FileInfo) error) error { +func (b *Backend) listv1(ctx context.Context, t backend.FileType, resp *http.Response, fn func(backend.FileInfo) error) error { debug.Log("parsing API v1 response") dec := json.NewDecoder(resp.Body) var list []string @@ -360,7 +363,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp } for _, m := range list { - fi, err := b.Stat(ctx, restic.Handle{Name: m, Type: t}) + fi, err := b.Stat(ctx, backend.Handle{Name: m, Type: t}) if err != nil { return err } @@ -385,7 +388,7 @@ func (b *Backend) listv1(ctx context.Context, t restic.FileType, resp *http.Resp // listv2 uses the REST protocol v2, where a list HTTP request (e.g. `GET // /data/`) returns the names and sizes of all files. -func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(restic.FileInfo) error) error { +func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(backend.FileInfo) error) error { debug.Log("parsing API v2 response") dec := json.NewDecoder(resp.Body) @@ -402,7 +405,7 @@ func (b *Backend) listv2(ctx context.Context, resp *http.Response, fn func(resti return ctx.Err() } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: item.Name, Size: item.Size, } @@ -429,5 +432,5 @@ func (b *Backend) Close() error { // Delete removes all data in the backend. func (b *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, b) + return util.DefaultDelete(ctx, b) } diff --git a/mover-restic/restic/internal/backend/rest/rest_int_test.go b/mover-restic/restic/internal/backend/rest/rest_int_test.go index e7810c5e3..853a852c7 100644 --- a/mover-restic/restic/internal/backend/rest/rest_int_test.go +++ b/mover-restic/restic/internal/backend/rest/rest_int_test.go @@ -10,8 +10,8 @@ import ( "strconv" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/rest" - "github.com/restic/restic/internal/restic" ) func TestListAPI(t *testing.T) { @@ -22,7 +22,7 @@ func TestListAPI(t *testing.T) { Data string // response data Requests int - Result []restic.FileInfo + Result []backend.FileInfo }{ { Name: "content-type-unknown", @@ -32,7 +32,7 @@ func TestListAPI(t *testing.T) { "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, @@ -47,7 +47,7 @@ func TestListAPI(t *testing.T) { "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b" ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 4386}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 15214}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 33393}, @@ -62,7 +62,7 @@ func TestListAPI(t *testing.T) { {"name": "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", "size": 1002}, {"name": "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", "size": 1003} ]`, - Result: []restic.FileInfo{ + Result: []backend.FileInfo{ {Name: "1122e6749358b057fa1ac6b580a0fbe7a9a5fbc92e82743ee21aaf829624a985", Size: 1001}, {Name: "3b6ec1af8d4f7099d0445b12fdb75b166ba19f789e5c48350c423dc3b3e68352", Size: 1002}, {Name: "8271d221a60e0058e6c624f248d0080fc04f4fac07a28584a9b89d0eb69e189b", Size: 1003}, @@ -122,8 +122,8 @@ func TestListAPI(t *testing.T) { t.Fatal(err) } - var list []restic.FileInfo - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + var list []backend.FileInfo + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { list = append(list, fi) return nil }) diff --git a/mover-restic/restic/internal/backend/rest/rest_test.go b/mover-restic/restic/internal/backend/rest/rest_test.go index 6a5b4f8a5..93b9a103e 100644 --- a/mover-restic/restic/internal/backend/rest/rest_test.go +++ b/mover-restic/restic/internal/backend/rest/rest_test.go @@ -1,11 +1,18 @@ +//go:build go1.20 +// +build go1.20 + package rest_test import ( + "bufio" "context" - "net" + "fmt" "net/url" "os" "os/exec" + "regexp" + "strings" + "syscall" "testing" "time" @@ -14,54 +21,133 @@ import ( rtest "github.com/restic/restic/internal/test" ) -func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, func()) { +var ( + serverStartedRE = regexp.MustCompile("^start server on (.*)$") +) + +func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string) (*url.URL, func()) { srv, err := exec.LookPath("rest-server") if err != nil { t.Skip(err) } - cmd := exec.CommandContext(ctx, srv, "--no-auth", "--path", dir) + // create our own context, so that our cleanup can cancel and wait for completion + // this will ensure any open ports, open unix sockets etc are properly closed + processCtx, cancel := context.WithCancel(ctx) + cmd := exec.CommandContext(processCtx, srv, "--no-auth", "--path", dir, "--listen", reqListenAddr) + + // this cancel func is called by when the process context is done + cmd.Cancel = func() error { + // we execute in a Go-routine as we know the caller will + // be waiting on a .Wait() regardless + go func() { + // try to send a graceful termination signal + if cmd.Process.Signal(syscall.SIGTERM) == nil { + // if we succeed, then wait a few seconds + time.Sleep(2 * time.Second) + } + // and then make sure it's killed either way, ignoring any error code + _ = cmd.Process.Kill() + }() + return nil + } + + // this is the cleanup function that we return the caller, + // which will cancel our process context, and then wait for it to finish + cleanup := func() { + cancel() + _ = cmd.Wait() + } + + // but in-case we don't finish this method, e.g. by calling t.Fatal() + // we also defer a call to clean it up ourselves, guarded by a flag to + // indicate that we returned the function to the caller to deal with. + callerWillCleanUp := false + defer func() { + if !callerWillCleanUp { + cleanup() + } + }() + + // send stdout to our std out cmd.Stdout = os.Stdout - cmd.Stderr = os.Stdout - if err := cmd.Start(); err != nil { + + // capture stderr with a pipe, as we want to examine this output + // to determine when the server is started and listening. + cmdErr, err := cmd.StderrPipe() + if err != nil { t.Fatal(err) } - // wait until the TCP port is reachable - var success bool - for i := 0; i < 10; i++ { - time.Sleep(200 * time.Millisecond) + // start the rest-server + if err := cmd.Start(); err != nil { + t.Fatal(err) + } - c, err := net.Dial("tcp", "localhost:8000") - if err != nil { - continue + // create a channel to receive the actual listen address on + listenAddrCh := make(chan string) + go func() { + defer close(listenAddrCh) + matched := false + br := bufio.NewReader(cmdErr) + for { + line, err := br.ReadString('\n') + if err != nil { + // we ignore errors, as code that relies on this + // will happily fail via timeout and empty closed + // channel. + return + } + + line = strings.Trim(line, "\r\n") + if !matched { + // look for the server started message, and return the address + // that it's listening on + matchedServerListen := serverStartedRE.FindSubmatch([]byte(line)) + if len(matchedServerListen) == 2 { + listenAddrCh <- string(matchedServerListen[1]) + matched = true + } + } + fmt.Fprintln(os.Stdout, line) // print all output to console } + }() - success = true - if err := c.Close(); err != nil { - t.Fatal(err) + // wait for us to get an address, + // or the parent context to cancel, + // or for us to timeout + var actualListenAddr string + select { + case <-processCtx.Done(): + t.Fatal(context.Canceled) + case <-time.NewTimer(2 * time.Second).C: + t.Fatal(context.DeadlineExceeded) + case a, ok := <-listenAddrCh: + if !ok { + t.Fatal(context.Canceled) } + actualListenAddr = a } - if !success { - t.Fatal("unable to connect to rest server") - return nil, nil + // this translate the address that the server is listening on + // to a URL suitable for us to connect to + var addrToConnectTo string + if strings.HasPrefix(reqListenAddr, "unix:") { + addrToConnectTo = fmt.Sprintf("http+unix://%s:/restic-test/", actualListenAddr) + } else { + // while we may listen on 0.0.0.0, we connect to localhost + addrToConnectTo = fmt.Sprintf("http://%s/restic-test/", strings.Replace(actualListenAddr, "0.0.0.0", "localhost", 1)) } - url, err := url.Parse("http://localhost:8000/restic-test/") + // parse to a URL + url, err := url.Parse(addrToConnectTo) if err != nil { t.Fatal(err) } - cleanup := func() { - if err := cmd.Process.Kill(); err != nil { - t.Fatal(err) - } - - // ignore errors, we've killed the process - _ = cmd.Wait() - } - + // indicate that we've completed successfully, and that the caller + // is responsible for calling cleanup + callerWillCleanUp = true return url, cleanup } @@ -91,7 +177,7 @@ func TestBackendREST(t *testing.T) { defer cancel() dir := rtest.TempDir(t) - serverURL, cleanup := runRESTServer(ctx, t, dir) + serverURL, cleanup := runRESTServer(ctx, t, dir, ":0") defer cleanup() newTestSuite(serverURL, false).RunTests(t) @@ -116,7 +202,7 @@ func BenchmarkBackendREST(t *testing.B) { defer cancel() dir := rtest.TempDir(t) - serverURL, cleanup := runRESTServer(ctx, t, dir) + serverURL, cleanup := runRESTServer(ctx, t, dir, ":0") defer cleanup() newTestSuite(serverURL, false).RunBenchmarks(t) diff --git a/mover-restic/restic/internal/backend/rest/rest_unix_test.go b/mover-restic/restic/internal/backend/rest/rest_unix_test.go new file mode 100644 index 000000000..85ef7a73d --- /dev/null +++ b/mover-restic/restic/internal/backend/rest/rest_unix_test.go @@ -0,0 +1,30 @@ +//go:build !windows && go1.20 +// +build !windows,go1.20 + +package rest_test + +import ( + "context" + "fmt" + "path" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func TestBackendRESTWithUnixSocket(t *testing.T) { + defer func() { + if t.Skipped() { + rtest.SkipDisallowed(t, "restic/backend/rest.TestBackendREST") + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dir := rtest.TempDir(t) + serverURL, cleanup := runRESTServer(ctx, t, path.Join(dir, "data"), fmt.Sprintf("unix:%s", path.Join(dir, "sock"))) + defer cleanup() + + newTestSuite(serverURL, false).RunTests(t) +} diff --git a/mover-restic/restic/internal/backend/retry/backend_retry.go b/mover-restic/restic/internal/backend/retry/backend_retry.go index 9c51efedc..8d0f42bfd 100644 --- a/mover-restic/restic/internal/backend/retry/backend_retry.go +++ b/mover-restic/restic/internal/backend/retry/backend_retry.go @@ -2,57 +2,94 @@ package retry import ( "context" + "errors" "fmt" "io" + "sync" "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/feature" ) // Backend retries operations on the backend in case of an error with a // backoff. type Backend struct { - restic.Backend - MaxTries int - Report func(string, error, time.Duration) - Success func(string, int) + backend.Backend + MaxElapsedTime time.Duration + Report func(string, error, time.Duration) + Success func(string, int) + + failedLoads sync.Map } -// statically ensure that RetryBackend implements restic.Backend. -var _ restic.Backend = &Backend{} +// statically ensure that RetryBackend implements backend.Backend. +var _ backend.Backend = &Backend{} // New wraps be with a backend that retries operations after a // backoff. report is called with a description and the error, if one occurred. // success is called with the number of retries before a successful operation // (it is not called if it succeeded on the first try) -func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend { +func New(be backend.Backend, maxElapsedTime time.Duration, report func(string, error, time.Duration), success func(string, int)) *Backend { return &Backend{ - Backend: be, - MaxTries: maxTries, - Report: report, - Success: success, + Backend: be, + MaxElapsedTime: maxElapsedTime, + Report: report, + Success: success, } } // retryNotifyErrorWithSuccess is an extension of backoff.RetryNotify with notification of success after an error. // success is NOT notified on the first run of operation (only after an error). -func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOff, notify backoff.Notify, success func(retries int)) error { +func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOffContext, notify backoff.Notify, success func(retries int)) error { + var operationWrapper backoff.Operation if success == nil { - return backoff.RetryNotify(operation, b, notify) - } - retries := 0 - operationWrapper := func() error { - err := operation() - if err != nil { - retries++ - } else if retries > 0 { - success(retries) + operationWrapper = operation + } else { + retries := 0 + operationWrapper = func() error { + err := operation() + if err != nil { + retries++ + } else if retries > 0 { + success(retries) + } + return err } - return err } - return backoff.RetryNotify(operationWrapper, b, notify) + err := backoff.RetryNotify(operationWrapper, b, notify) + + if err != nil && notify != nil && b.Context().Err() == nil { + // log final error, unless the context was canceled + notify(err, -1) + } + return err +} + +func withRetryAtLeastOnce(delegate *backoff.ExponentialBackOff) *retryAtLeastOnce { + return &retryAtLeastOnce{delegate: delegate} +} + +type retryAtLeastOnce struct { + delegate *backoff.ExponentialBackOff + numTries uint64 +} + +func (b *retryAtLeastOnce) NextBackOff() time.Duration { + delay := b.delegate.NextBackOff() + + b.numTries++ + if b.numTries == 1 && b.delegate.Stop == delay { + return b.delegate.InitialInterval + } + return delay +} + +func (b *retryAtLeastOnce) Reset() { + b.numTries = 0 + b.delegate.Reset() } var fastRetries = false @@ -69,13 +106,38 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error } bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = be.MaxElapsedTime + + if feature.Flag.Enabled(feature.BackendErrorRedesign) { + bo.InitialInterval = 1 * time.Second + bo.Multiplier = 2 + } if fastRetries { // speed up integration tests bo.InitialInterval = 1 * time.Millisecond + maxElapsedTime := 200 * time.Millisecond + if bo.MaxElapsedTime > maxElapsedTime { + bo.MaxElapsedTime = maxElapsedTime + } + } + + var b backoff.BackOff = withRetryAtLeastOnce(bo) + if !feature.Flag.Enabled(feature.BackendErrorRedesign) { + // deprecated behavior + b = backoff.WithMaxRetries(b, 10) } - err := retryNotifyErrorWithSuccess(f, - backoff.WithContext(backoff.WithMaxRetries(bo, uint64(be.MaxTries)), ctx), + err := retryNotifyErrorWithSuccess( + func() error { + err := f() + // don't retry permanent errors as those very likely cannot be fixed by retrying + // TODO remove IsNotExist(err) special cases when removing the feature flag + if feature.Flag.Enabled(feature.BackendErrorRedesign) && !errors.Is(err, &backoff.PermanentError{}) && be.Backend.IsPermanentError(err) { + return backoff.Permanent(err) + } + return err + }, + backoff.WithContext(b, ctx), func(err error, d time.Duration) { if be.Report != nil { be.Report(msg, err, d) @@ -92,7 +154,7 @@ func (be *Backend) retry(ctx context.Context, msg string, f func() error) error } // Save stores the data in the backend under the given handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error { err := rd.Rewind() if err != nil { @@ -121,19 +183,43 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe }) } +// Failed loads expire after an hour +var failedLoadExpiry = time.Hour + // Load returns a reader that yields the contents of the file at h at the // given offset. If length is larger than zero, only a portion of the file // is returned. rd must be closed after use. If an error is returned, the // ReadCloser must be nil. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { - return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) { + key := h + key.IsMetadata = false + + // Implement the circuit breaker pattern for files that exhausted all retries due to a non-permanent error + if v, ok := be.failedLoads.Load(key); ok { + if time.Since(v.(time.Time)) > failedLoadExpiry { + be.failedLoads.Delete(key) + } else { + // fail immediately if the file was already problematic during the last hour + return fmt.Errorf("circuit breaker open for file %v", h) + } + } + + err = be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset), func() error { return be.Backend.Load(ctx, h, length, offset, consumer) }) + + if feature.Flag.Enabled(feature.BackendErrorRedesign) && err != nil && !be.IsPermanentError(err) { + // We've exhausted the retries, the file is likely inaccessible. By excluding permanent + // errors, not found or truncated files are not recorded. + be.failedLoads.LoadOrStore(key, time.Now()) + } + + return err } // Stat returns information about the File identified by h. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (fi backend.FileInfo, err error) { err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h), func() error { var innerError error @@ -149,7 +235,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInf } // Remove removes a File with type t and name. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) (err error) { return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error { return be.Backend.Remove(ctx, h) }) @@ -159,7 +245,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) { // error is returned by the underlying backend, the request is retried. When fn // returns an error, the operation is aborted and the error is returned to the // caller. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { // create a new context that we can cancel when fn returns an error, so // that listing is aborted listCtx, cancel := context.WithCancel(ctx) @@ -169,7 +255,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F var innerErr error // remember when fn returned an error, so we can return that to the caller err := be.retry(listCtx, fmt.Sprintf("List(%v)", t), func() error { - return be.Backend.List(ctx, t, func(fi restic.FileInfo) error { + return be.Backend.List(ctx, t, func(fi backend.FileInfo) error { if _, ok := listed[fi.Name]; ok { return nil } @@ -192,6 +278,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F return err } -func (be *Backend) Unwrap() restic.Backend { +func (be *Backend) Unwrap() backend.Backend { return be.Backend } diff --git a/mover-restic/restic/internal/backend/retry/backend_retry_test.go b/mover-restic/restic/internal/backend/retry/backend_retry_test.go index 9f2f39589..fd76200d4 100644 --- a/mover-restic/restic/internal/backend/retry/backend_retry_test.go +++ b/mover-restic/restic/internal/backend/retry/backend_retry_test.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "io" + "strings" "testing" "time" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" @@ -18,7 +20,7 @@ func TestBackendSaveRetry(t *testing.T) { buf := bytes.NewBuffer(nil) errcount := 0 be := &mock.Backend{ - SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if errcount == 0 { errcount++ _, err := io.CopyN(io.Discard, rd, 120) @@ -38,7 +40,7 @@ func TestBackendSaveRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) - err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) + err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } @@ -56,14 +58,14 @@ func TestBackendSaveRetryAtomic(t *testing.T) { errcount := 0 calledRemove := false be := &mock.Backend{ - SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + SaveFn: func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if errcount == 0 { errcount++ return errors.New("injected error") } return nil }, - RemoveFn: func(ctx context.Context, h restic.Handle) error { + RemoveFn: func(ctx context.Context, h backend.Handle) error { calledRemove = true return nil }, @@ -74,7 +76,7 @@ func TestBackendSaveRetryAtomic(t *testing.T) { retryBackend := New(be, 10, nil, nil) data := test.Random(23, 5*1024*1024+11241) - err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher())) + err := retryBackend.Save(context.TODO(), backend.Handle{}, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatal(err) } @@ -91,15 +93,15 @@ func TestBackendListRetry(t *testing.T) { retry := 0 be := &mock.Backend{ - ListFn: func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { // fail during first retry, succeed during second retry++ if retry == 1 { - _ = fn(restic.FileInfo{Name: ID1}) + _ = fn(backend.FileInfo{Name: ID1}) return errors.New("test list error") } - _ = fn(restic.FileInfo{Name: ID1}) - _ = fn(restic.FileInfo{Name: ID2}) + _ = fn(backend.FileInfo{Name: ID1}) + _ = fn(backend.FileInfo{Name: ID2}) return nil }, } @@ -108,7 +110,7 @@ func TestBackendListRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) var listed []string - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { listed = append(listed, fi.Name) return nil }) @@ -121,10 +123,10 @@ func TestBackendListRetryErrorFn(t *testing.T) { var names = []string{"id1", "id2", "foo", "bar"} be := &mock.Backend{ - ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error { t.Logf("List called for %v", tpe) for _, name := range names { - err := fn(restic.FileInfo{Name: name}) + err := fn(backend.FileInfo{Name: name}) if err != nil { return err } @@ -141,7 +143,7 @@ func TestBackendListRetryErrorFn(t *testing.T) { var listed []string run := 0 - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { t.Logf("fn called for %v", fi.Name) run++ // return an error for the third item in the list @@ -172,7 +174,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { retries := 0 be := &mock.Backend{ - ListFn: func(ctx context.Context, tpe restic.FileType, fn func(restic.FileInfo) error) error { + ListFn: func(ctx context.Context, tpe backend.FileType, fn func(backend.FileInfo) error) error { t.Logf("List called for %v, retries %v", tpe, retries) retries++ for i, name := range names { @@ -180,7 +182,7 @@ func TestBackendListRetryErrorBackend(t *testing.T) { return ErrBackendTest } - err := fn(restic.FileInfo{Name: name}) + err := fn(backend.FileInfo{Name: name}) if err != nil { return err } @@ -191,11 +193,12 @@ func TestBackendListRetryErrorBackend(t *testing.T) { } TestFastRetries(t) - const maxRetries = 2 - retryBackend := New(be, maxRetries, nil, nil) + const maxElapsedTime = 10 * time.Millisecond + now := time.Now() + retryBackend := New(be, maxElapsedTime, nil, nil) var listed []string - err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := retryBackend.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { t.Logf("fn called for %v", fi.Name) listed = append(listed, fi.Name) return nil @@ -205,8 +208,9 @@ func TestBackendListRetryErrorBackend(t *testing.T) { t.Fatalf("wrong error returned, want %v, got %v", ErrBackendTest, err) } - if retries != maxRetries+1 { - t.Fatalf("List was called %d times, wanted %v", retries, maxRetries+1) + duration := time.Since(now) + if duration > 100*time.Millisecond { + t.Fatalf("list retries took %v, expected at most 10ms", duration) } test.Equals(t, names[:2], listed) @@ -252,7 +256,7 @@ func TestBackendLoadRetry(t *testing.T) { attempt := 0 be := mock.NewBackend() - be.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { // returns failing reader on first invocation, good reader on subsequent invocations attempt++ if attempt > 1 { @@ -265,7 +269,7 @@ func TestBackendLoadRetry(t *testing.T) { retryBackend := New(be, 10, nil, nil) var buf []byte - err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { buf, err = io.ReadAll(rd) return err }) @@ -274,19 +278,98 @@ func TestBackendLoadRetry(t *testing.T) { test.Equals(t, 2, attempt) } +func TestBackendLoadNotExists(t *testing.T) { + // load should not retry if the error matches IsNotExist + notFound := errors.New("not found") + attempt := 0 + + be := mock.NewBackend() + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + attempt++ + if attempt > 1 { + t.Fail() + return nil, errors.New("must not retry") + } + return nil, notFound + } + be.IsPermanentErrorFn = func(err error) bool { + return errors.Is(err, notFound) + } + + TestFastRetries(t) + retryBackend := New(be, 10, nil, nil) + + err := retryBackend.Load(context.TODO(), backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { + return nil + }) + test.Assert(t, be.IsPermanentErrorFn(err), "unexpected error %v", err) + test.Equals(t, 1, attempt) +} + +func TestBackendLoadCircuitBreaker(t *testing.T) { + // retry should not retry if the error matches IsPermanentError + notFound := errors.New("not found") + otherError := errors.New("something") + attempt := 0 + + be := mock.NewBackend() + be.IsPermanentErrorFn = func(err error) bool { + return errors.Is(err, notFound) + } + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + attempt++ + return nil, otherError + } + nilRd := func(rd io.Reader) (err error) { + return nil + } + + TestFastRetries(t) + retryBackend := New(be, 2, nil, nil) + // trip the circuit breaker for file "other" + err := retryBackend.Load(context.TODO(), backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Equals(t, otherError, err, "unexpected error") + test.Equals(t, 2, attempt) + + attempt = 0 + err = retryBackend.Load(context.TODO(), backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Assert(t, strings.Contains(err.Error(), "circuit breaker open for file"), "expected circuit breaker error, got %v") + test.Equals(t, 0, attempt) + + // don't trip for permanent errors + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + attempt++ + return nil, notFound + } + err = retryBackend.Load(context.TODO(), backend.Handle{Name: "notfound"}, 0, 0, nilRd) + test.Equals(t, notFound, err, "expected circuit breaker to only affect other file, got %v") + err = retryBackend.Load(context.TODO(), backend.Handle{Name: "notfound"}, 0, 0, nilRd) + test.Equals(t, notFound, err, "persistent error must not trigger circuit breaker, got %v") + + // wait for circuit breaker to expire + time.Sleep(5 * time.Millisecond) + old := failedLoadExpiry + defer func() { + failedLoadExpiry = old + }() + failedLoadExpiry = 3 * time.Millisecond + err = retryBackend.Load(context.TODO(), backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Equals(t, notFound, err, "expected circuit breaker to reset, got %v") +} + func TestBackendStatNotExists(t *testing.T) { // stat should not retry if the error matches IsNotExist notFound := errors.New("not found") attempt := 0 be := mock.NewBackend() - be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + be.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { attempt++ if attempt > 1 { t.Fail() - return restic.FileInfo{}, errors.New("must not retry") + return backend.FileInfo{}, errors.New("must not retry") } - return restic.FileInfo{}, notFound + return backend.FileInfo{}, notFound } be.IsNotExistFn = func(err error) bool { return errors.Is(err, notFound) @@ -295,11 +378,41 @@ func TestBackendStatNotExists(t *testing.T) { TestFastRetries(t) retryBackend := New(be, 10, nil, nil) - _, err := retryBackend.Stat(context.TODO(), restic.Handle{}) + _, err := retryBackend.Stat(context.TODO(), backend.Handle{}) test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err) test.Equals(t, 1, attempt) } +func TestBackendRetryPermanent(t *testing.T) { + // retry should not retry if the error matches IsPermanentError + notFound := errors.New("not found") + attempt := 0 + + be := mock.NewBackend() + be.IsPermanentErrorFn = func(err error) bool { + return errors.Is(err, notFound) + } + + TestFastRetries(t) + retryBackend := New(be, 2, nil, nil) + err := retryBackend.retry(context.TODO(), "test", func() error { + attempt++ + return notFound + }) + + test.Assert(t, be.IsPermanentErrorFn(err), "unexpected error %v", err) + test.Equals(t, 1, attempt) + + attempt = 0 + err = retryBackend.retry(context.TODO(), "test", func() error { + attempt++ + return errors.New("something") + }) + test.Assert(t, !be.IsPermanentErrorFn(err), "error unexpectedly considered permanent %v", err) + test.Equals(t, 2, attempt) + +} + func assertIsCanceled(t *testing.T, err error) { test.Assert(t, err == context.Canceled, "got unexpected err %v", err) } @@ -309,7 +422,7 @@ func TestBackendCanceledContext(t *testing.T) { // check that we received the expected context canceled error instead TestFastRetries(t) retryBackend := New(mock.NewBackend(), 2, nil, nil) - h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()} + h := backend.Handle{Type: backend.PackFile, Name: restic.NewRandomID().String()} // create an already canceled context ctx, cancel := context.WithCancel(context.Background()) @@ -318,15 +431,15 @@ func TestBackendCanceledContext(t *testing.T) { _, err := retryBackend.Stat(ctx, h) assertIsCanceled(t, err) - err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil)) + err = retryBackend.Save(ctx, h, backend.NewByteReader([]byte{}, nil)) assertIsCanceled(t, err) err = retryBackend.Remove(ctx, h) assertIsCanceled(t, err) - err = retryBackend.Load(ctx, restic.Handle{}, 0, 0, func(rd io.Reader) (err error) { + err = retryBackend.Load(ctx, backend.Handle{}, 0, 0, func(rd io.Reader) (err error) { return nil }) assertIsCanceled(t, err) - err = retryBackend.List(ctx, restic.PackFile, func(restic.FileInfo) error { + err = retryBackend.List(ctx, backend.PackFile, func(backend.FileInfo) error { return nil }) assertIsCanceled(t, err) @@ -347,7 +460,7 @@ func TestNotifyWithSuccessIsNotCalled(t *testing.T) { t.Fatal("Success should not have been called") } - err := retryNotifyErrorWithSuccess(operation, &backoff.ZeroBackOff{}, notify, success) + err := retryNotifyErrorWithSuccess(operation, backoff.WithContext(&backoff.ZeroBackOff{}, context.Background()), notify, success) if err != nil { t.Fatal("retry should not have returned an error") } @@ -373,7 +486,7 @@ func TestNotifyWithSuccessIsCalled(t *testing.T) { successCalled++ } - err := retryNotifyErrorWithSuccess(operation, &backoff.ZeroBackOff{}, notify, success) + err := retryNotifyErrorWithSuccess(operation, backoff.WithContext(&backoff.ZeroBackOff{}, context.Background()), notify, success) if err != nil { t.Fatal("retry should not have returned an error") } @@ -386,3 +499,83 @@ func TestNotifyWithSuccessIsCalled(t *testing.T) { t.Fatalf("Success should have been called only once, but was called %d times instead", successCalled) } } + +func TestNotifyWithSuccessFinalError(t *testing.T) { + operation := func() error { + return errors.New("expected error in test") + } + + notifyCalled := 0 + notify := func(error, time.Duration) { + notifyCalled++ + } + + successCalled := 0 + success := func(retries int) { + successCalled++ + } + + err := retryNotifyErrorWithSuccess(operation, backoff.WithContext(backoff.WithMaxRetries(&backoff.ZeroBackOff{}, 5), context.Background()), notify, success) + test.Assert(t, err.Error() == "expected error in test", "wrong error message %v", err) + test.Equals(t, 6, notifyCalled, "notify should have been called 6 times") + test.Equals(t, 0, successCalled, "success should not have been called") +} + +func TestNotifyWithCancelError(t *testing.T) { + operation := func() error { + return errors.New("expected error in test") + } + + notify := func(error, time.Duration) { + t.Error("unexpected call to notify") + } + + success := func(retries int) { + t.Error("unexpected call to success") + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := retryNotifyErrorWithSuccess(operation, backoff.WithContext(&backoff.ZeroBackOff{}, ctx), notify, success) + test.Assert(t, err == context.Canceled, "wrong error message %v", err) +} + +type testClock struct { + Time time.Time +} + +func (c *testClock) Now() time.Time { + return c.Time +} + +func TestRetryAtLeastOnce(t *testing.T) { + expBackOff := backoff.NewExponentialBackOff() + expBackOff.InitialInterval = 500 * time.Millisecond + expBackOff.RandomizationFactor = 0 + expBackOff.MaxElapsedTime = 5 * time.Second + expBackOff.Multiplier = 2 // guarantee numerical stability + clock := &testClock{Time: time.Now()} + expBackOff.Clock = clock + expBackOff.Reset() + + retry := withRetryAtLeastOnce(expBackOff) + + // expire backoff + clock.Time = clock.Time.Add(10 * time.Second) + delay := retry.NextBackOff() + test.Equals(t, expBackOff.InitialInterval, delay, "must retry at least once") + + delay = retry.NextBackOff() + test.Equals(t, expBackOff.Stop, delay, "must not retry more than once") + + // test reset behavior + retry.Reset() + test.Equals(t, uint64(0), retry.numTries, "numTries should be reset to 0") + + // Verify that after reset, NextBackOff returns the initial interval again + delay = retry.NextBackOff() + test.Equals(t, expBackOff.InitialInterval, delay, "retries must work after reset") + + delay = retry.NextBackOff() + test.Equals(t, expBackOff.InitialInterval*time.Duration(expBackOff.Multiplier), delay, "retries must work after reset") +} diff --git a/mover-restic/restic/internal/restic/rewind_reader.go b/mover-restic/restic/internal/backend/rewind_reader.go similarity index 99% rename from mover-restic/restic/internal/restic/rewind_reader.go rename to mover-restic/restic/internal/backend/rewind_reader.go index c27724e02..762b530aa 100644 --- a/mover-restic/restic/internal/restic/rewind_reader.go +++ b/mover-restic/restic/internal/backend/rewind_reader.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "bytes" diff --git a/mover-restic/restic/internal/restic/rewind_reader_test.go b/mover-restic/restic/internal/backend/rewind_reader_test.go similarity index 99% rename from mover-restic/restic/internal/restic/rewind_reader_test.go rename to mover-restic/restic/internal/backend/rewind_reader_test.go index 8ec79ddcd..2ee287596 100644 --- a/mover-restic/restic/internal/restic/rewind_reader_test.go +++ b/mover-restic/restic/internal/backend/rewind_reader_test.go @@ -1,4 +1,4 @@ -package restic +package backend import ( "bytes" diff --git a/mover-restic/restic/internal/backend/s3/config.go b/mover-restic/restic/internal/backend/s3/config.go index 8dcad9eee..be2a78ce5 100644 --- a/mover-restic/restic/internal/backend/s3/config.go +++ b/mover-restic/restic/internal/backend/s3/config.go @@ -6,9 +6,9 @@ import ( "path" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains all configuration necessary to connect to an s3 compatible @@ -20,14 +20,15 @@ type Config struct { Secret options.SecretString Bucket string Prefix string - Layout string `option:"layout" help:"use this backend layout (default: auto-detect)"` + Layout string `option:"layout" help:"use this backend layout (default: auto-detect) (deprecated)"` StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"` - Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` - MaxRetries uint `option:"retries" help:"set the number of retries attempted"` - Region string `option:"region" help:"set region"` - BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"` - ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"` + Connections uint `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"` + MaxRetries uint `option:"retries" help:"set the number of retries attempted"` + Region string `option:"region" help:"set region"` + BucketLookup string `option:"bucket-lookup" help:"bucket lookup style: 'auto', 'dns', or 'path'"` + ListObjectsV1 bool `option:"list-objects-v1" help:"use deprecated V1 api for ListObjects calls"` + UnsafeAnonymousAuth bool `option:"unsafe-anonymous-auth" help:"use anonymous authentication"` } // NewConfig returns a new Config with the default values filled in. @@ -94,7 +95,7 @@ func createConfig(endpoint, bucket, prefix string, useHTTP bool) (*Config, error return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/mover-restic/restic/internal/backend/s3/s3.go b/mover-restic/restic/internal/backend/s3/s3.go index 3fe32d215..019f8471b 100644 --- a/mover-restic/restic/internal/backend/s3/s3.go +++ b/mover-restic/restic/internal/backend/s3/s3.go @@ -14,9 +14,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/feature" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -30,7 +31,7 @@ type Backend struct { } // make sure that *Backend implements backend.Backend -var _ restic.Backend = &Backend{} +var _ backend.Backend = &Backend{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) @@ -51,6 +52,56 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro minio.MaxRetry = int(cfg.MaxRetries) } + creds, err := getCredentials(cfg, rt) + if err != nil { + return nil, errors.Wrap(err, "s3.getCredentials") + } + + options := &minio.Options{ + Creds: creds, + Secure: !cfg.UseHTTP, + Region: cfg.Region, + Transport: rt, + } + + switch strings.ToLower(cfg.BucketLookup) { + case "", "auto": + options.BucketLookup = minio.BucketLookupAuto + case "dns": + options.BucketLookup = minio.BucketLookupDNS + case "path": + options.BucketLookup = minio.BucketLookupPath + default: + return nil, fmt.Errorf(`bad bucket-lookup style %q must be "auto", "path" or "dns"`, cfg.BucketLookup) + } + + client, err := minio.New(cfg.Endpoint, options) + if err != nil { + return nil, errors.Wrap(err, "minio.New") + } + + be := &Backend{ + client: client, + cfg: cfg, + } + + l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) + if err != nil { + return nil, err + } + + be.Layout = l + + return be, nil +} + +// getCredentials -- runs through the various credential types and returns the first one that works. +// additionally if the user has specified a role to assume, it will do that as well. +func getCredentials(cfg Config, tr http.RoundTripper) (*credentials.Credentials, error) { + if cfg.UnsafeAnonymousAuth { + return credentials.New(&credentials.Static{}), nil + } + // Chains all credential types, in the following order: // - Static credentials provided by user // - AWS env vars (i.e. AWS_ACCESS_KEY_ID) @@ -73,7 +124,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro &credentials.FileMinioClient{}, &credentials.IAM{ Client: &http.Client{ - Transport: http.DefaultTransport, + Transport: tr, }, }, }) @@ -84,56 +135,72 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro } if c.SignerType == credentials.SignatureAnonymous { - debug.Log("using anonymous access for %#v", cfg.Endpoint) - } + // Fail if no credentials were found to prevent repeated attempts to (unsuccessfully) retrieve new credentials. + // The first attempt still has to timeout which slows down restic usage considerably. Thus, migrate towards forcing + // users to explicitly decide between authenticated and anonymous access. + if feature.Flag.Enabled(feature.ExplicitS3AnonymousAuth) { + return nil, fmt.Errorf("no credentials found. Use `-o s3.unsafe-anonymous-auth=true` for anonymous authentication") + } - options := &minio.Options{ - Creds: creds, - Secure: !cfg.UseHTTP, - Region: cfg.Region, - Transport: rt, + debug.Log("using anonymous access for %#v", cfg.Endpoint) + creds = credentials.New(&credentials.Static{}) } - switch strings.ToLower(cfg.BucketLookup) { - case "", "auto": - options.BucketLookup = minio.BucketLookupAuto - case "dns": - options.BucketLookup = minio.BucketLookupDNS - case "path": - options.BucketLookup = minio.BucketLookupPath - default: - return nil, fmt.Errorf(`bad bucket-lookup style %q must be "auto", "path" or "dns"`, cfg.BucketLookup) - } + roleArn := os.Getenv("RESTIC_AWS_ASSUME_ROLE_ARN") + if roleArn != "" { + // use the region provided by the configuration by default + awsRegion := cfg.Region + // allow the region to be overridden if for some reason it is required + if os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") != "" { + awsRegion = os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") + } - client, err := minio.New(cfg.Endpoint, options) - if err != nil { - return nil, errors.Wrap(err, "minio.New") - } + sessionName := os.Getenv("RESTIC_AWS_ASSUME_ROLE_SESSION_NAME") + externalID := os.Getenv("RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID") + policy := os.Getenv("RESTIC_AWS_ASSUME_ROLE_POLICY") + stsEndpoint := os.Getenv("RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT") + + if stsEndpoint == "" { + if awsRegion != "" { + if strings.HasPrefix(awsRegion, "cn-") { + stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com.cn" + } else { + stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com" + } + } else { + stsEndpoint = "https://sts.amazonaws.com" + } + } - be := &Backend{ - client: client, - cfg: cfg, - } + opts := credentials.STSAssumeRoleOptions{ + RoleARN: roleArn, + AccessKey: c.AccessKeyID, + SecretKey: c.SecretAccessKey, + SessionToken: c.SessionToken, + RoleSessionName: sessionName, + ExternalID: externalID, + Policy: policy, + Location: awsRegion, + } - l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) - if err != nil { - return nil, err + creds, err = credentials.NewSTSAssumeRole(stsEndpoint, opts) + if err != nil { + return nil, errors.Wrap(err, "creds.AssumeRole") + } } - be.Layout = l - - return be, nil + return creds, nil } // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { return open(ctx, cfg, rt) } // Create opens the S3 backend at bucket and region and creates the bucket if // it does not exist yet. -func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { be, err := open(ctx, cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") @@ -175,6 +242,21 @@ func (be *Backend) IsNotExist(err error) bool { return errors.As(err, &e) && e.Code == "NoSuchKey" } +func (be *Backend) IsPermanentError(err error) bool { + if be.IsNotExist(err) { + return true + } + + var merr minio.ErrorResponse + if errors.As(err, &merr) { + if merr.Code == "InvalidRange" || merr.Code == "AccessDenied" { + return true + } + } + + return false +} + // Join combines path components with slashes. func (be *Backend) Join(p ...string) string { return path.Join(p...) @@ -251,11 +333,6 @@ func (be *Backend) Connections() uint { return be.cfg.Connections } -// Location returns this backend's location (the bucket name). -func (be *Backend) Location() string { - return be.Join(be.cfg.Bucket, be.cfg.Prefix) -} - // Hasher may return a hash function for calculating a content hash for the backend func (be *Backend) Hasher() hash.Hash { return nil @@ -271,16 +348,29 @@ func (be *Backend) Path() string { return be.cfg.Prefix } +// useStorageClass returns whether file should be saved in the provided Storage Class +// For archive storage classes, only data files are stored using that class; metadata +// must remain instantly accessible. +func (be *Backend) useStorageClass(h backend.Handle) bool { + notArchiveClass := be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" + isDataFile := h.Type == backend.PackFile && !h.IsMetadata + return isDataFile || notArchiveClass +} + // Save stores data in the backend at the handle. -func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) - opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} - opts.ContentType = "application/octet-stream" - // the only option with the high-level api is to let the library handle the checksum computation - opts.SendContentMd5 = true - // only use multipart uploads for very large files - opts.PartSize = 200 * 1024 * 1024 + opts := minio.PutObjectOptions{ + ContentType: "application/octet-stream", + // the only option with the high-level api is to let the library handle the checksum computation + SendContentMd5: true, + // only use multipart uploads for very large files + PartSize: 200 * 1024 * 1024, + } + if be.useStorageClass(h) { + opts.StorageClass = be.cfg.StorageClass + } info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts) @@ -294,14 +384,14 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) opts := minio.GetObjectOptions{} @@ -317,16 +407,23 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int, } coreClient := minio.Core{Client: be.client} - rd, _, _, err := coreClient.GetObject(ctx, be.cfg.Bucket, objName, opts) + rd, info, _, err := coreClient.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { return nil, err } + if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 { + if info.Size > 0 && info.Size != int64(length) { + _ = rd.Close() + return nil, minio.ErrorResponse{Code: "InvalidRange", Message: "restic-file-too-short"} + } + } + return rd, err } // Stat returns information about a blob. -func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) var obj *minio.Object @@ -334,7 +431,7 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf obj, err = be.client.GetObject(ctx, be.cfg.Bucket, objName, opts) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "client.GetObject") + return backend.FileInfo{}, errors.Wrap(err, "client.GetObject") } // make sure that the object is closed properly. @@ -347,14 +444,14 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf fi, err := obj.Stat() if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Stat") + return backend.FileInfo{}, errors.Wrap(err, "Stat") } - return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(ctx, be.cfg.Bucket, objName, minio.RemoveObjectOptions{}) @@ -368,7 +465,7 @@ func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *Backend) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, recursive := be.Basedir(t) // make sure prefix ends with a slash @@ -400,7 +497,7 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: path.Base(m), Size: obj.Size, } @@ -424,14 +521,14 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F // Delete removes all restic keys in the bucket. It will not remove the bucket itself. func (be *Backend) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing func (be *Backend) Close() error { return nil } // Rename moves a file based on the new layout l. -func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error { +func (be *Backend) Rename(ctx context.Context, h backend.Handle, l layout.Layout) error { debug.Log("Rename %v to %v", h, l) oldname := be.Filename(h) newname := l.Filename(h) diff --git a/mover-restic/restic/internal/backend/s3/s3_test.go b/mover-restic/restic/internal/backend/s3/s3_test.go index 3051d8ddb..470088e07 100644 --- a/mover-restic/restic/internal/backend/s3/s3_test.go +++ b/mover-restic/restic/internal/backend/s3/s3_test.go @@ -14,11 +14,11 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -117,7 +117,7 @@ func newMinioTestSuite(t testing.TB) (*test.Suite[s3.Config], func()) { return &cfg, nil }, - Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be restic.Backend, err error) { + Factory: location.NewHTTPBackendFactory("s3", s3.ParseConfig, location.NoPassword, func(ctx context.Context, cfg s3.Config, rt http.RoundTripper) (be backend.Backend, err error) { for i := 0; i < 10; i++ { be, err = s3.Create(ctx, cfg, rt) if err != nil { diff --git a/mover-restic/restic/internal/backend/sema/backend.go b/mover-restic/restic/internal/backend/sema/backend.go index d60788f26..1d69c52ac 100644 --- a/mover-restic/restic/internal/backend/sema/backend.go +++ b/mover-restic/restic/internal/backend/sema/backend.go @@ -6,22 +6,22 @@ import ( "sync" "github.com/cenkalti/backoff/v4" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" ) -// make sure that connectionLimitedBackend implements restic.Backend -var _ restic.Backend = &connectionLimitedBackend{} +// make sure that connectionLimitedBackend implements backend.Backend +var _ backend.Backend = &connectionLimitedBackend{} // connectionLimitedBackend limits the number of concurrent operations. type connectionLimitedBackend struct { - restic.Backend + backend.Backend sem semaphore freezeLock sync.Mutex } // NewBackend creates a backend that limits the concurrent operations on the underlying backend -func NewBackend(be restic.Backend) restic.Backend { +func NewBackend(be backend.Backend) backend.Backend { sem, err := newSemaphore(be.Connections()) if err != nil { panic(err) @@ -35,9 +35,9 @@ func NewBackend(be restic.Backend) restic.Backend { // typeDependentLimit acquire a token unless the FileType is a lock file. The returned function // must be called to release the token. -func (be *connectionLimitedBackend) typeDependentLimit(t restic.FileType) func() { +func (be *connectionLimitedBackend) typeDependentLimit(t backend.FileType) func() { // allow concurrent lock file operations to ensure that the lock refresh is always possible - if t == restic.LockFile { + if t == backend.LockFile { return func() {} } be.sem.GetToken() @@ -59,7 +59,7 @@ func (be *connectionLimitedBackend) Unfreeze() { } // Save adds new Data to the backend. -func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *connectionLimitedBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -75,7 +75,7 @@ func (be *connectionLimitedBackend) Save(ctx context.Context, h restic.Handle, r // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *connectionLimitedBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -96,22 +96,22 @@ func (be *connectionLimitedBackend) Load(ctx context.Context, h restic.Handle, l } // Stat returns information about a file in the backend. -func (be *connectionLimitedBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { +func (be *connectionLimitedBackend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { if err := h.Valid(); err != nil { - return restic.FileInfo{}, backoff.Permanent(err) + return backend.FileInfo{}, backoff.Permanent(err) } defer be.typeDependentLimit(h.Type)() if ctx.Err() != nil { - return restic.FileInfo{}, ctx.Err() + return backend.FileInfo{}, ctx.Err() } return be.Backend.Stat(ctx, h) } // Remove deletes a file from the backend. -func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) error { +func (be *connectionLimitedBackend) Remove(ctx context.Context, h backend.Handle) error { if err := h.Valid(); err != nil { return backoff.Permanent(err) } @@ -125,6 +125,6 @@ func (be *connectionLimitedBackend) Remove(ctx context.Context, h restic.Handle) return be.Backend.Remove(ctx, h) } -func (be *connectionLimitedBackend) Unwrap() restic.Backend { +func (be *connectionLimitedBackend) Unwrap() backend.Backend { return be.Backend } diff --git a/mover-restic/restic/internal/backend/sema/backend_test.go b/mover-restic/restic/internal/backend/sema/backend_test.go index a1dd16187..d220f48a3 100644 --- a/mover-restic/restic/internal/backend/sema/backend_test.go +++ b/mover-restic/restic/internal/backend/sema/backend_test.go @@ -8,37 +8,37 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mock" "github.com/restic/restic/internal/backend/sema" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) func TestParameterValidationSave(t *testing.T) { m := mock.NewBackend() - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { return nil } be := sema.NewBackend(m) - err := be.Save(context.TODO(), restic.Handle{}, nil) + err := be.Save(context.TODO(), backend.Handle{}, nil) test.Assert(t, err != nil, "Save() with invalid handle did not return an error") } func TestParameterValidationLoad(t *testing.T) { m := mock.NewBackend() - m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { return io.NopCloser(nil), nil } be := sema.NewBackend(m) nilCb := func(rd io.Reader) error { return nil } - err := be.Load(context.TODO(), restic.Handle{}, 10, 0, nilCb) + err := be.Load(context.TODO(), backend.Handle{}, 10, 0, nilCb) test.Assert(t, err != nil, "Load() with invalid handle did not return an error") - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} err = be.Load(context.TODO(), h, 10, -1, nilCb) test.Assert(t, err != nil, "Save() with negative offset did not return an error") err = be.Load(context.TODO(), h, -1, 0, nilCb) @@ -47,23 +47,23 @@ func TestParameterValidationLoad(t *testing.T) { func TestParameterValidationStat(t *testing.T) { m := mock.NewBackend() - m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { - return restic.FileInfo{}, nil + m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { + return backend.FileInfo{}, nil } be := sema.NewBackend(m) - _, err := be.Stat(context.TODO(), restic.Handle{}) + _, err := be.Stat(context.TODO(), backend.Handle{}) test.Assert(t, err != nil, "Stat() with invalid handle did not return an error") } func TestParameterValidationRemove(t *testing.T) { m := mock.NewBackend() - m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + m.RemoveFn = func(ctx context.Context, h backend.Handle) error { return nil } be := sema.NewBackend(m) - err := be.Remove(context.TODO(), restic.Handle{}) + err := be.Remove(context.TODO(), backend.Handle{}) test.Assert(t, err != nil, "Remove() with invalid handle did not return an error") } @@ -71,7 +71,7 @@ func TestUnwrap(t *testing.T) { m := mock.NewBackend() be := sema.NewBackend(m) - unwrapper := be.(restic.BackendUnwrapper) + unwrapper := be.(backend.Unwrapper) test.Assert(t, unwrapper.Unwrap() == m, "Unwrap() returned wrong backend") } @@ -100,7 +100,7 @@ func countingBlocker() (func(), func(int) int) { return wait, unblock } -func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be restic.Backend) func() error, unblock func(int) int, isUnlimited bool) { +func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(be backend.Backend) func() error, unblock func(int) int, isUnlimited bool) { expectBlocked := int(2) workerCount := expectBlocked + 1 @@ -125,13 +125,13 @@ func concurrencyTester(t *testing.T, setup func(m *mock.Backend), handler func(b func TestConcurrencyLimitSave(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } }, unblock, false) @@ -140,13 +140,13 @@ func TestConcurrencyLimitSave(t *testing.T) { func TestConcurrencyLimitLoad(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { + m.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { wait() return io.NopCloser(nil), nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} nilCb := func(rd io.Reader) error { return nil } return be.Load(context.TODO(), h, 10, 0, nilCb) } @@ -156,13 +156,13 @@ func TestConcurrencyLimitLoad(t *testing.T) { func TestConcurrencyLimitStat(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) { + m.StatFn = func(ctx context.Context, h backend.Handle) (backend.FileInfo, error) { wait() - return restic.FileInfo{}, nil + return backend.FileInfo{}, nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} _, err := be.Stat(context.TODO(), h) return err } @@ -172,13 +172,13 @@ func TestConcurrencyLimitStat(t *testing.T) { func TestConcurrencyLimitDelete(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.RemoveFn = func(ctx context.Context, h restic.Handle) error { + m.RemoveFn = func(ctx context.Context, h backend.Handle) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} return be.Remove(context.TODO(), h) } }, unblock, false) @@ -187,13 +187,13 @@ func TestConcurrencyLimitDelete(t *testing.T) { func TestConcurrencyUnlimitedLockSave(t *testing.T) { wait, unblock := countingBlocker() concurrencyTester(t, func(m *mock.Backend) { - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { wait() return nil } - }, func(be restic.Backend) func() error { + }, func(be backend.Backend) func() error { return func() error { - h := restic.Handle{Type: restic.LockFile, Name: "foobar"} + h := backend.Handle{Type: backend.LockFile, Name: "foobar"} return be.Save(context.TODO(), h, nil) } }, unblock, true) @@ -202,13 +202,13 @@ func TestConcurrencyUnlimitedLockSave(t *testing.T) { func TestFreeze(t *testing.T) { var counter int64 m := mock.NewBackend() - m.SaveFn = func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { + m.SaveFn = func(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { atomic.AddInt64(&counter, 1) return nil } m.ConnectionsFn = func() uint { return 2 } be := sema.NewBackend(m) - fb := be.(restic.FreezeBackend) + fb := be.(backend.FreezeBackend) // Freeze backend fb.Freeze() @@ -218,7 +218,7 @@ func TestFreeze(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - h := restic.Handle{Type: restic.PackFile, Name: "foobar"} + h := backend.Handle{Type: backend.PackFile, Name: "foobar"} test.OK(t, be.Save(context.TODO(), h, nil)) }() diff --git a/mover-restic/restic/internal/backend/sftp/config.go b/mover-restic/restic/internal/backend/sftp/config.go index 65af50d19..aa8ac7bff 100644 --- a/mover-restic/restic/internal/backend/sftp/config.go +++ b/mover-restic/restic/internal/backend/sftp/config.go @@ -13,7 +13,7 @@ import ( type Config struct { User, Host, Port, Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"` + Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"` Command string `option:"command" help:"specify command to create sftp connection"` Args string `option:"args" help:"specify arguments for ssh"` diff --git a/mover-restic/restic/internal/backend/sftp/layout_test.go b/mover-restic/restic/internal/backend/sftp/layout_test.go index fc8d80928..8bb7eac01 100644 --- a/mover-restic/restic/internal/backend/sftp/layout_test.go +++ b/mover-restic/restic/internal/backend/sftp/layout_test.go @@ -6,8 +6,9 @@ import ( "path/filepath" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/feature" rtest "github.com/restic/restic/internal/test" ) @@ -16,6 +17,7 @@ func TestLayout(t *testing.T) { t.Skip("sftp server binary not available") } + defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() path := rtest.TempDir(t) var tests = []struct { @@ -56,7 +58,7 @@ func TestLayout(t *testing.T) { } packs := make(map[string]bool) - err = be.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err = be.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { packs[fi.Name] = false return nil }) diff --git a/mover-restic/restic/internal/backend/sftp/sftp.go b/mover-restic/restic/internal/backend/sftp/sftp.go index 735991eb4..70fc30a62 100644 --- a/mover-restic/restic/internal/backend/sftp/sftp.go +++ b/mover-restic/restic/internal/backend/sftp/sftp.go @@ -17,9 +17,10 @@ import ( "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/limiter" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/feature" "github.com/cenkalti/backoff/v4" "github.com/pkg/sftp" @@ -38,10 +39,12 @@ type SFTP struct { layout.Layout Config - backend.Modes + util.Modes } -var _ restic.Backend = &SFTP{} +var _ backend.Backend = &SFTP{} + +var errTooShort = fmt.Errorf("file is too short") func NewFactory() location.Factory { return location.NewLimitedBackendFactory("sftp", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) @@ -83,9 +86,9 @@ func startClient(cfg Config) (*SFTP, error) { return nil, errors.Wrap(err, "cmd.StdoutPipe") } - bg, err := backend.StartForeground(cmd) + bg, err := util.StartForeground(cmd) if err != nil { - if backend.IsErrDot(err) { + if util.IsErrDot(err) { return nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o sftp.command=./ to override", cmd.Path) } return nil, err @@ -102,7 +105,12 @@ func startClient(cfg Config) (*SFTP, error) { }() // open the SFTP session - client, err := sftp.NewClientPipe(rd, wr) + client, err := sftp.NewClientPipe(rd, wr, + // write multiple packets (32kb) in parallel per file + // not strictly necessary as we use ReadFromWithConcurrency + sftp.UseConcurrentWrites(true), + // increase send buffer per file to 4MB + sftp.MaxConcurrentRequestsPerFile(128)) if err != nil { return nil, errors.Errorf("unable to start the sftp session, error: %v", err) } @@ -152,8 +160,8 @@ func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { debug.Log("layout: %v\n", sftp.Layout) - fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) - m := backend.DeriveModesFromFileInfo(fi, err) + fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) + m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) sftp.Config = cfg @@ -207,6 +215,10 @@ func (r *SFTP) IsNotExist(err error) bool { return errors.Is(err, os.ErrNotExist) } +func (r *SFTP) IsPermanentError(err error) bool { + return r.IsNotExist(err) || errors.Is(err, errTooShort) || errors.Is(err, os.ErrPermission) +} + func buildSSHCommand(cfg Config) (cmd string, args []string, err error) { if cfg.Command != "" { args, err := backend.SplitShellStrings(cfg.Command) @@ -259,10 +271,10 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { return nil, err } - sftp.Modes = backend.DefaultModes + sftp.Modes = util.DefaultModes // test if config file already exists - _, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile})) + _, err = sftp.c.Lstat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) if err == nil { return nil, errors.New("config file already exists") } @@ -280,11 +292,6 @@ func (r *SFTP) Connections() uint { return r.Config.Connections } -// Location returns this backend's location (the directory name). -func (r *SFTP) Location() string { - return r.p -} - // Hasher may return a hash function for calculating a content hash for the backend func (r *SFTP) Hasher() hash.Hash { return nil @@ -302,7 +309,7 @@ func Join(parts ...string) string { } // tempSuffix generates a random string suffix that should be sufficiently long -// to avoid accidential conflicts +// to avoid accidental conflicts func tempSuffix() string { var nonce [16]byte _, err := rand.Read(nonce[:]) @@ -313,7 +320,7 @@ func tempSuffix() string { } // Save stores data in the backend at the handle. -func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) error { +func (r *SFTP) Save(_ context.Context, h backend.Handle, rd backend.RewindReader) error { if err := r.clientError(); err != nil { return err } @@ -359,7 +366,7 @@ func (r *SFTP) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) }() // save data, make sure to use the optimized sftp upload method - wbytes, err := f.ReadFrom(rd) + wbytes, err := f.ReadFromWithConcurrency(rd, 0) if err != nil { _ = f.Close() err = r.checkNoSpace(dirname, rd.Length(), err) @@ -413,11 +420,28 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, r.openReader, fn) +func (r *SFTP) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, r.openReader, func(rd io.Reader) error { + if length == 0 || !feature.Flag.Enabled(feature.BackendErrorRedesign) { + return fn(rd) + } + + // there is no direct way to efficiently check whether the file is too short + // rd is already a LimitedReader which can be used to track the number of bytes read + err := fn(rd) + + // check the underlying reader to be agnostic to however fn() handles the returned error + _, rderr := rd.Read([]byte{0}) + if rderr == io.EOF && rd.(*util.LimitedReadCloser).N != 0 { + // file is too short + return fmt.Errorf("%w: %v", errTooShort, err) + } + + return err + }) } -func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (r *SFTP) openReader(_ context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { f, err := r.c.Open(r.Filename(h)) if err != nil { return nil, err @@ -434,28 +458,28 @@ func (r *SFTP) openReader(_ context.Context, h restic.Handle, length int, offset if length > 0 { // unlimited reads usually use io.Copy which needs WriteTo support at the underlying reader // limited reads are usually combined with io.ReadFull which reads all required bytes into a buffer in one go - return backend.LimitReadCloser(f, int64(length)), nil + return util.LimitReadCloser(f, int64(length)), nil } return f, nil } // Stat returns information about a blob. -func (r *SFTP) Stat(_ context.Context, h restic.Handle) (restic.FileInfo, error) { +func (r *SFTP) Stat(_ context.Context, h backend.Handle) (backend.FileInfo, error) { if err := r.clientError(); err != nil { - return restic.FileInfo{}, err + return backend.FileInfo{}, err } fi, err := r.c.Lstat(r.Filename(h)) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "Lstat") + return backend.FileInfo{}, errors.Wrap(err, "Lstat") } - return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil + return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil } // Remove removes the content stored at name. -func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { +func (r *SFTP) Remove(_ context.Context, h backend.Handle) error { if err := r.clientError(); err != nil { return err } @@ -465,7 +489,7 @@ func (r *SFTP) Remove(_ context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (r *SFTP) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { basedir, subdirs := r.Basedir(t) walker := r.c.Walk(basedir) for { @@ -498,7 +522,7 @@ func (r *SFTP) List(ctx context.Context, t restic.FileType, fn func(restic.FileI debug.Log("send %v\n", path.Base(walker.Path())) - rfi := restic.FileInfo{ + rfi := backend.FileInfo{ Name: path.Base(walker.Path()), Size: fi.Size(), } diff --git a/mover-restic/restic/internal/backend/shell_split.go b/mover-restic/restic/internal/backend/shell_split.go index eff527616..888c993a0 100644 --- a/mover-restic/restic/internal/backend/shell_split.go +++ b/mover-restic/restic/internal/backend/shell_split.go @@ -6,7 +6,7 @@ import ( "github.com/restic/restic/internal/errors" ) -// shellSplitter splits a command string into separater arguments. It supports +// shellSplitter splits a command string into separated arguments. It supports // single and double quoted strings. type shellSplitter struct { quote rune diff --git a/mover-restic/restic/internal/backend/swift/config.go b/mover-restic/restic/internal/backend/swift/config.go index 5be2d9ce0..9adb80522 100644 --- a/mover-restic/restic/internal/backend/swift/config.go +++ b/mover-restic/restic/internal/backend/swift/config.go @@ -4,9 +4,9 @@ import ( "os" "strings" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/options" - "github.com/restic/restic/internal/restic" ) // Config contains basic configuration needed to specify swift location for a swift server @@ -74,7 +74,7 @@ func ParseConfig(s string) (*Config, error) { return &cfg, nil } -var _ restic.ApplyEnvironmenter = &Config{} +var _ backend.ApplyEnvironmenter = &Config{} // ApplyEnvironment saves values from the environment to the config. func (cfg *Config) ApplyEnvironment(prefix string) { diff --git a/mover-restic/restic/internal/backend/swift/swift.go b/mover-restic/restic/internal/backend/swift/swift.go index 1cfc0a65b..e6412d0bf 100644 --- a/mover-restic/restic/internal/backend/swift/swift.go +++ b/mover-restic/restic/internal/backend/swift/swift.go @@ -16,9 +16,10 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend/util" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/feature" "github.com/ncw/swift/v2" ) @@ -32,8 +33,8 @@ type beSwift struct { layout.Layout } -// ensure statically that *beSwift implements restic.Backend. -var _ restic.Backend = &beSwift{} +// ensure statically that *beSwift implements backend.Backend. +var _ backend.Backend = &beSwift{} func NewFactory() location.Factory { return location.NewHTTPBackendFactory("swift", ParseConfig, location.NoPassword, Open, Open) @@ -41,7 +42,7 @@ func NewFactory() location.Factory { // Open opens the swift backend at a container in region. The container is // created if it does not exist yet. -func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend, error) { +func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { debug.Log("config %#v", cfg) be := &beSwift{ @@ -117,11 +118,6 @@ func (be *beSwift) Connections() uint { return be.connections } -// Location returns this backend's location (the container name). -func (be *beSwift) Location() string { - return be.container -} - // Hasher may return a hash function for calculating a content hash for the backend func (be *beSwift) Hasher() hash.Hash { return md5.New() @@ -134,11 +130,11 @@ func (be *beSwift) HasAtomicReplace() bool { // Load runs fn with a reader that yields the contents of the file at h at the // given offset. -func (be *beSwift) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return backend.DefaultLoad(ctx, h, length, offset, be.openReader, fn) +func (be *beSwift) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return util.DefaultLoad(ctx, h, length, offset, be.openReader, fn) } -func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *beSwift) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { objName := be.Filename(h) @@ -153,14 +149,25 @@ func (be *beSwift) openReader(ctx context.Context, h restic.Handle, length int, obj, _, err := be.conn.ObjectOpen(ctx, be.container, objName, false, headers) if err != nil { - return nil, errors.Wrap(err, "conn.ObjectOpen") + return nil, fmt.Errorf("conn.ObjectOpen: %w", err) + } + + if feature.Flag.Enabled(feature.BackendErrorRedesign) && length > 0 { + // get response length, but don't cause backend calls + cctx, cancel := context.WithCancel(context.Background()) + cancel() + objLength, e := obj.Length(cctx) + if e == nil && objLength != int64(length) { + _ = obj.Close() + return nil, &swift.Error{StatusCode: http.StatusRequestedRangeNotSatisfiable, Text: "restic-file-too-short"} + } } return obj, nil } // Save stores data in the backend at the handle. -func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (be *beSwift) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) encoding := "binary/octet-stream" @@ -174,19 +181,19 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe } // Stat returns information about a blob. -func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *beSwift) Stat(ctx context.Context, h backend.Handle) (bi backend.FileInfo, err error) { objName := be.Filename(h) obj, _, err := be.conn.Object(ctx, be.container, objName) if err != nil { - return restic.FileInfo{}, errors.Wrap(err, "conn.Object") + return backend.FileInfo{}, errors.Wrap(err, "conn.Object") } - return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil + return backend.FileInfo{Size: obj.Bytes, Name: h.Name}, nil } // Remove removes the blob with the given name and type. -func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { +func (be *beSwift) Remove(ctx context.Context, h backend.Handle) error { objName := be.Filename(h) err := be.conn.ObjectDelete(ctx, be.container, objName) @@ -195,7 +202,7 @@ func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (be *beSwift) List(ctx context.Context, t backend.FileType, fn func(backend.FileInfo) error) error { prefix, _ := be.Basedir(t) prefix += "/" @@ -212,7 +219,7 @@ func (be *beSwift) List(ctx context.Context, t restic.FileType, fn func(restic.F continue } - fi := restic.FileInfo{ + fi := backend.FileInfo{ Name: m, Size: obj.Bytes, } @@ -242,10 +249,25 @@ func (be *beSwift) IsNotExist(err error) bool { return errors.As(err, &e) && e.StatusCode == http.StatusNotFound } +func (be *beSwift) IsPermanentError(err error) bool { + if be.IsNotExist(err) { + return true + } + + var serr *swift.Error + if errors.As(err, &serr) { + if serr.StatusCode == http.StatusRequestedRangeNotSatisfiable || serr.StatusCode == http.StatusUnauthorized || serr.StatusCode == http.StatusForbidden { + return true + } + } + + return false +} + // Delete removes all restic objects in the container. // It will not remove the container itself. func (be *beSwift) Delete(ctx context.Context) error { - return backend.DefaultDelete(ctx, be) + return util.DefaultDelete(ctx, be) } // Close does nothing diff --git a/mover-restic/restic/internal/backend/swift/swift_test.go b/mover-restic/restic/internal/backend/swift/swift_test.go index 98ee5b1c1..355947cc7 100644 --- a/mover-restic/restic/internal/backend/swift/swift_test.go +++ b/mover-restic/restic/internal/backend/swift/swift_test.go @@ -6,9 +6,9 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/swift" "github.com/restic/restic/internal/backend/test" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -20,7 +20,7 @@ func newSwiftTestSuite(t testing.TB) *test.Suite[swift.Config] { // wait for removals for at least 5m WaitForDelayedRemoval: 5 * time.Minute, - ErrorHandler: func(t testing.TB, be restic.Backend, err error) error { + ErrorHandler: func(t testing.TB, be backend.Backend, err error) error { if err == nil { return nil } diff --git a/mover-restic/restic/internal/backend/test/benchmarks.go b/mover-restic/restic/internal/backend/test/benchmarks.go index 150ef3987..e4271a386 100644 --- a/mover-restic/restic/internal/backend/test/benchmarks.go +++ b/mover-restic/restic/internal/backend/test/benchmarks.go @@ -6,22 +6,23 @@ import ( "io" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) -func saveRandomFile(t testing.TB, be restic.Backend, length int) ([]byte, restic.Handle) { +func saveRandomFile(t testing.TB, be backend.Backend, length int) ([]byte, backend.Handle) { data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := be.Save(context.TODO(), handle, restic.NewByteReader(data, be.Hasher())) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := be.Save(context.TODO(), handle, backend.NewByteReader(data, be.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } return data, handle } -func remove(t testing.TB, be restic.Backend, h restic.Handle) { +func remove(t testing.TB, be backend.Backend, h backend.Handle) { if err := be.Remove(context.TODO(), h); err != nil { t.Fatalf("Remove() returned error: %v", err) } @@ -146,9 +147,9 @@ func (s *Suite[C]) BenchmarkSave(t *testing.B) { length := 1<<24 + 2123 data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} - rd := restic.NewByteReader(data, be.Hasher()) + rd := backend.NewByteReader(data, be.Hasher()) t.SetBytes(int64(length)) t.ResetTimer() diff --git a/mover-restic/restic/internal/backend/test/doc.go b/mover-restic/restic/internal/backend/test/doc.go index 25bdf0417..c15ed4d82 100644 --- a/mover-restic/restic/internal/backend/test/doc.go +++ b/mover-restic/restic/internal/backend/test/doc.go @@ -17,7 +17,7 @@ // // func newTestSuite(t testing.TB) *test.Suite { // return &test.Suite{ -// Create: func(cfg interface{}) (restic.Backend, error) { +// Create: func(cfg interface{}) (backend.Backend, error) { // [...] // }, // [...] diff --git a/mover-restic/restic/internal/backend/test/suite.go b/mover-restic/restic/internal/backend/test/suite.go index bb77124d7..ad8eb4c5d 100644 --- a/mover-restic/restic/internal/backend/test/suite.go +++ b/mover-restic/restic/internal/backend/test/suite.go @@ -11,7 +11,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/location" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" ) @@ -35,7 +34,7 @@ type Suite[C any] struct { WaitForDelayedRemoval time.Duration // ErrorHandler allows ignoring certain errors. - ErrorHandler func(testing.TB, restic.Backend, error) error + ErrorHandler func(testing.TB, backend.Backend, error) error } // RunTests executes all defined tests as subtests of t. @@ -156,7 +155,7 @@ func (s *Suite[C]) RunBenchmarks(b *testing.B) { s.cleanup(b) } -func (s *Suite[C]) createOrError() (restic.Backend, error) { +func (s *Suite[C]) createOrError() (backend.Backend, error) { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { return nil, fmt.Errorf("cannot create transport for tests: %v", err) @@ -167,7 +166,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) { return nil, err } - _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile}) + _, err = be.Stat(context.TODO(), backend.Handle{Type: backend.ConfigFile}) if err != nil && !be.IsNotExist(err) { return nil, err } @@ -179,7 +178,7 @@ func (s *Suite[C]) createOrError() (restic.Backend, error) { return be, nil } -func (s *Suite[C]) create(t testing.TB) restic.Backend { +func (s *Suite[C]) create(t testing.TB) backend.Backend { be, err := s.createOrError() if err != nil { t.Fatal(err) @@ -187,7 +186,7 @@ func (s *Suite[C]) create(t testing.TB) restic.Backend { return be } -func (s *Suite[C]) open(t testing.TB) restic.Backend { +func (s *Suite[C]) open(t testing.TB) backend.Backend { tr, err := backend.Transport(backend.TransportOptions{}) if err != nil { t.Fatalf("cannot create transport for tests: %v", err) @@ -208,7 +207,7 @@ func (s *Suite[C]) cleanup(t testing.TB) { s.close(t, be) } -func (s *Suite[C]) close(t testing.TB, be restic.Backend) { +func (s *Suite[C]) close(t testing.TB, be backend.Backend) { err := be.Close() if err != nil { t.Fatal(err) diff --git a/mover-restic/restic/internal/backend/test/tests.go b/mover-restic/restic/internal/backend/test/tests.go index 5796e0e6a..9320dcad9 100644 --- a/mover-restic/restic/internal/backend/test/tests.go +++ b/mover-restic/restic/internal/backend/test/tests.go @@ -27,7 +27,7 @@ func seedRand(t testing.TB) { t.Logf("rand initialized with seed %d", seed) } -func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) { +func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, error) { _, err := be.Stat(ctx, h) if err != nil && be.IsNotExist(err) { return false, nil @@ -36,6 +36,19 @@ func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, erro return err == nil, err } +func LoadAll(ctx context.Context, be backend.Backend, h backend.Handle) ([]byte, error) { + var buf []byte + err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error { + var err error + buf, err = io.ReadAll(rd) + return err + }) + if err != nil { + return nil, err + } + return buf, nil +} + // TestStripPasswordCall tests that the StripPassword method of a factory can be called without crashing. // It does not verify whether passwords are removed correctly func (s *Suite[C]) TestStripPasswordCall(_ *testing.T) { @@ -49,7 +62,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { defer s.close(t, b) // remove a config if present - cfgHandle := restic.Handle{Type: restic.ConfigFile} + cfgHandle := backend.Handle{Type: backend.ConfigFile} cfgPresent, err := beTest(context.TODO(), b, cfgHandle) if err != nil { t.Fatalf("unable to test for config: %+v", err) @@ -60,7 +73,7 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { } // save a config - store(t, b, restic.ConfigFile, []byte("test config")) + store(t, b, backend.ConfigFile, []byte("test config")) // now create the backend again, this must fail _, err = s.createOrError() @@ -69,23 +82,12 @@ func (s *Suite[C]) TestCreateWithConfig(t *testing.T) { } // remove config - err = b.Remove(context.TODO(), restic.Handle{Type: restic.ConfigFile, Name: ""}) + err = b.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile, Name: ""}) if err != nil { t.Fatalf("unexpected error removing config: %+v", err) } } -// TestLocation tests that a location string is returned. -func (s *Suite[C]) TestLocation(t *testing.T) { - b := s.open(t) - defer s.close(t, b) - - l := b.Location() - if l == "" { - t.Fatalf("invalid location string %q", l) - } -} - // TestConfig saves and loads a config from the backend. func (s *Suite[C]) TestConfig(t *testing.T) { b := s.open(t) @@ -94,13 +96,14 @@ func (s *Suite[C]) TestConfig(t *testing.T) { var testString = "Config" // create config and read it back - _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.ConfigFile}) + _, err := LoadAll(context.TODO(), b, backend.Handle{Type: backend.ConfigFile}) if err == nil { t.Fatalf("did not get expected error for non-existing config") } test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize error from LoadAll(): %v", err) - err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher())) + err = b.Save(context.TODO(), backend.Handle{Type: backend.ConfigFile}, backend.NewByteReader([]byte(testString), b.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -108,8 +111,8 @@ func (s *Suite[C]) TestConfig(t *testing.T) { // try accessing the config with different names, should all return the // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - h := restic.Handle{Type: restic.ConfigFile, Name: name} - buf, err := backend.LoadAll(context.TODO(), nil, b, h) + h := backend.Handle{Type: backend.ConfigFile, Name: name} + buf, err := LoadAll(context.TODO(), b, h) if err != nil { t.Fatalf("unable to read config with name %q: %+v", name, err) } @@ -120,7 +123,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) { } // remove the config - remove(t, b, restic.Handle{Type: restic.ConfigFile}) + remove(t, b, backend.Handle{Type: backend.ConfigFile}) } // TestLoad tests the backend's Load function. @@ -130,19 +133,20 @@ func (s *Suite[C]) TestLoad(t *testing.T) { b := s.open(t) defer s.close(t, b) - err := testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}) + err := testLoad(b, backend.Handle{Type: backend.PackFile, Name: "foobar"}) if err == nil { t.Fatalf("Load() did not return an error for non-existing blob") } test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize non-existing blob: %v", err) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize non-existing blob: %v", err) length := rand.Intn(1<<24) + 2000 data := test.Random(23, length) id := restic.Hash(data) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - err = b.Save(context.TODO(), handle, restic.NewByteReader(data, b.Hasher())) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + err = b.Save(context.TODO(), handle, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatalf("Save() error: %+v", err) } @@ -181,8 +185,12 @@ func (s *Suite[C]) TestLoad(t *testing.T) { } getlen := l - if l >= len(d) && rand.Float32() >= 0.5 { - getlen = 0 + if l >= len(d) { + if rand.Float32() >= 0.5 { + getlen = 0 + } else { + getlen = len(d) + } } if l > 0 && l < len(d) { @@ -225,6 +233,18 @@ func (s *Suite[C]) TestLoad(t *testing.T) { } } + // test error checking for partial and fully out of bounds read + // only test for length > 0 as we currently do not need strict out of bounds handling for length==0 + for _, offset := range []int{length - 99, length - 50, length, length + 100} { + err = b.Load(context.TODO(), handle, 100, int64(offset), func(rd io.Reader) (ierr error) { + _, ierr = io.ReadAll(rd) + return ierr + }) + test.Assert(t, err != nil, "Load() did not return error on out of bounds read! o %v, l %v, filelength %v", offset, 100, length) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize out of range read: %v", err) + test.Assert(t, !b.IsNotExist(err), "IsNotExist() must not recognize out of range read: %v", err) + } + test.OK(t, b.Remove(context.TODO(), handle)) } @@ -243,7 +263,7 @@ func (s *Suite[C]) TestList(t *testing.T) { // Check that the backend is empty to start with var found []string - err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { found = append(found, fi.Name) return nil }) @@ -259,8 +279,8 @@ func (s *Suite[C]) TestList(t *testing.T) { for i := 0; i < numTestFiles; i++ { data := test.Random(rand.Int(), rand.Intn(100)+55) id := restic.Hash(data) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatal(err) } @@ -284,7 +304,7 @@ func (s *Suite[C]) TestList(t *testing.T) { s.SetListMaxItems(test.maxItems) } - err := b.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), backend.PackFile, func(fi backend.FileInfo) error { id, err := restic.ParseID(fi.Name) if err != nil { t.Fatal(err) @@ -320,9 +340,9 @@ func (s *Suite[C]) TestList(t *testing.T) { } t.Logf("remove %d files", numTestFiles) - handles := make([]restic.Handle, 0, len(list1)) + handles := make([]backend.Handle, 0, len(list1)) for id := range list1 { - handles = append(handles, restic.Handle{Type: restic.PackFile, Name: id.String()}) + handles = append(handles, backend.Handle{Type: backend.PackFile, Name: id.String()}) } err = s.delayedRemove(t, b, handles...) @@ -340,13 +360,13 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { b := s.open(t) defer s.close(t, b) - testFiles := make([]restic.Handle, 0, numTestFiles) + testFiles := make([]backend.Handle, 0, numTestFiles) for i := 0; i < numTestFiles; i++ { data := []byte(fmt.Sprintf("random test blob %v", i)) id := restic.Hash(data) - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) if err != nil { t.Fatal(err) } @@ -358,7 +378,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { cancel() // pass in a cancelled context - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { t.Errorf("got FileInfo %v for cancelled context", fi) return nil }) @@ -373,7 +393,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { defer cancel() i := 0 - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { i++ // cancel the context on the first file if i == 1 { @@ -396,7 +416,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { defer cancel() i := 0 - err := b.List(ctx, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctx, backend.PackFile, func(fi backend.FileInfo) error { // cancel the context at the last file i++ if i == numTestFiles { @@ -423,7 +443,7 @@ func (s *Suite[C]) TestListCancel(t *testing.T) { i := 0 // pass in a context with a timeout - err := b.List(ctxTimeout, restic.PackFile, func(fi restic.FileInfo) error { + err := b.List(ctxTimeout, backend.PackFile, func(fi backend.FileInfo) error { i++ // wait until the context is cancelled @@ -494,14 +514,14 @@ func (s *Suite[C]) TestSave(t *testing.T) { data := test.Random(23, length) id = sha256.Sum256(data) - h := restic.Handle{ - Type: restic.PackFile, + h := backend.Handle{ + Type: backend.PackFile, Name: id.String(), } - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) test.OK(t, err) - buf, err := backend.LoadAll(context.TODO(), nil, b, h) + buf, err := LoadAll(context.TODO(), b, h) test.OK(t, err) if len(buf) != len(data) { t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) @@ -546,7 +566,7 @@ func (s *Suite[C]) TestSave(t *testing.T) { t.Fatal(err) } - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: backend.PackFile, Name: id.String()} // wrap the tempfile in an errorCloser, so we can detect if the backend // closes the reader @@ -585,7 +605,7 @@ func (s *Suite[C]) TestSave(t *testing.T) { } type incompleteByteReader struct { - restic.ByteReader + backend.ByteReader } func (r *incompleteByteReader) Length() int64 { @@ -609,8 +629,8 @@ func (s *Suite[C]) TestSaveError(t *testing.T) { copy(id[:], data) // test that incomplete uploads fail - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, &incompleteByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())}) // try to delete possible leftovers _ = s.delayedRemove(t, b, h) if err == nil { @@ -619,7 +639,7 @@ func (s *Suite[C]) TestSaveError(t *testing.T) { } type wrongByteReader struct { - restic.ByteReader + backend.ByteReader } func (b *wrongByteReader) Hash() []byte { @@ -648,8 +668,8 @@ func (s *Suite[C]) TestSaveWrongHash(t *testing.T) { copy(id[:], data) // test that upload with hash mismatch fails - h := restic.Handle{Type: restic.PackFile, Name: id.String()} - err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())}) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *backend.NewByteReader(data, b.Hasher())}) exists, err2 := beTest(context.TODO(), b, h) if err2 != nil { t.Fatal(err2) @@ -674,23 +694,23 @@ var testStrings = []struct { {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, } -func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) restic.Handle { +func store(t testing.TB, b backend.Backend, tpe backend.FileType, data []byte) backend.Handle { id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: tpe} - err := b.Save(context.TODO(), h, restic.NewByteReader([]byte(data), b.Hasher())) + h := backend.Handle{Name: id.String(), Type: tpe} + err := b.Save(context.TODO(), h, backend.NewByteReader([]byte(data), b.Hasher())) test.OK(t, err) return h } // testLoad loads a blob (but discards its contents). -func testLoad(b restic.Backend, h restic.Handle) error { +func testLoad(b backend.Backend, h backend.Handle) error { return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) { _, ierr = io.Copy(io.Discard, rd) return ierr }) } -func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...restic.Handle) error { +func (s *Suite[C]) delayedRemove(t testing.TB, be backend.Backend, handles ...backend.Handle) error { // Some backend (swift, I'm looking at you) may implement delayed // removal of data. Let's wait a bit if this happens. @@ -734,11 +754,11 @@ func (s *Suite[C]) delayedRemove(t testing.TB, be restic.Backend, handles ...res return nil } -func delayedList(t testing.TB, b restic.Backend, tpe restic.FileType, max int, maxwait time.Duration) restic.IDs { +func delayedList(t testing.TB, b backend.Backend, tpe backend.FileType, max int, maxwait time.Duration) restic.IDs { list := restic.NewIDSet() start := time.Now() for i := 0; i < max; i++ { - err := b.List(context.TODO(), tpe, func(fi restic.FileInfo) error { + err := b.List(context.TODO(), tpe, func(fi backend.FileInfo) error { id := restic.TestParseID(fi.Name) list.Insert(id) return nil @@ -762,10 +782,11 @@ func (s *Suite[C]) TestBackend(t *testing.T) { defer s.close(t, b) test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error") + test.Assert(t, !b.IsPermanentError(nil), "IsPermanentError() recognized nil error") - for _, tpe := range []restic.FileType{ - restic.PackFile, restic.KeyFile, restic.LockFile, - restic.SnapshotFile, restic.IndexFile, + for _, tpe := range []backend.FileType{ + backend.PackFile, backend.KeyFile, backend.LockFile, + backend.SnapshotFile, backend.IndexFile, } { // detect non-existing files for _, ts := range testStrings { @@ -773,7 +794,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.OK(t, err) // test if blob is already in repository - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} ret, err := beTest(context.TODO(), b, h) test.OK(t, err) test.Assert(t, !ret, "blob was found to exist before creating") @@ -782,11 +803,13 @@ func (s *Suite[C]) TestBackend(t *testing.T) { _, err = b.Stat(context.TODO(), h) test.Assert(t, err != nil, "blob data could be extracted before creation") test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Stat() error: %v", err) // try to read not existing blob err = testLoad(b, h) test.Assert(t, err != nil, "blob could be read before creation") test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err) + test.Assert(t, b.IsPermanentError(err), "IsPermanentError() did not recognize Load() error: %v", err) // try to get string out, should fail ret, err = beTest(context.TODO(), b, h) @@ -799,8 +822,8 @@ func (s *Suite[C]) TestBackend(t *testing.T) { store(t, b, tpe, []byte(ts.data)) // test Load() - h := restic.Handle{Type: tpe, Name: ts.id} - buf, err := backend.LoadAll(context.TODO(), nil, b, h) + h := backend.Handle{Type: tpe, Name: ts.id} + buf, err := LoadAll(context.TODO(), b, h) test.OK(t, err) test.Equals(t, ts.data, string(buf)) @@ -823,7 +846,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { // test adding the first file again ts := testStrings[0] - h := restic.Handle{Type: tpe, Name: ts.id} + h := backend.Handle{Type: tpe, Name: ts.id} // remove and recreate err := s.delayedRemove(t, b, h) @@ -835,7 +858,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) { test.Assert(t, !ok, "removed blob still present") // create blob - err = b.Save(context.TODO(), h, restic.NewByteReader([]byte(ts.data), b.Hasher())) + err = b.Save(context.TODO(), h, backend.NewByteReader([]byte(ts.data), b.Hasher())) test.OK(t, err) // list items @@ -859,12 +882,12 @@ func (s *Suite[C]) TestBackend(t *testing.T) { t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) } - var handles []restic.Handle + var handles []backend.Handle for _, ts := range testStrings { id, err := restic.ParseID(ts.id) test.OK(t, err) - h := restic.Handle{Type: tpe, Name: id.String()} + h := backend.Handle{Type: tpe, Name: id.String()} found, err := beTest(context.TODO(), b, h) test.OK(t, err) diff --git a/mover-restic/restic/internal/backend/util/defaults.go b/mover-restic/restic/internal/backend/util/defaults.go new file mode 100644 index 000000000..e5b6fc456 --- /dev/null +++ b/mover-restic/restic/internal/backend/util/defaults.go @@ -0,0 +1,50 @@ +package util + +import ( + "context" + "io" + + "github.com/restic/restic/internal/backend" +) + +// DefaultLoad implements Backend.Load using lower-level openReader func +func DefaultLoad(ctx context.Context, h backend.Handle, length int, offset int64, + openReader func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error), + fn func(rd io.Reader) error) error { + + rd, err := openReader(ctx, h, length, offset) + if err != nil { + return err + } + err = fn(rd) + if err != nil { + _ = rd.Close() // ignore secondary errors closing the reader + return err + } + return rd.Close() +} + +// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. +func DefaultDelete(ctx context.Context, be backend.Backend) error { + alltypes := []backend.FileType{ + backend.PackFile, + backend.KeyFile, + backend.LockFile, + backend.SnapshotFile, + backend.IndexFile} + + for _, t := range alltypes { + err := be.List(ctx, t, func(fi backend.FileInfo) error { + return be.Remove(ctx, backend.Handle{Type: t, Name: fi.Name}) + }) + if err != nil { + return nil + } + } + err := be.Remove(ctx, backend.Handle{Type: backend.ConfigFile}) + if err != nil && be.IsNotExist(err) { + err = nil + } + + return err +} diff --git a/mover-restic/restic/internal/backend/util/defaults_test.go b/mover-restic/restic/internal/backend/util/defaults_test.go new file mode 100644 index 000000000..1dd79208f --- /dev/null +++ b/mover-restic/restic/internal/backend/util/defaults_test.go @@ -0,0 +1,64 @@ +package util_test + +import ( + "context" + "io" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/util" + "github.com/restic/restic/internal/errors" + + rtest "github.com/restic/restic/internal/test" +) + +type mockReader struct { + closed bool +} + +func (rd *mockReader) Read(_ []byte) (n int, err error) { + return 0, nil +} +func (rd *mockReader) Close() error { + rd.closed = true + return nil +} + +func TestDefaultLoad(t *testing.T) { + + h := backend.Handle{Name: "id", Type: backend.PackFile} + rd := &mockReader{} + + // happy case, assert correct parameters are passed around and content stream is closed + err := util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { + rtest.Equals(t, h, ih) + rtest.Equals(t, int(10), length) + rtest.Equals(t, int64(11), offset) + + return rd, nil + }, func(ird io.Reader) error { + rtest.Equals(t, rd, ird) + return nil + }) + rtest.OK(t, err) + rtest.Equals(t, true, rd.closed) + + // unhappy case, assert producer errors are handled correctly + err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { + return nil, errors.Errorf("producer error") + }, func(ird io.Reader) error { + t.Fatalf("unexpected consumer invocation") + return nil + }) + rtest.Equals(t, "producer error", err.Error()) + + // unhappy case, assert consumer errors are handled correctly + rd = &mockReader{} + err = util.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih backend.Handle, length int, offset int64) (io.ReadCloser, error) { + return rd, nil + }, func(ird io.Reader) error { + return errors.Errorf("consumer error") + }) + rtest.Equals(t, true, rd.closed) + rtest.Equals(t, "consumer error", err.Error()) +} diff --git a/mover-restic/restic/internal/backend/errdot_119.go b/mover-restic/restic/internal/backend/util/errdot_119.go similarity index 97% rename from mover-restic/restic/internal/backend/errdot_119.go rename to mover-restic/restic/internal/backend/util/errdot_119.go index 3676a099d..e20ed47b7 100644 --- a/mover-restic/restic/internal/backend/errdot_119.go +++ b/mover-restic/restic/internal/backend/util/errdot_119.go @@ -8,7 +8,7 @@ // Once the minimum Go version restic supports is 1.19, remove this file and // replace any calls to it with the corresponding code as per below. -package backend +package util import ( "errors" diff --git a/mover-restic/restic/internal/backend/errdot_old.go b/mover-restic/restic/internal/backend/util/errdot_old.go similarity index 95% rename from mover-restic/restic/internal/backend/errdot_old.go rename to mover-restic/restic/internal/backend/util/errdot_old.go index 92a58ad25..4f7a0b40b 100644 --- a/mover-restic/restic/internal/backend/errdot_old.go +++ b/mover-restic/restic/internal/backend/util/errdot_old.go @@ -6,7 +6,7 @@ // Once the minimum Go version restic supports is 1.19, remove this file // and perform the actions listed in errdot_119.go. -package backend +package util func IsErrDot(err error) bool { return false diff --git a/mover-restic/restic/internal/backend/foreground.go b/mover-restic/restic/internal/backend/util/foreground.go similarity index 97% rename from mover-restic/restic/internal/backend/foreground.go rename to mover-restic/restic/internal/backend/util/foreground.go index 7291dc8d6..35cbada1a 100644 --- a/mover-restic/restic/internal/backend/foreground.go +++ b/mover-restic/restic/internal/backend/util/foreground.go @@ -1,4 +1,4 @@ -package backend +package util import ( "os" diff --git a/mover-restic/restic/internal/backend/foreground_sysv.go b/mover-restic/restic/internal/backend/util/foreground_sysv.go similarity index 85% rename from mover-restic/restic/internal/backend/foreground_sysv.go rename to mover-restic/restic/internal/backend/util/foreground_sysv.go index 0e88a57a1..ec06aa677 100644 --- a/mover-restic/restic/internal/backend/foreground_sysv.go +++ b/mover-restic/restic/internal/backend/util/foreground_sysv.go @@ -1,7 +1,7 @@ //go:build aix || solaris // +build aix solaris -package backend +package util import ( "os/exec" @@ -11,7 +11,7 @@ import ( ) func startForeground(cmd *exec.Cmd) (bg func() error, err error) { - // run the command in it's own process group so that SIGINT + // run the command in its own process group so that SIGINT // is not sent to it. cmd.SysProcAttr = &syscall.SysProcAttr{ Setpgid: true, diff --git a/mover-restic/restic/internal/backend/foreground_test.go b/mover-restic/restic/internal/backend/util/foreground_test.go similarity index 85% rename from mover-restic/restic/internal/backend/foreground_test.go rename to mover-restic/restic/internal/backend/util/foreground_test.go index 4f701122d..c26861a6c 100644 --- a/mover-restic/restic/internal/backend/foreground_test.go +++ b/mover-restic/restic/internal/backend/util/foreground_test.go @@ -1,7 +1,7 @@ //go:build !windows // +build !windows -package backend_test +package util_test import ( "bufio" @@ -10,7 +10,7 @@ import ( "strings" "testing" - "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/util" rtest "github.com/restic/restic/internal/test" ) @@ -22,7 +22,7 @@ func TestForeground(t *testing.T) { stdout, err := cmd.StdoutPipe() rtest.OK(t, err) - bg, err := backend.StartForeground(cmd) + bg, err := util.StartForeground(cmd) rtest.OK(t, err) defer func() { rtest.OK(t, cmd.Wait()) diff --git a/mover-restic/restic/internal/backend/foreground_unix.go b/mover-restic/restic/internal/backend/util/foreground_unix.go similarity index 98% rename from mover-restic/restic/internal/backend/foreground_unix.go rename to mover-restic/restic/internal/backend/util/foreground_unix.go index fcc0dfe78..082b7f59b 100644 --- a/mover-restic/restic/internal/backend/foreground_unix.go +++ b/mover-restic/restic/internal/backend/util/foreground_unix.go @@ -1,7 +1,7 @@ //go:build !aix && !solaris && !windows // +build !aix,!solaris,!windows -package backend +package util import ( "os" diff --git a/mover-restic/restic/internal/backend/foreground_windows.go b/mover-restic/restic/internal/backend/util/foreground_windows.go similarity index 96% rename from mover-restic/restic/internal/backend/foreground_windows.go rename to mover-restic/restic/internal/backend/util/foreground_windows.go index 54883c30f..f9b753c35 100644 --- a/mover-restic/restic/internal/backend/foreground_windows.go +++ b/mover-restic/restic/internal/backend/util/foreground_windows.go @@ -1,4 +1,4 @@ -package backend +package util import ( "os/exec" diff --git a/mover-restic/restic/internal/backend/util/limited_reader.go b/mover-restic/restic/internal/backend/util/limited_reader.go new file mode 100644 index 000000000..fdee1c06a --- /dev/null +++ b/mover-restic/restic/internal/backend/util/limited_reader.go @@ -0,0 +1,15 @@ +package util + +import "io" + +// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method. +type LimitedReadCloser struct { + io.Closer + io.LimitedReader +} + +// LimitReadCloser returns a new reader wraps r in an io.LimitedReader, but also +// exposes the Close() method. +func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { + return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}} +} diff --git a/mover-restic/restic/internal/backend/paths.go b/mover-restic/restic/internal/backend/util/paths.go similarity index 97% rename from mover-restic/restic/internal/backend/paths.go rename to mover-restic/restic/internal/backend/util/paths.go index 7e511be9c..206fbb56d 100644 --- a/mover-restic/restic/internal/backend/paths.go +++ b/mover-restic/restic/internal/backend/util/paths.go @@ -1,4 +1,4 @@ -package backend +package util import "os" diff --git a/mover-restic/restic/internal/backend/utils.go b/mover-restic/restic/internal/backend/utils.go deleted file mode 100644 index cd6614f34..000000000 --- a/mover-restic/restic/internal/backend/utils.go +++ /dev/null @@ -1,142 +0,0 @@ -package backend - -import ( - "bytes" - "context" - "fmt" - "io" - - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" -) - -// LoadAll reads all data stored in the backend for the handle into the given -// buffer, which is truncated. If the buffer is not large enough or nil, a new -// one is allocated. -func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) { - retriedInvalidData := false - err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error { - // make sure this is idempotent, in case an error occurs this function may be called multiple times! - wr := bytes.NewBuffer(buf[:0]) - _, cerr := io.Copy(wr, rd) - if cerr != nil { - return cerr - } - buf = wr.Bytes() - - // retry loading damaged data only once. If a file fails to download correctly - // the second time, then it is likely corrupted at the backend. Return the data - // to the caller in that case to let it decide what to do with the data. - if !retriedInvalidData && h.Type != restic.ConfigFile { - id, err := restic.ParseID(h.Name) - if err == nil && !restic.Hash(buf).Equal(id) { - debug.Log("retry loading broken blob %v", h) - retriedInvalidData = true - return errors.Errorf("loadAll(%v): invalid data returned", h) - } - } - return nil - }) - - if err != nil { - return nil, err - } - - return buf, nil -} - -// LimitedReadCloser wraps io.LimitedReader and exposes the Close() method. -type LimitedReadCloser struct { - io.Closer - io.LimitedReader -} - -// LimitReadCloser returns a new reader wraps r in an io.LimitedReader, but also -// exposes the Close() method. -func LimitReadCloser(r io.ReadCloser, n int64) *LimitedReadCloser { - return &LimitedReadCloser{Closer: r, LimitedReader: io.LimitedReader{R: r, N: n}} -} - -// DefaultLoad implements Backend.Load using lower-level openReader func -func DefaultLoad(ctx context.Context, h restic.Handle, length int, offset int64, - openReader func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error), - fn func(rd io.Reader) error) error { - - rd, err := openReader(ctx, h, length, offset) - if err != nil { - return err - } - err = fn(rd) - if err != nil { - _ = rd.Close() // ignore secondary errors closing the reader - return err - } - return rd.Close() -} - -// DefaultDelete removes all restic keys in the bucket. It will not remove the bucket itself. -func DefaultDelete(ctx context.Context, be restic.Backend) error { - alltypes := []restic.FileType{ - restic.PackFile, - restic.KeyFile, - restic.LockFile, - restic.SnapshotFile, - restic.IndexFile} - - for _, t := range alltypes { - err := be.List(ctx, t, func(fi restic.FileInfo) error { - return be.Remove(ctx, restic.Handle{Type: t, Name: fi.Name}) - }) - if err != nil { - return nil - } - } - err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile}) - if err != nil && be.IsNotExist(err) { - err = nil - } - - return err -} - -type memorizedLister struct { - fileInfos []restic.FileInfo - tpe restic.FileType -} - -func (m *memorizedLister) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - if t != m.tpe { - return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) - } - for _, fi := range m.fileInfos { - if ctx.Err() != nil { - break - } - err := fn(fi) - if err != nil { - return err - } - } - return ctx.Err() -} - -func MemorizeList(ctx context.Context, be restic.Lister, t restic.FileType) (restic.Lister, error) { - if _, ok := be.(*memorizedLister); ok { - return be, nil - } - - var fileInfos []restic.FileInfo - err := be.List(ctx, t, func(fi restic.FileInfo) error { - fileInfos = append(fileInfos, fi) - return nil - }) - if err != nil { - return nil, err - } - - return &memorizedLister{ - fileInfos: fileInfos, - tpe: t, - }, nil -} diff --git a/mover-restic/restic/internal/backend/utils_test.go b/mover-restic/restic/internal/backend/utils_test.go deleted file mode 100644 index 8392bfa8f..000000000 --- a/mover-restic/restic/internal/backend/utils_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package backend_test - -import ( - "bytes" - "context" - "fmt" - "io" - "math/rand" - "testing" - - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/mem" - "github.com/restic/restic/internal/backend/mock" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/restic" - rtest "github.com/restic/restic/internal/test" -) - -const KiB = 1 << 10 -const MiB = 1 << 20 - -func TestLoadAll(t *testing.T) { - b := mem.New() - var buf []byte - - for i := 0; i < 20; i++ { - data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB) - - id := restic.Hash(data) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} - err := b.Save(context.TODO(), h, restic.NewByteReader(data, b.Hasher())) - rtest.OK(t, err) - - buf, err := backend.LoadAll(context.TODO(), buf, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) - rtest.OK(t, err) - - if len(buf) != len(data) { - t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf)) - continue - } - - if !bytes.Equal(buf, data) { - t.Errorf("wrong data returned") - continue - } - } -} - -func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle { - id := restic.Hash(buf) - h := restic.Handle{Name: id.String(), Type: restic.PackFile} - err := be.Save(context.TODO(), h, restic.NewByteReader(buf, be.Hasher())) - if err != nil { - t.Fatal(err) - } - return h -} - -type quickRetryBackend struct { - restic.Backend -} - -func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - err := be.Backend.Load(ctx, h, length, offset, fn) - if err != nil { - // retry - err = be.Backend.Load(ctx, h, length, offset, fn) - } - return err -} - -func TestLoadAllBroken(t *testing.T) { - b := mock.NewBackend() - - data := rtest.Random(23, rand.Intn(MiB)+500*KiB) - id := restic.Hash(data) - // damage buffer - data[0] ^= 0xff - - b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - return io.NopCloser(bytes.NewReader(data)), nil - } - - // must fail on first try - _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()}) - if err == nil { - t.Fatalf("missing expected error") - } - - // must return the broken data after a retry - be := &quickRetryBackend{Backend: b} - buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()}) - rtest.OK(t, err) - - if !bytes.Equal(buf, data) { - t.Fatalf("wrong data returned") - } -} - -func TestLoadAllAppend(t *testing.T) { - b := mem.New() - - h1 := save(t, b, []byte("foobar test string")) - randomData := rtest.Random(23, rand.Intn(MiB)+500*KiB) - h2 := save(t, b, randomData) - - var tests = []struct { - handle restic.Handle - buf []byte - want []byte - }{ - { - handle: h1, - buf: nil, - want: []byte("foobar test string"), - }, - { - handle: h1, - buf: []byte("xxx"), - want: []byte("foobar test string"), - }, - { - handle: h2, - buf: nil, - want: randomData, - }, - { - handle: h2, - buf: make([]byte, 0, 200), - want: randomData, - }, - { - handle: h2, - buf: []byte("foobarbaz"), - want: randomData, - }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - buf, err := backend.LoadAll(context.TODO(), test.buf, b, test.handle) - if err != nil { - t.Fatal(err) - } - - if !bytes.Equal(buf, test.want) { - t.Errorf("wrong data returned, want %q, got %q", test.want, buf) - } - }) - } -} - -type mockReader struct { - closed bool -} - -func (rd *mockReader) Read(_ []byte) (n int, err error) { - return 0, nil -} -func (rd *mockReader) Close() error { - rd.closed = true - return nil -} - -func TestDefaultLoad(t *testing.T) { - - h := restic.Handle{Name: "id", Type: restic.PackFile} - rd := &mockReader{} - - // happy case, assert correct parameters are passed around and content stream is closed - err := backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { - rtest.Equals(t, h, ih) - rtest.Equals(t, int(10), length) - rtest.Equals(t, int64(11), offset) - - return rd, nil - }, func(ird io.Reader) error { - rtest.Equals(t, rd, ird) - return nil - }) - rtest.OK(t, err) - rtest.Equals(t, true, rd.closed) - - // unhappy case, assert producer errors are handled correctly - err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { - return nil, errors.Errorf("producer error") - }, func(ird io.Reader) error { - t.Fatalf("unexpected consumer invocation") - return nil - }) - rtest.Equals(t, "producer error", err.Error()) - - // unhappy case, assert consumer errors are handled correctly - rd = &mockReader{} - err = backend.DefaultLoad(context.TODO(), h, 10, 11, func(ctx context.Context, ih restic.Handle, length int, offset int64) (io.ReadCloser, error) { - return rd, nil - }, func(ird io.Reader) error { - return errors.Errorf("consumer error") - }) - rtest.Equals(t, true, rd.closed) - rtest.Equals(t, "consumer error", err.Error()) -} - -func TestMemoizeList(t *testing.T) { - // setup backend to serve as data source for memoized list - be := mock.NewBackend() - files := []restic.FileInfo{ - {Size: 42, Name: restic.NewRandomID().String()}, - {Size: 45, Name: restic.NewRandomID().String()}, - } - be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - for _, fi := range files { - if err := fn(fi); err != nil { - return err - } - } - return nil - } - - mem, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) - rtest.OK(t, err) - - err = mem.List(context.TODO(), restic.IndexFile, func(fi restic.FileInfo) error { - t.Fatal("file type mismatch") - return nil // the memoized lister must return an error by itself - }) - rtest.Assert(t, err != nil, "missing error on file typ mismatch") - - var memFiles []restic.FileInfo - err = mem.List(context.TODO(), restic.SnapshotFile, func(fi restic.FileInfo) error { - memFiles = append(memFiles, fi) - return nil - }) - rtest.OK(t, err) - rtest.Equals(t, files, memFiles) -} - -func TestMemoizeListError(t *testing.T) { - // setup backend to serve as data source for memoized list - be := mock.NewBackend() - be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { - return fmt.Errorf("list error") - } - _, err := backend.MemorizeList(context.TODO(), be, restic.SnapshotFile) - rtest.Assert(t, err != nil, "missing error on list error") -} diff --git a/mover-restic/restic/internal/backend/watchdog_roundtriper.go b/mover-restic/restic/internal/backend/watchdog_roundtriper.go new file mode 100644 index 000000000..e3e10d7fe --- /dev/null +++ b/mover-restic/restic/internal/backend/watchdog_roundtriper.go @@ -0,0 +1,119 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "sync/atomic" + "time" +) + +var errRequestTimeout = fmt.Errorf("request timeout") + +// watchdogRoundtripper cancels an http request if an upload or download did not make progress +// within timeout. The time between fully sending the request and receiving an response is also +// limited by this timeout. This ensures that stuck requests are cancelled after some time. +// +// The roundtriper makes the assumption that the upload and download happen continuously. In particular, +// the caller must not make long pauses between individual read requests from the response body. +type watchdogRoundtripper struct { + rt http.RoundTripper + timeout time.Duration + chunkSize int +} + +var _ http.RoundTripper = &watchdogRoundtripper{} + +func newWatchdogRoundtripper(rt http.RoundTripper, timeout time.Duration, chunkSize int) *watchdogRoundtripper { + return &watchdogRoundtripper{ + rt: rt, + timeout: timeout, + chunkSize: chunkSize, + } +} + +func (w *watchdogRoundtripper) RoundTrip(req *http.Request) (*http.Response, error) { + timer := time.NewTimer(w.timeout) + ctx, cancel := context.WithCancel(req.Context()) + timedOut := &atomic.Bool{} + + // cancel context if timer expires + go func() { + defer timer.Stop() + select { + case <-timer.C: + timedOut.Store(true) + cancel() + case <-ctx.Done(): + } + }() + + kick := func() { + timer.Reset(w.timeout) + } + isTimeout := func(err error) bool { + return timedOut.Load() && errors.Is(err, context.Canceled) + } + + req = req.Clone(ctx) + if req.Body != nil { + // kick watchdog timer as long as uploading makes progress + req.Body = newWatchdogReadCloser(req.Body, w.chunkSize, kick, nil, isTimeout) + } + + resp, err := w.rt.RoundTrip(req) + if err != nil { + return nil, err + } + + // kick watchdog timer as long as downloading makes progress + // cancel context to stop goroutine once response body is closed + resp.Body = newWatchdogReadCloser(resp.Body, w.chunkSize, kick, cancel, isTimeout) + return resp, nil +} + +func newWatchdogReadCloser(rc io.ReadCloser, chunkSize int, kick func(), close func(), isTimeout func(err error) bool) *watchdogReadCloser { + return &watchdogReadCloser{ + rc: rc, + chunkSize: chunkSize, + kick: kick, + close: close, + isTimeout: isTimeout, + } +} + +type watchdogReadCloser struct { + rc io.ReadCloser + chunkSize int + kick func() + close func() + isTimeout func(err error) bool +} + +var _ io.ReadCloser = &watchdogReadCloser{} + +func (w *watchdogReadCloser) Read(p []byte) (n int, err error) { + w.kick() + + // Read is not required to fill the whole passed in byte slice + // Thus, keep things simple and just stay within our chunkSize. + if len(p) > w.chunkSize { + p = p[:w.chunkSize] + } + n, err = w.rc.Read(p) + w.kick() + + if err != nil && w.isTimeout(err) { + err = errRequestTimeout + } + return n, err +} + +func (w *watchdogReadCloser) Close() error { + if w.close != nil { + w.close() + } + return w.rc.Close() +} diff --git a/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go b/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go new file mode 100644 index 000000000..bc43447e1 --- /dev/null +++ b/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go @@ -0,0 +1,204 @@ +package backend + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + rtest "github.com/restic/restic/internal/test" +) + +func TestRead(t *testing.T) { + data := []byte("abcdef") + var ctr int + kick := func() { + ctr++ + } + var closed bool + onClose := func() { + closed = true + } + isTimeout := func(err error) bool { + return false + } + + wd := newWatchdogReadCloser(io.NopCloser(bytes.NewReader(data)), 1, kick, onClose, isTimeout) + + out, err := io.ReadAll(wd) + rtest.OK(t, err) + rtest.Equals(t, data, out, "data mismatch") + // the EOF read also triggers the kick function + rtest.Equals(t, len(data)*2+2, ctr, "unexpected number of kick calls") + + rtest.Equals(t, false, closed, "close function called too early") + rtest.OK(t, wd.Close()) + rtest.Equals(t, true, closed, "close function not called") +} + +func TestRoundtrip(t *testing.T) { + t.Parallel() + + // at the higher delay values, it takes longer to transmit the request/response body + // than the roundTripper timeout + for _, delay := range []int{0, 1, 10, 20} { + t.Run(fmt.Sprintf("%v", delay), func(t *testing.T) { + msg := []byte("ping-pong-data") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + w.WriteHeader(200) + + // slowly send the reply + for len(data) >= 2 { + _, _ = w.Write(data[:2]) + w.(http.Flusher).Flush() + data = data[2:] + time.Sleep(time.Duration(delay) * time.Millisecond) + } + _, _ = w.Write(data) + })) + defer srv.Close() + + rt := newWatchdogRoundtripper(http.DefaultTransport, 100*time.Millisecond, 2) + req, err := http.NewRequestWithContext(context.TODO(), "GET", srv.URL, io.NopCloser(newSlowReader(bytes.NewReader(msg), time.Duration(delay)*time.Millisecond))) + rtest.OK(t, err) + + resp, err := rt.RoundTrip(req) + rtest.OK(t, err) + rtest.Equals(t, 200, resp.StatusCode, "unexpected status code") + + response, err := io.ReadAll(resp.Body) + rtest.OK(t, err) + rtest.Equals(t, msg, response, "unexpected response") + + rtest.OK(t, resp.Body.Close()) + }) + } +} + +func TestCanceledRoundtrip(t *testing.T) { + rt := newWatchdogRoundtripper(http.DefaultTransport, time.Second, 2) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req, err := http.NewRequestWithContext(ctx, "GET", "http://some.random.url.dfdgsfg", nil) + rtest.OK(t, err) + + resp, err := rt.RoundTrip(req) + rtest.Equals(t, context.Canceled, err) + // make linter happy + if resp != nil { + rtest.OK(t, resp.Body.Close()) + } +} + +type slowReader struct { + data io.Reader + delay time.Duration +} + +func newSlowReader(data io.Reader, delay time.Duration) *slowReader { + return &slowReader{ + data: data, + delay: delay, + } +} + +func (s *slowReader) Read(p []byte) (n int, err error) { + time.Sleep(s.delay) + return s.data.Read(p) +} + +func TestUploadTimeout(t *testing.T) { + t.Parallel() + + msg := []byte("ping") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + t.Error("upload should have been canceled") + })) + defer srv.Close() + + rt := newWatchdogRoundtripper(http.DefaultTransport, 10*time.Millisecond, 1024) + req, err := http.NewRequestWithContext(context.TODO(), "GET", srv.URL, io.NopCloser(newSlowReader(bytes.NewReader(msg), 100*time.Millisecond))) + rtest.OK(t, err) + + resp, err := rt.RoundTrip(req) + rtest.Equals(t, context.Canceled, err) + // make linter happy + if resp != nil { + rtest.OK(t, resp.Body.Close()) + } +} + +func TestProcessingTimeout(t *testing.T) { + t.Parallel() + + msg := []byte("ping") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + time.Sleep(100 * time.Millisecond) + w.WriteHeader(200) + })) + defer srv.Close() + + rt := newWatchdogRoundtripper(http.DefaultTransport, 10*time.Millisecond, 1024) + req, err := http.NewRequestWithContext(context.TODO(), "GET", srv.URL, io.NopCloser(bytes.NewReader(msg))) + rtest.OK(t, err) + + resp, err := rt.RoundTrip(req) + rtest.Equals(t, context.Canceled, err) + // make linter happy + if resp != nil { + rtest.OK(t, resp.Body.Close()) + } +} + +func TestDownloadTimeout(t *testing.T) { + t.Parallel() + + msg := []byte("ping") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + data, err := io.ReadAll(r.Body) + if err != nil { + w.WriteHeader(500) + return + } + w.WriteHeader(200) + _, _ = w.Write(data[:2]) + w.(http.Flusher).Flush() + data = data[2:] + + time.Sleep(100 * time.Millisecond) + _, _ = w.Write(data) + + })) + defer srv.Close() + + rt := newWatchdogRoundtripper(http.DefaultTransport, 10*time.Millisecond, 1024) + req, err := http.NewRequestWithContext(context.TODO(), "GET", srv.URL, io.NopCloser(bytes.NewReader(msg))) + rtest.OK(t, err) + + resp, err := rt.RoundTrip(req) + rtest.OK(t, err) + rtest.Equals(t, 200, resp.StatusCode, "unexpected status code") + + _, err = io.ReadAll(resp.Body) + rtest.Equals(t, errRequestTimeout, err, "response download not canceled") + rtest.OK(t, resp.Body.Close()) +} diff --git a/mover-restic/restic/internal/bloblru/cache.go b/mover-restic/restic/internal/bloblru/cache.go index 302ecc769..9981f8a87 100644 --- a/mover-restic/restic/internal/bloblru/cache.go +++ b/mover-restic/restic/internal/bloblru/cache.go @@ -20,13 +20,15 @@ type Cache struct { c *simplelru.LRU[restic.ID, []byte] free, size int // Current and max capacity, in bytes. + inProgress map[restic.ID]chan struct{} } // New constructs a blob cache that stores at most size bytes worth of blobs. func New(size int) *Cache { c := &Cache{ - free: size, - size: size, + free: size, + size: size, + inProgress: make(map[restic.ID]chan struct{}), } // NewLRU wants us to specify some max. number of entries, else it errors. @@ -85,6 +87,57 @@ func (c *Cache) Get(id restic.ID) ([]byte, bool) { return blob, ok } +func (c *Cache) GetOrCompute(id restic.ID, compute func() ([]byte, error)) ([]byte, error) { + // check if already cached + blob, ok := c.Get(id) + if ok { + return blob, nil + } + + // check for parallel download or start our own + finish := make(chan struct{}) + c.mu.Lock() + waitForResult, isComputing := c.inProgress[id] + if !isComputing { + c.inProgress[id] = finish + } + c.mu.Unlock() + + if isComputing { + // wait for result of parallel download + <-waitForResult + } else { + // remove progress channel once finished here + defer func() { + c.mu.Lock() + delete(c.inProgress, id) + c.mu.Unlock() + close(finish) + }() + } + + // try again. This is necessary independent of whether isComputing is true or not. + // The calls to `c.Get()` and checking/adding the entry in `c.inProgress` are not atomic, + // thus the item might have been computed in the meantime. + // The following scenario would compute() the value multiple times otherwise: + // Goroutine A does not find a value in the initial call to `c.Get`, then goroutine B + // takes over, caches the computed value and cleans up its channel in c.inProgress. + // Then goroutine A continues, does not detect a parallel computation and would try + // to call compute() again. + blob, ok = c.Get(id) + if ok { + return blob, nil + } + + // download it + blob, err := compute() + if err == nil { + c.Add(id, blob) + } + + return blob, err +} + func (c *Cache) evict(key restic.ID, blob []byte) { debug.Log("bloblru.Cache: evict %v, %d bytes", key, cap(blob)) c.free += cap(blob) + overhead diff --git a/mover-restic/restic/internal/bloblru/cache_test.go b/mover-restic/restic/internal/bloblru/cache_test.go index aa6f4465c..d25daf764 100644 --- a/mover-restic/restic/internal/bloblru/cache_test.go +++ b/mover-restic/restic/internal/bloblru/cache_test.go @@ -1,11 +1,14 @@ package bloblru import ( + "context" + "fmt" "math/rand" "testing" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "golang.org/x/sync/errgroup" ) func TestCache(t *testing.T) { @@ -52,6 +55,70 @@ func TestCache(t *testing.T) { rtest.Equals(t, cacheSize, c.free) } +func TestCacheGetOrCompute(t *testing.T) { + var id1, id2 restic.ID + id1[0] = 1 + id2[0] = 2 + + const ( + kiB = 1 << 10 + cacheSize = 64*kiB + 3*overhead + ) + + c := New(cacheSize) + + e := fmt.Errorf("broken") + _, err := c.GetOrCompute(id1, func() ([]byte, error) { + return nil, e + }) + rtest.Equals(t, e, err, "expected error was not returned") + + // fill buffer + data1 := make([]byte, 10*kiB) + blob, err := c.GetOrCompute(id1, func() ([]byte, error) { + return data1, nil + }) + rtest.OK(t, err) + rtest.Equals(t, &data1[0], &blob[0], "wrong buffer returned") + + // now the buffer should be returned without calling the compute function + blob, err = c.GetOrCompute(id1, func() ([]byte, error) { + return nil, e + }) + rtest.OK(t, err) + rtest.Equals(t, &data1[0], &blob[0], "wrong buffer returned") + + // check concurrency + wg, _ := errgroup.WithContext(context.TODO()) + wait := make(chan struct{}) + calls := make(chan struct{}, 10) + + // start a bunch of blocking goroutines + for i := 0; i < 10; i++ { + wg.Go(func() error { + buf, err := c.GetOrCompute(id2, func() ([]byte, error) { + // block to ensure that multiple requests are waiting in parallel + <-wait + calls <- struct{}{} + return make([]byte, 42), nil + }) + if len(buf) != 42 { + return fmt.Errorf("wrong buffer") + } + return err + }) + } + + close(wait) + rtest.OK(t, wg.Wait()) + close(calls) + count := 0 + for range calls { + count++ + } + rtest.Equals(t, 1, count, "expected exactly one call of the compute function") +} + func BenchmarkAdd(b *testing.B) { const ( MiB = 1 << 20 diff --git a/mover-restic/restic/internal/cache/backend_test.go b/mover-restic/restic/internal/cache/backend_test.go deleted file mode 100644 index 930d853b2..000000000 --- a/mover-restic/restic/internal/cache/backend_test.go +++ /dev/null @@ -1,202 +0,0 @@ -package cache - -import ( - "bytes" - "context" - "io" - "math/rand" - "sync" - "testing" - "time" - - "github.com/pkg/errors" - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/mem" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" -) - -func loadAndCompare(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { - buf, err := backend.LoadAll(context.TODO(), nil, be, h) - if err != nil { - t.Fatal(err) - } - - if len(buf) != len(data) { - t.Fatalf("wrong number of bytes read, want %v, got %v", len(data), len(buf)) - } - - if !bytes.Equal(buf, data) { - t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[:16], buf[:16]) - } -} - -func save(t testing.TB, be restic.Backend, h restic.Handle, data []byte) { - err := be.Save(context.TODO(), h, restic.NewByteReader(data, be.Hasher())) - if err != nil { - t.Fatal(err) - } -} - -func remove(t testing.TB, be restic.Backend, h restic.Handle) { - err := be.Remove(context.TODO(), h) - if err != nil { - t.Fatal(err) - } -} - -func randomData(n int) (restic.Handle, []byte) { - data := test.Random(rand.Int(), n) - id := restic.Hash(data) - h := restic.Handle{ - Type: restic.IndexFile, - Name: id.String(), - } - return h, data -} - -func TestBackend(t *testing.T) { - be := mem.New() - c := TestNewCache(t) - wbe := c.Wrap(be) - - h, data := randomData(5234142) - - // save directly in backend - save(t, be, h, data) - if c.Has(h) { - t.Errorf("cache has file too early") - } - - // load data via cache - loadAndCompare(t, wbe, h, data) - if !c.Has(h) { - t.Errorf("cache doesn't have file after load") - } - - // remove via cache - remove(t, wbe, h) - if c.Has(h) { - t.Errorf("cache has file after remove") - } - - // save via cache - save(t, wbe, h, data) - if !c.Has(h) { - t.Errorf("cache doesn't have file after load") - } - - // load data directly from backend - loadAndCompare(t, be, h, data) - - // load data via cache - loadAndCompare(t, be, h, data) - - // remove directly - remove(t, be, h) - if !c.Has(h) { - t.Errorf("file not in cache any more") - } - - // run stat - _, err := wbe.Stat(context.TODO(), h) - if err == nil { - t.Errorf("expected error for removed file not found, got nil") - } - - if !wbe.IsNotExist(err) { - t.Errorf("Stat() returned error that does not match IsNotExist(): %v", err) - } - - if c.Has(h) { - t.Errorf("removed file still in cache after stat") - } -} - -type loadErrorBackend struct { - restic.Backend - loadError error -} - -func (be loadErrorBackend) Load(_ context.Context, _ restic.Handle, _ int, _ int64, _ func(rd io.Reader) error) error { - time.Sleep(10 * time.Millisecond) - return be.loadError -} - -func TestErrorBackend(t *testing.T) { - be := mem.New() - c := TestNewCache(t) - h, data := randomData(5234142) - - // save directly in backend - save(t, be, h, data) - - testErr := errors.New("test error") - errBackend := loadErrorBackend{ - Backend: be, - loadError: testErr, - } - - loadTest := func(wg *sync.WaitGroup, be restic.Backend) { - defer wg.Done() - - buf, err := backend.LoadAll(context.TODO(), nil, be, h) - if err == testErr { - return - } - - if err != nil { - t.Error(err) - return - } - - if !bytes.Equal(buf, data) { - t.Errorf("data does not match") - } - time.Sleep(time.Millisecond) - } - - wrappedBE := c.Wrap(errBackend) - var wg sync.WaitGroup - for i := 0; i < 5; i++ { - wg.Add(1) - go loadTest(&wg, wrappedBE) - } - - wg.Wait() -} - -func TestBackendRemoveBroken(t *testing.T) { - be := mem.New() - c := TestNewCache(t) - - h, data := randomData(5234142) - // save directly in backend - save(t, be, h, data) - - // prime cache with broken copy - broken := append([]byte{}, data...) - broken[0] ^= 0xff - err := c.Save(h, bytes.NewReader(broken)) - test.OK(t, err) - - // loadall retries if broken data was returned - buf, err := backend.LoadAll(context.TODO(), nil, c.Wrap(be), h) - test.OK(t, err) - - if !bytes.Equal(buf, data) { - t.Fatalf("wrong data returned") - } - - // check that the cache now contains the correct data - rd, err := c.load(h, 0, 0) - defer func() { - _ = rd.Close() - }() - test.OK(t, err) - cached, err := io.ReadAll(rd) - test.OK(t, err) - if !bytes.Equal(cached, data) { - t.Fatalf("wrong data cache") - } -} diff --git a/mover-restic/restic/internal/checker/checker.go b/mover-restic/restic/internal/checker/checker.go index 04b2bdf79..031e13807 100644 --- a/mover-restic/restic/internal/checker/checker.go +++ b/mover-restic/restic/internal/checker/checker.go @@ -2,24 +2,19 @@ package checker import ( "bufio" - "bytes" "context" "fmt" - "io" "runtime" - "sort" "sync" - "crypto/sha256" + "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/hashing" - "github.com/restic/restic/internal/index" - "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" @@ -90,25 +85,15 @@ func (err *ErrOldIndexFormat) Error() string { return fmt.Sprintf("index %v has old format", err.ID) } -// ErrPackData is returned if errors are discovered while verifying a packfile -type ErrPackData struct { - PackID restic.ID - errs []error -} - -func (e *ErrPackData) Error() string { - return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs) -} - func (c *Checker) LoadSnapshots(ctx context.Context) error { var err error - c.snapshots, err = backend.MemorizeList(ctx, c.repo.Backend(), restic.SnapshotFile) + c.snapshots, err = restic.MemorizeList(ctx, c.repo, restic.SnapshotFile) return err } -func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType { +func computePackTypes(ctx context.Context, idx restic.ListBlobser) (map[restic.ID]restic.BlobType, error) { packs := make(map[restic.ID]restic.BlobType) - idx.Each(ctx, func(pb restic.PackedBlob) { + err := idx.ListBlobs(ctx, func(pb restic.PackedBlob) { tpe, exists := packs[pb.PackID] if exists { if pb.Type != tpe { @@ -119,46 +104,17 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID } packs[pb.PackID] = tpe }) - return packs + return packs, err } // LoadIndex loads all index files. func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []error, errs []error) { debug.Log("Start") - indexList, err := backend.MemorizeList(ctx, c.repo.Backend(), restic.IndexFile) - if err != nil { - // abort if an error occurs while listing the indexes - return hints, append(errs, err) - } - - if p != nil { - var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { - _, err := restic.ParseID(fi.Name) - if err != nil { - debug.Log("unable to parse %v as an ID", fi.Name) - return nil - } - - numIndexFiles++ - return nil - }) - if err != nil { - return hints, append(errs, err) - } - p.SetMax(numIndexFiles) - defer p.Done() - } - packToIndex := make(map[restic.ID]restic.IDSet) - err = index.ForAllIndexes(ctx, indexList, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := c.masterIndex.Load(ctx, c.repo, p, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { debug.Log("process index %v, err %v", id, err) - if p != nil { - p.Add(1) - } - if oldFormat { debug.Log("index %v has old format", id) hints = append(hints, &ErrOldIndexFormat{id}) @@ -171,11 +127,9 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e return nil } - c.masterIndex.Insert(index) - debug.Log("process blobs") cnt := 0 - index.Each(ctx, func(blob restic.PackedBlob) { + err = idx.Each(ctx, func(blob restic.PackedBlob) { cnt++ if _, ok := packToIndex[blob.PackID]; !ok { @@ -185,22 +139,28 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e }) debug.Log("%d blobs processed", cnt) - return nil + return err }) if err != nil { - errs = append(errs, err) + // failed to load the index + return hints, append(errs, err) } - // Merge index before computing pack sizes, as this needs removed duplicates - err = c.masterIndex.MergeFinalIndexes() + err = c.repo.SetIndex(c.masterIndex) if err != nil { - // abort if an error occurs merging the indexes - return hints, append(errs, err) + debug.Log("SetIndex returned error: %v", err) + errs = append(errs, err) } // compute pack size using index entries - c.packs = pack.Size(ctx, c.masterIndex, false) - packTypes := computePackTypes(ctx, c.masterIndex) + c.packs, err = pack.Size(ctx, c.repo, false) + if err != nil { + return hints, append(errs, err) + } + packTypes, err := computePackTypes(ctx, c.repo) + if err != nil { + return hints, append(errs, err) + } debug.Log("checking for duplicate packs") for packID := range c.packs { @@ -218,45 +178,24 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e } } - err = c.repo.SetIndex(c.masterIndex) - if err != nil { - debug.Log("SetIndex returned error: %v", err) - errs = append(errs, err) - } - return hints, errs } // PackError describes an error with a specific pack. type PackError struct { - ID restic.ID - Orphaned bool - Err error + ID restic.ID + Orphaned bool + Truncated bool + Err error } func (e *PackError) Error() string { return "pack " + e.ID.String() + ": " + e.Err.Error() } -// IsOrphanedPack returns true if the error describes a pack which is not -// contained in any index. -func IsOrphanedPack(err error) bool { - var e *PackError - return errors.As(err, &e) && e.Orphaned -} - -func isS3Legacy(b restic.Backend) bool { - // unwrap cache - if be, ok := b.(*cache.Backend); ok { - b = be.Backend - } - - be, ok := b.(*s3.Backend) - if !ok { - return false - } - - return be.Layout.Name() == "s3legacy" +func isS3Legacy(b backend.Backend) bool { + be := backend.AsBackend[*s3.Backend](b) + return be != nil && be.Layout.Name() == "s3legacy" } // Packs checks that all packs referenced in the index are still available and @@ -265,8 +204,10 @@ func isS3Legacy(b restic.Backend) bool { func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { defer close(errChan) - if isS3Legacy(c.repo.Backend()) { - errChan <- ErrLegacyLayout + if r, ok := c.repo.(*repository.Repository); ok { + if isS3Legacy(repository.AsS3Backend(r)) { + errChan <- ErrLegacyLayout + } } debug.Log("checking for %d packs", len(c.packs)) @@ -303,7 +244,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { select { case <-ctx.Done(): return - case errChan <- &PackError{ID: id, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}: + case errChan <- &PackError{ID: id, Truncated: true, Err: errors.Errorf("unexpected file size: got %d, expected %d", reposize, size)}: } } } @@ -367,7 +308,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI } } -func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { +func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { errs = append(errs, err) @@ -448,10 +389,10 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { } // Note that we do not use the blob size. The "obvious" check // whether the sum of the blob sizes matches the file size - // unfortunately fails in some cases that are not resolveable + // unfortunately fails in some cases that are not resolvable // by users, so we omit this check, see #1887 - _, found := c.repo.LookupBlobSize(blobID, restic.DataBlob) + _, found := c.repo.LookupBlobSize(restic.DataBlob, blobID) if !found { debug.Log("tree %v references blob %v which isn't contained in index", id, blobID) errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("file %q blob %v not found in index", node.Name, blobID)}) @@ -499,7 +440,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) { } // UnusedBlobs returns all blobs that have never been referenced. -func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) { +func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, err error) { if !c.trackUnused { panic("only works when tracking blob references") } @@ -510,7 +451,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) { ctx, cancel := context.WithCancel(ctx) defer cancel() - c.repo.Index().Each(ctx, func(blob restic.PackedBlob) { + err = c.repo.ListBlobs(ctx, func(blob restic.PackedBlob) { h := restic.BlobHandle{ID: blob.ID, Type: blob.Type} if !c.blobRefs.M.Has(h) { debug.Log("blob %v not referenced", h) @@ -518,7 +459,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) { } }) - return blobs + return blobs, err } // CountPacks returns the number of packs in the repository. @@ -531,131 +472,13 @@ func (c *Checker) GetPacks() map[restic.ID]int64 { return c.packs } -// checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader) error { - debug.Log("checking pack %v", id.String()) - - if len(blobs) == 0 { - return errors.Errorf("pack %v is empty or not indexed", id) - } - - // sanity check blobs in index - sort.Slice(blobs, func(i, j int) bool { - return blobs[i].Offset < blobs[j].Offset - }) - idxHdrSize := pack.CalculateHeaderSize(blobs) - lastBlobEnd := 0 - nonContinuousPack := false - for _, blob := range blobs { - if lastBlobEnd != int(blob.Offset) { - nonContinuousPack = true - } - lastBlobEnd = int(blob.Offset + blob.Length) - } - // size was calculated by masterindex.PackSize, thus there's no need to recalculate it here - - var errs []error - if nonContinuousPack { - debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs) - errs = append(errs, errors.New("Index for pack contains gaps / overlapping blobs")) - } - - // calculate hash on-the-fly while reading the pack and capture pack header - var hash restic.ID - var hdrBuf []byte - hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { - hrd := hashing.NewReader(rd, sha256.New()) - bufRd.Reset(hrd) - - // skip to start of first blob, offset == 0 for correct pack files - _, err := bufRd.Discard(int(offset)) - if err != nil { - return err - } - - err = fn(bufRd) - if err != nil { - return err - } - - // skip enough bytes until we reach the possible header start - curPos := length + int(offset) - minHdrStart := int(size) - pack.MaxHeaderSize - if minHdrStart > curPos { - _, err := bufRd.Discard(minHdrStart - curPos) - if err != nil { - return err - } - } - - // read remainder, which should be the pack header - hdrBuf, err = io.ReadAll(bufRd) - if err != nil { - return err - } - - hash = restic.IDFromHash(hrd.Sum(nil)) - return nil - }) - } - - err := repository.StreamPack(ctx, hashingLoader, r.Key(), id, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - debug.Log(" check blob %v: %v", blob.ID, blob) - if err != nil { - debug.Log(" error verifying blob %v: %v", blob.ID, err) - errs = append(errs, errors.Errorf("blob %v: %v", blob.ID, err)) - } - return nil - }) - if err != nil { - // failed to load the pack file, return as further checks cannot succeed anyways - debug.Log(" error streaming pack: %v", err) - return errors.Errorf("pack %v failed to download: %v", id, err) - } - if !hash.Equal(id) { - debug.Log("Pack ID does not match, want %v, got %v", id, hash) - return errors.Errorf("Pack ID does not match, want %v, got %v", id, hash) - } - - blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf))) - if err != nil { - return err - } - - if uint32(idxHdrSize) != hdrSize { - debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize) - errs = append(errs, errors.Errorf("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) - } - - idx := r.Index() - for _, blob := range blobs { - // Check if blob is contained in index and position is correct - idxHas := false - for _, pb := range idx.Lookup(blob.BlobHandle) { - if pb.PackID == id && pb.Blob == blob { - idxHas = true - break - } - } - if !idxHas { - errs = append(errs, errors.Errorf("Blob %v is not contained in index or position is incorrect", blob.ID)) - continue - } - } - - if len(errs) > 0 { - return &ErrPackData{PackID: id, errs: errs} - } - - return nil -} - // ReadData loads all data from the repository and checks the integrity. func (c *Checker) ReadData(ctx context.Context, errChan chan<- error) { c.ReadPacks(ctx, c.packs, nil, errChan) } +const maxStreamBufferSize = 4 * 1024 * 1024 + // ReadPacks loads data from specified packs and checks the integrity. func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *progress.Counter, errChan chan<- error) { defer close(errChan) @@ -673,9 +496,12 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p // run workers for i := 0; i < workerCount; i++ { g.Go(func() error { - // create a buffer that is large enough to be reused by repository.StreamPack - // this ensures that we can read the pack header later on - bufRd := bufio.NewReaderSize(nil, repository.MaxStreamBufferSize) + bufRd := bufio.NewReaderSize(nil, maxStreamBufferSize) + dec, err := zstd.NewReader(nil) + if err != nil { + panic(dec) + } + defer dec.Close() for { var ps checkTask var ok bool @@ -689,7 +515,7 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p } } - err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd) + err := repository.CheckPack(ctx, c.repo.(*repository.Repository), ps.id, ps.blobs, ps.size, bufRd, dec) p.Add(1) if err == nil { continue @@ -710,7 +536,7 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p } // push packs to ch - for pbs := range c.repo.Index().ListPacks(ctx, packSet) { + for pbs := range c.repo.ListPacksFromIndex(ctx, packSet) { size := packs[pbs.PackID] debug.Log("listed %v", pbs.PackID) select { diff --git a/mover-restic/restic/internal/checker/checker_test.go b/mover-restic/restic/internal/checker/checker_test.go index ee7e2867c..5eaf550ba 100644 --- a/mover-restic/restic/internal/checker/checker_test.go +++ b/mover-restic/restic/internal/checker/checker_test.go @@ -8,15 +8,17 @@ import ( "path/filepath" "sort" "strconv" + "strings" "sync" "testing" "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/hashing" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" @@ -71,11 +73,9 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) { } func TestCheckRepo(t *testing.T) { - repodir, cleanup := test.Env(t, checkerTestData) + repo, _, cleanup := repository.TestFromFixture(t, checkerTestData) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - chkr := checker.New(repo, false) hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(errs) > 0 { @@ -91,16 +91,11 @@ func TestCheckRepo(t *testing.T) { } func TestMissingPack(t *testing.T) { - repodir, cleanup := test.Env(t, checkerTestData) + repo, be, cleanup := repository.TestFromFixture(t, checkerTestData) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - - packHandle := restic.Handle{ - Type: restic.PackFile, - Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6", - } - test.OK(t, repo.Backend().Remove(context.TODO(), packHandle)) + packID := restic.TestParseID("657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6") + test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: packID.String()})) chkr := checker.New(repo, false) hints, errs := chkr.LoadIndex(context.TODO(), nil) @@ -115,25 +110,20 @@ func TestMissingPack(t *testing.T) { "expected exactly one error, got %v", len(errs)) if err, ok := errs[0].(*checker.PackError); ok { - test.Equals(t, packHandle.Name, err.ID.String()) + test.Equals(t, packID, err.ID) } else { t.Errorf("expected error returned by checker.Packs() to be PackError, got %v", err) } } func TestUnreferencedPack(t *testing.T) { - repodir, cleanup := test.Env(t, checkerTestData) + repo, be, cleanup := repository.TestFromFixture(t, checkerTestData) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - // index 3f1a only references pack 60e0 packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e" - indexHandle := restic.Handle{ - Type: restic.IndexFile, - Name: "3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44", - } - test.OK(t, repo.Backend().Remove(context.TODO(), indexHandle)) + indexID := restic.TestParseID("3f1abfcb79c6f7d0a3be517d2c83c8562fba64ef2c8e9a3544b4edaf8b5e3b44") + test.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: indexID.String()})) chkr := checker.New(repo, false) hints, errs := chkr.LoadIndex(context.TODO(), nil) @@ -155,16 +145,11 @@ func TestUnreferencedPack(t *testing.T) { } func TestUnreferencedBlobs(t *testing.T) { - repodir, cleanup := test.Env(t, checkerTestData) + repo, _, cleanup := repository.TestFromFixture(t, checkerTestData) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - - snapshotHandle := restic.Handle{ - Type: restic.SnapshotFile, - Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02", - } - test.OK(t, repo.Backend().Remove(context.TODO(), snapshotHandle)) + snapshotID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02") + test.OK(t, repo.RemoveUnpacked(context.TODO(), restic.SnapshotFile, snapshotID)) unusedBlobsBySnapshot := restic.BlobHandles{ restic.TestParseHandle("58c748bbe2929fdf30c73262bd8313fe828f8925b05d1d4a87fe109082acb849", restic.DataBlob), @@ -187,22 +172,21 @@ func TestUnreferencedBlobs(t *testing.T) { test.OKs(t, checkPacks(chkr)) test.OKs(t, checkStruct(chkr)) - blobs := chkr.UnusedBlobs(context.TODO()) + blobs, err := chkr.UnusedBlobs(context.TODO()) + test.OK(t, err) sort.Sort(blobs) test.Equals(t, unusedBlobsBySnapshot, blobs) } func TestModifiedIndex(t *testing.T) { - repodir, cleanup := test.Env(t, checkerTestData) + repo, be, cleanup := repository.TestFromFixture(t, checkerTestData) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - done := make(chan struct{}) defer close(done) - h := restic.Handle{ + h := backend.Handle{ Type: restic.IndexFile, Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } @@ -224,13 +208,13 @@ func TestModifiedIndex(t *testing.T) { }() wr := io.Writer(tmpfile) var hw *hashing.Writer - if repo.Backend().Hasher() != nil { - hw = hashing.NewWriter(wr, repo.Backend().Hasher()) + if be.Hasher() != nil { + hw = hashing.NewWriter(wr, be.Hasher()) wr = hw } // read the file from the backend - err = repo.Backend().Load(context.TODO(), h, 0, 0, func(rd io.Reader) error { + err = be.Load(context.TODO(), h, 0, 0, func(rd io.Reader) error { _, err := io.Copy(wr, rd) return err }) @@ -238,7 +222,7 @@ func TestModifiedIndex(t *testing.T) { // save the index again with a modified name so that the hash doesn't match // the content any more - h2 := restic.Handle{ + h2 := backend.Handle{ Type: restic.IndexFile, Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd", } @@ -247,12 +231,12 @@ func TestModifiedIndex(t *testing.T) { if hw != nil { hash = hw.Sum(nil) } - rd, err := restic.NewFileReader(tmpfile, hash) + rd, err := backend.NewFileReader(tmpfile, hash) if err != nil { t.Fatal(err) } - err = repo.Backend().Save(context.TODO(), h2, rd) + err = be.Save(context.TODO(), h2, rd) if err != nil { t.Fatal(err) } @@ -273,11 +257,9 @@ func TestModifiedIndex(t *testing.T) { var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz") func TestDuplicatePacksInIndex(t *testing.T) { - repodir, cleanup := test.Env(t, checkerDuplicateIndexTestData) + repo, _, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - chkr := checker.New(repo, false) hints, errs := chkr.LoadIndex(context.TODO(), nil) if len(hints) == 0 { @@ -304,11 +286,11 @@ func TestDuplicatePacksInIndex(t *testing.T) { // errorBackend randomly modifies data after reading. type errorBackend struct { - restic.Backend + backend.Backend ProduceErrors bool } -func (b errorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { +func (b errorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { if b.ProduceErrors { return consumer(errorReadCloser{rd}) @@ -335,44 +317,91 @@ func induceError(data []byte) { data[pos] ^= 1 } +// errorOnceBackend randomly modifies data when reading a file for the first time. +type errorOnceBackend struct { + backend.Backend + m sync.Map +} + +func (b *errorOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error { + _, isRetry := b.m.LoadOrStore(h, struct{}{}) + return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error { + if !isRetry && h.Type != restic.ConfigFile { + return consumer(errorReadCloser{rd}) + } + return consumer(rd) + }) +} + func TestCheckerModifiedData(t *testing.T) { - repo := repository.TestRepository(t) + repo, be := repository.TestRepositoryWithVersion(t, 0) sn := archiver.TestSnapshot(t, repo, ".", nil) t.Logf("archived as %v", sn.ID().Str()) - beError := &errorBackend{Backend: repo.Backend()} - checkRepo, err := repository.New(beError, repository.Options{}) - test.OK(t, err) - test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5, "")) - - chkr := checker.New(checkRepo, false) - - hints, errs := chkr.LoadIndex(context.TODO(), nil) - if len(errs) > 0 { - t.Fatalf("expected no errors, got %v: %v", len(errs), errs) - } - - if len(hints) > 0 { - t.Errorf("expected no hints, got %v: %v", len(hints), hints) - } - - beError.ProduceErrors = true - errFound := false - for _, err := range checkPacks(chkr) { - t.Logf("pack error: %v", err) - } - - for _, err := range checkStruct(chkr) { - t.Logf("struct error: %v", err) - } - - for _, err := range checkData(chkr) { - t.Logf("data error: %v", err) - errFound = true - } - - if !errFound { - t.Fatal("no error found, checker is broken") + errBe := &errorBackend{Backend: be} + + for _, test := range []struct { + name string + be backend.Backend + damage func() + check func(t *testing.T, err error) + }{ + { + "errorBackend", + errBe, + func() { + errBe.ProduceErrors = true + }, + func(t *testing.T, err error) { + if err == nil { + t.Fatal("no error found, checker is broken") + } + }, + }, + { + "errorOnceBackend", + &errorOnceBackend{Backend: be}, + func() {}, + func(t *testing.T, err error) { + if !strings.Contains(err.Error(), "check successful on second attempt, original error pack") { + t.Fatalf("wrong error found, got %v", err) + } + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + checkRepo := repository.TestOpenBackend(t, test.be) + + chkr := checker.New(checkRepo, false) + + hints, errs := chkr.LoadIndex(context.TODO(), nil) + if len(errs) > 0 { + t.Fatalf("expected no errors, got %v: %v", len(errs), errs) + } + + if len(hints) > 0 { + t.Errorf("expected no hints, got %v: %v", len(hints), hints) + } + + test.damage() + var err error + for _, err := range checkPacks(chkr) { + t.Logf("pack error: %v", err) + } + + for _, err := range checkStruct(chkr) { + t.Logf("struct error: %v", err) + } + + for _, cerr := range checkData(chkr) { + t.Logf("data error: %v", cerr) + if err == nil { + err = cerr + } + } + + test.check(t, err) + }) } } @@ -398,10 +427,8 @@ func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (* } func TestCheckerNoDuplicateTreeDecodes(t *testing.T) { - repodir, cleanup := test.Env(t, checkerTestData) + repo, _, cleanup := repository.TestFromFixture(t, checkerTestData) defer cleanup() - - repo := repository.TestOpenLocal(t, repodir) checkRepo := &loadTreesOnceRepository{ Repository: repo, loadedTrees: restic.NewIDSet(), @@ -434,11 +461,11 @@ func (r *delayRepository) LoadTree(ctx context.Context, id restic.ID) (*restic.T return restic.LoadTree(ctx, r.Repository, id) } -func (r *delayRepository) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, bool) { +func (r *delayRepository) LookupBlobSize(t restic.BlobType, id restic.ID) (uint, bool) { if id == r.DelayTree && t == restic.DataBlob { r.Unblock() } - return r.Repository.LookupBlobSize(id, t) + return r.Repository.LookupBlobSize(t, id) } func (r *delayRepository) Unblock() { @@ -548,9 +575,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) { } func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) { - repodir, cleanup := test.Env(t, checkerTestData) - - repo := repository.TestOpenLocal(t, repodir) + repo, _, cleanup := repository.TestFromFixture(t, checkerTestData) chkr := checker.New(repo, false) hints, errs := chkr.LoadIndex(context.TODO(), nil) diff --git a/mover-restic/restic/internal/checker/testing.go b/mover-restic/restic/internal/checker/testing.go index fe1679393..d0014398f 100644 --- a/mover-restic/restic/internal/checker/testing.go +++ b/mover-restic/restic/internal/checker/testing.go @@ -8,7 +8,7 @@ import ( ) // TestCheckRepo runs the checker on repo. -func TestCheckRepo(t testing.TB, repo restic.Repository) { +func TestCheckRepo(t testing.TB, repo restic.Repository, skipStructure bool) { chkr := New(repo, true) hints, errs := chkr.LoadIndex(context.TODO(), nil) @@ -33,18 +33,23 @@ func TestCheckRepo(t testing.TB, repo restic.Repository) { t.Error(err) } - // structure - errChan = make(chan error) - go chkr.Structure(context.TODO(), nil, errChan) - - for err := range errChan { - t.Error(err) - } - - // unused blobs - blobs := chkr.UnusedBlobs(context.TODO()) - if len(blobs) > 0 { - t.Errorf("unused blobs found: %v", blobs) + if !skipStructure { + // structure + errChan = make(chan error) + go chkr.Structure(context.TODO(), nil, errChan) + + for err := range errChan { + t.Error(err) + } + + // unused blobs + blobs, err := chkr.UnusedBlobs(context.TODO()) + if err != nil { + t.Error(err) + } + if len(blobs) > 0 { + t.Errorf("unused blobs found: %v", blobs) + } } // read data diff --git a/mover-restic/restic/internal/crypto/crypto.go b/mover-restic/restic/internal/crypto/crypto.go index 752d886e3..d7ac9c3d4 100644 --- a/mover-restic/restic/internal/crypto/crypto.go +++ b/mover-restic/restic/internal/crypto/crypto.go @@ -27,7 +27,7 @@ const ( var ( // ErrUnauthenticated is returned when ciphertext verification has failed. - ErrUnauthenticated = errors.New("ciphertext verification failed") + ErrUnauthenticated = fmt.Errorf("ciphertext verification failed") ) // Key holds encryption and message authentication keys for a repository. It is stored @@ -45,28 +45,6 @@ type EncryptionKey [32]byte type MACKey struct { K [16]byte // for AES-128 R [16]byte // for Poly1305 - - masked bool // remember if the MAC key has already been masked -} - -// mask for key, (cf. http://cr.yp.to/mac/poly1305-20050329.pdf) -var poly1305KeyMask = [16]byte{ - 0xff, - 0xff, - 0xff, - 0x0f, // 3: top four bits zero - 0xfc, // 4: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 7: top four bits zero - 0xfc, // 8: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 11: top four bits zero - 0xfc, // 12: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 15: top four bits zero } func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte { @@ -78,32 +56,16 @@ func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte { return out[:] } -// mask poly1305 key -func maskKey(k *MACKey) { - if k == nil || k.masked { - return - } - - for i := 0; i < poly1305.TagSize; i++ { - k.R[i] = k.R[i] & poly1305KeyMask[i] - } - - k.masked = true -} - // construct mac key from slice (k||r), with masking func macKeyFromSlice(mk *MACKey, data []byte) { copy(mk.K[:], data[:16]) copy(mk.R[:], data[16:32]) - maskKey(mk) } // prepare key for low-level poly1305.Sum(): r||n func poly1305PrepareKey(nonce []byte, key *MACKey) [32]byte { var k [32]byte - maskKey(key) - cipher, err := aes.NewCipher(key.K[:]) if err != nil { panic(err) @@ -143,7 +105,6 @@ func NewRandomKey() *Key { panic("unable to read enough random bytes for MAC key") } - maskKey(&k.MACKey) return k } @@ -338,7 +299,7 @@ func (k *Key) Open(dst, nonce, ciphertext, _ []byte) ([]byte, error) { // check for plausible length if len(ciphertext) < k.Overhead() { - return nil, errors.Errorf("trying to decrypt invalid data: ciphertext too small") + return nil, errors.Errorf("trying to decrypt invalid data: ciphertext too short") } l := len(ciphertext) - macSize diff --git a/mover-restic/restic/internal/debug/debug.go b/mover-restic/restic/internal/debug/debug.go index 62c145e1a..7bc3291d1 100644 --- a/mover-restic/restic/internal/debug/debug.go +++ b/mover-restic/restic/internal/debug/debug.go @@ -8,8 +8,6 @@ import ( "path/filepath" "runtime" "strings" - - "github.com/restic/restic/internal/fs" ) var opts struct { @@ -46,7 +44,7 @@ func initDebugLogger() { fmt.Fprintf(os.Stderr, "debug log file %v\n", debugfile) - f, err := fs.OpenFile(debugfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + f, err := os.OpenFile(debugfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { fmt.Fprintf(os.Stderr, "unable to open debug log file: %v\n", err) os.Exit(2) diff --git a/mover-restic/restic/internal/dump/common.go b/mover-restic/restic/internal/dump/common.go index c3ba69431..62145ba9c 100644 --- a/mover-restic/restic/internal/dump/common.go +++ b/mover-restic/restic/internal/dump/common.go @@ -9,6 +9,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/walker" + "golang.org/x/sync/errgroup" ) // A Dumper writes trees and files from a repository to a Writer @@ -16,11 +17,11 @@ import ( type Dumper struct { cache *bloblru.Cache format string - repo restic.Repository + repo restic.Loader w io.Writer } -func New(format string, repo restic.Repository, w io.Writer) *Dumper { +func New(format string, repo restic.Loader, w io.Writer) *Dumper { return &Dumper{ cache: bloblru.New(64 << 20), format: format, @@ -47,7 +48,7 @@ func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath strin } } -func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, ch chan *restic.Node) { +func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *restic.Tree, rootPath string, ch chan *restic.Node) { defer close(ch) for _, root := range tree.Nodes { @@ -58,7 +59,7 @@ func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, r } } -func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, ch chan *restic.Node) error { +func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, ch chan *restic.Node) error { select { case ch <- root: case <-ctx.Done(): @@ -70,28 +71,28 @@ func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, c return nil } - err := walker.Walk(ctx, repo, *root.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { - return false, err + return err } if node == nil { - return false, nil + return nil } node.Path = path.Join(root.Path, nodepath) if !IsFile(node) && !IsDir(node) && !IsLink(node) { - return false, nil + return nil } select { case ch <- node: case <-ctx.Done(): - return false, ctx.Err() + return ctx.Err() } - return false, nil - }) + return nil + }}) return err } @@ -103,27 +104,77 @@ func (d *Dumper) WriteNode(ctx context.Context, node *restic.Node) error { } func (d *Dumper) writeNode(ctx context.Context, w io.Writer, node *restic.Node) error { - var ( - buf []byte - err error - ) - for _, id := range node.Content { - blob, ok := d.cache.Get(id) - if !ok { - blob, err = d.repo.LoadBlob(ctx, restic.DataBlob, id, buf) - if err != nil { - return err + type loadTask struct { + id restic.ID + out chan<- []byte + } + type writeTask struct { + data <-chan []byte + } + + loaderCh := make(chan loadTask) + // per worker: allows for one blob that gets download + one blob thats queue for writing + writerCh := make(chan writeTask, d.repo.Connections()*2) + + wg, ctx := errgroup.WithContext(ctx) + + wg.Go(func() error { + defer close(loaderCh) + defer close(writerCh) + for _, id := range node.Content { + // non-blocking blob handover to allow the loader to load the next blob + // while the old one is still written + ch := make(chan []byte, 1) + select { + case loaderCh <- loadTask{id: id, out: ch}: + case <-ctx.Done(): + return ctx.Err() } - buf = d.cache.Add(id, blob) // Reuse evicted buffer. + select { + case writerCh <- writeTask{data: ch}: + case <-ctx.Done(): + return ctx.Err() + } } + return nil + }) - if _, err := w.Write(blob); err != nil { - return errors.Wrap(err, "Write") - } + for i := uint(0); i < d.repo.Connections(); i++ { + wg.Go(func() error { + for task := range loaderCh { + blob, err := d.cache.GetOrCompute(task.id, func() ([]byte, error) { + return d.repo.LoadBlob(ctx, restic.DataBlob, task.id, nil) + }) + if err != nil { + return err + } + + select { + case task.out <- blob: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) } - return nil + wg.Go(func() error { + for result := range writerCh { + select { + case data := <-result.data: + if _, err := w.Write(data); err != nil { + return errors.Wrap(err, "Write") + } + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + + return wg.Wait() } // IsDir checks if the given node is a directory. diff --git a/mover-restic/restic/internal/dump/common_test.go b/mover-restic/restic/internal/dump/common_test.go index 3ee9112af..afd19df63 100644 --- a/mover-restic/restic/internal/dump/common_test.go +++ b/mover-restic/restic/internal/dump/common_test.go @@ -78,7 +78,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) { back := rtest.Chdir(t, tmpdir) defer back() - sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{}) + sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{}) rtest.OK(t, err) tree, err := restic.LoadTree(ctx, repo, *sn.Tree) diff --git a/mover-restic/restic/internal/errors/errors.go b/mover-restic/restic/internal/errors/errors.go index 0327ea0da..ca36611eb 100644 --- a/mover-restic/restic/internal/errors/errors.go +++ b/mover-restic/restic/internal/errors/errors.go @@ -2,6 +2,7 @@ package errors import ( stderrors "errors" + "fmt" "github.com/pkg/errors" ) @@ -22,12 +23,48 @@ var Wrap = errors.Wrap // nil, Wrapf returns nil. var Wrapf = errors.Wrapf +// WithStack annotates err with a stack trace at the point WithStack was called. +// If err is nil, WithStack returns nil. var WithStack = errors.WithStack // Go 1.13-style error handling. +// As finds the first error in err's tree that matches target, and if one is found, +// sets target to that error value and returns true. Otherwise, it returns false. func As(err error, tgt interface{}) bool { return stderrors.As(err, tgt) } +// Is reports whether any error in err's tree matches target. func Is(x, y error) bool { return stderrors.Is(x, y) } +// Unwrap returns the result of calling the Unwrap method on err, if err's type contains +// an Unwrap method returning error. Otherwise, Unwrap returns nil. +// +// Unwrap only calls a method of the form "Unwrap() error". In particular Unwrap does not +// unwrap errors returned by [Join]. func Unwrap(err error) error { return stderrors.Unwrap(err) } + +// CombineErrors combines multiple errors into a single error after filtering out any nil values. +// If no errors are passed, it returns nil. +// If one error is passed, it simply returns that same error. +func CombineErrors(errors ...error) (err error) { + var combinedErrorMsg string + var multipleErrors bool + for _, errVal := range errors { + if errVal != nil { + if combinedErrorMsg != "" { + combinedErrorMsg += "; " // Separate error messages with a delimiter + multipleErrors = true + } else { + // Set the first error + err = errVal + } + combinedErrorMsg += errVal.Error() + } + } + if combinedErrorMsg == "" { + return nil // If no errors, return nil + } else if !multipleErrors { + return err // If only one error, return that first error + } + return fmt.Errorf("multiple errors occurred: [%s]", combinedErrorMsg) +} diff --git a/mover-restic/restic/internal/feature/features.go b/mover-restic/restic/internal/feature/features.go new file mode 100644 index 000000000..e3b625e92 --- /dev/null +++ b/mover-restic/restic/internal/feature/features.go @@ -0,0 +1,140 @@ +package feature + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +type state string +type FlagName string + +const ( + // Alpha features are disabled by default. They do not guarantee any backwards compatibility and may change in arbitrary ways between restic versions. + Alpha state = "alpha" + // Beta features are enabled by default. They may still change, but incompatible changes should be avoided. + Beta state = "beta" + // Stable features are always enabled + Stable state = "stable" + // Deprecated features are always disabled + Deprecated state = "deprecated" +) + +type FlagDesc struct { + Type state + Description string +} + +type FlagSet struct { + flags map[FlagName]*FlagDesc + enabled map[FlagName]bool +} + +func New() *FlagSet { + return &FlagSet{} +} + +func getDefault(phase state) bool { + switch phase { + case Alpha, Deprecated: + return false + case Beta, Stable: + return true + default: + panic("unknown feature phase") + } +} + +func (f *FlagSet) SetFlags(flags map[FlagName]FlagDesc) { + f.flags = map[FlagName]*FlagDesc{} + f.enabled = map[FlagName]bool{} + + for name, flag := range flags { + fcopy := flag + f.flags[name] = &fcopy + f.enabled[name] = getDefault(fcopy.Type) + } +} + +func (f *FlagSet) Apply(flags string, logWarning func(string)) error { + if flags == "" { + return nil + } + + selection := make(map[string]bool) + + for _, flag := range strings.Split(flags, ",") { + parts := strings.SplitN(flag, "=", 2) + + name := parts[0] + value := "true" + if len(parts) == 2 { + value = parts[1] + } + + isEnabled, err := strconv.ParseBool(value) + if err != nil { + return fmt.Errorf("failed to parse value %q for feature flag %v: %w", value, name, err) + } + + selection[name] = isEnabled + } + + for name, value := range selection { + fname := FlagName(name) + flag := f.flags[fname] + if flag == nil { + return fmt.Errorf("unknown feature flag %q", name) + } + + switch flag.Type { + case Alpha, Beta: + f.enabled[fname] = value + case Stable: + logWarning(fmt.Sprintf("feature flag %q is always enabled and will be removed in a future release", fname)) + case Deprecated: + logWarning(fmt.Sprintf("feature flag %q is always disabled and will be removed in a future release", fname)) + default: + panic("unknown feature phase") + } + } + + return nil +} + +func (f *FlagSet) Enabled(name FlagName) bool { + isEnabled, ok := f.enabled[name] + if !ok { + panic(fmt.Sprintf("unknown feature flag %v", name)) + } + + return isEnabled +} + +// Help contains information about a feature. +type Help struct { + Name string + Type string + Default bool + Description string +} + +func (f *FlagSet) List() []Help { + var help []Help + + for name, flag := range f.flags { + help = append(help, Help{ + Name: string(name), + Type: string(flag.Type), + Default: getDefault(flag.Type), + Description: flag.Description, + }) + } + + sort.Slice(help, func(i, j int) bool { + return strings.Compare(help[i].Name, help[j].Name) < 0 + }) + + return help +} diff --git a/mover-restic/restic/internal/feature/features_test.go b/mover-restic/restic/internal/feature/features_test.go new file mode 100644 index 000000000..f5d405fa7 --- /dev/null +++ b/mover-restic/restic/internal/feature/features_test.go @@ -0,0 +1,151 @@ +package feature_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/restic/restic/internal/feature" + rtest "github.com/restic/restic/internal/test" +) + +var ( + alpha = feature.FlagName("alpha-feature") + beta = feature.FlagName("beta-feature") + stable = feature.FlagName("stable-feature") + deprecated = feature.FlagName("deprecated-feature") +) + +var testFlags = map[feature.FlagName]feature.FlagDesc{ + alpha: { + Type: feature.Alpha, + Description: "alpha", + }, + beta: { + Type: feature.Beta, + Description: "beta", + }, + stable: { + Type: feature.Stable, + Description: "stable", + }, + deprecated: { + Type: feature.Deprecated, + Description: "deprecated", + }, +} + +func buildTestFlagSet() *feature.FlagSet { + flags := feature.New() + flags.SetFlags(testFlags) + return flags +} + +func TestFeatureDefaults(t *testing.T) { + flags := buildTestFlagSet() + for _, exp := range []struct { + flag feature.FlagName + value bool + }{ + {alpha, false}, + {beta, true}, + {stable, true}, + {deprecated, false}, + } { + rtest.Assert(t, flags.Enabled(exp.flag) == exp.value, "expected flag %v to have value %v got %v", exp.flag, exp.value, flags.Enabled(exp.flag)) + } +} + +func panicIfCalled(msg string) { + panic(msg) +} + +func TestEmptyApply(t *testing.T) { + flags := buildTestFlagSet() + rtest.OK(t, flags.Apply("", panicIfCalled)) + + rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled") + rtest.Assert(t, flags.Enabled(beta), "expected beta feature to be enabled") +} + +func TestFeatureApply(t *testing.T) { + flags := buildTestFlagSet() + rtest.OK(t, flags.Apply(string(alpha), panicIfCalled)) + rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled") + + rtest.OK(t, flags.Apply(fmt.Sprintf("%s=false", alpha), panicIfCalled)) + rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled") + + rtest.OK(t, flags.Apply(fmt.Sprintf("%s=true", alpha), panicIfCalled)) + rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled again") + + rtest.OK(t, flags.Apply(fmt.Sprintf("%s=false", beta), panicIfCalled)) + rtest.Assert(t, !flags.Enabled(beta), "expected beta feature to be disabled") + + logMsg := "" + log := func(msg string) { + logMsg = msg + } + + rtest.OK(t, flags.Apply(fmt.Sprintf("%s=false", stable), log)) + rtest.Assert(t, flags.Enabled(stable), "expected stable feature to remain enabled") + rtest.Assert(t, strings.Contains(logMsg, string(stable)), "unexpected log message for stable flag: %v", logMsg) + + logMsg = "" + rtest.OK(t, flags.Apply(fmt.Sprintf("%s=true", deprecated), log)) + rtest.Assert(t, !flags.Enabled(deprecated), "expected deprecated feature to remain disabled") + rtest.Assert(t, strings.Contains(logMsg, string(deprecated)), "unexpected log message for deprecated flag: %v", logMsg) +} + +func TestFeatureMultipleApply(t *testing.T) { + flags := buildTestFlagSet() + + rtest.OK(t, flags.Apply(fmt.Sprintf("%s=true,%s=false", alpha, beta), panicIfCalled)) + rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled") + rtest.Assert(t, !flags.Enabled(beta), "expected beta feature to be disabled") +} + +func TestFeatureApplyInvalid(t *testing.T) { + flags := buildTestFlagSet() + + err := flags.Apply("invalid-flag", panicIfCalled) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "unknown feature flag"), "expected unknown feature flag error, got: %v", err) + + err = flags.Apply(fmt.Sprintf("%v=invalid", alpha), panicIfCalled) + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "failed to parse value"), "expected parsing error, got: %v", err) +} + +func assertPanic(t *testing.T) { + if r := recover(); r == nil { + t.Fatal("should have panicked") + } +} + +func TestFeatureQueryInvalid(t *testing.T) { + defer assertPanic(t) + + flags := buildTestFlagSet() + flags.Enabled("invalid-flag") +} + +func TestFeatureSetInvalidPhase(t *testing.T) { + defer assertPanic(t) + + flags := feature.New() + flags.SetFlags(map[feature.FlagName]feature.FlagDesc{ + "invalid": { + Type: "invalid", + }, + }) +} + +func TestFeatureList(t *testing.T) { + flags := buildTestFlagSet() + + rtest.Equals(t, []feature.Help{ + {string(alpha), string(feature.Alpha), false, "alpha"}, + {string(beta), string(feature.Beta), true, "beta"}, + {string(deprecated), string(feature.Deprecated), false, "deprecated"}, + {string(stable), string(feature.Stable), true, "stable"}, + }, flags.List()) +} diff --git a/mover-restic/restic/internal/feature/registry.go b/mover-restic/restic/internal/feature/registry.go new file mode 100644 index 000000000..6b8f6b397 --- /dev/null +++ b/mover-restic/restic/internal/feature/registry.go @@ -0,0 +1,25 @@ +package feature + +// Flag is named such that checking for a feature uses `feature.Flag.Enabled(feature.ExampleFeature)`. +var Flag = New() + +// flag names are written in kebab-case +const ( + BackendErrorRedesign FlagName = "backend-error-redesign" + DeprecateLegacyIndex FlagName = "deprecate-legacy-index" + DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout" + DeviceIDForHardlinks FlagName = "device-id-for-hardlinks" + ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth" + SafeForgetKeepTags FlagName = "safe-forget-keep-tags" +) + +func init() { + Flag.SetFlags(map[FlagName]FlagDesc{ + BackendErrorRedesign: {Type: Beta, Description: "enforce timeouts for stuck HTTP requests and use new backend error handling design."}, + DeprecateLegacyIndex: {Type: Beta, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."}, + DeprecateS3LegacyLayout: {Type: Beta, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your S3 repository if necessary."}, + DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"}, + ExplicitS3AnonymousAuth: {Type: Beta, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"}, + SafeForgetKeepTags: {Type: Beta, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"}, + }) +} diff --git a/mover-restic/restic/internal/feature/testing.go b/mover-restic/restic/internal/feature/testing.go new file mode 100644 index 000000000..b796e89b5 --- /dev/null +++ b/mover-restic/restic/internal/feature/testing.go @@ -0,0 +1,33 @@ +package feature + +import ( + "fmt" + "testing" +) + +// TestSetFlag temporarily sets a feature flag to the given value until the +// returned function is called. +// +// Usage +// ``` +// defer TestSetFlag(t, features.Flags, features.ExampleFlag, true)() +// ``` +func TestSetFlag(t *testing.T, f *FlagSet, flag FlagName, value bool) func() { + current := f.Enabled(flag) + + panicIfCalled := func(msg string) { + panic(msg) + } + + if err := f.Apply(fmt.Sprintf("%s=%v", flag, value), panicIfCalled); err != nil { + // not reachable + panic(err) + } + + return func() { + if err := f.Apply(fmt.Sprintf("%s=%v", flag, current), panicIfCalled); err != nil { + // not reachable + panic(err) + } + } +} diff --git a/mover-restic/restic/internal/feature/testing_test.go b/mover-restic/restic/internal/feature/testing_test.go new file mode 100644 index 000000000..f11b4bae4 --- /dev/null +++ b/mover-restic/restic/internal/feature/testing_test.go @@ -0,0 +1,19 @@ +package feature_test + +import ( + "testing" + + "github.com/restic/restic/internal/feature" + rtest "github.com/restic/restic/internal/test" +) + +func TestSetFeatureFlag(t *testing.T) { + flags := buildTestFlagSet() + rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled") + + restore := feature.TestSetFlag(t, flags, alpha, true) + rtest.Assert(t, flags.Enabled(alpha), "expected alpha feature to be enabled") + + restore() + rtest.Assert(t, !flags.Enabled(alpha), "expected alpha feature to be disabled again") +} diff --git a/mover-restic/restic/internal/fs/ea_windows.go b/mover-restic/restic/internal/fs/ea_windows.go new file mode 100644 index 000000000..08466c33f --- /dev/null +++ b/mover-restic/restic/internal/fs/ea_windows.go @@ -0,0 +1,285 @@ +//go:build windows +// +build windows + +package fs + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +// The code below was adapted from https://github.com/microsoft/go-winio under MIT license. + +// The MIT License (MIT) + +// Copyright (c) 2015 Microsoft + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// The code below was copied over from https://github.com/microsoft/go-winio/blob/main/ea.go under MIT license. + +type fileFullEaInformation struct { + NextEntryOffset uint32 + Flags uint8 + NameLength uint8 + ValueLength uint16 +} + +var ( + fileFullEaInformationSize = binary.Size(&fileFullEaInformation{}) + + errInvalidEaBuffer = errors.New("invalid extended attribute buffer") + errEaNameTooLarge = errors.New("extended attribute name too large") + errEaValueTooLarge = errors.New("extended attribute value too large") +) + +// ExtendedAttribute represents a single Windows EA. +type ExtendedAttribute struct { + Name string + Value []byte + Flags uint8 +} + +func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) { + var info fileFullEaInformation + err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info) + if err != nil { + err = errInvalidEaBuffer + return ea, nb, err + } + + nameOffset := fileFullEaInformationSize + nameLen := int(info.NameLength) + valueOffset := nameOffset + int(info.NameLength) + 1 + valueLen := int(info.ValueLength) + nextOffset := int(info.NextEntryOffset) + if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) { + err = errInvalidEaBuffer + return ea, nb, err + } + + ea.Name = string(b[nameOffset : nameOffset+nameLen]) + ea.Value = b[valueOffset : valueOffset+valueLen] + ea.Flags = info.Flags + if info.NextEntryOffset != 0 { + nb = b[info.NextEntryOffset:] + } + return ea, nb, err +} + +// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION +// buffer retrieved from BackupRead, ZwQueryEaFile, etc. +func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) { + for len(b) != 0 { + ea, nb, err := parseEa(b) + if err != nil { + return nil, err + } + + eas = append(eas, ea) + b = nb + } + return eas, err +} + +func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error { + if int(uint8(len(ea.Name))) != len(ea.Name) { + return errEaNameTooLarge + } + if int(uint16(len(ea.Value))) != len(ea.Value) { + return errEaValueTooLarge + } + entrySize := uint32(fileFullEaInformationSize + len(ea.Name) + 1 + len(ea.Value)) + withPadding := (entrySize + 3) &^ 3 + nextOffset := uint32(0) + if !last { + nextOffset = withPadding + } + info := fileFullEaInformation{ + NextEntryOffset: nextOffset, + Flags: ea.Flags, + NameLength: uint8(len(ea.Name)), + ValueLength: uint16(len(ea.Value)), + } + + err := binary.Write(buf, binary.LittleEndian, &info) + if err != nil { + return err + } + + _, err = buf.Write([]byte(ea.Name)) + if err != nil { + return err + } + + err = buf.WriteByte(0) + if err != nil { + return err + } + + _, err = buf.Write(ea.Value) + if err != nil { + return err + } + + _, err = buf.Write([]byte{0, 0, 0}[0 : withPadding-entrySize]) + if err != nil { + return err + } + + return nil +} + +// EncodeExtendedAttributes encodes a list of EAs into a FILE_FULL_EA_INFORMATION +// buffer for use with BackupWrite, ZwSetEaFile, etc. +func EncodeExtendedAttributes(eas []ExtendedAttribute) ([]byte, error) { + var buf bytes.Buffer + for i := range eas { + last := false + if i == len(eas)-1 { + last = true + } + + err := writeEa(&buf, &eas[i], last) + if err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// The code below was copied over from https://github.com/microsoft/go-winio/blob/main/pipe.go under MIT license. + +type ntStatus int32 + +func (status ntStatus) Err() error { + if status >= 0 { + return nil + } + return rtlNtStatusToDosError(status) +} + +// The code below was copied over from https://github.com/microsoft/go-winio/blob/main/zsyscall_windows.go under MIT license. + +// ioStatusBlock represents the IO_STATUS_BLOCK struct defined here: +// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_io_status_block +type ioStatusBlock struct { + Status, Information uintptr +} + +var ( + modntdll = windows.NewLazySystemDLL("ntdll.dll") + procRtlNtStatusToDosErrorNoTeb = modntdll.NewProc("RtlNtStatusToDosErrorNoTeb") +) + +func rtlNtStatusToDosError(status ntStatus) (winerr error) { + r0, _, _ := syscall.SyscallN(procRtlNtStatusToDosErrorNoTeb.Addr(), uintptr(status)) + if r0 != 0 { + winerr = syscall.Errno(r0) + } + return +} + +// The code below was adapted from https://github.com/ambarve/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea.go +// under MIT license. + +var ( + procNtQueryEaFile = modntdll.NewProc("NtQueryEaFile") + procNtSetEaFile = modntdll.NewProc("NtSetEaFile") +) + +const ( + // STATUS_NO_EAS_ON_FILE is a constant value which indicates EAs were requested for the file but it has no EAs. + // Windows NTSTATUS value: STATUS_NO_EAS_ON_FILE=0xC0000052 + STATUS_NO_EAS_ON_FILE = -1073741742 +) + +// GetFileEA retrieves the extended attributes for the file represented by `handle`. The +// `handle` must have been opened with file access flag FILE_READ_EA (0x8). +// The extended file attribute names in windows are case-insensitive and when fetching +// the attributes the names are generally returned in UPPER case. +func GetFileEA(handle windows.Handle) ([]ExtendedAttribute, error) { + // default buffer size to start with + bufLen := 1024 + buf := make([]byte, bufLen) + var iosb ioStatusBlock + // keep increasing the buffer size until it is large enough + for { + status := getFileEA(handle, &iosb, &buf[0], uint32(bufLen), false, 0, 0, nil, true) + + if status == STATUS_NO_EAS_ON_FILE { + //If status is -1073741742, no extended attributes were found + return nil, nil + } + err := status.Err() + if err != nil { + // convert ntstatus code to windows error + if err == windows.ERROR_INSUFFICIENT_BUFFER || err == windows.ERROR_MORE_DATA { + bufLen *= 2 + buf = make([]byte, bufLen) + continue + } + return nil, fmt.Errorf("get file EA failed with: %w", err) + } + break + } + return DecodeExtendedAttributes(buf) +} + +// SetFileEA sets the extended attributes for the file represented by `handle`. The +// handle must have been opened with the file access flag FILE_WRITE_EA(0x10). +func SetFileEA(handle windows.Handle, attrs []ExtendedAttribute) error { + encodedEA, err := EncodeExtendedAttributes(attrs) + if err != nil { + return fmt.Errorf("failed to encoded extended attributes: %w", err) + } + + var iosb ioStatusBlock + + return setFileEA(handle, &iosb, &encodedEA[0], uint32(len(encodedEA))).Err() +} + +// The code below was adapted from https://github.com/ambarve/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/zsyscall_windows.go +// under MIT license. + +func getFileEA(handle windows.Handle, iosb *ioStatusBlock, buf *uint8, bufLen uint32, returnSingleEntry bool, eaList uintptr, eaListLen uint32, eaIndex *uint32, restartScan bool) (status ntStatus) { + var _p0 uint32 + if returnSingleEntry { + _p0 = 1 + } + var _p1 uint32 + if restartScan { + _p1 = 1 + } + r0, _, _ := syscall.SyscallN(procNtQueryEaFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(buf)), uintptr(bufLen), uintptr(_p0), uintptr(eaList), uintptr(eaListLen), uintptr(unsafe.Pointer(eaIndex)), uintptr(_p1)) + status = ntStatus(r0) + return +} + +func setFileEA(handle windows.Handle, iosb *ioStatusBlock, buf *uint8, bufLen uint32) (status ntStatus) { + r0, _, _ := syscall.SyscallN(procNtSetEaFile.Addr(), uintptr(handle), uintptr(unsafe.Pointer(iosb)), uintptr(unsafe.Pointer(buf)), uintptr(bufLen)) + status = ntStatus(r0) + return +} diff --git a/mover-restic/restic/internal/fs/ea_windows_test.go b/mover-restic/restic/internal/fs/ea_windows_test.go new file mode 100644 index 000000000..b249f43c4 --- /dev/null +++ b/mover-restic/restic/internal/fs/ea_windows_test.go @@ -0,0 +1,247 @@ +//go:build windows +// +build windows + +package fs + +import ( + "crypto/rand" + "fmt" + "math/big" + "os" + "path/filepath" + "reflect" + "syscall" + "testing" + "unsafe" + + "golang.org/x/sys/windows" +) + +// The code below was adapted from github.com/Microsoft/go-winio under MIT license. + +// The MIT License (MIT) + +// Copyright (c) 2015 Microsoft + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +// The code below was adapted from https://github.com/ambarve/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go +// under MIT license. + +var ( + testEas = []ExtendedAttribute{ + {Name: "foo", Value: []byte("bar")}, + {Name: "fizz", Value: []byte("buzz")}, + } + + testEasEncoded = []byte{16, 0, 0, 0, 0, 3, 3, 0, 102, 111, 111, 0, 98, 97, 114, 0, 0, + 0, 0, 0, 0, 4, 4, 0, 102, 105, 122, 122, 0, 98, 117, 122, 122, 0, 0, 0} + testEasNotPadded = testEasEncoded[0 : len(testEasEncoded)-3] + testEasTruncated = testEasEncoded[0:20] +) + +func TestRoundTripEas(t *testing.T) { + b, err := EncodeExtendedAttributes(testEas) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEasEncoded, b) { + t.Fatalf("Encoded mismatch %v %v", testEasEncoded, b) + } + eas, err := DecodeExtendedAttributes(b) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEas, eas) { + t.Fatalf("mismatch %+v %+v", testEas, eas) + } +} + +func TestEasDontNeedPaddingAtEnd(t *testing.T) { + eas, err := DecodeExtendedAttributes(testEasNotPadded) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(testEas, eas) { + t.Fatalf("mismatch %+v %+v", testEas, eas) + } +} + +func TestTruncatedEasFailCorrectly(t *testing.T) { + _, err := DecodeExtendedAttributes(testEasTruncated) + if err == nil { + t.Fatal("expected error") + } +} + +func TestNilEasEncodeAndDecodeAsNil(t *testing.T) { + b, err := EncodeExtendedAttributes(nil) + if err != nil { + t.Fatal(err) + } + if len(b) != 0 { + t.Fatal("expected empty") + } + eas, err := DecodeExtendedAttributes(nil) + if err != nil { + t.Fatal(err) + } + if len(eas) != 0 { + t.Fatal("expected empty") + } +} + +// TestSetFileEa makes sure that the test buffer is actually parsable by NtSetEaFile. +func TestSetFileEa(t *testing.T) { + f, err := os.CreateTemp("", "testea") + if err != nil { + t.Fatal(err) + } + defer func() { + err := os.Remove(f.Name()) + if err != nil { + t.Logf("Error removing file %s: %v\n", f.Name(), err) + } + err = f.Close() + if err != nil { + t.Logf("Error closing file %s: %v\n", f.Name(), err) + } + }() + ntdll := syscall.MustLoadDLL("ntdll.dll") + ntSetEaFile := ntdll.MustFindProc("NtSetEaFile") + var iosb [2]uintptr + r, _, _ := ntSetEaFile.Call(f.Fd(), + uintptr(unsafe.Pointer(&iosb[0])), + uintptr(unsafe.Pointer(&testEasEncoded[0])), + uintptr(len(testEasEncoded))) + if r != 0 { + t.Fatalf("NtSetEaFile failed with %08x", r) + } +} + +// The code below was refactored from github.com/Microsoft/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go +// under MIT license. +func TestSetGetFileEA(t *testing.T) { + testFilePath, testFile := setupTestFile(t) + testEAs := generateTestEAs(t, 3, testFilePath) + fileHandle := openFile(t, testFilePath, windows.FILE_ATTRIBUTE_NORMAL) + defer closeFileHandle(t, testFilePath, testFile, fileHandle) + + testSetGetEA(t, testFilePath, fileHandle, testEAs) +} + +// The code is new code and reuses code refactored from github.com/Microsoft/go-winio/blob/a7564fd482feb903f9562a135f1317fd3b480739/ea_test.go +// under MIT license. +func TestSetGetFolderEA(t *testing.T) { + testFolderPath := setupTestFolder(t) + + testEAs := generateTestEAs(t, 3, testFolderPath) + fileHandle := openFile(t, testFolderPath, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS) + defer closeFileHandle(t, testFolderPath, nil, fileHandle) + + testSetGetEA(t, testFolderPath, fileHandle, testEAs) +} + +func setupTestFile(t *testing.T) (testFilePath string, testFile *os.File) { + tempDir := t.TempDir() + testFilePath = filepath.Join(tempDir, "testfile.txt") + var err error + if testFile, err = os.Create(testFilePath); err != nil { + t.Fatalf("failed to create temporary file: %s", err) + } + return testFilePath, testFile +} + +func setupTestFolder(t *testing.T) string { + tempDir := t.TempDir() + testfolderPath := filepath.Join(tempDir, "testfolder") + if err := os.Mkdir(testfolderPath, os.ModeDir); err != nil { + t.Fatalf("failed to create temporary folder: %s", err) + } + return testfolderPath +} + +func generateTestEAs(t *testing.T, nAttrs int, path string) []ExtendedAttribute { + testEAs := make([]ExtendedAttribute, nAttrs) + for i := 0; i < nAttrs; i++ { + testEAs[i].Name = fmt.Sprintf("TESTEA%d", i+1) + testEAs[i].Value = make([]byte, getRandomInt()) + if _, err := rand.Read(testEAs[i].Value); err != nil { + t.Logf("Error reading rand for path %s: %v\n", path, err) + } + } + return testEAs +} + +func getRandomInt() int64 { + nBig, err := rand.Int(rand.Reader, big.NewInt(27)) + if err != nil { + panic(err) + } + n := nBig.Int64() + if n == 0 { + n = getRandomInt() + } + return n +} + +func openFile(t *testing.T, path string, attributes uint32) windows.Handle { + utf16Path := windows.StringToUTF16Ptr(path) + fileAccessRightReadWriteEA := uint32(0x8 | 0x10) + fileHandle, err := windows.CreateFile(utf16Path, fileAccessRightReadWriteEA, 0, nil, windows.OPEN_EXISTING, attributes, 0) + if err != nil { + t.Fatalf("open file failed with: %s", err) + } + return fileHandle +} + +func closeFileHandle(t *testing.T, testfilePath string, testFile *os.File, handle windows.Handle) { + if testFile != nil { + err := testFile.Close() + if err != nil { + t.Logf("Error closing file %s: %v\n", testFile.Name(), err) + } + } + if err := windows.Close(handle); err != nil { + t.Logf("Error closing file handle %s: %v\n", testfilePath, err) + } + cleanupTestFile(t, testfilePath) +} + +func cleanupTestFile(t *testing.T, path string) { + if err := os.Remove(path); err != nil { + t.Logf("Error removing file/folder %s: %v\n", path, err) + } +} + +func testSetGetEA(t *testing.T, path string, handle windows.Handle, testEAs []ExtendedAttribute) { + if err := SetFileEA(handle, testEAs); err != nil { + t.Fatalf("set EA for path %s failed: %s", path, err) + } + + readEAs, err := GetFileEA(handle) + if err != nil { + t.Fatalf("get EA for path %s failed: %s", path, err) + } + + if !reflect.DeepEqual(readEAs, testEAs) { + t.Logf("expected: %+v, found: %+v\n", testEAs, readEAs) + t.Fatalf("EAs read from path %s don't match", path) + } +} diff --git a/mover-restic/restic/internal/fs/file.go b/mover-restic/restic/internal/fs/file.go index f35901c06..929195f1c 100644 --- a/mover-restic/restic/internal/fs/file.go +++ b/mover-restic/restic/internal/fs/file.go @@ -1,6 +1,7 @@ package fs import ( + "fmt" "os" "path/filepath" "time" @@ -124,3 +125,38 @@ func RemoveIfExists(filename string) error { func Chtimes(name string, atime time.Time, mtime time.Time) error { return os.Chtimes(fixpath(name), atime, mtime) } + +// IsAccessDenied checks if the error is due to permission error. +func IsAccessDenied(err error) bool { + return os.IsPermission(err) +} + +// ResetPermissions resets the permissions of the file at the specified path +func ResetPermissions(path string) error { + // Set the default file permissions + if err := os.Chmod(path, 0600); err != nil { + return err + } + return nil +} + +// Readdirnames returns a list of file in a directory. Flags are passed to fs.OpenFile. O_RDONLY is implied. +func Readdirnames(filesystem FS, dir string, flags int) ([]string, error) { + f, err := filesystem.OpenFile(dir, O_RDONLY|flags, 0) + if err != nil { + return nil, fmt.Errorf("openfile for readdirnames failed: %w", err) + } + + entries, err := f.Readdirnames(-1) + if err != nil { + _ = f.Close() + return nil, fmt.Errorf("readdirnames %v failed: %w", dir, err) + } + + err = f.Close() + if err != nil { + return nil, err + } + + return entries, nil +} diff --git a/mover-restic/restic/internal/fs/file_unix.go b/mover-restic/restic/internal/fs/file_unix.go index 65f10c844..b562d15b1 100644 --- a/mover-restic/restic/internal/fs/file_unix.go +++ b/mover-restic/restic/internal/fs/file_unix.go @@ -29,7 +29,7 @@ func TempFile(dir, prefix string) (f *os.File, err error) { return f, nil } -// isNotSuported returns true if the error is caused by an unsupported file system feature. +// isNotSupported returns true if the error is caused by an unsupported file system feature. func isNotSupported(err error) bool { if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ENOTSUP { return true diff --git a/mover-restic/restic/internal/fs/file_windows.go b/mover-restic/restic/internal/fs/file_windows.go index d19a744e1..b05068c42 100644 --- a/mover-restic/restic/internal/fs/file_windows.go +++ b/mover-restic/restic/internal/fs/file_windows.go @@ -77,3 +77,50 @@ func TempFile(dir, prefix string) (f *os.File, err error) { func Chmod(name string, mode os.FileMode) error { return os.Chmod(fixpath(name), mode) } + +// ClearSystem removes the system attribute from the file. +func ClearSystem(path string) error { + return ClearAttribute(path, windows.FILE_ATTRIBUTE_SYSTEM) +} + +// ClearAttribute removes the specified attribute from the file. +func ClearAttribute(path string, attribute uint32) error { + ptr, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + fileAttributes, err := windows.GetFileAttributes(ptr) + if err != nil { + return err + } + if fileAttributes&attribute != 0 { + // Clear the attribute + fileAttributes &= ^uint32(attribute) + err = windows.SetFileAttributes(ptr, fileAttributes) + if err != nil { + return err + } + } + return nil +} + +// OpenHandleForEA return a file handle for file or dir for setting/getting EAs +func OpenHandleForEA(nodeType, path string, writeAccess bool) (handle windows.Handle, err error) { + path = fixpath(path) + fileAccess := windows.FILE_READ_EA + if writeAccess { + fileAccess = fileAccess | windows.FILE_WRITE_EA + } + + switch nodeType { + case "file": + utf16Path := windows.StringToUTF16Ptr(path) + handle, err = windows.CreateFile(utf16Path, uint32(fileAccess), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + case "dir": + utf16Path := windows.StringToUTF16Ptr(path) + handle, err = windows.CreateFile(utf16Path, uint32(fileAccess), 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + default: + return 0, nil + } + return handle, err +} diff --git a/mover-restic/restic/internal/fs/fs_local_vss.go b/mover-restic/restic/internal/fs/fs_local_vss.go index aa3522aea..718dfc46d 100644 --- a/mover-restic/restic/internal/fs/fs_local_vss.go +++ b/mover-restic/restic/internal/fs/fs_local_vss.go @@ -3,41 +3,108 @@ package fs import ( "os" "path/filepath" + "runtime" "strings" "sync" + "time" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/options" ) -// ErrorHandler is used to report errors via callback -type ErrorHandler func(item string, err error) error +// VSSConfig holds extended options of windows volume shadow copy service. +type VSSConfig struct { + ExcludeAllMountPoints bool `option:"exclude-all-mount-points" help:"exclude mountpoints from snapshotting on all volumes"` + ExcludeVolumes string `option:"exclude-volumes" help:"semicolon separated list of volumes to exclude from snapshotting (ex. 'c:\\;e:\\mnt;\\\\?\\Volume{...}')"` + Timeout time.Duration `option:"timeout" help:"time that the VSS can spend creating snapshot before timing out"` + Provider string `option:"provider" help:"VSS provider identifier which will be used for snapshotting"` +} + +func init() { + if runtime.GOOS == "windows" { + options.Register("vss", VSSConfig{}) + } +} + +// NewVSSConfig returns a new VSSConfig with the default values filled in. +func NewVSSConfig() VSSConfig { + return VSSConfig{ + Timeout: time.Second * 120, + } +} + +// ParseVSSConfig parses a VSS extended options to VSSConfig struct. +func ParseVSSConfig(o options.Options) (VSSConfig, error) { + cfg := NewVSSConfig() + o = o.Extract("vss") + if err := o.Apply("vss", &cfg); err != nil { + return VSSConfig{}, err + } + + return cfg, nil +} + +// ErrorHandler is used to report errors via callback. +type ErrorHandler func(item string, err error) // MessageHandler is used to report errors/messages via callbacks. type MessageHandler func(msg string, args ...interface{}) +// VolumeFilter is used to filter volumes by it's mount point or GUID path. +type VolumeFilter func(volume string) bool + // LocalVss is a wrapper around the local file system which uses windows volume // shadow copy service (VSS) in a transparent way. type LocalVss struct { FS - snapshots map[string]VssSnapshot - failedSnapshots map[string]struct{} - mutex sync.RWMutex - msgError ErrorHandler - msgMessage MessageHandler + snapshots map[string]VssSnapshot + failedSnapshots map[string]struct{} + mutex sync.RWMutex + msgError ErrorHandler + msgMessage MessageHandler + excludeAllMountPoints bool + excludeVolumes map[string]struct{} + timeout time.Duration + provider string } // statically ensure that LocalVss implements FS. var _ FS = &LocalVss{} +// parseMountPoints try to convert semicolon separated list of mount points +// to map of lowercased volume GUID paths. Mountpoints already in volume +// GUID path format will be validated and normalized. +func parseMountPoints(list string, msgError ErrorHandler) (volumes map[string]struct{}) { + if list == "" { + return + } + for _, s := range strings.Split(list, ";") { + if v, err := GetVolumeNameForVolumeMountPoint(s); err != nil { + msgError(s, errors.Errorf("failed to parse vss.exclude-volumes [%s]: %s", s, err)) + } else { + if volumes == nil { + volumes = make(map[string]struct{}) + } + volumes[strings.ToLower(v)] = struct{}{} + } + } + + return +} + // NewLocalVss creates a new wrapper around the windows filesystem using volume // shadow copy service to access locked files. -func NewLocalVss(msgError ErrorHandler, msgMessage MessageHandler) *LocalVss { +func NewLocalVss(msgError ErrorHandler, msgMessage MessageHandler, cfg VSSConfig) *LocalVss { return &LocalVss{ - FS: Local{}, - snapshots: make(map[string]VssSnapshot), - failedSnapshots: make(map[string]struct{}), - msgError: msgError, - msgMessage: msgMessage, + FS: Local{}, + snapshots: make(map[string]VssSnapshot), + failedSnapshots: make(map[string]struct{}), + msgError: msgError, + msgMessage: msgMessage, + excludeAllMountPoints: cfg.ExcludeAllMountPoints, + excludeVolumes: parseMountPoints(cfg.ExcludeVolumes, msgError), + timeout: cfg.Timeout, + provider: cfg.Provider, } } @@ -50,7 +117,7 @@ func (fs *LocalVss) DeleteSnapshots() { for volumeName, snapshot := range fs.snapshots { if err := snapshot.Delete(); err != nil { - _ = fs.msgError(volumeName, errors.Errorf("failed to delete VSS snapshot: %s", err)) + fs.msgError(volumeName, errors.Errorf("failed to delete VSS snapshot: %s", err)) activeSnapshots[volumeName] = snapshot } } @@ -78,12 +145,27 @@ func (fs *LocalVss) Lstat(name string) (os.FileInfo, error) { return os.Lstat(fs.snapshotPath(name)) } +// isMountPointIncluded is true if given mountpoint included by user. +func (fs *LocalVss) isMountPointIncluded(mountPoint string) bool { + if fs.excludeVolumes == nil { + return true + } + + volume, err := GetVolumeNameForVolumeMountPoint(mountPoint) + if err != nil { + fs.msgError(mountPoint, errors.Errorf("failed to get volume from mount point [%s]: %s", mountPoint, err)) + return true + } + + _, ok := fs.excludeVolumes[strings.ToLower(volume)] + return !ok +} + // snapshotPath returns the path inside a VSS snapshots if it already exists. // If the path is not yet available as a snapshot, a snapshot is created. // If creation of a snapshot fails the file's original path is returned as // a fallback. func (fs *LocalVss) snapshotPath(path string) string { - fixPath := fixpath(path) if strings.HasPrefix(fixPath, `\\?\UNC\`) { @@ -114,23 +196,36 @@ func (fs *LocalVss) snapshotPath(path string) string { if !snapshotExists && !snapshotFailed { vssVolume := volumeNameLower + string(filepath.Separator) - fs.msgMessage("creating VSS snapshot for [%s]\n", vssVolume) - if snapshot, err := NewVssSnapshot(vssVolume, 120, fs.msgError); err != nil { - _ = fs.msgError(vssVolume, errors.Errorf("failed to create snapshot for [%s]: %s", - vssVolume, err)) + if !fs.isMountPointIncluded(vssVolume) { + fs.msgMessage("snapshots for [%s] excluded by user\n", vssVolume) fs.failedSnapshots[volumeNameLower] = struct{}{} } else { - fs.snapshots[volumeNameLower] = snapshot - fs.msgMessage("successfully created snapshot for [%s]\n", vssVolume) - if len(snapshot.mountPointInfo) > 0 { - fs.msgMessage("mountpoints in snapshot volume [%s]:\n", vssVolume) - for mp, mpInfo := range snapshot.mountPointInfo { - info := "" - if !mpInfo.IsSnapshotted() { - info = " (not snapshotted)" + fs.msgMessage("creating VSS snapshot for [%s]\n", vssVolume) + + var includeVolume VolumeFilter + if !fs.excludeAllMountPoints { + includeVolume = func(volume string) bool { + return fs.isMountPointIncluded(volume) + } + } + + if snapshot, err := NewVssSnapshot(fs.provider, vssVolume, fs.timeout, includeVolume, fs.msgError); err != nil { + fs.msgError(vssVolume, errors.Errorf("failed to create snapshot for [%s]: %s", + vssVolume, err)) + fs.failedSnapshots[volumeNameLower] = struct{}{} + } else { + fs.snapshots[volumeNameLower] = snapshot + fs.msgMessage("successfully created snapshot for [%s]\n", vssVolume) + if len(snapshot.mountPointInfo) > 0 { + fs.msgMessage("mountpoints in snapshot volume [%s]:\n", vssVolume) + for mp, mpInfo := range snapshot.mountPointInfo { + info := "" + if !mpInfo.IsSnapshotted() { + info = " (not snapshotted)" + } + fs.msgMessage(" - %s%s\n", mp, info) } - fs.msgMessage(" - %s%s\n", mp, info) } } } @@ -173,9 +268,8 @@ func (fs *LocalVss) snapshotPath(path string) string { snapshotPath = fs.Join(snapshot.GetSnapshotDeviceObject(), strings.TrimPrefix(fixPath, volumeName)) if snapshotPath == snapshot.GetSnapshotDeviceObject() { - snapshotPath = snapshotPath + string(filepath.Separator) + snapshotPath += string(filepath.Separator) } - } else { // no snapshot is available for the requested path: // -> try to backup without a snapshot diff --git a/mover-restic/restic/internal/fs/fs_local_vss_test.go b/mover-restic/restic/internal/fs/fs_local_vss_test.go new file mode 100644 index 000000000..60262c873 --- /dev/null +++ b/mover-restic/restic/internal/fs/fs_local_vss_test.go @@ -0,0 +1,285 @@ +// +build windows + +package fs + +import ( + "fmt" + "regexp" + "strings" + "testing" + "time" + + ole "github.com/go-ole/go-ole" + "github.com/restic/restic/internal/options" +) + +func matchStrings(ptrs []string, strs []string) bool { + if len(ptrs) != len(strs) { + return false + } + + for i, p := range ptrs { + if p == "" { + return false + } + matched, err := regexp.MatchString(p, strs[i]) + if err != nil { + panic(err) + } + if !matched { + return false + } + } + + return true +} + +func matchMap(strs []string, m map[string]struct{}) bool { + if len(strs) != len(m) { + return false + } + + for _, s := range strs { + if _, ok := m[s]; !ok { + return false + } + } + + return true +} + +func TestVSSConfig(t *testing.T) { + type config struct { + excludeAllMountPoints bool + timeout time.Duration + provider string + } + setTests := []struct { + input options.Options + output config + }{ + { + options.Options{ + "vss.timeout": "6h38m42s", + "vss.provider": "Ms", + }, + config{ + timeout: 23922000000000, + provider: "Ms", + }, + }, + { + options.Options{ + "vss.exclude-all-mount-points": "t", + "vss.provider": "{b5946137-7b9f-4925-af80-51abd60b20d5}", + }, + config{ + excludeAllMountPoints: true, + timeout: 120000000000, + provider: "{b5946137-7b9f-4925-af80-51abd60b20d5}", + }, + }, + { + options.Options{ + "vss.exclude-all-mount-points": "0", + "vss.exclude-volumes": "", + "vss.timeout": "120s", + "vss.provider": "Microsoft Software Shadow Copy provider 1.0", + }, + config{ + timeout: 120000000000, + provider: "Microsoft Software Shadow Copy provider 1.0", + }, + }, + } + for i, test := range setTests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + cfg, err := ParseVSSConfig(test.input) + if err != nil { + t.Fatal(err) + } + + errorHandler := func(item string, err error) { + t.Fatalf("unexpected error (%v)", err) + } + messageHandler := func(msg string, args ...interface{}) { + t.Fatalf("unexpected message (%s)", fmt.Sprintf(msg, args)) + } + + dst := NewLocalVss(errorHandler, messageHandler, cfg) + + if dst.excludeAllMountPoints != test.output.excludeAllMountPoints || + dst.excludeVolumes != nil || dst.timeout != test.output.timeout || + dst.provider != test.output.provider { + t.Fatalf("wrong result, want:\n %#v\ngot:\n %#v", test.output, dst) + } + }) + } +} + +func TestParseMountPoints(t *testing.T) { + volumeMatch := regexp.MustCompile(`^\\\\\?\\Volume\{[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\}\\$`) + + // It's not a good idea to test functions based on GetVolumeNameForVolumeMountPoint by calling + // GetVolumeNameForVolumeMountPoint itself, but we have restricted test environment: + // cannot manage volumes and can only be sure that the mount point C:\ exists + sysVolume, err := GetVolumeNameForVolumeMountPoint("C:") + if err != nil { + t.Fatal(err) + } + // We don't know a valid volume GUID path for c:\, but we'll at least check its format + if !volumeMatch.MatchString(sysVolume) { + t.Fatalf("invalid volume GUID path: %s", sysVolume) + } + // Changing the case and removing trailing backslash allows tests + // the equality of different ways of writing a volume name + sysVolumeMutated := strings.ToUpper(sysVolume[:len(sysVolume)-1]) + sysVolumeMatch := strings.ToLower(sysVolume) + + type check struct { + volume string + result bool + } + setTests := []struct { + input options.Options + output []string + checks []check + errors []string + }{ + { + options.Options{ + "vss.exclude-volumes": `c:;c:\;` + sysVolume + `;` + sysVolumeMutated, + }, + []string{ + sysVolumeMatch, + }, + []check{ + {`c:\`, false}, + {`c:`, false}, + {sysVolume, false}, + {sysVolumeMutated, false}, + }, + []string{}, + }, + { + options.Options{ + "vss.exclude-volumes": `z:\nonexistent;c:;c:\windows\;\\?\Volume{39b9cac2-bcdb-4d51-97c8-0d0677d607fb}\`, + }, + []string{ + sysVolumeMatch, + }, + []check{ + {`c:\windows\`, true}, + {`\\?\Volume{39b9cac2-bcdb-4d51-97c8-0d0677d607fb}\`, true}, + {`c:`, false}, + {``, true}, + }, + []string{ + `failed to parse vss\.exclude-volumes \[z:\\nonexistent\]:.*`, + `failed to parse vss\.exclude-volumes \[c:\\windows\\\]:.*`, + `failed to parse vss\.exclude-volumes \[\\\\\?\\Volume\{39b9cac2-bcdb-4d51-97c8-0d0677d607fb\}\\\]:.*`, + `failed to get volume from mount point \[c:\\windows\\\]:.*`, + `failed to get volume from mount point \[\\\\\?\\Volume\{39b9cac2-bcdb-4d51-97c8-0d0677d607fb\}\\\]:.*`, + `failed to get volume from mount point \[\]:.*`, + }, + }, + } + + for i, test := range setTests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + cfg, err := ParseVSSConfig(test.input) + if err != nil { + t.Fatal(err) + } + + var log []string + errorHandler := func(item string, err error) { + log = append(log, strings.TrimSpace(err.Error())) + } + messageHandler := func(msg string, args ...interface{}) { + t.Fatalf("unexpected message (%s)", fmt.Sprintf(msg, args)) + } + + dst := NewLocalVss(errorHandler, messageHandler, cfg) + + if !matchMap(test.output, dst.excludeVolumes) { + t.Fatalf("wrong result, want:\n %#v\ngot:\n %#v", + test.output, dst.excludeVolumes) + } + + for _, c := range test.checks { + if dst.isMountPointIncluded(c.volume) != c.result { + t.Fatalf(`wrong check: isMountPointIncluded("%s") != %v`, c.volume, c.result) + } + } + + if !matchStrings(test.errors, log) { + t.Fatalf("wrong log, want:\n %#v\ngot:\n %#v", test.errors, log) + } + }) + } +} + +func TestParseProvider(t *testing.T) { + msProvider := ole.NewGUID("{b5946137-7b9f-4925-af80-51abd60b20d5}") + setTests := []struct { + provider string + id *ole.GUID + result string + }{ + { + "", + ole.IID_NULL, + "", + }, + { + "mS", + msProvider, + "", + }, + { + "{B5946137-7b9f-4925-Af80-51abD60b20d5}", + msProvider, + "", + }, + { + "Microsoft Software Shadow Copy provider 1.0", + msProvider, + "", + }, + { + "{04560982-3d7d-4bbc-84f7-0712f833a28f}", + nil, + `invalid VSS provider "{04560982-3d7d-4bbc-84f7-0712f833a28f}"`, + }, + { + "non-existent provider", + nil, + `invalid VSS provider "non-existent provider"`, + }, + } + + _ = ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED) + + for i, test := range setTests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + id, err := getProviderID(test.provider) + + if err != nil && id != nil { + t.Fatalf("err!=nil but id=%v", id) + } + + if test.result != "" || err != nil { + var result string + if err != nil { + result = err.Error() + } + if test.result != result || test.result == "" { + t.Fatalf("wrong result, want:\n %#v\ngot:\n %#v", test.result, result) + } + } else if !ole.IsEqualGUID(id, test.id) { + t.Fatalf("wrong id, want:\n %s\ngot:\n %s", test.id.String(), id.String()) + } + }) + } +} diff --git a/mover-restic/restic/internal/fs/fs_reader_command.go b/mover-restic/restic/internal/fs/fs_reader_command.go new file mode 100644 index 000000000..3830e5811 --- /dev/null +++ b/mover-restic/restic/internal/fs/fs_reader_command.go @@ -0,0 +1,97 @@ +package fs + +import ( + "bufio" + "context" + "fmt" + "io" + "os/exec" + + "github.com/restic/restic/internal/errors" +) + +// CommandReader wrap a command such that its standard output can be read using +// a io.ReadCloser. Close() waits for the command to terminate, reporting +// any error back to the caller. +type CommandReader struct { + cmd *exec.Cmd + stdout io.ReadCloser + + // cmd.Wait() must only be called once. Prevent duplicate executions in + // Read() and Close(). + waitHandled bool + + // alreadyClosedReadErr is the error that we should return if we try to + // read the pipe again after closing. This works around a Read() call that + // is issued after a previous Read() with `io.EOF` (but some bytes were + // read in the past). + alreadyClosedReadErr error +} + +func NewCommandReader(ctx context.Context, args []string, logOutput io.Writer) (*CommandReader, error) { + // Prepare command and stdout + command := exec.CommandContext(ctx, args[0], args[1:]...) + stdout, err := command.StdoutPipe() + if err != nil { + return nil, fmt.Errorf("failed to setup stdout pipe: %w", err) + } + + // Use a Go routine to handle the stderr to avoid deadlocks + stderr, err := command.StderrPipe() + if err != nil { + return nil, fmt.Errorf("failed to setup stderr pipe: %w", err) + } + go func() { + sc := bufio.NewScanner(stderr) + for sc.Scan() { + _, _ = fmt.Fprintf(logOutput, "subprocess %v: %v\n", command.Args[0], sc.Text()) + } + }() + + if err := command.Start(); err != nil { + return nil, fmt.Errorf("failed to start command: %w", err) + } + + return &CommandReader{ + cmd: command, + stdout: stdout, + }, nil +} + +// Read populate the array with data from the process stdout. +func (fp *CommandReader) Read(p []byte) (int, error) { + if fp.alreadyClosedReadErr != nil { + return 0, fp.alreadyClosedReadErr + } + b, err := fp.stdout.Read(p) + + // If the error is io.EOF, the program terminated. We need to check the + // exit code here because, if the program terminated with no output, the + // error in `Close()` is ignored. + if errors.Is(err, io.EOF) { + fp.waitHandled = true + // check if the command terminated successfully, If not return the error. + if errw := fp.wait(); errw != nil { + err = errw + } + } + fp.alreadyClosedReadErr = err + return b, err +} + +func (fp *CommandReader) wait() error { + err := fp.cmd.Wait() + if err != nil { + // Use a fatal error to abort the snapshot. + return errors.Fatal(fmt.Errorf("command failed: %w", err).Error()) + } + return nil +} + +func (fp *CommandReader) Close() error { + if fp.waitHandled { + return nil + } + + return fp.wait() +} diff --git a/mover-restic/restic/internal/fs/fs_reader_command_test.go b/mover-restic/restic/internal/fs/fs_reader_command_test.go new file mode 100644 index 000000000..a9028544c --- /dev/null +++ b/mover-restic/restic/internal/fs/fs_reader_command_test.go @@ -0,0 +1,48 @@ +package fs_test + +import ( + "bytes" + "context" + "io" + "strings" + "testing" + + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/test" +) + +func TestCommandReaderSuccess(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"true"}, io.Discard) + test.OK(t, err) + + _, err = io.Copy(io.Discard, reader) + test.OK(t, err) + + test.OK(t, reader.Close()) +} + +func TestCommandReaderFail(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"false"}, io.Discard) + test.OK(t, err) + + _, err = io.Copy(io.Discard, reader) + test.Assert(t, err != nil, "missing error") +} + +func TestCommandReaderInvalid(t *testing.T) { + _, err := fs.NewCommandReader(context.TODO(), []string{"w54fy098hj7fy5twijouytfrj098y645wr"}, io.Discard) + test.Assert(t, err != nil, "missing error") +} + +func TestCommandReaderOutput(t *testing.T) { + reader, err := fs.NewCommandReader(context.TODO(), []string{"echo", "hello world"}, io.Discard) + test.OK(t, err) + + var buf bytes.Buffer + + _, err = io.Copy(&buf, reader) + test.OK(t, err) + test.OK(t, reader.Close()) + + test.Equals(t, "hello world", strings.TrimSpace(buf.String())) +} diff --git a/mover-restic/restic/internal/fs/fs_track.go b/mover-restic/restic/internal/fs/fs_track.go index 319fbfaff..0c65a8564 100644 --- a/mover-restic/restic/internal/fs/fs_track.go +++ b/mover-restic/restic/internal/fs/fs_track.go @@ -41,7 +41,7 @@ type trackFile struct { func newTrackFile(stack []byte, filename string, file File) *trackFile { f := &trackFile{file} - runtime.SetFinalizer(f, func(f *trackFile) { + runtime.SetFinalizer(f, func(_ *trackFile) { fmt.Fprintf(os.Stderr, "file %s not closed\n\nStacktrack:\n%s\n", filename, stack) panic("file " + filename + " not closed") }) diff --git a/mover-restic/restic/internal/fs/sd_windows.go b/mover-restic/restic/internal/fs/sd_windows.go new file mode 100644 index 000000000..5d98b4ef4 --- /dev/null +++ b/mover-restic/restic/internal/fs/sd_windows.go @@ -0,0 +1,439 @@ +package fs + +import ( + "bytes" + "encoding/binary" + "fmt" + "sync" + "sync/atomic" + "syscall" + "unicode/utf16" + "unsafe" + + "github.com/restic/restic/internal/debug" + "golang.org/x/sys/windows" +) + +var ( + onceBackup sync.Once + onceRestore sync.Once + + // SeBackupPrivilege allows the application to bypass file and directory ACLs to back up files and directories. + SeBackupPrivilege = "SeBackupPrivilege" + // SeRestorePrivilege allows the application to bypass file and directory ACLs to restore files and directories. + SeRestorePrivilege = "SeRestorePrivilege" + // SeSecurityPrivilege allows read and write access to all SACLs. + SeSecurityPrivilege = "SeSecurityPrivilege" + // SeTakeOwnershipPrivilege allows the application to take ownership of files and directories, regardless of the permissions set on them. + SeTakeOwnershipPrivilege = "SeTakeOwnershipPrivilege" + + lowerPrivileges atomic.Bool +) + +// Flags for backup and restore with admin permissions +var highSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.SACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.BACKUP_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.PROTECTED_SACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_SACL_SECURITY_INFORMATION + +// Flags for backup without admin permissions. If there are no admin permissions, only the current user's owner, group and DACL will be backed up. +var lowBackupSecurityFlags windows.SECURITY_INFORMATION = windows.OWNER_SECURITY_INFORMATION | windows.GROUP_SECURITY_INFORMATION | windows.DACL_SECURITY_INFORMATION | windows.LABEL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.SCOPE_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION | windows.UNPROTECTED_DACL_SECURITY_INFORMATION + +// Flags for restore without admin permissions. If there are no admin permissions, only the DACL from the SD can be restored and owner and group will be set based on the current user. +var lowRestoreSecurityFlags windows.SECURITY_INFORMATION = windows.DACL_SECURITY_INFORMATION | windows.ATTRIBUTE_SECURITY_INFORMATION | windows.PROTECTED_DACL_SECURITY_INFORMATION + +// GetSecurityDescriptor takes the path of the file and returns the SecurityDescriptor for the file. +// This needs admin permissions or SeBackupPrivilege for getting the full SD. +// If there are no admin permissions, only the current user's owner, group and DACL will be got. +func GetSecurityDescriptor(filePath string) (securityDescriptor *[]byte, err error) { + onceBackup.Do(enableBackupPrivilege) + + var sd *windows.SECURITY_DESCRIPTOR + + if lowerPrivileges.Load() { + sd, err = getNamedSecurityInfoLow(filePath) + } else { + sd, err = getNamedSecurityInfoHigh(filePath) + } + if err != nil { + if !lowerPrivileges.Load() && isHandlePrivilegeNotHeldError(err) { + // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. + lowerPrivileges.Store(true) + sd, err = getNamedSecurityInfoLow(filePath) + if err != nil { + return nil, fmt.Errorf("get low-level named security info failed with: %w", err) + } + } else { + return nil, fmt.Errorf("get named security info failed with: %w", err) + } + } + + sdBytes, err := securityDescriptorStructToBytes(sd) + if err != nil { + return nil, fmt.Errorf("convert security descriptor to bytes failed: %w", err) + } + return &sdBytes, nil +} + +// SetSecurityDescriptor sets the SecurityDescriptor for the file at the specified path. +// This needs admin permissions or SeRestorePrivilege, SeSecurityPrivilege and SeTakeOwnershipPrivilege +// for setting the full SD. +// If there are no admin permissions/required privileges, only the DACL from the SD can be set and +// owner and group will be set based on the current user. +func SetSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { + onceRestore.Do(enableRestorePrivilege) + // Set the security descriptor on the file + sd, err := SecurityDescriptorBytesToStruct(*securityDescriptor) + if err != nil { + return fmt.Errorf("error converting bytes to security descriptor: %w", err) + } + + owner, _, err := sd.Owner() + if err != nil { + //Do not set partial values. + owner = nil + } + group, _, err := sd.Group() + if err != nil { + //Do not set partial values. + group = nil + } + dacl, _, err := sd.DACL() + if err != nil { + //Do not set partial values. + dacl = nil + } + sacl, _, err := sd.SACL() + if err != nil { + //Do not set partial values. + sacl = nil + } + + if lowerPrivileges.Load() { + err = setNamedSecurityInfoLow(filePath, dacl) + } else { + err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl) + } + + if err != nil { + if !lowerPrivileges.Load() && isHandlePrivilegeNotHeldError(err) { + // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. + lowerPrivileges.Store(true) + err = setNamedSecurityInfoLow(filePath, dacl) + if err != nil { + return fmt.Errorf("set low-level named security info failed with: %w", err) + } + } else { + return fmt.Errorf("set named security info failed with: %w", err) + } + } + return nil +} + +// getNamedSecurityInfoHigh gets the higher level SecurityDescriptor which requires admin permissions. +func getNamedSecurityInfoHigh(filePath string) (*windows.SECURITY_DESCRIPTOR, error) { + return windows.GetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, highSecurityFlags) +} + +// getNamedSecurityInfoLow gets the lower level SecurityDescriptor which requires no admin permissions. +func getNamedSecurityInfoLow(filePath string) (*windows.SECURITY_DESCRIPTOR, error) { + return windows.GetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, lowBackupSecurityFlags) +} + +// setNamedSecurityInfoHigh sets the higher level SecurityDescriptor which requires admin permissions. +func setNamedSecurityInfoHigh(filePath string, owner *windows.SID, group *windows.SID, dacl *windows.ACL, sacl *windows.ACL) error { + return windows.SetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, highSecurityFlags, owner, group, dacl, sacl) +} + +// setNamedSecurityInfoLow sets the lower level SecurityDescriptor which requires no admin permissions. +func setNamedSecurityInfoLow(filePath string, dacl *windows.ACL) error { + return windows.SetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, lowRestoreSecurityFlags, nil, nil, dacl, nil) +} + +// enableBackupPrivilege enables privilege for backing up security descriptors +func enableBackupPrivilege() { + err := enableProcessPrivileges([]string{SeBackupPrivilege}) + if err != nil { + debug.Log("error enabling backup privilege: %v", err) + } +} + +// enableBackupPrivilege enables privilege for restoring security descriptors +func enableRestorePrivilege() { + err := enableProcessPrivileges([]string{SeRestorePrivilege, SeSecurityPrivilege, SeTakeOwnershipPrivilege}) + if err != nil { + debug.Log("error enabling restore/security privilege: %v", err) + } +} + +// isHandlePrivilegeNotHeldError checks if the error is ERROR_PRIVILEGE_NOT_HELD +func isHandlePrivilegeNotHeldError(err error) bool { + // Use a type assertion to check if the error is of type syscall.Errno + if errno, ok := err.(syscall.Errno); ok { + // Compare the error code to the expected value + return errno == windows.ERROR_PRIVILEGE_NOT_HELD + } + return false +} + +// SecurityDescriptorBytesToStruct converts the security descriptor bytes representation +// into a pointer to windows SECURITY_DESCRIPTOR. +func SecurityDescriptorBytesToStruct(sd []byte) (*windows.SECURITY_DESCRIPTOR, error) { + if l := int(unsafe.Sizeof(windows.SECURITY_DESCRIPTOR{})); len(sd) < l { + return nil, fmt.Errorf("securityDescriptor (%d) smaller than expected (%d): %w", len(sd), l, windows.ERROR_INCORRECT_SIZE) + } + s := (*windows.SECURITY_DESCRIPTOR)(unsafe.Pointer(&sd[0])) + return s, nil +} + +// securityDescriptorStructToBytes converts the pointer to windows SECURITY_DESCRIPTOR +// into a security descriptor bytes representation. +func securityDescriptorStructToBytes(sd *windows.SECURITY_DESCRIPTOR) ([]byte, error) { + b := unsafe.Slice((*byte)(unsafe.Pointer(sd)), sd.Length()) + return b, nil +} + +// The code below was adapted from +// https://github.com/microsoft/go-winio/blob/3c9576c9346a1892dee136329e7e15309e82fb4f/privilege.go +// under MIT license. + +// The MIT License (MIT) + +// Copyright (c) 2015 Microsoft + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: + +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. + +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +var ( + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + + procLookupPrivilegeValueW = modadvapi32.NewProc("LookupPrivilegeValueW") + procAdjustTokenPrivileges = modadvapi32.NewProc("AdjustTokenPrivileges") + procLookupPrivilegeDisplayNameW = modadvapi32.NewProc("LookupPrivilegeDisplayNameW") + procLookupPrivilegeNameW = modadvapi32.NewProc("LookupPrivilegeNameW") +) + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoErrorIOPending = 997 + + //revive:disable-next-line:var-naming ALL_CAPS + SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED + + //revive:disable-next-line:var-naming ALL_CAPS + ERROR_NOT_ALL_ASSIGNED windows.Errno = windows.ERROR_NOT_ALL_ASSIGNED +) + +var ( + errErrorIOPending error = syscall.Errno(errnoErrorIOPending) + errErrorEinval error = syscall.EINVAL + + privNames = make(map[string]uint64) + privNameMutex sync.Mutex +) + +// PrivilegeError represents an error enabling privileges. +type PrivilegeError struct { + privileges []uint64 +} + +// Error returns the string message for the error. +func (e *PrivilegeError) Error() string { + s := "Could not enable privilege " + if len(e.privileges) > 1 { + s = "Could not enable privileges " + } + for i, p := range e.privileges { + if i != 0 { + s += ", " + } + s += `"` + s += getPrivilegeName(p) + s += `"` + } + return s +} + +func mapPrivileges(names []string) ([]uint64, error) { + privileges := make([]uint64, 0, len(names)) + privNameMutex.Lock() + defer privNameMutex.Unlock() + for _, name := range names { + p, ok := privNames[name] + if !ok { + err := lookupPrivilegeValue("", name, &p) + if err != nil { + return nil, err + } + privNames[name] = p + } + privileges = append(privileges, p) + } + return privileges, nil +} + +// enableProcessPrivileges enables privileges globally for the process. +func enableProcessPrivileges(names []string) error { + return enableDisableProcessPrivilege(names, SE_PRIVILEGE_ENABLED) +} + +func enableDisableProcessPrivilege(names []string, action uint32) error { + privileges, err := mapPrivileges(names) + if err != nil { + return err + } + + p := windows.CurrentProcess() + var token windows.Token + err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token) + if err != nil { + return err + } + + defer func() { + _ = token.Close() + }() + return adjustPrivileges(token, privileges, action) +} + +func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error { + var b bytes.Buffer + _ = binary.Write(&b, binary.LittleEndian, uint32(len(privileges))) + for _, p := range privileges { + _ = binary.Write(&b, binary.LittleEndian, p) + _ = binary.Write(&b, binary.LittleEndian, action) + } + prevState := make([]byte, b.Len()) + reqSize := uint32(0) + success, err := adjustTokenPrivileges(token, false, &b.Bytes()[0], uint32(len(prevState)), &prevState[0], &reqSize) + if !success { + return err + } + if err == ERROR_NOT_ALL_ASSIGNED { //nolint:errorlint // err is Errno + debug.Log("Not all requested privileges were fully set: %v. AdjustTokenPrivileges returned warning: %v", privileges, err) + } + return nil +} + +func getPrivilegeName(luid uint64) string { + var nameBuffer [256]uint16 + bufSize := uint32(len(nameBuffer)) + err := lookupPrivilegeName("", &luid, &nameBuffer[0], &bufSize) + if err != nil { + return fmt.Sprintf("", luid) + } + + var displayNameBuffer [256]uint16 + displayBufSize := uint32(len(displayNameBuffer)) + var langID uint32 + err = lookupPrivilegeDisplayName("", &nameBuffer[0], &displayNameBuffer[0], &displayBufSize, &langID) + if err != nil { + return fmt.Sprintf("", string(utf16.Decode(nameBuffer[:bufSize]))) + } + + return string(utf16.Decode(displayNameBuffer[:displayBufSize])) +} + +// The functions below are copied over from https://github.com/microsoft/go-winio/blob/main/zsyscall_windows.go under MIT license. + +// This windows api always returns an error even in case of success, warnings (partial success) and error cases. +// +// Full success - When we call this with admin permissions, it returns DNS_ERROR_RCODE_NO_ERROR (0). +// This gets translated to errErrorEinval and ultimately in adjustTokenPrivileges, it gets ignored. +// +// Partial success - If we call this api without admin privileges, privileges related to SACLs do not get set and +// though the api returns success, it returns an error - golang.org/x/sys/windows.ERROR_NOT_ALL_ASSIGNED (1300) +func adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) { + var _p0 uint32 + if releaseAll { + _p0 = 1 + } + r0, _, e1 := syscall.SyscallN(procAdjustTokenPrivileges.Addr(), uintptr(token), uintptr(_p0), uintptr(unsafe.Pointer(input)), uintptr(outputSize), uintptr(unsafe.Pointer(output)), uintptr(unsafe.Pointer(requiredSize))) + success = r0 != 0 + if true { + err = errnoErr(e1) + } + return +} + +func lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageID *uint32) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(systemName) + if err != nil { + return + } + return _lookupPrivilegeDisplayName(_p0, name, buffer, size, languageID) +} + +func _lookupPrivilegeDisplayName(systemName *uint16, name *uint16, buffer *uint16, size *uint32, languageID *uint32) (err error) { + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeDisplayNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(languageID))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(systemName) + if err != nil { + return + } + return _lookupPrivilegeName(_p0, luid, buffer, size) +} + +func _lookupPrivilegeName(systemName *uint16, luid *uint64, buffer *uint16, size *uint32) (err error) { + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeNameW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(luid)), uintptr(unsafe.Pointer(buffer)), uintptr(unsafe.Pointer(size))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +func lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) { + var _p0 *uint16 + _p0, err = syscall.UTF16PtrFromString(systemName) + if err != nil { + return + } + var _p1 *uint16 + _p1, err = syscall.UTF16PtrFromString(name) + if err != nil { + return + } + return _lookupPrivilegeValue(_p0, _p1, luid) +} + +func _lookupPrivilegeValue(systemName *uint16, name *uint16, luid *uint64) (err error) { + r1, _, e1 := syscall.SyscallN(procLookupPrivilegeValueW.Addr(), uintptr(unsafe.Pointer(systemName)), uintptr(unsafe.Pointer(name)), uintptr(unsafe.Pointer(luid))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + +// The code below was copied from https://github.com/microsoft/go-winio/blob/main/tools/mkwinsyscall/mkwinsyscall.go under MIT license. + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return errErrorEinval + case errnoErrorIOPending: + return errErrorIOPending + } + return e +} diff --git a/mover-restic/restic/internal/fs/sd_windows_test.go b/mover-restic/restic/internal/fs/sd_windows_test.go new file mode 100644 index 000000000..e78241ed3 --- /dev/null +++ b/mover-restic/restic/internal/fs/sd_windows_test.go @@ -0,0 +1,60 @@ +//go:build windows +// +build windows + +package fs + +import ( + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" +) + +func TestSetGetFileSecurityDescriptors(t *testing.T) { + tempDir := t.TempDir() + testfilePath := filepath.Join(tempDir, "testfile.txt") + // create temp file + testfile, err := os.Create(testfilePath) + if err != nil { + t.Fatalf("failed to create temporary file: %s", err) + } + defer func() { + err := testfile.Close() + if err != nil { + t.Logf("Error closing file %s: %v\n", testfilePath, err) + } + }() + + testSecurityDescriptors(t, TestFileSDs, testfilePath) +} + +func TestSetGetFolderSecurityDescriptors(t *testing.T) { + tempDir := t.TempDir() + testfolderPath := filepath.Join(tempDir, "testfolder") + // create temp folder + err := os.Mkdir(testfolderPath, os.ModeDir) + if err != nil { + t.Fatalf("failed to create temporary file: %s", err) + } + + testSecurityDescriptors(t, TestDirSDs, testfolderPath) +} + +func testSecurityDescriptors(t *testing.T, testSDs []string, testPath string) { + for _, testSD := range testSDs { + sdInputBytes, err := base64.StdEncoding.DecodeString(testSD) + test.OK(t, errors.Wrapf(err, "Error decoding SD: %s", testPath)) + + err = SetSecurityDescriptor(testPath, &sdInputBytes) + test.OK(t, errors.Wrapf(err, "Error setting file security descriptor for: %s", testPath)) + + var sdOutputBytes *[]byte + sdOutputBytes, err = GetSecurityDescriptor(testPath) + test.OK(t, errors.Wrapf(err, "Error getting file security descriptor for: %s", testPath)) + + CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdOutputBytes) + } +} diff --git a/mover-restic/restic/internal/fs/sd_windows_test_helpers.go b/mover-restic/restic/internal/fs/sd_windows_test_helpers.go new file mode 100644 index 000000000..8b3be5fd7 --- /dev/null +++ b/mover-restic/restic/internal/fs/sd_windows_test_helpers.go @@ -0,0 +1,126 @@ +//go:build windows +// +build windows + +package fs + +import ( + "os/user" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" + "golang.org/x/sys/windows" +) + +var ( + TestFileSDs = []string{"AQAUvBQAAAAwAAAAAAAAAEwAAAABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvqAwAAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSarAQIAAAIAfAAEAAAAAAAkAKkAEgABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvtAwAAABAUAP8BHwABAQAAAAAABRIAAAAAEBgA/wEfAAECAAAAAAAFIAAAACACAAAAECQA/wEfAAEFAAAAAAAFFQAAAIifWK5WoILqzL0mq+oDAAA=", + "AQAUvBQAAAAwAAAAAAAAAEwAAAABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvqAwAAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSarAQIAAAIAyAAHAAAAAAAUAKkAEgABAQAAAAAABQcAAAAAABQAiQASAAEBAAAAAAAFBwAAAAAAJACpABIAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSar7QMAAAAAJAC/ARMAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSar6gMAAAAAFAD/AR8AAQEAAAAAAAUSAAAAAAAYAP8BHwABAgAAAAAABSAAAAAgAgAAAAAkAP8BHwABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvqAwAA", + "AQAUvBQAAAAwAAAA7AAAAEwAAAABBQAAAAAABRUAAAAvr7t03PyHGk2FokNHCAAAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSarAQIAAAIAoAAFAAAAAAAkAP8BHwABBQAAAAAABRUAAAAvr7t03PyHGk2FokNHCAAAAAAkAKkAEgABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvtAwAAABAUAP8BHwABAQAAAAAABRIAAAAAEBgA/wEfAAECAAAAAAAFIAAAACACAAAAECQA/wEfAAEFAAAAAAAFFQAAAIifWK5WoILqzL0mq+oDAAACAHQAAwAAAAKAJAC/AQIAAQUAAAAAAAUVAAAAL6+7dNz8hxpNhaJDtgQAAALAJAC/AQMAAQUAAAAAAAUVAAAAL6+7dNz8hxpNhaJDPgkAAAJAJAD/AQ8AAQUAAAAAAAUVAAAAL6+7dNz8hxpNhaJDtQQAAA==", + } + TestDirSDs = []string{"AQAUvBQAAAAwAAAAAAAAAEwAAAABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvqAwAAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSarAQIAAAIAfAAEAAAAAAAkAKkAEgABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvtAwAAABMUAP8BHwABAQAAAAAABRIAAAAAExgA/wEfAAECAAAAAAAFIAAAACACAAAAEyQA/wEfAAEFAAAAAAAFFQAAAIifWK5WoILqzL0mq+oDAAA=", + "AQAUvBQAAAAwAAAAAAAAAEwAAAABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvqAwAAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSarAQIAAAIA3AAIAAAAAAIUAKkAEgABAQAAAAAABQcAAAAAAxQAiQASAAEBAAAAAAAFBwAAAAAAJACpABIAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSar7QMAAAAAJAC/ARMAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSar6gMAAAALFAC/ARMAAQEAAAAAAAMAAAAAABMUAP8BHwABAQAAAAAABRIAAAAAExgA/wEfAAECAAAAAAAFIAAAACACAAAAEyQA/wEfAAEFAAAAAAAFFQAAAIifWK5WoILqzL0mq+oDAAA=", + "AQAUvBQAAAAwAAAA7AAAAEwAAAABBQAAAAAABRUAAAAvr7t03PyHGk2FokNHCAAAAQUAAAAAAAUVAAAAiJ9YrlaggurMvSarAQIAAAIAoAAFAAAAAAAkAP8BHwABBQAAAAAABRUAAAAvr7t03PyHGk2FokNHCAAAAAAkAKkAEgABBQAAAAAABRUAAACIn1iuVqCC6sy9JqvtAwAAABMUAP8BHwABAQAAAAAABRIAAAAAExgA/wEfAAECAAAAAAAFIAAAACACAAAAEyQA/wEfAAEFAAAAAAAFFQAAAIifWK5WoILqzL0mq+oDAAACAHQAAwAAAAKAJAC/AQIAAQUAAAAAAAUVAAAAL6+7dNz8hxpNhaJDtgQAAALAJAC/AQMAAQUAAAAAAAUVAAAAL6+7dNz8hxpNhaJDPgkAAAJAJAD/AQ8AAQUAAAAAAAUVAAAAL6+7dNz8hxpNhaJDtQQAAA==", + } +) + +// IsAdmin checks if current user is an administrator. +func IsAdmin() (isAdmin bool, err error) { + var sid *windows.SID + err = windows.AllocateAndInitializeSid(&windows.SECURITY_NT_AUTHORITY, 2, windows.SECURITY_BUILTIN_DOMAIN_RID, windows.DOMAIN_ALIAS_RID_ADMINS, + 0, 0, 0, 0, 0, 0, &sid) + if err != nil { + return false, errors.Errorf("sid error: %s", err) + } + windows.GetCurrentProcessToken() + token := windows.Token(0) + member, err := token.IsMember(sid) + if err != nil { + return false, errors.Errorf("token membership error: %s", err) + } + return member, nil +} + +// CompareSecurityDescriptors runs tests for comparing 2 security descriptors in []byte format. +func CompareSecurityDescriptors(t *testing.T, testPath string, sdInputBytes, sdOutputBytes []byte) { + sdInput, err := SecurityDescriptorBytesToStruct(sdInputBytes) + test.OK(t, errors.Wrapf(err, "Error converting SD to struct for: %s", testPath)) + + sdOutput, err := SecurityDescriptorBytesToStruct(sdOutputBytes) + test.OK(t, errors.Wrapf(err, "Error converting SD to struct for: %s", testPath)) + + isAdmin, err := IsAdmin() + test.OK(t, errors.Wrapf(err, "Error checking if user is admin: %s", testPath)) + + var ownerExpected *windows.SID + var defaultedOwnerExpected bool + var groupExpected *windows.SID + var defaultedGroupExpected bool + var daclExpected *windows.ACL + var defaultedDaclExpected bool + var saclExpected *windows.ACL + var defaultedSaclExpected bool + + // The Dacl is set correctly whether or not application is running as admin. + daclExpected, defaultedDaclExpected, err = sdInput.DACL() + test.OK(t, errors.Wrapf(err, "Error getting input dacl for: %s", testPath)) + + if isAdmin { + // If application is running as admin, all sd values including owner, group, dacl, sacl are set correctly during restore. + // Hence we will use the input values for comparison with the output values. + ownerExpected, defaultedOwnerExpected, err = sdInput.Owner() + test.OK(t, errors.Wrapf(err, "Error getting input owner for: %s", testPath)) + groupExpected, defaultedGroupExpected, err = sdInput.Group() + test.OK(t, errors.Wrapf(err, "Error getting input group for: %s", testPath)) + saclExpected, defaultedSaclExpected, err = sdInput.SACL() + test.OK(t, errors.Wrapf(err, "Error getting input sacl for: %s", testPath)) + } else { + // If application is not running as admin, owner and group are set as current user's SID/GID during restore and sacl is empty. + // Get the current user + user, err := user.Current() + test.OK(t, errors.Wrapf(err, "Could not get current user for: %s", testPath)) + // Get current user's SID + currentUserSID, err := windows.StringToSid(user.Uid) + test.OK(t, errors.Wrapf(err, "Error getting output group for: %s", testPath)) + // Get current user's Group SID + currentGroupSID, err := windows.StringToSid(user.Gid) + test.OK(t, errors.Wrapf(err, "Error getting output group for: %s", testPath)) + + // Set owner and group as current user's SID and GID during restore. + ownerExpected = currentUserSID + defaultedOwnerExpected = false + groupExpected = currentGroupSID + defaultedGroupExpected = false + + // If application is not running as admin, SACL is returned empty. + saclExpected = nil + defaultedSaclExpected = false + } + // Now do all the comparisons + // Get owner SID from output file + ownerOut, defaultedOwnerOut, err := sdOutput.Owner() + test.OK(t, errors.Wrapf(err, "Error getting output owner for: %s", testPath)) + // Compare owner SIDs. We must use the Equals method for comparison as a syscall is made for comparing SIDs. + test.Assert(t, ownerExpected.Equals(ownerOut), "Owner from SDs read from test path don't match: %s, cur:%s, exp: %s", testPath, ownerExpected.String(), ownerOut.String()) + test.Equals(t, defaultedOwnerExpected, defaultedOwnerOut, "Defaulted for owner from SDs read from test path don't match: %s", testPath) + + // Get group SID from output file + groupOut, defaultedGroupOut, err := sdOutput.Group() + test.OK(t, errors.Wrapf(err, "Error getting output group for: %s", testPath)) + // Compare group SIDs. We must use the Equals method for comparison as a syscall is made for comparing SIDs. + test.Assert(t, groupExpected.Equals(groupOut), "Group from SDs read from test path don't match: %s, cur:%s, exp: %s", testPath, groupExpected.String(), groupOut.String()) + test.Equals(t, defaultedGroupExpected, defaultedGroupOut, "Defaulted for group from SDs read from test path don't match: %s", testPath) + + // Get dacl from output file + daclOut, defaultedDaclOut, err := sdOutput.DACL() + test.OK(t, errors.Wrapf(err, "Error getting output dacl for: %s", testPath)) + // Compare dacls + test.Equals(t, daclExpected, daclOut, "DACL from SDs read from test path don't match: %s", testPath) + test.Equals(t, defaultedDaclExpected, defaultedDaclOut, "Defaulted for DACL from SDs read from test path don't match: %s", testPath) + + // Get sacl from output file + saclOut, defaultedSaclOut, err := sdOutput.SACL() + test.OK(t, errors.Wrapf(err, "Error getting output sacl for: %s", testPath)) + // Compare sacls + test.Equals(t, saclExpected, saclOut, "DACL from SDs read from test path don't match: %s", testPath) + test.Equals(t, defaultedSaclExpected, defaultedSaclOut, "Defaulted for SACL from SDs read from test path don't match: %s", testPath) +} diff --git a/mover-restic/restic/internal/fs/stat_test.go b/mover-restic/restic/internal/fs/stat_test.go index a5ec77c7a..d52415c1d 100644 --- a/mover-restic/restic/internal/fs/stat_test.go +++ b/mover-restic/restic/internal/fs/stat_test.go @@ -5,11 +5,11 @@ import ( "path/filepath" "testing" - restictest "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" ) func TestExtendedStat(t *testing.T) { - tempdir := restictest.TempDir(t) + tempdir := rtest.TempDir(t) filename := filepath.Join(tempdir, "file") err := os.WriteFile(filename, []byte("foobar"), 0640) if err != nil { diff --git a/mover-restic/restic/internal/fs/vss.go b/mover-restic/restic/internal/fs/vss.go index 5f0ea36d9..8bfffab71 100644 --- a/mover-restic/restic/internal/fs/vss.go +++ b/mover-restic/restic/internal/fs/vss.go @@ -4,6 +4,8 @@ package fs import ( + "time" + "github.com/restic/restic/internal/errors" ) @@ -31,10 +33,16 @@ func HasSufficientPrivilegesForVSS() error { return errors.New("VSS snapshots are only supported on windows") } +// GetVolumeNameForVolumeMountPoint add trailing backslash to input parameter +// and calls the equivalent windows api. +func GetVolumeNameForVolumeMountPoint(mountPoint string) (string, error) { + return mountPoint, nil +} + // NewVssSnapshot creates a new vss snapshot. If creating the snapshots doesn't // finish within the timeout an error is returned. -func NewVssSnapshot( - _ string, _ uint, _ ErrorHandler) (VssSnapshot, error) { +func NewVssSnapshot(_ string, + _ string, _ time.Duration, _ VolumeFilter, _ ErrorHandler) (VssSnapshot, error) { return VssSnapshot{}, errors.New("VSS snapshots are only supported on windows") } diff --git a/mover-restic/restic/internal/fs/vss_windows.go b/mover-restic/restic/internal/fs/vss_windows.go index bd82f4405..0b51b00f3 100644 --- a/mover-restic/restic/internal/fs/vss_windows.go +++ b/mover-restic/restic/internal/fs/vss_windows.go @@ -5,10 +5,12 @@ package fs import ( "fmt" + "math" "path/filepath" "runtime" "strings" "syscall" + "time" "unsafe" ole "github.com/go-ole/go-ole" @@ -20,8 +22,10 @@ import ( type HRESULT uint // HRESULT constant values necessary for using VSS api. +//nolint:golint const ( S_OK HRESULT = 0x00000000 + S_FALSE HRESULT = 0x00000001 E_ACCESSDENIED HRESULT = 0x80070005 E_OUTOFMEMORY HRESULT = 0x8007000E E_INVALIDARG HRESULT = 0x80070057 @@ -166,7 +170,7 @@ func (h HRESULT) Str() string { return "UNKNOWN" } -// VssError encapsulates errors retruned from calling VSS api. +// VssError encapsulates errors returned from calling VSS api. type vssError struct { text string hresult HRESULT @@ -190,7 +194,7 @@ func (e *vssError) Error() string { return fmt.Sprintf("VSS error: %s: %s (%#x)", e.text, e.hresult.Str(), e.hresult) } -// VssError encapsulates errors retruned from calling VSS api. +// vssTextError encapsulates errors returned from calling VSS api. type vssTextError struct { text string } @@ -255,6 +259,7 @@ type IVssBackupComponents struct { } // IVssBackupComponentsVTable is the vtable for IVssBackupComponents. +// nolint:structcheck type IVssBackupComponentsVTable struct { ole.IUnknownVtbl getWriterComponentsCount uintptr @@ -364,7 +369,7 @@ func (vss *IVssBackupComponents) convertToVSSAsync( } // IsVolumeSupported calls the equivalent VSS api. -func (vss *IVssBackupComponents) IsVolumeSupported(volumeName string) (bool, error) { +func (vss *IVssBackupComponents) IsVolumeSupported(providerID *ole.GUID, volumeName string) (bool, error) { volumeNamePointer, err := syscall.UTF16PtrFromString(volumeName) if err != nil { panic(err) @@ -374,7 +379,7 @@ func (vss *IVssBackupComponents) IsVolumeSupported(volumeName string) (bool, err var result uintptr if runtime.GOARCH == "386" { - id := (*[4]uintptr)(unsafe.Pointer(ole.IID_NULL)) + id := (*[4]uintptr)(unsafe.Pointer(providerID)) result, _, _ = syscall.Syscall9(vss.getVTable().isVolumeSupported, 7, uintptr(unsafe.Pointer(vss)), id[0], id[1], id[2], id[3], @@ -382,7 +387,7 @@ func (vss *IVssBackupComponents) IsVolumeSupported(volumeName string) (bool, err 0) } else { result, _, _ = syscall.Syscall6(vss.getVTable().isVolumeSupported, 4, - uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(ole.IID_NULL)), + uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(providerID)), uintptr(unsafe.Pointer(volumeNamePointer)), uintptr(unsafe.Pointer(&isSupportedRaw)), 0, 0) } @@ -408,24 +413,24 @@ func (vss *IVssBackupComponents) StartSnapshotSet() (ole.GUID, error) { } // AddToSnapshotSet calls the equivalent VSS api. -func (vss *IVssBackupComponents) AddToSnapshotSet(volumeName string, idSnapshot *ole.GUID) error { +func (vss *IVssBackupComponents) AddToSnapshotSet(volumeName string, providerID *ole.GUID, idSnapshot *ole.GUID) error { volumeNamePointer, err := syscall.UTF16PtrFromString(volumeName) if err != nil { panic(err) } - var result uintptr = 0 + var result uintptr if runtime.GOARCH == "386" { - id := (*[4]uintptr)(unsafe.Pointer(ole.IID_NULL)) + id := (*[4]uintptr)(unsafe.Pointer(providerID)) result, _, _ = syscall.Syscall9(vss.getVTable().addToSnapshotSet, 7, - uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(volumeNamePointer)), id[0], id[1], - id[2], id[3], uintptr(unsafe.Pointer(idSnapshot)), 0, 0) + uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(volumeNamePointer)), + id[0], id[1], id[2], id[3], uintptr(unsafe.Pointer(idSnapshot)), 0, 0) } else { result, _, _ = syscall.Syscall6(vss.getVTable().addToSnapshotSet, 4, uintptr(unsafe.Pointer(vss)), uintptr(unsafe.Pointer(volumeNamePointer)), - uintptr(unsafe.Pointer(ole.IID_NULL)), uintptr(unsafe.Pointer(idSnapshot)), 0, 0) + uintptr(unsafe.Pointer(providerID)), uintptr(unsafe.Pointer(idSnapshot)), 0, 0) } return newVssErrorIfResultNotOK("AddToSnapshotSet() failed", HRESULT(result)) @@ -478,9 +483,9 @@ func (vss *IVssBackupComponents) DoSnapshotSet() (*IVSSAsync, error) { // DeleteSnapshots calls the equivalent VSS api. func (vss *IVssBackupComponents) DeleteSnapshots(snapshotID ole.GUID) (int32, ole.GUID, error) { - var deletedSnapshots int32 = 0 + var deletedSnapshots int32 var nondeletedSnapshotID ole.GUID - var result uintptr = 0 + var result uintptr if runtime.GOARCH == "386" { id := (*[4]uintptr)(unsafe.Pointer(&snapshotID)) @@ -504,7 +509,7 @@ func (vss *IVssBackupComponents) DeleteSnapshots(snapshotID ole.GUID) (int32, ol // GetSnapshotProperties calls the equivalent VSS api. func (vss *IVssBackupComponents) GetSnapshotProperties(snapshotID ole.GUID, properties *VssSnapshotProperties) error { - var result uintptr = 0 + var result uintptr if runtime.GOARCH == "386" { id := (*[4]uintptr)(unsafe.Pointer(&snapshotID)) @@ -527,8 +532,8 @@ func vssFreeSnapshotProperties(properties *VssSnapshotProperties) error { if err != nil { return err } - - proc.Call(uintptr(unsafe.Pointer(properties))) + // this function always succeeds and returns no value + _, _, _ = proc.Call(uintptr(unsafe.Pointer(properties))) return nil } @@ -543,6 +548,7 @@ func (vss *IVssBackupComponents) BackupComplete() (*IVSSAsync, error) { } // VssSnapshotProperties defines the properties of a VSS snapshot as part of the VSS api. +// nolint:structcheck type VssSnapshotProperties struct { snapshotID ole.GUID snapshotSetID ole.GUID @@ -559,6 +565,24 @@ type VssSnapshotProperties struct { status uint } +// VssProviderProperties defines the properties of a VSS provider as part of the VSS api. +// nolint:structcheck +type VssProviderProperties struct { + providerID ole.GUID + providerName *uint16 + providerType uint32 + providerVersion *uint16 + providerVersionID ole.GUID + classID ole.GUID +} + +func vssFreeProviderProperties(p *VssProviderProperties) { + ole.CoTaskMemFree(uintptr(unsafe.Pointer(p.providerName))) + p.providerName = nil + ole.CoTaskMemFree(uintptr(unsafe.Pointer(p.providerVersion))) + p.providerVersion = nil +} + // GetSnapshotDeviceObject returns root path to access the snapshot files // and folders. func (p *VssSnapshotProperties) GetSnapshotDeviceObject() string { @@ -615,10 +639,15 @@ func (vssAsync *IVSSAsync) QueryStatus() (HRESULT, uint32) { return HRESULT(result), state } -// WaitUntilAsyncFinished waits until either the async call is finshed or +// WaitUntilAsyncFinished waits until either the async call is finished or // the given timeout is reached. -func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(millis uint32) error { - hresult := vssAsync.Wait(millis) +func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(timeout time.Duration) error { + const maxTimeout = math.MaxInt32 * time.Millisecond + if timeout > maxTimeout { + timeout = maxTimeout + } + + hresult := vssAsync.Wait(uint32(timeout.Milliseconds())) err := newVssErrorIfResultNotOK("Wait() failed", hresult) if err != nil { vssAsync.Cancel() @@ -651,6 +680,75 @@ func (vssAsync *IVSSAsync) WaitUntilAsyncFinished(millis uint32) error { return nil } +// UIID_IVSS_ADMIN defines the GUID of IVSSAdmin. +var ( + UIID_IVSS_ADMIN = ole.NewGUID("{77ED5996-2F63-11d3-8A39-00C04F72D8E3}") + CLSID_VSS_COORDINATOR = ole.NewGUID("{E579AB5F-1CC4-44b4-BED9-DE0991FF0623}") +) + +// IVSSAdmin VSS api interface. +type IVSSAdmin struct { + ole.IUnknown +} + +// IVSSAdminVTable is the vtable for IVSSAdmin. +// nolint:structcheck +type IVSSAdminVTable struct { + ole.IUnknownVtbl + registerProvider uintptr + unregisterProvider uintptr + queryProviders uintptr + abortAllSnapshotsInProgress uintptr +} + +// getVTable returns the vtable for IVSSAdmin. +func (vssAdmin *IVSSAdmin) getVTable() *IVSSAdminVTable { + return (*IVSSAdminVTable)(unsafe.Pointer(vssAdmin.RawVTable)) +} + +// QueryProviders calls the equivalent VSS api. +func (vssAdmin *IVSSAdmin) QueryProviders() (*IVssEnumObject, error) { + var enum *IVssEnumObject + + result, _, _ := syscall.Syscall(vssAdmin.getVTable().queryProviders, 2, + uintptr(unsafe.Pointer(vssAdmin)), uintptr(unsafe.Pointer(&enum)), 0) + + return enum, newVssErrorIfResultNotOK("QueryProviders() failed", HRESULT(result)) +} + +// IVssEnumObject VSS api interface. +type IVssEnumObject struct { + ole.IUnknown +} + +// IVssEnumObjectVTable is the vtable for IVssEnumObject. +// nolint:structcheck +type IVssEnumObjectVTable struct { + ole.IUnknownVtbl + next uintptr + skip uintptr + reset uintptr + clone uintptr +} + +// getVTable returns the vtable for IVssEnumObject. +func (vssEnum *IVssEnumObject) getVTable() *IVssEnumObjectVTable { + return (*IVssEnumObjectVTable)(unsafe.Pointer(vssEnum.RawVTable)) +} + +// Next calls the equivalent VSS api. +func (vssEnum *IVssEnumObject) Next(count uint, props unsafe.Pointer) (uint, error) { + var fetched uint32 + result, _, _ := syscall.Syscall6(vssEnum.getVTable().next, 4, + uintptr(unsafe.Pointer(vssEnum)), uintptr(count), uintptr(props), + uintptr(unsafe.Pointer(&fetched)), 0, 0) + if HRESULT(result) == S_FALSE { + return uint(fetched), nil + } + + return uint(fetched), newVssErrorIfResultNotOK("Next() failed", HRESULT(result)) +} + // MountPoint wraps all information of a snapshot of a mountpoint on a volume. type MountPoint struct { isSnapshotted bool @@ -677,7 +775,7 @@ type VssSnapshot struct { snapshotProperties VssSnapshotProperties snapshotDeviceObject string mountPointInfo map[string]MountPoint - timeoutInMillis uint32 + timeout time.Duration } // GetSnapshotDeviceObject returns root path to access the snapshot files @@ -694,7 +792,12 @@ func initializeVssCOMInterface() (*ole.IUnknown, error) { } // ensure COM is initialized before use - ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED) + if err = ole.CoInitializeEx(0, ole.COINIT_MULTITHREADED); err != nil { + // CoInitializeEx returns S_FALSE if COM is already initialized + if oleErr, ok := err.(*ole.OleError); !ok || HRESULT(oleErr.Code()) != S_FALSE { + return nil, err + } + } var oleIUnknown *ole.IUnknown result, _, _ := vssInstance.Call(uintptr(unsafe.Pointer(&oleIUnknown))) @@ -727,12 +830,34 @@ func HasSufficientPrivilegesForVSS() error { return err } +// GetVolumeNameForVolumeMountPoint add trailing backslash to input parameter +// and calls the equivalent windows api. +func GetVolumeNameForVolumeMountPoint(mountPoint string) (string, error) { + if mountPoint != "" && mountPoint[len(mountPoint)-1] != filepath.Separator { + mountPoint += string(filepath.Separator) + } + + mountPointPointer, err := syscall.UTF16PtrFromString(mountPoint) + if err != nil { + return mountPoint, err + } + + // A reasonable size for the buffer to accommodate the largest possible + // volume GUID path is 50 characters. + volumeNameBuffer := make([]uint16, 50) + if err := windows.GetVolumeNameForVolumeMountPoint( + mountPointPointer, &volumeNameBuffer[0], 50); err != nil { + return mountPoint, err + } + + return syscall.UTF16ToString(volumeNameBuffer), nil +} + // NewVssSnapshot creates a new vss snapshot. If creating the snapshots doesn't // finish within the timeout an error is returned. -func NewVssSnapshot( - volume string, timeoutInSeconds uint, msgError ErrorHandler) (VssSnapshot, error) { +func NewVssSnapshot(provider string, + volume string, timeout time.Duration, filter VolumeFilter, msgError ErrorHandler) (VssSnapshot, error) { is64Bit, err := isRunningOn64BitWindows() - if err != nil { return VssSnapshot{}, newVssTextError(fmt.Sprintf( "Failed to detect windows architecture: %s", err.Error())) @@ -744,7 +869,7 @@ func NewVssSnapshot( runtime.GOARCH)) } - timeoutInMillis := uint32(timeoutInSeconds * 1000) + deadline := time.Now().Add(timeout) oleIUnknown, err := initializeVssCOMInterface() if oleIUnknown != nil { @@ -778,6 +903,12 @@ func NewVssSnapshot( iVssBackupComponents := (*IVssBackupComponents)(unsafe.Pointer(comInterface)) + providerID, err := getProviderID(provider) + if err != nil { + iVssBackupComponents.Release() + return VssSnapshot{}, err + } + if err := iVssBackupComponents.InitializeForBackup(); err != nil { iVssBackupComponents.Release() return VssSnapshot{}, err @@ -796,13 +927,13 @@ func NewVssSnapshot( } err = callAsyncFunctionAndWait(iVssBackupComponents.GatherWriterMetadata, - "GatherWriterMetadata", timeoutInMillis) + "GatherWriterMetadata", deadline) if err != nil { iVssBackupComponents.Release() return VssSnapshot{}, err } - if isSupported, err := iVssBackupComponents.IsVolumeSupported(volume); err != nil { + if isSupported, err := iVssBackupComponents.IsVolumeSupported(providerID, volume); err != nil { iVssBackupComponents.Release() return VssSnapshot{}, err } else if !isSupported { @@ -817,57 +948,66 @@ func NewVssSnapshot( return VssSnapshot{}, err } - if err := iVssBackupComponents.AddToSnapshotSet(volume, &snapshotSetID); err != nil { + if err := iVssBackupComponents.AddToSnapshotSet(volume, providerID, &snapshotSetID); err != nil { iVssBackupComponents.Release() return VssSnapshot{}, err } - mountPoints, err := enumerateMountedFolders(volume) - if err != nil { - iVssBackupComponents.Release() - return VssSnapshot{}, newVssTextError(fmt.Sprintf( - "failed to enumerate mount points for volume %s: %s", volume, err)) - } - mountPointInfo := make(map[string]MountPoint) - for _, mountPoint := range mountPoints { - // ensure every mountpoint is available even without a valid - // snapshot because we need to consider this when backing up files - mountPointInfo[mountPoint] = MountPoint{isSnapshotted: false} - - if isSupported, err := iVssBackupComponents.IsVolumeSupported(mountPoint); err != nil { - continue - } else if !isSupported { - continue - } - - var mountPointSnapshotSetID ole.GUID - err := iVssBackupComponents.AddToSnapshotSet(mountPoint, &mountPointSnapshotSetID) + // if filter==nil just don't process mount points for this volume at all + if filter != nil { + mountPoints, err := enumerateMountedFolders(volume) if err != nil { iVssBackupComponents.Release() - return VssSnapshot{}, err + + return VssSnapshot{}, newVssTextError(fmt.Sprintf( + "failed to enumerate mount points for volume %s: %s", volume, err)) } - mountPointInfo[mountPoint] = MountPoint{isSnapshotted: true, - snapshotSetID: mountPointSnapshotSetID} + for _, mountPoint := range mountPoints { + // ensure every mountpoint is available even without a valid + // snapshot because we need to consider this when backing up files + mountPointInfo[mountPoint] = MountPoint{isSnapshotted: false} + + if !filter(mountPoint) { + continue + } else if isSupported, err := iVssBackupComponents.IsVolumeSupported(providerID, mountPoint); err != nil { + continue + } else if !isSupported { + continue + } + + var mountPointSnapshotSetID ole.GUID + err := iVssBackupComponents.AddToSnapshotSet(mountPoint, providerID, &mountPointSnapshotSetID) + if err != nil { + iVssBackupComponents.Release() + + return VssSnapshot{}, err + } + + mountPointInfo[mountPoint] = MountPoint{ + isSnapshotted: true, + snapshotSetID: mountPointSnapshotSetID, + } + } } err = callAsyncFunctionAndWait(iVssBackupComponents.PrepareForBackup, "PrepareForBackup", - timeoutInMillis) + deadline) if err != nil { // After calling PrepareForBackup one needs to call AbortBackup() before releasing the VSS // instance for proper cleanup. - // It is not neccessary to call BackupComplete before releasing the VSS instance afterwards. + // It is not necessary to call BackupComplete before releasing the VSS instance afterwards. iVssBackupComponents.AbortBackup() iVssBackupComponents.Release() return VssSnapshot{}, err } err = callAsyncFunctionAndWait(iVssBackupComponents.DoSnapshotSet, "DoSnapshotSet", - timeoutInMillis) + deadline) if err != nil { - iVssBackupComponents.AbortBackup() + _ = iVssBackupComponents.AbortBackup() iVssBackupComponents.Release() return VssSnapshot{}, err } @@ -875,13 +1015,12 @@ func NewVssSnapshot( var snapshotProperties VssSnapshotProperties err = iVssBackupComponents.GetSnapshotProperties(snapshotSetID, &snapshotProperties) if err != nil { - iVssBackupComponents.AbortBackup() + _ = iVssBackupComponents.AbortBackup() iVssBackupComponents.Release() return VssSnapshot{}, err } for mountPoint, info := range mountPointInfo { - if !info.isSnapshotted { continue } @@ -900,8 +1039,10 @@ func NewVssSnapshot( mountPointInfo[mountPoint] = info } - return VssSnapshot{iVssBackupComponents, snapshotSetID, snapshotProperties, - snapshotProperties.GetSnapshotDeviceObject(), mountPointInfo, timeoutInMillis}, nil + return VssSnapshot{ + iVssBackupComponents, snapshotSetID, snapshotProperties, + snapshotProperties.GetSnapshotDeviceObject(), mountPointInfo, time.Until(deadline), + }, nil } // Delete deletes the created snapshot. @@ -922,15 +1063,17 @@ func (p *VssSnapshot) Delete() error { if p.iVssBackupComponents != nil { defer p.iVssBackupComponents.Release() + deadline := time.Now().Add(p.timeout) + err = callAsyncFunctionAndWait(p.iVssBackupComponents.BackupComplete, "BackupComplete", - p.timeoutInMillis) + deadline) if err != nil { return err } if _, _, e := p.iVssBackupComponents.DeleteSnapshots(p.snapshotID); e != nil { err = newVssTextError(fmt.Sprintf("Failed to delete snapshot: %s", e.Error())) - p.iVssBackupComponents.AbortBackup() + _ = p.iVssBackupComponents.AbortBackup() if err != nil { return err } @@ -940,12 +1083,61 @@ func (p *VssSnapshot) Delete() error { return nil } +func getProviderID(provider string) (*ole.GUID, error) { + providerLower := strings.ToLower(provider) + switch providerLower { + case "": + return ole.IID_NULL, nil + case "ms": + return ole.NewGUID("{b5946137-7b9f-4925-af80-51abd60b20d5}"), nil + } + + comInterface, err := ole.CreateInstance(CLSID_VSS_COORDINATOR, UIID_IVSS_ADMIN) + if err != nil { + return nil, err + } + defer comInterface.Release() + + vssAdmin := (*IVSSAdmin)(unsafe.Pointer(comInterface)) + + enum, err := vssAdmin.QueryProviders() + if err != nil { + return nil, err + } + defer enum.Release() + + id := ole.NewGUID(provider) + + var props struct { + objectType uint32 + provider VssProviderProperties + } + for { + count, err := enum.Next(1, unsafe.Pointer(&props)) + if err != nil { + return nil, err + } + + if count < 1 { + return nil, errors.Errorf(`invalid VSS provider "%s"`, provider) + } + + name := ole.UTF16PtrToString(props.provider.providerName) + vssFreeProviderProperties(&props.provider) + + if id != nil && *id == props.provider.providerID || + id == nil && providerLower == strings.ToLower(name) { + return &props.provider.providerID, nil + } + } +} + // asyncCallFunc is the callback type for callAsyncFunctionAndWait. type asyncCallFunc func() (*IVSSAsync, error) // callAsyncFunctionAndWait calls an async functions and waits for it to either // finish or timeout. -func callAsyncFunctionAndWait(function asyncCallFunc, name string, timeoutInMillis uint32) error { +func callAsyncFunctionAndWait(function asyncCallFunc, name string, deadline time.Time) error { iVssAsync, err := function() if err != nil { return err @@ -955,7 +1147,12 @@ func callAsyncFunctionAndWait(function asyncCallFunc, name string, timeoutInMill return newVssTextError(fmt.Sprintf("%s() returned nil", name)) } - err = iVssAsync.WaitUntilAsyncFinished(timeoutInMillis) + timeout := time.Until(deadline) + if timeout <= 0 { + return newVssTextError(fmt.Sprintf("%s() deadline exceeded", name)) + } + + err = iVssAsync.WaitUntilAsyncFinished(timeout) iVssAsync.Release() return err } @@ -1036,6 +1233,7 @@ func enumerateMountedFolders(volume string) ([]string, error) { return mountedFolders, nil } + // nolint:errcheck defer windows.FindVolumeMountPointClose(handle) volumeMountPoint := syscall.UTF16ToString(volumeMountPointBuffer) diff --git a/mover-restic/restic/internal/fuse/dir.go b/mover-restic/restic/internal/fuse/dir.go index 242b4b03e..763a9640c 100644 --- a/mover-restic/restic/internal/fuse/dir.go +++ b/mover-restic/restic/internal/fuse/dir.go @@ -46,7 +46,7 @@ func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, err }, nil } -// returing a wrapped context.Canceled error will instead result in returing +// returning a wrapped context.Canceled error will instead result in returning // an input / output error to the user. Thus unwrap the error to match the // expectations of bazil/fuse func unwrapCtxCanceled(err error) error { @@ -58,7 +58,7 @@ func unwrapCtxCanceled(err error) error { // replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // Otherwise, the node is returned. -func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *restic.Node) ([]*restic.Node, error) { +func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *restic.Node) ([]*restic.Node, error) { if node.Type != "dir" || node.Subtree == nil { return []*restic.Node{node}, nil } diff --git a/mover-restic/restic/internal/fuse/file.go b/mover-restic/restic/internal/fuse/file.go index 2fedf30bf..e2e0cf9a0 100644 --- a/mover-restic/restic/internal/fuse/file.go +++ b/mover-restic/restic/internal/fuse/file.go @@ -72,7 +72,7 @@ func (f *file) Open(_ context.Context, _ *fuse.OpenRequest, _ *fuse.OpenResponse var bytes uint64 cumsize := make([]uint64, 1+len(f.node.Content)) for i, id := range f.node.Content { - size, found := f.root.repo.LookupBlobSize(id, restic.DataBlob) + size, found := f.root.repo.LookupBlobSize(restic.DataBlob, id) if !found { return nil, errors.Errorf("id %v not found in repository", id) } @@ -96,20 +96,14 @@ func (f *file) Open(_ context.Context, _ *fuse.OpenRequest, _ *fuse.OpenResponse } func (f *openFile) getBlobAt(ctx context.Context, i int) (blob []byte, err error) { - - blob, ok := f.root.blobCache.Get(f.node.Content[i]) - if ok { - return blob, nil - } - - blob, err = f.root.repo.LoadBlob(ctx, restic.DataBlob, f.node.Content[i], nil) + blob, err = f.root.blobCache.GetOrCompute(f.node.Content[i], func() ([]byte, error) { + return f.root.repo.LoadBlob(ctx, restic.DataBlob, f.node.Content[i], nil) + }) if err != nil { debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err) return nil, unwrapCtxCanceled(err) } - f.root.blobCache.Add(f.node.Content[i], blob) - return blob, nil } @@ -142,7 +136,7 @@ func (f *openFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.R // Multiple goroutines may call service methods simultaneously; // the methods being called are responsible for appropriate synchronization. // - // However, no lock needed here as getBlobAt can be called conurrently + // However, no lock needed here as getBlobAt can be called concurrently // (blobCache has its own locking) for i := startContent; remainingBytes > 0 && i < len(f.cumsize)-1; i++ { blob, err := f.getBlobAt(ctx, i) diff --git a/mover-restic/restic/internal/fuse/fuse_test.go b/mover-restic/restic/internal/fuse/fuse_test.go index 0a121b986..aebcb1272 100644 --- a/mover-restic/restic/internal/fuse/fuse_test.go +++ b/mover-restic/restic/internal/fuse/fuse_test.go @@ -37,7 +37,7 @@ func testRead(t testing.TB, f fs.Handle, offset, length int, data []byte) { rtest.OK(t, fr.Read(ctx, req, resp)) } -func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { +func firstSnapshotID(t testing.TB, repo restic.Lister) (first restic.ID) { err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error { if first.IsNull() { first = id @@ -52,14 +52,14 @@ func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { return first } -func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { +func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *restic.Snapshot { id := firstSnapshotID(t, repo) sn, err := restic.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) return sn } -func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { +func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *restic.Tree { tree, err := restic.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) return tree @@ -89,7 +89,7 @@ func TestFuseFile(t *testing.T) { memfile []byte ) for _, id := range content { - size, found := repo.LookupBlobSize(id, restic.DataBlob) + size, found := repo.LookupBlobSize(restic.DataBlob, id) rtest.Assert(t, found, "Expected to find blob id %v", id) filesize += uint64(size) diff --git a/mover-restic/restic/internal/fuse/snapshots_dirstruct.go b/mover-restic/restic/internal/fuse/snapshots_dirstruct.go index 4d2c24b79..99a45a9bf 100644 --- a/mover-restic/restic/internal/fuse/snapshots_dirstruct.go +++ b/mover-restic/restic/internal/fuse/snapshots_dirstruct.go @@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { } var snapshots restic.Snapshots - err := d.root.cfg.Filter.FindAll(ctx, d.root.repo.Backend(), d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(_ string, sn *restic.Snapshot, _ error) error { if sn != nil { snapshots = append(snapshots, sn) } diff --git a/mover-restic/restic/internal/migrations/s3_layout.go b/mover-restic/restic/internal/migrations/s3_layout.go index 9effaee70..8b994b8fc 100644 --- a/mover-restic/restic/internal/migrations/s3_layout.go +++ b/mover-restic/restic/internal/migrations/s3_layout.go @@ -6,10 +6,12 @@ import ( "os" "path" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/layout" "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -23,7 +25,7 @@ type S3Layout struct{} // Check tests whether the migration can be applied. func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) { - be := restic.AsBackend[*s3.Backend](repo.Backend()) + be := repository.AsS3Backend(repo.(*repository.Repository)) if be == nil { debug.Log("backend is not s3") return false, "backend is not s3", nil @@ -63,8 +65,8 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) } - return be.List(ctx, t, func(fi restic.FileInfo) error { - h := restic.Handle{Type: t, Name: fi.Name} + return be.List(ctx, t, func(fi backend.FileInfo) error { + h := backend.Handle{Type: t, Name: fi.Name} debug.Log("move %v", h) return retry(maxErrors, printErr, func() error { @@ -75,7 +77,7 @@ func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layou // Apply runs the migration. func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be := restic.AsBackend[*s3.Backend](repo.Backend()) + be := repository.AsS3Backend(repo.(*repository.Repository)) if be == nil { debug.Log("backend is not s3") return errors.New("backend is not s3") diff --git a/mover-restic/restic/internal/migrations/upgrade_repo_v2.go b/mover-restic/restic/internal/migrations/upgrade_repo_v2.go index a81abc0e3..23a7f1ff0 100644 --- a/mover-restic/restic/internal/migrations/upgrade_repo_v2.go +++ b/mover-restic/restic/internal/migrations/upgrade_repo_v2.go @@ -3,10 +3,8 @@ package migrations import ( "context" "fmt" - "io" - "os" - "path/filepath" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -14,26 +12,6 @@ func init() { register(&UpgradeRepoV2{}) } -type UpgradeRepoV2Error struct { - UploadNewConfigError error - ReuploadOldConfigError error - - BackupFilePath string -} - -func (err *UpgradeRepoV2Error) Error() string { - if err.ReuploadOldConfigError != nil { - return fmt.Sprintf("error uploading config (%v), re-uploading old config filed failed as well (%v), but there is a backup of the config file in %v", err.UploadNewConfigError, err.ReuploadOldConfigError, err.BackupFilePath) - } - - return fmt.Sprintf("error uploading config (%v), re-uploaded old config was successful, there is a backup of the config file in %v", err.UploadNewConfigError, err.BackupFilePath) -} - -func (err *UpgradeRepoV2Error) Unwrap() error { - // consider the original upload error as the primary cause - return err.UploadNewConfigError -} - type UpgradeRepoV2 struct{} func (*UpgradeRepoV2) Name() string { @@ -56,74 +34,7 @@ func (*UpgradeRepoV2) Check(_ context.Context, repo restic.Repository) (bool, st func (*UpgradeRepoV2) RepoCheck() bool { return true } -func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error { - h := restic.Handle{Type: restic.ConfigFile} - - if !repo.Backend().HasAtomicReplace() { - // remove the original file for backends which do not support atomic overwriting - err := repo.Backend().Remove(ctx, h) - if err != nil { - return fmt.Errorf("remove config failed: %w", err) - } - } - - // upgrade config - cfg := repo.Config() - cfg.Version = 2 - - err := restic.SaveConfig(ctx, repo, cfg) - if err != nil { - return fmt.Errorf("save new config file failed: %w", err) - } - - return nil -} func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error { - tempdir, err := os.MkdirTemp("", "restic-migrate-upgrade-repo-v2-") - if err != nil { - return fmt.Errorf("create temp dir failed: %w", err) - } - - h := restic.Handle{Type: restic.ConfigFile} - - // read raw config file and save it to a temp dir, just in case - var rawConfigFile []byte - err = repo.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (err error) { - rawConfigFile, err = io.ReadAll(rd) - return err - }) - if err != nil { - return fmt.Errorf("load config file failed: %w", err) - } - - backupFileName := filepath.Join(tempdir, "config") - err = os.WriteFile(backupFileName, rawConfigFile, 0600) - if err != nil { - return fmt.Errorf("write config file backup to %v failed: %w", tempdir, err) - } - - // run the upgrade - err = m.upgrade(ctx, repo) - if err != nil { - - // build an error we can return to the caller - repoError := &UpgradeRepoV2Error{ - UploadNewConfigError: err, - BackupFilePath: backupFileName, - } - - // try contingency methods, reupload the original file - _ = repo.Backend().Remove(ctx, h) - err = repo.Backend().Save(ctx, h, restic.NewByteReader(rawConfigFile, nil)) - if err != nil { - repoError.ReuploadOldConfigError = err - } - - return repoError - } - - _ = os.Remove(backupFileName) - _ = os.Remove(tempdir) - return nil + return repository.UpgradeRepo(ctx, repo.(*repository.Repository)) } diff --git a/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go b/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go index 7f251de93..44a39b6c5 100644 --- a/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go +++ b/mover-restic/restic/internal/migrations/upgrade_repo_v2_test.go @@ -2,19 +2,13 @@ package migrations import ( "context" - "os" - "path/filepath" - "sync" "testing" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" ) func TestUpgradeRepoV2(t *testing.T) { - repo := repository.TestRepositoryWithVersion(t, 1) + repo, _ := repository.TestRepositoryWithVersion(t, 1) if repo.Config().Version != 1 { t.Fatal("test repo has wrong version") } @@ -35,73 +29,3 @@ func TestUpgradeRepoV2(t *testing.T) { t.Fatal(err) } } - -type failBackend struct { - restic.Backend - - mu sync.Mutex - ConfigFileSavesUntilError uint -} - -func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { - if h.Type != restic.ConfigFile { - return be.Backend.Save(ctx, h, rd) - } - - be.mu.Lock() - if be.ConfigFileSavesUntilError == 0 { - be.mu.Unlock() - return errors.New("failure induced for testing") - } - - be.ConfigFileSavesUntilError-- - be.mu.Unlock() - - return be.Backend.Save(ctx, h, rd) -} - -func TestUpgradeRepoV2Failure(t *testing.T) { - be := repository.TestBackend(t) - - // wrap backend so that it fails upgrading the config after the initial write - be = &failBackend{ - ConfigFileSavesUntilError: 1, - Backend: be, - } - - repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{}) - if repo.Config().Version != 1 { - t.Fatal("test repo has wrong version") - } - - m := &UpgradeRepoV2{} - - ok, _, err := m.Check(context.Background(), repo) - if err != nil { - t.Fatal(err) - } - - if !ok { - t.Fatal("migration check returned false") - } - - err = m.Apply(context.Background(), repo) - if err == nil { - t.Fatal("expected error returned from Apply(), got nil") - } - - upgradeErr := err.(*UpgradeRepoV2Error) - if upgradeErr.UploadNewConfigError == nil { - t.Fatal("expected upload error, got nil") - } - - if upgradeErr.ReuploadOldConfigError == nil { - t.Fatal("expected reupload error, got nil") - } - - if upgradeErr.BackupFilePath == "" { - t.Fatal("no backup file path found") - } - test.OK(t, os.Remove(upgradeErr.BackupFilePath)) - test.OK(t, os.Remove(filepath.Dir(upgradeErr.BackupFilePath))) -} diff --git a/mover-restic/restic/internal/repository/check.go b/mover-restic/restic/internal/repository/check.go new file mode 100644 index 000000000..9f571cb4f --- /dev/null +++ b/mover-restic/restic/internal/repository/check.go @@ -0,0 +1,209 @@ +package repository + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io" + "sort" + + "github.com/klauspost/compress/zstd" + "crypto/sha256" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository/hashing" + "github.com/restic/restic/internal/repository/pack" + "github.com/restic/restic/internal/restic" +) + +// ErrPackData is returned if errors are discovered while verifying a packfile +type ErrPackData struct { + PackID restic.ID + errs []error +} + +func (e *ErrPackData) Error() string { + return fmt.Sprintf("pack %v contains %v errors: %v", e.PackID, len(e.errs), e.errs) +} + +type partialReadError struct { + err error +} + +func (e *partialReadError) Error() string { + return e.err.Error() +} + +// CheckPack reads a pack and checks the integrity of all blobs. +func CheckPack(ctx context.Context, r *Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { + err := checkPackInner(ctx, r, id, blobs, size, bufRd, dec) + if err != nil { + if r.Cache != nil { + // ignore error as there's not much we can do here + _ = r.Cache.Forget(backend.Handle{Type: restic.PackFile, Name: id.String()}) + } + + // retry pack verification to detect transient errors + err2 := checkPackInner(ctx, r, id, blobs, size, bufRd, dec) + if err2 != nil { + err = err2 + } else { + err = fmt.Errorf("check successful on second attempt, original error %w", err) + } + } + return err +} + +func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { + + debug.Log("checking pack %v", id.String()) + + if len(blobs) == 0 { + return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}} + } + + // sanity check blobs in index + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + idxHdrSize := pack.CalculateHeaderSize(blobs) + lastBlobEnd := 0 + nonContinuousPack := false + for _, blob := range blobs { + if lastBlobEnd != int(blob.Offset) { + nonContinuousPack = true + } + lastBlobEnd = int(blob.Offset + blob.Length) + } + // size was calculated by masterindex.PackSize, thus there's no need to recalculate it here + + var errs []error + if nonContinuousPack { + debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs) + errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs")) + } + + // calculate hash on-the-fly while reading the pack and capture pack header + var hash restic.ID + var hdrBuf []byte + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := r.be.Load(ctx, h, int(size), 0, func(rd io.Reader) error { + hrd := hashing.NewReader(rd, sha256.New()) + bufRd.Reset(hrd) + + it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec) + for { + val, err := it.Next() + if err == errPackEOF { + break + } else if err != nil { + return &partialReadError{err} + } + debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle) + if val.Err != nil { + debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err) + errs = append(errs, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err)) + } + } + + // skip enough bytes until we reach the possible header start + curPos := lastBlobEnd + minHdrStart := int(size) - pack.MaxHeaderSize + if minHdrStart > curPos { + _, err := bufRd.Discard(minHdrStart - curPos) + if err != nil { + return &partialReadError{err} + } + curPos += minHdrStart - curPos + } + + // read remainder, which should be the pack header + var err error + hdrBuf = make([]byte, int(size-int64(curPos))) + _, err = io.ReadFull(bufRd, hdrBuf) + if err != nil { + return &partialReadError{err} + } + + hash = restic.IDFromHash(hrd.Sum(nil)) + return nil + }) + if err != nil { + var e *partialReadError + isPartialReadError := errors.As(err, &e) + // failed to load the pack file, return as further checks cannot succeed anyways + debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err) + if isPartialReadError { + return &ErrPackData{PackID: id, errs: append(errs, fmt.Errorf("partial download error: %w", err))} + } + + // The check command suggests to repair files for which a `ErrPackData` is returned. However, this file + // completely failed to download such that there's no point in repairing anything. + return fmt.Errorf("download error: %w", err) + } + if !hash.Equal(id) { + debug.Log("pack ID does not match, want %v, got %v", id, hash) + return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))} + } + + blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf))) + if err != nil { + return &ErrPackData{PackID: id, errs: append(errs, err)} + } + + if uint32(idxHdrSize) != hdrSize { + debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize) + errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) + } + + for _, blob := range blobs { + // Check if blob is contained in index and position is correct + idxHas := false + for _, pb := range r.LookupBlob(blob.BlobHandle.Type, blob.BlobHandle.ID) { + if pb.PackID == id && pb.Blob == blob { + idxHas = true + break + } + } + if !idxHas { + errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID)) + continue + } + } + + if len(errs) > 0 { + return &ErrPackData{PackID: id, errs: errs} + } + + return nil +} + +type bufReader struct { + rd *bufio.Reader + buf []byte +} + +func newBufReader(rd *bufio.Reader) *bufReader { + return &bufReader{ + rd: rd, + } +} + +func (b *bufReader) Discard(n int) (discarded int, err error) { + return b.rd.Discard(n) +} + +func (b *bufReader) ReadFull(n int) (buf []byte, err error) { + if cap(b.buf) < n { + b.buf = make([]byte, n) + } + b.buf = b.buf[:n] + + _, err = io.ReadFull(b.rd, b.buf) + if err != nil { + return nil, err + } + return b.buf, nil +} diff --git a/mover-restic/restic/internal/repository/fuzz_test.go b/mover-restic/restic/internal/repository/fuzz_test.go index 80372f8e0..f1fb06157 100644 --- a/mover-restic/restic/internal/repository/fuzz_test.go +++ b/mover-restic/restic/internal/repository/fuzz_test.go @@ -18,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) { } id := restic.Hash(blob) - repo := TestRepositoryWithVersion(t, 2) + repo, _ := TestRepositoryWithVersion(t, 2) var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) diff --git a/mover-restic/restic/internal/hashing/reader.go b/mover-restic/restic/internal/repository/hashing/reader.go similarity index 100% rename from mover-restic/restic/internal/hashing/reader.go rename to mover-restic/restic/internal/repository/hashing/reader.go diff --git a/mover-restic/restic/internal/hashing/reader_test.go b/mover-restic/restic/internal/repository/hashing/reader_test.go similarity index 100% rename from mover-restic/restic/internal/hashing/reader_test.go rename to mover-restic/restic/internal/repository/hashing/reader_test.go diff --git a/mover-restic/restic/internal/hashing/writer.go b/mover-restic/restic/internal/repository/hashing/writer.go similarity index 100% rename from mover-restic/restic/internal/hashing/writer.go rename to mover-restic/restic/internal/repository/hashing/writer.go diff --git a/mover-restic/restic/internal/hashing/writer_test.go b/mover-restic/restic/internal/repository/hashing/writer_test.go similarity index 100% rename from mover-restic/restic/internal/hashing/writer_test.go rename to mover-restic/restic/internal/repository/hashing/writer_test.go diff --git a/mover-restic/restic/internal/repository/index/associated_data.go b/mover-restic/restic/internal/repository/index/associated_data.go new file mode 100644 index 000000000..ee58957e0 --- /dev/null +++ b/mover-restic/restic/internal/repository/index/associated_data.go @@ -0,0 +1,156 @@ +package index + +import ( + "context" + "sort" + + "github.com/restic/restic/internal/restic" +) + +type associatedSetSub[T any] struct { + value []T + isSet []bool +} + +// AssociatedSet is a memory efficient implementation of a BlobSet that can +// store a small data item for each BlobHandle. It relies on a special property +// of our MasterIndex implementation. A BlobHandle can be permanently identified +// using an offset that never changes as MasterIndex entries cannot be modified (only added). +// +// The AssociatedSet thus can use an array with the size of the MasterIndex to store +// its data. Access to an individual entry is possible by looking up the BlobHandle's +// offset from the MasterIndex. +// +// BlobHandles that are not part of the MasterIndex can be stored by placing them in +// an overflow set that is expected to be empty in the normal case. +type AssociatedSet[T any] struct { + byType [restic.NumBlobTypes]associatedSetSub[T] + overflow map[restic.BlobHandle]T + idx *MasterIndex +} + +func NewAssociatedSet[T any](mi *MasterIndex) *AssociatedSet[T] { + a := AssociatedSet[T]{ + overflow: make(map[restic.BlobHandle]T), + idx: mi, + } + + for typ := range a.byType { + if typ == 0 { + continue + } + // index starts counting at 1 + count := mi.stableLen(restic.BlobType(typ)) + 1 + a.byType[typ].value = make([]T, count) + a.byType[typ].isSet = make([]bool, count) + } + + return &a +} + +func (a *AssociatedSet[T]) Get(bh restic.BlobHandle) (T, bool) { + if val, ok := a.overflow[bh]; ok { + return val, true + } + + idx := a.idx.blobIndex(bh) + bt := &a.byType[bh.Type] + if idx >= len(bt.value) || idx == -1 { + var zero T + return zero, false + } + + has := bt.isSet[idx] + if has { + return bt.value[idx], has + } + var zero T + return zero, false +} + +func (a *AssociatedSet[T]) Has(bh restic.BlobHandle) bool { + _, ok := a.Get(bh) + return ok +} + +func (a *AssociatedSet[T]) Set(bh restic.BlobHandle, val T) { + if _, ok := a.overflow[bh]; ok { + a.overflow[bh] = val + return + } + + idx := a.idx.blobIndex(bh) + bt := &a.byType[bh.Type] + if idx >= len(bt.value) || idx == -1 { + a.overflow[bh] = val + } else { + bt.value[idx] = val + bt.isSet[idx] = true + } +} + +func (a *AssociatedSet[T]) Insert(bh restic.BlobHandle) { + var zero T + a.Set(bh, zero) +} + +func (a *AssociatedSet[T]) Delete(bh restic.BlobHandle) { + if _, ok := a.overflow[bh]; ok { + delete(a.overflow, bh) + return + } + + idx := a.idx.blobIndex(bh) + bt := &a.byType[bh.Type] + if idx < len(bt.value) && idx != -1 { + bt.isSet[idx] = false + } +} + +func (a *AssociatedSet[T]) Len() int { + count := 0 + a.For(func(_ restic.BlobHandle, _ T) { + count++ + }) + return count +} + +func (a *AssociatedSet[T]) For(cb func(bh restic.BlobHandle, val T)) { + for k, v := range a.overflow { + cb(k, v) + } + + _ = a.idx.Each(context.Background(), func(pb restic.PackedBlob) { + if _, ok := a.overflow[pb.BlobHandle]; ok { + // already reported via overflow set + return + } + + val, known := a.Get(pb.BlobHandle) + if known { + cb(pb.BlobHandle, val) + } + }) +} + +// List returns a sorted slice of all BlobHandle in the set. +func (a *AssociatedSet[T]) List() restic.BlobHandles { + list := make(restic.BlobHandles, 0) + a.For(func(bh restic.BlobHandle, _ T) { + list = append(list, bh) + }) + + return list +} + +func (a *AssociatedSet[T]) String() string { + list := a.List() + sort.Sort(list) + + str := list.String() + if len(str) < 2 { + return "{}" + } + + return "{" + str[1:len(str)-1] + "}" +} diff --git a/mover-restic/restic/internal/repository/index/associated_data_test.go b/mover-restic/restic/internal/repository/index/associated_data_test.go new file mode 100644 index 000000000..82dd9908d --- /dev/null +++ b/mover-restic/restic/internal/repository/index/associated_data_test.go @@ -0,0 +1,154 @@ +package index + +import ( + "context" + "testing" + + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" +) + +type noopSaver struct{} + +func (n *noopSaver) Connections() uint { + return 2 +} +func (n *noopSaver) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (restic.ID, error) { + return restic.Hash(buf), nil +} + +func makeFakePackedBlob() (restic.BlobHandle, restic.PackedBlob) { + bh := restic.NewRandomBlobHandle() + blob := restic.PackedBlob{ + PackID: restic.NewRandomID(), + Blob: restic.Blob{ + BlobHandle: bh, + Length: uint(crypto.CiphertextLength(10)), + Offset: 0, + }, + } + return bh, blob +} + +func TestAssociatedSet(t *testing.T) { + bh, blob := makeFakePackedBlob() + + mi := NewMasterIndex() + mi.StorePack(blob.PackID, []restic.Blob{blob.Blob}) + test.OK(t, mi.SaveIndex(context.TODO(), &noopSaver{})) + + bs := NewAssociatedSet[uint8](mi) + test.Equals(t, bs.Len(), 0) + test.Equals(t, bs.List(), restic.BlobHandles{}) + + // check non existent + test.Equals(t, bs.Has(bh), false) + _, ok := bs.Get(bh) + test.Equals(t, false, ok) + + // test insert + bs.Insert(bh) + test.Equals(t, bs.Has(bh), true) + test.Equals(t, bs.Len(), 1) + test.Equals(t, bs.List(), restic.BlobHandles{bh}) + test.Equals(t, 0, len(bs.overflow)) + + // test set + bs.Set(bh, 42) + test.Equals(t, bs.Has(bh), true) + test.Equals(t, bs.Len(), 1) + val, ok := bs.Get(bh) + test.Equals(t, true, ok) + test.Equals(t, uint8(42), val) + + s := bs.String() + test.Assert(t, len(s) > 10, "invalid string: %v", s) + + // test remove + bs.Delete(bh) + test.Equals(t, bs.Len(), 0) + test.Equals(t, bs.Has(bh), false) + test.Equals(t, bs.List(), restic.BlobHandles{}) + + test.Equals(t, "{}", bs.String()) + + // test set + bs.Set(bh, 43) + test.Equals(t, bs.Has(bh), true) + test.Equals(t, bs.Len(), 1) + val, ok = bs.Get(bh) + test.Equals(t, true, ok) + test.Equals(t, uint8(43), val) + test.Equals(t, 0, len(bs.overflow)) + // test update + bs.Set(bh, 44) + val, ok = bs.Get(bh) + test.Equals(t, true, ok) + test.Equals(t, uint8(44), val) + test.Equals(t, 0, len(bs.overflow)) + + // test overflow blob + of := restic.NewRandomBlobHandle() + test.Equals(t, false, bs.Has(of)) + // set + bs.Set(of, 7) + test.Equals(t, 1, len(bs.overflow)) + test.Equals(t, bs.Len(), 2) + // get + val, ok = bs.Get(of) + test.Equals(t, true, ok) + test.Equals(t, uint8(7), val) + test.Equals(t, bs.List(), restic.BlobHandles{of, bh}) + // update + bs.Set(of, 8) + val, ok = bs.Get(of) + test.Equals(t, true, ok) + test.Equals(t, uint8(8), val) + test.Equals(t, 1, len(bs.overflow)) + // delete + bs.Delete(of) + test.Equals(t, bs.Len(), 1) + test.Equals(t, bs.Has(of), false) + test.Equals(t, bs.List(), restic.BlobHandles{bh}) + test.Equals(t, 0, len(bs.overflow)) +} + +func TestAssociatedSetWithExtendedIndex(t *testing.T) { + _, blob := makeFakePackedBlob() + + mi := NewMasterIndex() + mi.StorePack(blob.PackID, []restic.Blob{blob.Blob}) + test.OK(t, mi.SaveIndex(context.TODO(), &noopSaver{})) + + bs := NewAssociatedSet[uint8](mi) + + // add new blobs to index after building the set + of, blob2 := makeFakePackedBlob() + mi.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) + test.OK(t, mi.SaveIndex(context.TODO(), &noopSaver{})) + + // non-existent + test.Equals(t, false, bs.Has(of)) + // set + bs.Set(of, 5) + test.Equals(t, 1, len(bs.overflow)) + test.Equals(t, bs.Len(), 1) + // get + val, ok := bs.Get(of) + test.Equals(t, true, ok) + test.Equals(t, uint8(5), val) + test.Equals(t, bs.List(), restic.BlobHandles{of}) + // update + bs.Set(of, 8) + val, ok = bs.Get(of) + test.Equals(t, true, ok) + test.Equals(t, uint8(8), val) + test.Equals(t, 1, len(bs.overflow)) + // delete + bs.Delete(of) + test.Equals(t, bs.Len(), 0) + test.Equals(t, bs.Has(of), false) + test.Equals(t, bs.List(), restic.BlobHandles{}) + test.Equals(t, 0, len(bs.overflow)) +} diff --git a/mover-restic/restic/internal/index/index.go b/mover-restic/restic/internal/repository/index/index.go similarity index 86% rename from mover-restic/restic/internal/index/index.go rename to mover-restic/restic/internal/repository/index/index.go index ecd481594..36ac2560f 100644 --- a/mover-restic/restic/internal/index/index.go +++ b/mover-restic/restic/internal/repository/index/index.go @@ -1,14 +1,18 @@ package index import ( + "bytes" "context" "encoding/json" + "fmt" "io" + "math" "sync" "time" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/debug" @@ -43,14 +47,13 @@ import ( // Index holds lookup tables for id -> pack. type Index struct { - m sync.Mutex + m sync.RWMutex byType [restic.NumBlobTypes]indexMap packs restic.IDs - final bool // set to true for all indexes read from the backend ("finalized") - ids restic.IDs // set to the IDs of the contained finalized indexes - supersedes restic.IDs - created time.Time + final bool // set to true for all indexes read from the backend ("finalized") + ids restic.IDs // set to the IDs of the contained finalized indexes + created time.Time } // NewIndex returns a new index. @@ -67,11 +70,9 @@ func (idx *Index) addToPacks(id restic.ID) int { return len(idx.packs) - 1 } -const maxuint32 = 1<<32 - 1 - func (idx *Index) store(packIndex int, blob restic.Blob) { // assert that offset and length fit into uint32! - if blob.Offset > maxuint32 || blob.Length > maxuint32 || blob.UncompressedLength > maxuint32 { + if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 { panic("offset or length does not fit in uint32. You have packs > 4GB!") } @@ -82,22 +83,21 @@ func (idx *Index) store(packIndex int, blob restic.Blob) { // Final returns true iff the index is already written to the repository, it is // finalized. func (idx *Index) Final() bool { - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() return idx.final } const ( - indexMaxBlobs = 50000 - indexMaxBlobsCompressed = 3 * indexMaxBlobs - indexMaxAge = 10 * time.Minute + indexMaxBlobs = 50000 + indexMaxAge = 10 * time.Minute ) // IndexFull returns true iff the index is "full enough" to be saved as a preliminary index. -var IndexFull = func(idx *Index, compress bool) bool { - idx.m.Lock() - defer idx.m.Unlock() +var IndexFull = func(idx *Index) bool { + idx.m.RLock() + defer idx.m.RUnlock() debug.Log("checking whether index %p is full", idx) @@ -106,18 +106,12 @@ var IndexFull = func(idx *Index, compress bool) bool { blobs += idx.byType[typ].len() } age := time.Since(idx.created) - var maxBlobs uint - if compress { - maxBlobs = indexMaxBlobsCompressed - } else { - maxBlobs = indexMaxBlobs - } switch { case age >= indexMaxAge: debug.Log("index %p is old enough", idx, age) return true - case blobs >= maxBlobs: + case blobs >= indexMaxBlobs: debug.Log("index %p has %d blobs", idx, blobs) return true } @@ -162,8 +156,8 @@ func (idx *Index) toPackedBlob(e *indexEntry, t restic.BlobType) restic.PackedBl // Lookup queries the index for the blob ID and returns all entries including // duplicates. Adds found entries to blobs and returns the result. func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic.PackedBlob { - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() idx.byType[bh.Type].foreachWithID(bh.ID, func(e *indexEntry) { pbs = append(pbs, idx.toPackedBlob(e, bh.Type)) @@ -174,8 +168,8 @@ func (idx *Index) Lookup(bh restic.BlobHandle, pbs []restic.PackedBlob) []restic // Has returns true iff the id is listed in the index. func (idx *Index) Has(bh restic.BlobHandle) bool { - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() return idx.byType[bh.Type].get(bh.ID) != nil } @@ -183,8 +177,8 @@ func (idx *Index) Has(bh restic.BlobHandle) bool { // LookupSize returns the length of the plaintext content of the blob with the // given id. func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found bool) { - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() e := idx.byType[bh.Type].get(bh.ID) if e == nil { @@ -196,30 +190,11 @@ func (idx *Index) LookupSize(bh restic.BlobHandle) (plaintextLength uint, found return uint(crypto.PlaintextLength(int(e.length))), true } -// Supersedes returns the list of indexes this index supersedes, if any. -func (idx *Index) Supersedes() restic.IDs { - return idx.supersedes -} - -// AddToSupersedes adds the ids to the list of indexes superseded by this -// index. If the index has already been finalized, an error is returned. -func (idx *Index) AddToSupersedes(ids ...restic.ID) error { - idx.m.Lock() - defer idx.m.Unlock() - - if idx.final { - return errors.New("index already finalized") - } - - idx.supersedes = append(idx.supersedes, ids...) - return nil -} - // Each passes all blobs known to the index to the callback fn. This blocks any // modification of the index. -func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) { - idx.m.Lock() - defer idx.m.Unlock() +func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) error { + idx.m.RLock() + defer idx.m.RUnlock() for typ := range idx.byType { m := &idx.byType[typ] @@ -231,6 +206,7 @@ func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) { return true }) } + return ctx.Err() } type EachByPackResult struct { @@ -246,12 +222,12 @@ type EachByPackResult struct { // When the context is cancelled, the background goroutine // terminates. This blocks any modification of the index. func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-chan EachByPackResult { - idx.m.Lock() + idx.m.RLock() ch := make(chan EachByPackResult) go func() { - defer idx.m.Unlock() + defer idx.m.RUnlock() defer close(ch) byPack := make(map[restic.ID][restic.NumBlobTypes][]*indexEntry) @@ -292,8 +268,8 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <- // Packs returns all packs in this index func (idx *Index) Packs() restic.IDSet { - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() packs := restic.NewIDSet() for _, packID := range idx.packs { @@ -354,15 +330,15 @@ func (idx *Index) generatePackList() ([]packJSON, error) { } type jsonIndex struct { - Supersedes restic.IDs `json:"supersedes,omitempty"` - Packs []packJSON `json:"packs"` + // removed: Supersedes restic.IDs `json:"supersedes,omitempty"` + Packs []packJSON `json:"packs"` } // Encode writes the JSON serialization of the index to the writer w. func (idx *Index) Encode(w io.Writer) error { debug.Log("encoding index") - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() list, err := idx.generatePackList() if err != nil { @@ -371,12 +347,29 @@ func (idx *Index) Encode(w io.Writer) error { enc := json.NewEncoder(w) idxJSON := jsonIndex{ - Supersedes: idx.supersedes, - Packs: list, + Packs: list, } return enc.Encode(idxJSON) } +// SaveIndex saves an index in the repository. +func (idx *Index) SaveIndex(ctx context.Context, repo restic.SaverUnpacked) (restic.ID, error) { + buf := bytes.NewBuffer(nil) + + err := idx.Encode(buf) + if err != nil { + return restic.ID{}, err + } + + id, err := repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) + ierr := idx.SetID(id) + if ierr != nil { + // logic bug + panic(ierr) + } + return id, err +} + // Finalize sets the index to final. func (idx *Index) Finalize() { debug.Log("finalizing index") @@ -389,8 +382,8 @@ func (idx *Index) Finalize() { // IDs returns the IDs of the index, if available. If the index is not yet // finalized, an error is returned. func (idx *Index) IDs() (restic.IDs, error) { - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() if !idx.final { return nil, errors.New("index not finalized") @@ -422,8 +415,8 @@ func (idx *Index) SetID(id restic.ID) error { // Dump writes the pretty-printed JSON representation of the index to w. func (idx *Index) Dump(w io.Writer) error { debug.Log("dumping index") - idx.m.Lock() - defer idx.m.Unlock() + idx.m.RLock() + defer idx.m.RUnlock() list, err := idx.generatePackList() if err != nil { @@ -431,8 +424,7 @@ func (idx *Index) Dump(w io.Writer) error { } outer := jsonIndex{ - Supersedes: idx.Supersedes(), - Packs: list, + Packs: list, } buf, err := json.MarshalIndent(outer, "", " ") @@ -493,7 +485,6 @@ func (idx *Index) merge(idx2 *Index) error { } idx.ids = append(idx.ids, idx2.ids...) - idx.supersedes = append(idx.supersedes, idx2.supersedes...) return nil } @@ -515,8 +506,13 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro debug.Log("Error %v", err) if isErrOldIndex(err) { + if feature.Flag.Enabled(feature.DeprecateLegacyIndex) { + return nil, false, fmt.Errorf("index seems to use the legacy format. update it using `restic repair index`") + } + debug.Log("index is probably old format, trying that") idx, err = decodeOldIndex(buf) + idx.ids = append(idx.ids, id) return idx, err == nil, err } @@ -538,7 +534,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro }) } } - idx.supersedes = idxJSON.Supersedes idx.ids = append(idx.ids, id) idx.final = true @@ -577,3 +572,17 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) { debug.Log("done") return idx, nil } + +func (idx *Index) BlobIndex(bh restic.BlobHandle) int { + idx.m.RLock() + defer idx.m.RUnlock() + + return idx.byType[bh.Type].firstIndex(bh.ID) +} + +func (idx *Index) Len(t restic.BlobType) uint { + idx.m.RLock() + defer idx.m.RUnlock() + + return idx.byType[t].len() +} diff --git a/mover-restic/restic/internal/index/index_parallel.go b/mover-restic/restic/internal/repository/index/index_parallel.go similarity index 92% rename from mover-restic/restic/internal/index/index_parallel.go rename to mover-restic/restic/internal/repository/index/index_parallel.go index d505d756e..3d5621a2d 100644 --- a/mover-restic/restic/internal/index/index_parallel.go +++ b/mover-restic/restic/internal/repository/index/index_parallel.go @@ -11,7 +11,7 @@ import ( // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound @@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Reposi workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) var m sync.Mutex - return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, _ int64) error { var err error var idx *Index oldFormat := false diff --git a/mover-restic/restic/internal/index/index_parallel_test.go b/mover-restic/restic/internal/repository/index/index_parallel_test.go similarity index 64% rename from mover-restic/restic/internal/index/index_parallel_test.go rename to mover-restic/restic/internal/repository/index/index_parallel_test.go index 86be46473..38dafb507 100644 --- a/mover-restic/restic/internal/index/index_parallel_test.go +++ b/mover-restic/restic/internal/repository/index/index_parallel_test.go @@ -6,20 +6,18 @@ import ( "testing" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -var repoFixture = filepath.Join("..", "repository", "testdata", "test-repo.tar.gz") +var repoFixture = filepath.Join("..", "testdata", "test-repo.tar.gz") func TestRepositoryForAllIndexes(t *testing.T) { - repodir, cleanup := rtest.Env(t, repoFixture) + repo, _, cleanup := repository.TestFromFixture(t, repoFixture) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) - expectedIndexIDs := restic.NewIDSet() rtest.OK(t, repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { expectedIndexIDs.Insert(id) @@ -29,7 +27,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // check that all expected indexes are loaded without errors indexIDs := restic.NewIDSet() var indexErr error - rtest.OK(t, index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { if err != nil { indexErr = err } @@ -42,7 +40,7 @@ func TestRepositoryForAllIndexes(t *testing.T) { // must failed with the returned error iterErr := errors.New("error to pass upwards") - err := index.ForAllIndexes(context.TODO(), repo.Backend(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(context.TODO(), repo, repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error { return iterErr }) diff --git a/mover-restic/restic/internal/index/index_test.go b/mover-restic/restic/internal/repository/index/index_test.go similarity index 96% rename from mover-restic/restic/internal/index/index_test.go rename to mover-restic/restic/internal/repository/index/index_test.go index 4f0dbd2a0..bf752d3d3 100644 --- a/mover-restic/restic/internal/index/index_test.go +++ b/mover-restic/restic/internal/repository/index/index_test.go @@ -8,7 +8,8 @@ import ( "sync" "testing" - "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/feature" + "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -171,6 +172,9 @@ func TestIndexSize(t *testing.T) { err := idx.Encode(wr) rtest.OK(t, err) + rtest.Equals(t, uint(packs*blobCount), idx.Len(restic.DataBlob)) + rtest.Equals(t, uint(0), idx.Len(restic.TreeBlob)) + t.Logf("Index file size for %d blobs in %d packs is %d", blobCount*packs, packs, wr.Len()) } @@ -308,8 +312,6 @@ func TestIndexUnserialize(t *testing.T) { {docExampleV1, 1}, {docExampleV2, 2}, } { - oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")} - idx, oldFormat, err := index.DecodeIndex(task.idxBytes, restic.NewRandomID()) rtest.OK(t, err) rtest.Assert(t, !oldFormat, "new index format recognized as old format") @@ -336,9 +338,7 @@ func TestIndexUnserialize(t *testing.T) { } } - rtest.Equals(t, oldIdx, idx.Supersedes()) - - blobs := listPack(idx, exampleLookupTest.packID) + blobs := listPack(t, idx, exampleLookupTest.packID) if len(blobs) != len(exampleLookupTest.blobs) { t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs)) } @@ -355,12 +355,12 @@ func TestIndexUnserialize(t *testing.T) { } } -func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) { - idx.Each(context.TODO(), func(pb restic.PackedBlob) { +func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) { + rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) { if pb.PackID.Equal(id) { pbs = append(pbs, pb) } - }) + })) return pbs } @@ -427,6 +427,8 @@ func BenchmarkEncodeIndex(b *testing.B) { } func TestIndexUnserializeOld(t *testing.T) { + defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateLegacyIndex, false)() + idx, oldFormat, err := index.DecodeIndex(docOldExample, restic.NewRandomID()) rtest.OK(t, err) rtest.Assert(t, oldFormat, "old index format recognized as new format") @@ -443,8 +445,6 @@ func TestIndexUnserializeOld(t *testing.T) { rtest.Equals(t, test.offset, blob.Offset) rtest.Equals(t, test.length, blob.Length) } - - rtest.Equals(t, 0, len(idx.Supersedes())) } func TestIndexPacks(t *testing.T) { diff --git a/mover-restic/restic/internal/index/indexmap.go b/mover-restic/restic/internal/repository/index/indexmap.go similarity index 88% rename from mover-restic/restic/internal/index/indexmap.go rename to mover-restic/restic/internal/repository/index/indexmap.go index 2386e01b6..6db523633 100644 --- a/mover-restic/restic/internal/index/indexmap.go +++ b/mover-restic/restic/internal/repository/index/indexmap.go @@ -99,6 +99,32 @@ func (m *indexMap) get(id restic.ID) *indexEntry { return nil } +// firstIndex returns the index of the first entry for ID id. +// This index is guaranteed to never change. +func (m *indexMap) firstIndex(id restic.ID) int { + if len(m.buckets) == 0 { + return -1 + } + + idx := -1 + h := m.hash(id) + ei := m.buckets[h] + for ei != 0 { + e := m.resolve(ei) + cur := ei + ei = e.next + if e.id != id { + continue + } + if int(cur) < idx || idx == -1 { + // casting from uint to int is unproblematic as we'd run out of memory + // before this can result in an overflow. + idx = int(cur) + } + } + return idx +} + func (m *indexMap) grow() { m.buckets = make([]uint, growthFactor*len(m.buckets)) @@ -118,9 +144,10 @@ func (m *indexMap) hash(id restic.ID) uint { // While SHA-256 should be collision-resistant, for hash table indices // we use only a few bits of it and finding collisions for those is // much easier than breaking the whole algorithm. - m.mh.Reset() - _, _ = m.mh.Write(id[:]) - h := uint(m.mh.Sum64()) + mh := maphash.Hash{} + mh.SetSeed(m.mh.Seed()) + _, _ = mh.Write(id[:]) + h := uint(mh.Sum64()) return h & uint(len(m.buckets)-1) } @@ -204,7 +231,7 @@ func (h *hashedArrayTree) Size() uint { func (h *hashedArrayTree) grow() { idx, subIdx := h.index(h.size) if int(idx) == len(h.blockList) { - // blockList is too small -> double list and block size + // blockList is too short -> double list and block size h.blockSize *= 2 h.mask = h.mask*2 + 1 h.maskShift++ diff --git a/mover-restic/restic/internal/index/indexmap_test.go b/mover-restic/restic/internal/repository/index/indexmap_test.go similarity index 77% rename from mover-restic/restic/internal/index/indexmap_test.go rename to mover-restic/restic/internal/repository/index/indexmap_test.go index a16670c7d..f34e6a1d3 100644 --- a/mover-restic/restic/internal/index/indexmap_test.go +++ b/mover-restic/restic/internal/repository/index/indexmap_test.go @@ -143,3 +143,45 @@ func BenchmarkIndexMapHash(b *testing.B) { } } } + +func TestIndexMapFirstIndex(t *testing.T) { + t.Parallel() + + var ( + id restic.ID + m indexMap + r = rand.New(rand.NewSource(98765)) + fi = make(map[restic.ID]int) + ) + + for i := 1; i <= 400; i++ { + r.Read(id[:]) + rtest.Equals(t, -1, m.firstIndex(id), "wrong firstIndex for nonexistent id") + + m.add(id, 0, 0, 0, 0) + idx := m.firstIndex(id) + rtest.Equals(t, i, idx, "unexpected index for id") + fi[id] = idx + } + // iterate over blobs, as this is a hashmap the order is effectively random + for id, idx := range fi { + rtest.Equals(t, idx, m.firstIndex(id), "wrong index returned") + } +} + +func TestIndexMapFirstIndexDuplicates(t *testing.T) { + t.Parallel() + + var ( + id restic.ID + m indexMap + r = rand.New(rand.NewSource(98765)) + ) + + r.Read(id[:]) + for i := 1; i <= 10; i++ { + m.add(id, 0, 0, 0, 0) + } + idx := m.firstIndex(id) + rtest.Equals(t, 1, idx, "unexpected index for id") +} diff --git a/mover-restic/restic/internal/index/master_index.go b/mover-restic/restic/internal/repository/index/master_index.go similarity index 52% rename from mover-restic/restic/internal/index/master_index.go rename to mover-restic/restic/internal/repository/index/master_index.go index ca7c16135..f8e776b23 100644 --- a/mover-restic/restic/internal/index/master_index.go +++ b/mover-restic/restic/internal/repository/index/master_index.go @@ -1,7 +1,6 @@ package index import ( - "bytes" "context" "fmt" "runtime" @@ -18,21 +17,19 @@ type MasterIndex struct { idx []*Index pendingBlobs restic.BlobSet idxMutex sync.RWMutex - compress bool } // NewMasterIndex creates a new master index. func NewMasterIndex() *MasterIndex { - // Always add an empty final index, such that MergeFinalIndexes can merge into this. - // Note that removing this index could lead to a race condition in the rare - // sitation that only two indexes exist which are saved and merged concurrently. - idx := []*Index{NewIndex()} - idx[0].Finalize() - return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()} + mi := &MasterIndex{pendingBlobs: restic.NewBlobSet()} + mi.clear() + return mi } -func (mi *MasterIndex) MarkCompressed() { - mi.compress = true +func (mi *MasterIndex) clear() { + // Always add an empty final index, such that MergeFinalIndexes can merge into this. + mi.idx = []*Index{NewIndex()} + mi.idx[0].Finalize() } // Lookup queries all known Indexes for the ID and returns all matches. @@ -209,7 +206,7 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index { continue } - if IndexFull(idx, mi.compress) { + if IndexFull(idx) { debug.Log("index %p is full", idx) idx.Finalize() list = append(list, idx) @@ -224,13 +221,16 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index { // Each runs fn on all blobs known to the index. When the context is cancelled, // the index iteration return immediately. This blocks any modification of the index. -func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) { +func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) error { mi.idxMutex.RLock() defer mi.idxMutex.RUnlock() for _, idx := range mi.idx { - idx.Each(ctx, fn) + if err := idx.Each(ctx, fn); err != nil { + return err + } } + return nil } // MergeFinalIndexes merges all final indexes together. @@ -265,74 +265,280 @@ func (mi *MasterIndex) MergeFinalIndexes() error { return nil } -// Save saves all known indexes to index files, leaving out any +func (mi *MasterIndex) Load(ctx context.Context, r restic.ListerLoaderUnpacked, p *progress.Counter, cb func(id restic.ID, idx *Index, oldFormat bool, err error) error) error { + indexList, err := restic.MemorizeList(ctx, r, restic.IndexFile) + if err != nil { + return err + } + + if p != nil { + var numIndexFiles uint64 + err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error { + numIndexFiles++ + return nil + }) + if err != nil { + return err + } + p.SetMax(numIndexFiles) + defer p.Done() + } + + err = ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error { + if p != nil { + p.Add(1) + } + if cb != nil { + err = cb(id, idx, oldFormat, err) + } + if err != nil { + return err + } + // special case to allow check to ignore index loading errors + if idx == nil { + return nil + } + mi.Insert(idx) + return nil + }) + + if err != nil { + return err + } + + return mi.MergeFinalIndexes() +} + +type MasterIndexRewriteOpts struct { + SaveProgress *progress.Counter + DeleteProgress func() *progress.Counter + DeleteReport func(id restic.ID, err error) +} + +// Rewrite removes packs whose ID is in excludePacks from all known indexes. +// It also removes the rewritten index files and those listed in extraObsolete. +// If oldIndexes is not nil, then only the indexes in this set are processed. +// This is used by repair index to only rewrite and delete the old indexes. +// +// Must not be called concurrently to any other MasterIndex operation. +func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked, excludePacks restic.IDSet, oldIndexes restic.IDSet, extraObsolete restic.IDs, opts MasterIndexRewriteOpts) error { + for _, idx := range mi.idx { + if !idx.Final() { + panic("internal error - index must be saved before calling MasterIndex.Rewrite") + } + } + + var indexes restic.IDSet + if oldIndexes != nil { + // repair index adds new index entries for already existing pack files + // only remove the old (possibly broken) entries by only processing old indexes + indexes = oldIndexes + } else { + indexes = mi.IDs() + } + + p := opts.SaveProgress + p.SetMax(uint64(len(indexes))) + + // reset state which is not necessary for Rewrite and just consumes a lot of memory + // the index state would be invalid after Rewrite completes anyways + mi.clear() + runtime.GC() + + // copy excludePacks to prevent unintended sideeffects + excludePacks = excludePacks.Clone() + debug.Log("start rebuilding index of %d indexes, excludePacks: %v", len(indexes), excludePacks) + wg, wgCtx := errgroup.WithContext(ctx) + + idxCh := make(chan restic.ID) + wg.Go(func() error { + defer close(idxCh) + for id := range indexes { + select { + case idxCh <- id: + case <-wgCtx.Done(): + return wgCtx.Err() + } + } + return nil + }) + + var rewriteWg sync.WaitGroup + type rewriteTask struct { + idx *Index + oldFormat bool + } + rewriteCh := make(chan rewriteTask) + loader := func() error { + defer rewriteWg.Done() + for id := range idxCh { + buf, err := repo.LoadUnpacked(wgCtx, restic.IndexFile, id) + if err != nil { + return fmt.Errorf("LoadUnpacked(%v): %w", id.Str(), err) + } + idx, oldFormat, err := DecodeIndex(buf, id) + if err != nil { + return err + } + + select { + case rewriteCh <- rewriteTask{idx, oldFormat}: + case <-wgCtx.Done(): + return wgCtx.Err() + } + + } + return nil + } + // loading an index can take quite some time such that this is probably CPU-bound + // the index files are probably already cached at this point + loaderCount := runtime.GOMAXPROCS(0) + // run workers on ch + for i := 0; i < loaderCount; i++ { + rewriteWg.Add(1) + wg.Go(loader) + } + wg.Go(func() error { + rewriteWg.Wait() + close(rewriteCh) + return nil + }) + + obsolete := restic.NewIDSet(extraObsolete...) + saveCh := make(chan *Index) + + wg.Go(func() error { + defer close(saveCh) + newIndex := NewIndex() + for task := range rewriteCh { + // always rewrite indexes using the old format, that include a pack that must be removed or that are not full + if !task.oldFormat && len(task.idx.Packs().Intersect(excludePacks)) == 0 && IndexFull(task.idx) { + // make sure that each pack is only stored exactly once in the index + excludePacks.Merge(task.idx.Packs()) + // index is already up to date + p.Add(1) + continue + } + + ids, err := task.idx.IDs() + if err != nil || len(ids) != 1 { + panic("internal error, index has no ID") + } + obsolete.Merge(restic.NewIDSet(ids...)) + + for pbs := range task.idx.EachByPack(wgCtx, excludePacks) { + newIndex.StorePack(pbs.PackID, pbs.Blobs) + if IndexFull(newIndex) { + select { + case saveCh <- newIndex: + case <-wgCtx.Done(): + return wgCtx.Err() + } + newIndex = NewIndex() + } + } + if wgCtx.Err() != nil { + return wgCtx.Err() + } + // make sure that each pack is only stored exactly once in the index + excludePacks.Merge(task.idx.Packs()) + p.Add(1) + } + + select { + case saveCh <- newIndex: + case <-wgCtx.Done(): + } + return nil + }) + + // a worker receives an index from ch, and saves the index + worker := func() error { + for idx := range saveCh { + idx.Finalize() + if _, err := idx.SaveIndex(wgCtx, repo); err != nil { + return err + } + } + return nil + } + + // encoding an index can take quite some time such that this can be CPU- or IO-bound + // do not add repo.Connections() here as there are already the loader goroutines. + workerCount := runtime.GOMAXPROCS(0) + // run workers on ch + for i := 0; i < workerCount; i++ { + wg.Go(worker) + } + err := wg.Wait() + p.Done() + if err != nil { + return fmt.Errorf("failed to rewrite indexes: %w", err) + } + + p = nil + if opts.DeleteProgress != nil { + p = opts.DeleteProgress() + } + defer p.Done() + return restic.ParallelRemove(ctx, repo, obsolete, restic.IndexFile, func(id restic.ID, err error) error { + if opts.DeleteReport != nil { + opts.DeleteReport(id, err) + } + return err + }, p) +} + +// SaveFallback saves all known indexes to index files, leaving out any // packs whose ID is contained in packBlacklist from finalized indexes. -// The new index contains the IDs of all known indexes in the "supersedes" -// field. The IDs are also returned in the IDSet obsolete. -// After calling this function, you should remove the obsolete index files. -func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, packBlacklist restic.IDSet, extraObsolete restic.IDs, p *progress.Counter) (obsolete restic.IDSet, err error) { - p.SetMax(uint64(len(mi.Packs(packBlacklist)))) +// It is only intended for use by prune with the UnsafeRecovery option. +// +// Must not be called concurrently to any other MasterIndex operation. +func (mi *MasterIndex) SaveFallback(ctx context.Context, repo restic.SaverRemoverUnpacked, excludePacks restic.IDSet, p *progress.Counter) error { + p.SetMax(uint64(len(mi.Packs(excludePacks)))) mi.idxMutex.Lock() defer mi.idxMutex.Unlock() - debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist) + debug.Log("start rebuilding index of %d indexes, excludePacks: %v", len(mi.idx), excludePacks) - newIndex := NewIndex() - obsolete = restic.NewIDSet() - - // track spawned goroutines using wg, create a new context which is - // cancelled as soon as an error occurs. - wg, ctx := errgroup.WithContext(ctx) + obsolete := restic.NewIDSet() + wg, wgCtx := errgroup.WithContext(ctx) ch := make(chan *Index) - wg.Go(func() error { defer close(ch) - for i, idx := range mi.idx { + newIndex := NewIndex() + for _, idx := range mi.idx { if idx.Final() { ids, err := idx.IDs() if err != nil { - debug.Log("index %d does not have an ID: %v", err) - return err + panic("internal error - finalized index without ID") } - debug.Log("adding index ids %v to supersedes field", ids) - - err = newIndex.AddToSupersedes(ids...) - if err != nil { - return err - } obsolete.Merge(restic.NewIDSet(ids...)) - } else { - debug.Log("index %d isn't final, don't add to supersedes field", i) } - debug.Log("adding index %d", i) - - for pbs := range idx.EachByPack(ctx, packBlacklist) { + for pbs := range idx.EachByPack(wgCtx, excludePacks) { newIndex.StorePack(pbs.PackID, pbs.Blobs) p.Add(1) - if IndexFull(newIndex, mi.compress) { + if IndexFull(newIndex) { select { case ch <- newIndex: - case <-ctx.Done(): - return ctx.Err() + case <-wgCtx.Done(): + return wgCtx.Err() } newIndex = NewIndex() } } + if wgCtx.Err() != nil { + return wgCtx.Err() + } } - err = newIndex.AddToSupersedes(extraObsolete...) - if err != nil { - return err - } - obsolete.Merge(restic.NewIDSet(extraObsolete...)) - select { case ch <- newIndex: - case <-ctx.Done(): + case <-wgCtx.Done(): } return nil }) @@ -341,40 +547,25 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack worker := func() error { for idx := range ch { idx.Finalize() - if _, err := SaveIndex(ctx, repo, idx); err != nil { + if _, err := idx.SaveIndex(wgCtx, repo); err != nil { return err } } return nil } - // encoding an index can take quite some time such that this can be both CPU- or IO-bound - workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0) + // keep concurrency bounded as we're on a fallback path + workerCount := int(repo.Connections()) // run workers on ch for i := 0; i < workerCount; i++ { wg.Go(worker) } - err = wg.Wait() + err := wg.Wait() + p.Done() + // the index no longer matches to stored state + mi.clear() - return obsolete, err -} - -// SaveIndex saves an index in the repository. -func SaveIndex(ctx context.Context, repo restic.SaverUnpacked, index *Index) (restic.ID, error) { - buf := bytes.NewBuffer(nil) - - err := index.Encode(buf) - if err != nil { - return restic.ID{}, err - } - - id, err := repo.SaveUnpacked(ctx, restic.IndexFile, buf.Bytes()) - ierr := index.SetID(id) - if ierr != nil { - // logic bug - panic(ierr) - } - return id, err + return err } // saveIndex saves all indexes in the backend. @@ -382,7 +573,7 @@ func (mi *MasterIndex) saveIndex(ctx context.Context, r restic.SaverUnpacked, in for i, idx := range indexes { debug.Log("Saving index %d", i) - sid, err := SaveIndex(ctx, r, idx) + sid, err := idx.SaveIndex(ctx, r) if err != nil { return err } @@ -410,10 +601,6 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan defer close(out) // only resort a part of the index to keep the memory overhead bounded for i := byte(0); i < 16; i++ { - if ctx.Err() != nil { - return - } - packBlob := make(map[restic.ID][]restic.Blob) for pack := range packs { if pack[0]&0xf == i { @@ -423,11 +610,14 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan if len(packBlob) == 0 { continue } - mi.Each(ctx, func(pb restic.PackedBlob) { + err := mi.Each(ctx, func(pb restic.PackedBlob) { if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i { packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob) } }) + if err != nil { + return + } // pass on packs for packID, pbs := range packBlob { @@ -443,3 +633,21 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan }() return out } + +// Only for use by AssociatedSet +func (mi *MasterIndex) blobIndex(h restic.BlobHandle) int { + mi.idxMutex.RLock() + defer mi.idxMutex.RUnlock() + + // other indexes are ignored as their ids can change when merged into the main index + return mi.idx[0].BlobIndex(h) +} + +// Only for use by AssociatedSet +func (mi *MasterIndex) stableLen(t restic.BlobType) uint { + mi.idxMutex.RLock() + defer mi.idxMutex.RUnlock() + + // other indexes are ignored as their ids can change when merged into the main index + return mi.idx[0].Len(t) +} diff --git a/mover-restic/restic/internal/index/master_index_test.go b/mover-restic/restic/internal/repository/index/master_index_test.go similarity index 68% rename from mover-restic/restic/internal/index/master_index_test.go rename to mover-restic/restic/internal/repository/index/master_index_test.go index bf8ec3f41..23185962e 100644 --- a/mover-restic/restic/internal/index/master_index_test.go +++ b/mover-restic/restic/internal/repository/index/master_index_test.go @@ -10,8 +10,8 @@ import ( "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" - "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -161,14 +161,17 @@ func TestMasterMergeFinalIndexes(t *testing.T) { mIdx.Insert(idx1) mIdx.Insert(idx2) - finalIndexes, idxCount := index.TestMergeIndex(t, mIdx) + rtest.Equals(t, restic.NewIDSet(), mIdx.IDs()) + + finalIndexes, idxCount, ids := index.TestMergeIndex(t, mIdx) rtest.Equals(t, []*index.Index{idx1, idx2}, finalIndexes) rtest.Equals(t, 1, idxCount) + rtest.Equals(t, ids, mIdx.IDs()) blobCount := 0 - mIdx.Each(context.TODO(), func(pb restic.PackedBlob) { + rtest.OK(t, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) { blobCount++ - }) + })) rtest.Equals(t, 2, blobCount) blobs := mIdx.Lookup(bhInIdx1) @@ -186,9 +189,11 @@ func TestMasterMergeFinalIndexes(t *testing.T) { idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob}) mIdx.Insert(idx3) - finalIndexes, idxCount = index.TestMergeIndex(t, mIdx) + finalIndexes, idxCount, newIDs := index.TestMergeIndex(t, mIdx) rtest.Equals(t, []*index.Index{idx3}, finalIndexes) rtest.Equals(t, 1, idxCount) + ids.Merge(newIDs) + rtest.Equals(t, ids, mIdx.IDs()) // Index should have same entries as before! blobs = mIdx.Lookup(bhInIdx1) @@ -198,9 +203,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) { rtest.Equals(t, []restic.PackedBlob{blob2}, blobs) blobCount = 0 - mIdx.Each(context.TODO(), func(pb restic.PackedBlob) { + rtest.OK(t, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) { blobCount++ - }) + })) rtest.Equals(t, 2, blobCount) } @@ -319,9 +324,9 @@ func BenchmarkMasterIndexEach(b *testing.B) { for i := 0; i < b.N; i++ { entries := 0 - mIdx.Each(context.TODO(), func(pb restic.PackedBlob) { + rtest.OK(b, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) { entries++ - }) + })) } } @@ -342,7 +347,7 @@ var ( ) func createFilledRepo(t testing.TB, snapshots int, version uint) restic.Repository { - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) for i := 0; i < snapshots; i++ { restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth) @@ -355,55 +360,102 @@ func TestIndexSave(t *testing.T) { } func testIndexSave(t *testing.T, version uint) { - repo := createFilledRepo(t, 3, version) + for _, test := range []struct { + name string + saver func(idx *index.MasterIndex, repo restic.Repository) error + }{ + {"rewrite no-op", func(idx *index.MasterIndex, repo restic.Repository) error { + return idx.Rewrite(context.TODO(), repo, nil, nil, nil, index.MasterIndexRewriteOpts{}) + }}, + {"rewrite skip-all", func(idx *index.MasterIndex, repo restic.Repository) error { + return idx.Rewrite(context.TODO(), repo, nil, restic.NewIDSet(), nil, index.MasterIndexRewriteOpts{}) + }}, + {"SaveFallback", func(idx *index.MasterIndex, repo restic.Repository) error { + err := restic.ParallelRemove(context.TODO(), repo, idx.IDs(), restic.IndexFile, nil, nil) + if err != nil { + return nil + } + return idx.SaveFallback(context.TODO(), repo, restic.NewIDSet(), nil) + }}, + } { + t.Run(test.name, func(t *testing.T) { + repo := createFilledRepo(t, 3, version) + + idx := index.NewMasterIndex() + rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) + blobs := make(map[restic.PackedBlob]struct{}) + rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) { + blobs[pb] = struct{}{} + })) + + rtest.OK(t, test.saver(idx, repo)) + idx = index.NewMasterIndex() + rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) + + rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) { + if _, ok := blobs[pb]; ok { + delete(blobs, pb) + } else { + t.Fatalf("unexpected blobs %v", pb) + } + })) + rtest.Equals(t, 0, len(blobs), "saved index is missing blobs") - err := repo.LoadIndex(context.TODO(), nil) - if err != nil { - t.Fatal(err) + checker.TestCheckRepo(t, repo, false) + }) } +} - obsoletes, err := repo.Index().Save(context.TODO(), repo, nil, nil, nil) - if err != nil { - t.Fatalf("unable to save new index: %v", err) - } +func TestIndexSavePartial(t *testing.T) { + repository.TestAllVersions(t, testIndexSavePartial) +} - for id := range obsoletes { - t.Logf("remove index %v", id.Str()) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(context.TODO(), h) - if err != nil { - t.Errorf("error removing index %v: %v", id, err) - } - } +func testIndexSavePartial(t *testing.T, version uint) { + repo := createFilledRepo(t, 3, version) - checker := checker.New(repo, false) - err = checker.LoadSnapshots(context.TODO()) - if err != nil { - t.Error(err) - } + // capture blob list before adding fourth snapshot + idx := index.NewMasterIndex() + rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) + blobs := make(map[restic.PackedBlob]struct{}) + rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) { + blobs[pb] = struct{}{} + })) + + // add+remove new snapshot and track its pack files + packsBefore := listPacks(t, repo) + sn := restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(4)*time.Second), depth) + rtest.OK(t, repo.RemoveUnpacked(context.TODO(), restic.SnapshotFile, *sn.ID())) + packsAfter := listPacks(t, repo) + newPacks := packsAfter.Sub(packsBefore) + + // rewrite index and remove pack files of new snapshot + idx = index.NewMasterIndex() + rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) + rtest.OK(t, idx.Rewrite(context.TODO(), repo, newPacks, nil, nil, index.MasterIndexRewriteOpts{})) + + // check blobs + idx = index.NewMasterIndex() + rtest.OK(t, idx.Load(context.TODO(), repo, nil, nil)) + rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) { + if _, ok := blobs[pb]; ok { + delete(blobs, pb) + } else { + t.Fatalf("unexpected blobs %v", pb) + } + })) + rtest.Equals(t, 0, len(blobs), "saved index is missing blobs") - hints, errs := checker.LoadIndex(context.TODO(), nil) - for _, h := range hints { - t.Logf("hint: %v\n", h) - } + // remove pack files to make check happy + rtest.OK(t, restic.ParallelRemove(context.TODO(), repo, newPacks, restic.PackFile, nil, nil)) - for _, err := range errs { - t.Errorf("checker found error: %v", err) - } + checker.TestCheckRepo(t, repo, false) +} - ctx, cancel := context.WithCancel(context.TODO()) - defer cancel() - - errCh := make(chan error) - go checker.Structure(ctx, nil, errCh) - i := 0 - for err := range errCh { - t.Errorf("checker returned error: %v", err) - i++ - if i == 10 { - t.Errorf("more than 10 errors returned, skipping the rest") - cancel() - break - } - } +func listPacks(t testing.TB, repo restic.Lister) restic.IDSet { + s := restic.NewIDSet() + rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, _ int64) error { + s.Insert(id) + return nil + })) + return s } diff --git a/mover-restic/restic/internal/index/testing.go b/mover-restic/restic/internal/repository/index/testing.go similarity index 66% rename from mover-restic/restic/internal/index/testing.go rename to mover-restic/restic/internal/repository/index/testing.go index 7c05ac651..0b5084bb0 100644 --- a/mover-restic/restic/internal/index/testing.go +++ b/mover-restic/restic/internal/repository/index/testing.go @@ -7,12 +7,15 @@ import ( "github.com/restic/restic/internal/test" ) -func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) { +func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int, restic.IDSet) { finalIndexes := mi.finalizeNotFinalIndexes() + ids := restic.NewIDSet() for _, idx := range finalIndexes { - test.OK(t, idx.SetID(restic.NewRandomID())) + id := restic.NewRandomID() + ids.Insert(id) + test.OK(t, idx.SetID(id)) } test.OK(t, mi.MergeFinalIndexes()) - return finalIndexes, len(mi.idx) + return finalIndexes, len(mi.idx), ids } diff --git a/mover-restic/restic/internal/repository/key.go b/mover-restic/restic/internal/repository/key.go index fd20b8e5f..08f997544 100644 --- a/mover-restic/restic/internal/repository/key.go +++ b/mover-restic/restic/internal/repository/key.go @@ -43,11 +43,11 @@ type Key struct { id restic.ID } -// Params tracks the parameters used for the KDF. If not set, it will be +// params tracks the parameters used for the KDF. If not set, it will be // calibrated on the first run of AddKey(). -var Params *crypto.Params +var params *crypto.Params -var ( +const ( // KDFTimeout specifies the maximum runtime for the KDF. KDFTimeout = 500 * time.Millisecond @@ -116,7 +116,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, checked := 0 if len(keyHint) > 0 { - id, err := restic.Find(ctx, s.Backend(), restic.KeyFile, keyHint) + id, err := restic.Find(ctx, s, restic.KeyFile, keyHint) if err == nil { key, err := OpenKey(ctx, s, id, password) @@ -136,7 +136,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, defer cancel() // try at most maxKeys keys in repo - err = s.List(listCtx, restic.KeyFile, func(id restic.ID, size int64) error { + err = s.List(listCtx, restic.KeyFile, func(id restic.ID, _ int64) error { checked++ if maxKeys > 0 && checked > maxKeys { return ErrMaxKeysReached @@ -178,8 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, // LoadKey loads a key from the backend. func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) { - h := restic.Handle{Type: restic.KeyFile, Name: id.String()} - data, err := backend.LoadAll(ctx, nil, s.be, h) + data, err := s.LoadRaw(ctx, restic.KeyFile, id) if err != nil { return nil, err } @@ -196,13 +195,13 @@ func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err erro // AddKey adds a new key to an already existing repository. func AddKey(ctx context.Context, s *Repository, password, username, hostname string, template *crypto.Key) (*Key, error) { // make sure we have valid KDF parameters - if Params == nil { + if params == nil { p, err := crypto.Calibrate(KDFTimeout, KDFMemory) if err != nil { return nil, errors.Wrap(err, "Calibrate") } - Params = &p + params = &p debug.Log("calibrated KDF parameters are %v", p) } @@ -213,9 +212,9 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str Hostname: hostname, KDF: "scrypt", - N: Params.N, - R: Params.R, - P: Params.P, + N: params.N, + R: params.R, + P: params.P, } if newkey.Hostname == "" { @@ -237,7 +236,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str } // call KDF to derive user key - newkey.user, err = crypto.KDF(*Params, newkey.Salt, password) + newkey.user, err = crypto.KDF(*params, newkey.Salt, password) if err != nil { return nil, err } @@ -270,12 +269,12 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str id := restic.Hash(buf) // store in repository and return - h := restic.Handle{ + h := backend.Handle{ Type: restic.KeyFile, Name: id.String(), } - err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher())) + err = s.be.Save(ctx, h, backend.NewByteReader(buf, s.be.Hasher())) if err != nil { return nil, err } @@ -285,6 +284,15 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str return newkey, nil } +func RemoveKey(ctx context.Context, repo *Repository, id restic.ID) error { + if id == repo.KeyID() { + return errors.New("refusing to remove key currently used to access repository") + } + + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} + return repo.be.Remove(ctx, h) +} + func (k *Key) String() string { if k == nil { return "" diff --git a/mover-restic/restic/internal/repository/lock.go b/mover-restic/restic/internal/repository/lock.go new file mode 100644 index 000000000..fd46066d1 --- /dev/null +++ b/mover-restic/restic/internal/repository/lock.go @@ -0,0 +1,274 @@ +package repository + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +type lockContext struct { + lock *restic.Lock + cancel context.CancelFunc + refreshWG sync.WaitGroup +} + +type locker struct { + retrySleepStart time.Duration + retrySleepMax time.Duration + refreshInterval time.Duration + refreshabilityTimeout time.Duration +} + +const defaultRefreshInterval = 5 * time.Minute + +var lockerInst = &locker{ + retrySleepStart: 5 * time.Second, + retrySleepMax: 60 * time.Second, + refreshInterval: defaultRefreshInterval, + // consider a lock refresh failed a bit before the lock actually becomes stale + // the difference allows to compensate for a small time drift between clients. + refreshabilityTimeout: restic.StaleLockTimeout - defaultRefreshInterval*3/2, +} + +func Lock(ctx context.Context, repo *Repository, exclusive bool, retryLock time.Duration, printRetry func(msg string), logger func(format string, args ...interface{})) (*Unlocker, context.Context, error) { + return lockerInst.Lock(ctx, repo, exclusive, retryLock, printRetry, logger) +} + +// Lock wraps the ctx such that it is cancelled when the repository is unlocked +// cancelling the original context also stops the lock refresh +func (l *locker) Lock(ctx context.Context, repo *Repository, exclusive bool, retryLock time.Duration, printRetry func(msg string), logger func(format string, args ...interface{})) (*Unlocker, context.Context, error) { + + lockFn := restic.NewLock + if exclusive { + lockFn = restic.NewExclusiveLock + } + + var lock *restic.Lock + var err error + + retrySleep := minDuration(l.retrySleepStart, retryLock) + retryMessagePrinted := false + retryTimeout := time.After(retryLock) + +retryLoop: + for { + lock, err = lockFn(ctx, repo) + if err != nil && restic.IsAlreadyLocked(err) { + + if !retryMessagePrinted { + printRetry(fmt.Sprintf("repo already locked, waiting up to %s for the lock\n", retryLock)) + retryMessagePrinted = true + } + + debug.Log("repo already locked, retrying in %v", retrySleep) + retrySleepCh := time.After(retrySleep) + + select { + case <-ctx.Done(): + return nil, ctx, ctx.Err() + case <-retryTimeout: + debug.Log("repo already locked, timeout expired") + // Last lock attempt + lock, err = lockFn(ctx, repo) + break retryLoop + case <-retrySleepCh: + retrySleep = minDuration(retrySleep*2, l.retrySleepMax) + } + } else { + // anything else, either a successful lock or another error + break retryLoop + } + } + if restic.IsInvalidLock(err) { + return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err) + } + if err != nil { + return nil, ctx, fmt.Errorf("unable to create lock in backend: %w", err) + } + debug.Log("create lock %p (exclusive %v)", lock, exclusive) + + ctx, cancel := context.WithCancel(ctx) + lockInfo := &lockContext{ + lock: lock, + cancel: cancel, + } + lockInfo.refreshWG.Add(2) + refreshChan := make(chan struct{}) + forceRefreshChan := make(chan refreshLockRequest) + + go l.refreshLocks(ctx, repo.be, lockInfo, refreshChan, forceRefreshChan, logger) + go l.monitorLockRefresh(ctx, lockInfo, refreshChan, forceRefreshChan, logger) + + return &Unlocker{lockInfo}, ctx, nil +} + +func minDuration(a, b time.Duration) time.Duration { + if a <= b { + return a + } + return b +} + +type refreshLockRequest struct { + result chan bool +} + +func (l *locker) refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest, logger func(format string, args ...interface{})) { + debug.Log("start") + lock := lockInfo.lock + ticker := time.NewTicker(l.refreshInterval) + lastRefresh := lock.Time + + defer func() { + ticker.Stop() + // ensure that the context was cancelled before removing the lock + lockInfo.cancel() + + // remove the lock from the repo + debug.Log("unlocking repository with lock %v", lock) + if err := lock.Unlock(ctx); err != nil { + debug.Log("error while unlocking: %v", err) + logger("error while unlocking: %v", err) + } + + lockInfo.refreshWG.Done() + }() + + for { + select { + case <-ctx.Done(): + debug.Log("terminate") + return + + case req := <-forceRefresh: + debug.Log("trying to refresh stale lock") + // keep on going if our current lock still exists + success := tryRefreshStaleLock(ctx, backend, lock, lockInfo.cancel, logger) + // inform refresh goroutine about forced refresh + select { + case <-ctx.Done(): + case req.result <- success: + } + + if success { + // update lock refresh time + lastRefresh = lock.Time + } + + case <-ticker.C: + if time.Since(lastRefresh) > l.refreshabilityTimeout { + // the lock is too old, wait until the expiry monitor cancels the context + continue + } + + debug.Log("refreshing locks") + err := lock.Refresh(context.TODO()) + if err != nil { + logger("unable to refresh lock: %v\n", err) + } else { + lastRefresh = lock.Time + // inform monitor goroutine about successful refresh + select { + case <-ctx.Done(): + case refreshed <- struct{}{}: + } + } + } + } +} + +func (l *locker) monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-chan struct{}, forceRefresh chan<- refreshLockRequest, logger func(format string, args ...interface{})) { + // time.Now() might use a monotonic timer which is paused during standby + // convert to unix time to ensure we compare real time values + lastRefresh := time.Now().UnixNano() + pollDuration := 1 * time.Second + if l.refreshInterval < pollDuration { + // required for TestLockFailedRefresh + pollDuration = l.refreshInterval / 5 + } + // timers are paused during standby, which is a problem as the refresh timeout + // _must_ expire if the host was too long in standby. Thus fall back to periodic checks + // https://github.com/golang/go/issues/35012 + ticker := time.NewTicker(pollDuration) + defer func() { + ticker.Stop() + lockInfo.cancel() + lockInfo.refreshWG.Done() + }() + + var refreshStaleLockResult chan bool + + for { + select { + case <-ctx.Done(): + debug.Log("terminate expiry monitoring") + return + case <-refreshed: + if refreshStaleLockResult != nil { + // ignore delayed refresh notifications while the stale lock is refreshed + continue + } + lastRefresh = time.Now().UnixNano() + case <-ticker.C: + if time.Now().UnixNano()-lastRefresh < l.refreshabilityTimeout.Nanoseconds() || refreshStaleLockResult != nil { + continue + } + + debug.Log("trying to refreshStaleLock") + // keep on going if our current lock still exists + refreshReq := refreshLockRequest{ + result: make(chan bool), + } + refreshStaleLockResult = refreshReq.result + + // inform refresh goroutine about forced refresh + select { + case <-ctx.Done(): + case forceRefresh <- refreshReq: + } + case success := <-refreshStaleLockResult: + if success { + lastRefresh = time.Now().UnixNano() + refreshStaleLockResult = nil + continue + } + + logger("Fatal: failed to refresh lock in time\n") + return + } + } +} + +func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc, logger func(format string, args ...interface{})) bool { + freeze := backend.AsBackend[backend.FreezeBackend](be) + if freeze != nil { + debug.Log("freezing backend") + freeze.Freeze() + defer freeze.Unfreeze() + } + + err := lock.RefreshStaleLock(ctx) + if err != nil { + logger("failed to refresh stale lock: %v\n", err) + // cancel context while the backend is still frozen to prevent accidental modifications + cancel() + return false + } + + return true +} + +type Unlocker struct { + info *lockContext +} + +func (l *Unlocker) Unlock() { + l.info.cancel() + l.info.refreshWG.Wait() +} diff --git a/mover-restic/restic/cmd/restic/lock_test.go b/mover-restic/restic/internal/repository/lock_test.go similarity index 53% rename from mover-restic/restic/cmd/restic/lock_test.go rename to mover-restic/restic/internal/repository/lock_test.go index 2f8420853..bd7cbd5e2 100644 --- a/mover-restic/restic/cmd/restic/lock_test.go +++ b/mover-restic/restic/internal/repository/lock_test.go @@ -1,4 +1,4 @@ -package main +package repository import ( "context" @@ -9,94 +9,77 @@ import ( "testing" "time" - "github.com/restic/restic/internal/backend/location" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" ) -func openLockTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) { - env, cleanup := withTestEnvironment(t) +type backendWrapper func(r backend.Backend) (backend.Backend, error) - reg := location.NewRegistry() - reg.Register(mem.NewFactory()) - env.gopts.backends = reg - env.gopts.Repo = "mem:" +func openLockTestRepo(t *testing.T, wrapper backendWrapper) (*Repository, backend.Backend) { + be := backend.Backend(mem.New()) + // initialize repo + TestRepositoryWithBackend(t, be, 0, Options{}) + // reopen repository to allow injecting a backend wrapper if wrapper != nil { - env.gopts.backendTestHook = wrapper + var err error + be, err = wrapper(be) + rtest.OK(t, err) } - testRunInit(t, env.gopts) - repo, err := OpenRepository(context.TODO(), env.gopts) - test.OK(t, err) - return repo, cleanup, env + return TestOpenBackend(t, be), be } -func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository, env *testEnvironment) (*restic.Lock, context.Context) { - lock, wrappedCtx, err := lockRepo(ctx, repo, env.gopts.RetryLock, env.gopts.JSON) +func checkedLockRepo(ctx context.Context, t *testing.T, repo *Repository, lockerInst *locker, retryLock time.Duration) (*Unlocker, context.Context) { + lock, wrappedCtx, err := lockerInst.Lock(ctx, repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {}) test.OK(t, err) test.OK(t, wrappedCtx.Err()) - if lock.Stale() { + if lock.info.lock.Stale() { t.Fatal("lock returned stale lock") } return lock, wrappedCtx } func TestLock(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() + t.Parallel() + repo, _ := openLockTestRepo(t, nil) - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) - unlockRepo(lock) + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, lockerInst, 0) + lock.Unlock() if wrappedCtx.Err() == nil { t.Fatal("unlock did not cancel context") } } func TestLockCancel(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() + t.Parallel() + repo, _ := openLockTestRepo(t, nil) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - lock, wrappedCtx := checkedLockRepo(ctx, t, repo, env) + lock, wrappedCtx := checkedLockRepo(ctx, t, repo, lockerInst, 0) cancel() if wrappedCtx.Err() == nil { t.Fatal("canceled parent context did not cancel context") } - // unlockRepo should not crash - unlockRepo(lock) -} - -func TestLockUnlockAll(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() - - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) - _, err := unlockAll(0) - test.OK(t, err) - if wrappedCtx.Err() == nil { - t.Fatal("canceled parent context did not cancel context") - } - - // unlockRepo should not crash - unlockRepo(lock) + // Unlock should not crash + lock.Unlock() } func TestLockConflict(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() - repo2, err := OpenRepository(context.TODO(), env.gopts) - test.OK(t, err) + t.Parallel() + repo, be := openLockTestRepo(t, nil) + repo2 := TestOpenBackend(t, be) - lock, _, err := lockRepoExclusive(context.Background(), repo, env.gopts.RetryLock, env.gopts.JSON) + lock, _, err := Lock(context.Background(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {}) test.OK(t, err) - defer unlockRepo(lock) - _, _, err = lockRepo(context.Background(), repo2, env.gopts.RetryLock, env.gopts.JSON) + defer lock.Unlock() + _, _, err = Lock(context.Background(), repo2, false, 0, func(msg string) {}, func(format string, args ...interface{}) {}) if err == nil { t.Fatal("second lock should have failed") } @@ -104,11 +87,11 @@ func TestLockConflict(t *testing.T) { } type writeOnceBackend struct { - restic.Backend + backend.Backend written bool } -func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *writeOnceBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { if b.written { return fmt.Errorf("fail after first write") } @@ -117,20 +100,19 @@ func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic. } func TestLockFailedRefresh(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + t.Parallel() + repo, _ := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { return &writeOnceBackend{Backend: r}, nil }) - defer cleanup() // reduce locking intervals to be suitable for testing - ri, rt := refreshInterval, refreshabilityTimeout - refreshInterval = 20 * time.Millisecond - refreshabilityTimeout = 100 * time.Millisecond - defer func() { - refreshInterval, refreshabilityTimeout = ri, rt - }() - - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) + li := &locker{ + retrySleepStart: lockerInst.retrySleepStart, + retrySleepMax: lockerInst.retrySleepMax, + refreshInterval: 20 * time.Millisecond, + refreshabilityTimeout: 100 * time.Millisecond, + } + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, li, 0) select { case <-wrappedCtx.Done(): @@ -138,16 +120,16 @@ func TestLockFailedRefresh(t *testing.T) { case <-time.After(time.Second): t.Fatal("failed lock refresh did not cause context cancellation") } - // unlockRepo should not crash - unlockRepo(lock) + // Unlock should not crash + lock.Unlock() } type loggingBackend struct { - restic.Backend + backend.Backend t *testing.T } -func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *loggingBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { b.t.Logf("save %v @ %v", h, time.Now()) err := b.Backend.Save(ctx, h, rd) b.t.Logf("save finished %v @ %v", h, time.Now()) @@ -155,24 +137,23 @@ func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re } func TestLockSuccessfulRefresh(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + t.Parallel() + repo, _ := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { return &loggingBackend{ Backend: r, t: t, }, nil }) - defer cleanup() t.Logf("test for successful lock refresh %v", time.Now()) // reduce locking intervals to be suitable for testing - ri, rt := refreshInterval, refreshabilityTimeout - refreshInterval = 60 * time.Millisecond - refreshabilityTimeout = 500 * time.Millisecond - defer func() { - refreshInterval, refreshabilityTimeout = ri, rt - }() - - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) + li := &locker{ + retrySleepStart: lockerInst.retrySleepStart, + retrySleepMax: lockerInst.retrySleepMax, + refreshInterval: 60 * time.Millisecond, + refreshabilityTimeout: 500 * time.Millisecond, + } + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, li, 0) select { case <-wrappedCtx.Done(): @@ -185,20 +166,20 @@ func TestLockSuccessfulRefresh(t *testing.T) { buf = buf[:n] t.Log(string(buf)) - case <-time.After(2 * refreshabilityTimeout): + case <-time.After(2 * li.refreshabilityTimeout): // expected lock refresh to work } - // unlockRepo should not crash - unlockRepo(lock) + // Unlock should not crash + lock.Unlock() } type slowBackend struct { - restic.Backend + backend.Backend m sync.Mutex sleep time.Duration } -func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error { +func (b *slowBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { b.m.Lock() sleep := b.sleep b.m.Unlock() @@ -207,26 +188,26 @@ func (b *slowBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewin } func TestLockSuccessfulStaleRefresh(t *testing.T) { + t.Parallel() var sb *slowBackend - repo, cleanup, env := openLockTestRepo(t, func(r restic.Backend) (restic.Backend, error) { + repo, _ := openLockTestRepo(t, func(r backend.Backend) (backend.Backend, error) { sb = &slowBackend{Backend: r} return sb, nil }) - defer cleanup() t.Logf("test for successful lock refresh %v", time.Now()) // reduce locking intervals to be suitable for testing - ri, rt := refreshInterval, refreshabilityTimeout - refreshInterval = 10 * time.Millisecond - refreshabilityTimeout = 50 * time.Millisecond - defer func() { - refreshInterval, refreshabilityTimeout = ri, rt - }() - - lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, env) + li := &locker{ + retrySleepStart: lockerInst.retrySleepStart, + retrySleepMax: lockerInst.retrySleepMax, + refreshInterval: 10 * time.Millisecond, + refreshabilityTimeout: 50 * time.Millisecond, + } + + lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo, li, 0) // delay lock refreshing long enough that the lock would expire sb.m.Lock() - sb.sleep = refreshabilityTimeout + refreshInterval + sb.sleep = li.refreshabilityTimeout + li.refreshInterval sb.m.Unlock() select { @@ -234,7 +215,7 @@ func TestLockSuccessfulStaleRefresh(t *testing.T) { // don't call t.Fatal to allow the lock to be properly cleaned up t.Error("lock refresh failed", time.Now()) - case <-time.After(refreshabilityTimeout): + case <-time.After(li.refreshabilityTimeout): } // reset slow backend sb.m.Lock() @@ -247,25 +228,26 @@ func TestLockSuccessfulStaleRefresh(t *testing.T) { // don't call t.Fatal to allow the lock to be properly cleaned up t.Error("lock refresh failed", time.Now()) - case <-time.After(3 * refreshabilityTimeout): + case <-time.After(3 * li.refreshabilityTimeout): // expected lock refresh to work } - // unlockRepo should not crash - unlockRepo(lock) + // Unlock should not crash + lock.Unlock() } func TestLockWaitTimeout(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() + t.Parallel() + repo, _ := openLockTestRepo(t, nil) - elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + elock, _, err := Lock(context.TODO(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {}) test.OK(t, err) + defer elock.Unlock() retryLock := 200 * time.Millisecond start := time.Now() - lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) + _, _, err = Lock(context.TODO(), repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {}) duration := time.Since(start) test.Assert(t, err != nil, @@ -274,17 +256,15 @@ func TestLockWaitTimeout(t *testing.T) { "create normal lock with exclusively locked repo didn't return the correct error") test.Assert(t, retryLock <= duration && duration < retryLock*3/2, "create normal lock with exclusively locked repo didn't wait for the specified timeout") - - test.OK(t, lock.Unlock()) - test.OK(t, elock.Unlock()) } func TestLockWaitCancel(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() + t.Parallel() + repo, _ := openLockTestRepo(t, nil) - elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + elock, _, err := Lock(context.TODO(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {}) test.OK(t, err) + defer elock.Unlock() retryLock := 200 * time.Millisecond cancelAfter := 40 * time.Millisecond @@ -293,7 +273,7 @@ func TestLockWaitCancel(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) time.AfterFunc(cancelAfter, cancel) - lock, _, err := lockRepo(ctx, repo, retryLock, env.gopts.JSON) + _, _, err = Lock(ctx, repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {}) duration := time.Since(start) test.Assert(t, err != nil, @@ -302,27 +282,23 @@ func TestLockWaitCancel(t *testing.T) { "create normal lock with exclusively locked repo didn't return the correct error") test.Assert(t, cancelAfter <= duration && duration < retryLock-10*time.Millisecond, "create normal lock with exclusively locked repo didn't return in time, duration %v", duration) - - test.OK(t, lock.Unlock()) - test.OK(t, elock.Unlock()) } func TestLockWaitSuccess(t *testing.T) { - repo, cleanup, env := openLockTestRepo(t, nil) - defer cleanup() + t.Parallel() + repo, _ := openLockTestRepo(t, nil) - elock, _, err := lockRepoExclusive(context.TODO(), repo, env.gopts.RetryLock, env.gopts.JSON) + elock, _, err := Lock(context.TODO(), repo, true, 0, func(msg string) {}, func(format string, args ...interface{}) {}) test.OK(t, err) retryLock := 200 * time.Millisecond unlockAfter := 40 * time.Millisecond time.AfterFunc(unlockAfter, func() { - test.OK(t, elock.Unlock()) + elock.Unlock() }) - lock, _, err := lockRepo(context.TODO(), repo, retryLock, env.gopts.JSON) + lock, _, err := Lock(context.TODO(), repo, false, retryLock, func(msg string) {}, func(format string, args ...interface{}) {}) test.OK(t, err) - - test.OK(t, lock.Unlock()) + lock.Unlock() } diff --git a/mover-restic/restic/internal/pack/doc.go b/mover-restic/restic/internal/repository/pack/doc.go similarity index 100% rename from mover-restic/restic/internal/pack/doc.go rename to mover-restic/restic/internal/repository/pack/doc.go diff --git a/mover-restic/restic/internal/pack/pack.go b/mover-restic/restic/internal/repository/pack/pack.go similarity index 96% rename from mover-restic/restic/internal/pack/pack.go rename to mover-restic/restic/internal/repository/pack/pack.go index f9e7896e0..57957ce91 100644 --- a/mover-restic/restic/internal/pack/pack.go +++ b/mover-restic/restic/internal/repository/pack/pack.go @@ -211,7 +211,7 @@ const ( // MaxHeaderSize is the max size of header including header-length field MaxHeaderSize = 16*1024*1024 + headerLengthSize - // number of header enries to download as part of header-length request + // number of header entries to download as part of header-length request eagerEntries = 15 ) @@ -239,7 +239,7 @@ func readRecords(rd io.ReaderAt, size int64, bufsize int) ([]byte, int, error) { case hlen == 0: err = InvalidFileError{Message: "header length is zero"} case hlen < crypto.Extension: - err = InvalidFileError{Message: "header length is too small"} + err = InvalidFileError{Message: "header length is too short"} case int64(hlen) > size-int64(headerLengthSize): err = InvalidFileError{Message: "header is larger than file"} case int64(hlen) > MaxHeaderSize-int64(headerLengthSize): @@ -263,7 +263,7 @@ func readRecords(rd io.ReaderAt, size int64, bufsize int) ([]byte, int, error) { func readHeader(rd io.ReaderAt, size int64) ([]byte, error) { debug.Log("size: %v", size) if size < int64(minFileSize) { - err := InvalidFileError{Message: "file is too small"} + err := InvalidFileError{Message: "file is too short"} return nil, errors.Wrap(err, "readHeader") } @@ -305,7 +305,7 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, hdr } if len(buf) < crypto.CiphertextLength(0) { - return nil, 0, errors.New("invalid header, too small") + return nil, 0, errors.New("invalid header, too short") } hdrSize = headerLengthSize + uint32(len(buf)) @@ -389,10 +389,10 @@ func CalculateHeaderSize(blobs []restic.Blob) int { // If onlyHdr is set to true, only the size of the header is returned // Note that this function only gives correct sizes, if there are no // duplicates in the index. -func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 { +func Size(ctx context.Context, mi restic.ListBlobser, onlyHdr bool) (map[restic.ID]int64, error) { packSize := make(map[restic.ID]int64) - mi.Each(ctx, func(blob restic.PackedBlob) { + err := mi.ListBlobs(ctx, func(blob restic.PackedBlob) { size, ok := packSize[blob.PackID] if !ok { size = headerSize @@ -403,5 +403,5 @@ func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.I packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob)) }) - return packSize + return packSize, err } diff --git a/mover-restic/restic/internal/pack/pack_internal_test.go b/mover-restic/restic/internal/repository/pack/pack_internal_test.go similarity index 100% rename from mover-restic/restic/internal/pack/pack_internal_test.go rename to mover-restic/restic/internal/repository/pack/pack_internal_test.go diff --git a/mover-restic/restic/internal/pack/pack_test.go b/mover-restic/restic/internal/repository/pack/pack_test.go similarity index 89% rename from mover-restic/restic/internal/pack/pack_test.go rename to mover-restic/restic/internal/repository/pack/pack_test.go index 3f7077390..5ac146348 100644 --- a/mover-restic/restic/internal/pack/pack_test.go +++ b/mover-restic/restic/internal/repository/pack/pack_test.go @@ -12,7 +12,7 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/crypto" - "github.com/restic/restic/internal/pack" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -127,8 +127,8 @@ func TestUnpackReadSeeker(t *testing.T) { b := mem.New() id := restic.Hash(packData) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher()))) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } @@ -140,7 +140,7 @@ func TestShortPack(t *testing.T) { b := mem.New() id := restic.Hash(packData) - handle := restic.Handle{Type: restic.PackFile, Name: id.String()} - rtest.OK(t, b.Save(context.TODO(), handle, restic.NewByteReader(packData, b.Hasher()))) + handle := backend.Handle{Type: backend.PackFile, Name: id.String()} + rtest.OK(t, b.Save(context.TODO(), handle, backend.NewByteReader(packData, b.Hasher()))) verifyBlobs(t, bufs, k, backend.ReaderAt(context.TODO(), b, handle), packSize) } diff --git a/mover-restic/restic/internal/repository/packer_manager.go b/mover-restic/restic/internal/repository/packer_manager.go index df90d5ac0..20e363577 100644 --- a/mover-restic/restic/internal/repository/packer_manager.go +++ b/mover-restic/restic/internal/repository/packer_manager.go @@ -8,20 +8,21 @@ import ( "runtime" "sync" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/hashing" + "github.com/restic/restic/internal/repository/hashing" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/pack" + "github.com/restic/restic/internal/repository/pack" "crypto/sha256" ) -// Packer holds a pack.Packer together with a hash writer. -type Packer struct { +// packer holds a pack.packer together with a hash writer. +type packer struct { *pack.Packer tmpfile *os.File bufWr *bufio.Writer @@ -31,16 +32,16 @@ type Packer struct { type packerManager struct { tpe restic.BlobType key *crypto.Key - queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error + queueFn func(ctx context.Context, t restic.BlobType, p *packer) error pm sync.Mutex - packer *Packer + packer *packer packSize uint } -// newPackerManager returns an new packer manager which writes temporary files +// newPackerManager returns a new packer manager which writes temporary files // to a temporary directory -func newPackerManager(key *crypto.Key, tpe restic.BlobType, packSize uint, queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error) *packerManager { +func newPackerManager(key *crypto.Key, tpe restic.BlobType, packSize uint, queueFn func(ctx context.Context, t restic.BlobType, p *packer) error) *packerManager { return &packerManager{ tpe: tpe, key: key, @@ -113,7 +114,7 @@ func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id rest // findPacker returns a packer for a new blob of size bytes. Either a new one is // created or one is returned that already has some blobs. -func (r *packerManager) newPacker() (packer *Packer, err error) { +func (r *packerManager) newPacker() (pck *packer, err error) { debug.Log("create new pack") tmpfile, err := fs.TempFile("", "restic-temp-pack-") if err != nil { @@ -122,17 +123,17 @@ func (r *packerManager) newPacker() (packer *Packer, err error) { bufWr := bufio.NewWriter(tmpfile) p := pack.NewPacker(r.key, bufWr) - packer = &Packer{ + pck = &packer{ Packer: p, tmpfile: tmpfile, bufWr: bufWr, } - return packer, nil + return pck, nil } // savePacker stores p in the backend. -func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packer) error { +func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *packer) error { debug.Log("save packer for %v with %d blobs (%d bytes)\n", t, p.Packer.Count(), p.Packer.Size()) err := p.Packer.Finalize() if err != nil { @@ -145,7 +146,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe // calculate sha256 hash in a second pass var rd io.Reader - rd, err = restic.NewFileReader(p.tmpfile, nil) + rd, err = backend.NewFileReader(p.tmpfile, nil) if err != nil { return err } @@ -163,12 +164,12 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe } id := restic.IDFromHash(hr.Sum(nil)) - h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t} + h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()} var beHash []byte if beHr != nil { beHash = beHr.Sum(nil) } - rrd, err := restic.NewFileReader(p.tmpfile, beHash) + rrd, err := backend.NewFileReader(p.tmpfile, beHash) if err != nil { return err } @@ -199,8 +200,5 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe r.idx.StorePack(id, p.Packer.Blobs()) // Save index if full - if r.noAutoIndexUpdate { - return nil - } return r.idx.SaveFullIndex(ctx, r) } diff --git a/mover-restic/restic/internal/repository/packer_manager_test.go b/mover-restic/restic/internal/repository/packer_manager_test.go index 8984073da..0f3aea05f 100644 --- a/mover-restic/restic/internal/repository/packer_manager_test.go +++ b/mover-restic/restic/internal/repository/packer_manager_test.go @@ -70,7 +70,7 @@ func testPackerManager(t testing.TB) int64 { rnd := rand.New(rand.NewSource(randomSeed)) savedBytes := int(0) - pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, DefaultPackSize, func(ctx context.Context, tp restic.BlobType, p *Packer) error { + pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, DefaultPackSize, func(ctx context.Context, tp restic.BlobType, p *packer) error { err := p.Finalize() if err != nil { return err @@ -92,7 +92,7 @@ func testPackerManager(t testing.TB) int64 { func TestPackerManagerWithOversizeBlob(t *testing.T) { packFiles := int(0) sizeLimit := uint(512 * 1024) - pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, sizeLimit, func(ctx context.Context, tp restic.BlobType, p *Packer) error { + pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, sizeLimit, func(ctx context.Context, tp restic.BlobType, p *packer) error { packFiles++ return nil }) @@ -122,7 +122,7 @@ func BenchmarkPackerManager(t *testing.B) { for i := 0; i < t.N; i++ { rnd.Seed(randomSeed) - pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, DefaultPackSize, func(ctx context.Context, t restic.BlobType, p *Packer) error { + pm := newPackerManager(crypto.NewRandomKey(), restic.DataBlob, DefaultPackSize, func(ctx context.Context, t restic.BlobType, p *packer) error { return nil }) fillPacks(t, rnd, pm, blobBuf) diff --git a/mover-restic/restic/internal/repository/packer_uploader.go b/mover-restic/restic/internal/repository/packer_uploader.go index 30c8f77af..936e7ea1d 100644 --- a/mover-restic/restic/internal/repository/packer_uploader.go +++ b/mover-restic/restic/internal/repository/packer_uploader.go @@ -7,13 +7,13 @@ import ( "golang.org/x/sync/errgroup" ) -// SavePacker implements saving a pack in the repository. -type SavePacker interface { - savePacker(ctx context.Context, t restic.BlobType, p *Packer) error +// savePacker implements saving a pack in the repository. +type savePacker interface { + savePacker(ctx context.Context, t restic.BlobType, p *packer) error } type uploadTask struct { - packer *Packer + packer *packer tpe restic.BlobType } @@ -21,7 +21,7 @@ type packerUploader struct { uploadQueue chan uploadTask } -func newPackerUploader(ctx context.Context, wg *errgroup.Group, repo SavePacker, connections uint) *packerUploader { +func newPackerUploader(ctx context.Context, wg *errgroup.Group, repo savePacker, connections uint) *packerUploader { pu := &packerUploader{ uploadQueue: make(chan uploadTask), } @@ -48,7 +48,7 @@ func newPackerUploader(ctx context.Context, wg *errgroup.Group, repo SavePacker, return pu } -func (pu *packerUploader) QueuePacker(ctx context.Context, t restic.BlobType, p *Packer) (err error) { +func (pu *packerUploader) QueuePacker(ctx context.Context, t restic.BlobType, p *packer) (err error) { select { case <-ctx.Done(): return ctx.Err() diff --git a/mover-restic/restic/internal/repository/prune.go b/mover-restic/restic/internal/repository/prune.go new file mode 100644 index 000000000..d5fdbba07 --- /dev/null +++ b/mover-restic/restic/internal/repository/prune.go @@ -0,0 +1,640 @@ +package repository + +import ( + "context" + "fmt" + "math" + "sort" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" +) + +var ErrIndexIncomplete = errors.Fatal("index is not complete") +var ErrPacksMissing = errors.Fatal("packs from index missing in repo") +var ErrSizeNotMatching = errors.Fatal("pack size does not match calculated size from index") + +// PruneOptions collects all options for the cleanup command. +type PruneOptions struct { + DryRun bool + UnsafeRecovery bool + + MaxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused + MaxRepackBytes uint64 + + RepackCacheableOnly bool + RepackSmall bool + RepackUncompressed bool +} + +type PruneStats struct { + Blobs struct { + Used uint + Duplicate uint + Unused uint + Remove uint + Repack uint + Repackrm uint + } + Size struct { + Used uint64 + Duplicate uint64 + Unused uint64 + Remove uint64 + Repack uint64 + Repackrm uint64 + Unref uint64 + Uncompressed uint64 + } + Packs struct { + Used uint + Unused uint + PartlyUsed uint + Unref uint + Keep uint + Repack uint + Remove uint + } +} + +type PrunePlan struct { + removePacksFirst restic.IDSet // packs to remove first (unreferenced packs) + repackPacks restic.IDSet // packs to repack + keepBlobs *index.AssociatedSet[uint8] // blobs to keep during repacking + removePacks restic.IDSet // packs to remove + ignorePacks restic.IDSet // packs to ignore when rebuilding the index + + repo *Repository + stats PruneStats + opts PruneOptions +} + +type packInfo struct { + usedBlobs uint + unusedBlobs uint + duplicateBlobs uint + usedSize uint64 + unusedSize uint64 + + tpe restic.BlobType + uncompressed bool +} + +type packInfoWithID struct { + ID restic.ID + packInfo + mustCompress bool +} + +// PlanPrune selects which files to rewrite and which to delete and which blobs to keep. +// Also some summary statistics are returned. +func PlanPrune(ctx context.Context, opts PruneOptions, repo *Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error, printer progress.Printer) (*PrunePlan, error) { + var stats PruneStats + + if opts.UnsafeRecovery { + // prevent repacking data to make sure users cannot get stuck. + opts.MaxRepackBytes = 0 + } + if repo.Connections() < 2 { + return nil, fmt.Errorf("prune requires a backend connection limit of at least two") + } + if repo.Config().Version < 2 && opts.RepackUncompressed { + return nil, fmt.Errorf("compression requires at least repository format version 2") + } + + usedBlobs := index.NewAssociatedSet[uint8](repo.idx) + err := getUsedBlobs(ctx, repo, usedBlobs) + if err != nil { + return nil, err + } + + printer.P("searching used packs...\n") + keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo, usedBlobs, &stats, printer) + if err != nil { + return nil, err + } + + printer.P("collecting packs for deletion and repacking\n") + plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, printer) + if err != nil { + return nil, err + } + + if len(plan.repackPacks) != 0 { + // when repacking, we do not want to keep blobs which are + // already contained in kept packs, so delete them from keepBlobs + err := repo.ListBlobs(ctx, func(blob restic.PackedBlob) { + if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) { + return + } + keepBlobs.Delete(blob.BlobHandle) + }) + if err != nil { + return nil, err + } + } else { + // keepBlobs is only needed if packs are repacked + keepBlobs = nil + } + plan.keepBlobs = keepBlobs + + plan.repo = repo + plan.stats = stats + plan.opts = opts + + return &plan, nil +} + +func packInfoFromIndex(ctx context.Context, idx restic.ListBlobser, usedBlobs *index.AssociatedSet[uint8], stats *PruneStats, printer progress.Printer) (*index.AssociatedSet[uint8], map[restic.ID]packInfo, error) { + // iterate over all blobs in index to find out which blobs are duplicates + // The counter in usedBlobs describes how many instances of the blob exist in the repository index + // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist + err := idx.ListBlobs(ctx, func(blob restic.PackedBlob) { + bh := blob.BlobHandle + count, ok := usedBlobs.Get(bh) + if ok { + if count < math.MaxUint8 { + // don't overflow, but saturate count at 255 + // this can lead to a non-optimal pack selection, but won't cause + // problems otherwise + count++ + } + + usedBlobs.Set(bh, count) + } + }) + if err != nil { + return nil, nil, err + } + + // Check if all used blobs have been found in index + missingBlobs := restic.NewBlobSet() + usedBlobs.For(func(bh restic.BlobHandle, count uint8) { + if count == 0 { + // blob does not exist in any pack files + missingBlobs.Insert(bh) + } + }) + + if len(missingBlobs) != 0 { + printer.E("%v not found in the index\n\n"+ + "Integrity check failed: Data seems to be missing.\n"+ + "Will not start prune to prevent (additional) data loss!\n"+ + "Please report this error (along with the output of the 'prune' run) at\n"+ + "https://github.com/restic/restic/issues/new/choose\n", missingBlobs) + return nil, nil, ErrIndexIncomplete + } + + indexPack := make(map[restic.ID]packInfo) + + // save computed pack header size + sz, err := pack.Size(ctx, idx, true) + if err != nil { + return nil, nil, err + } + for pid, hdrSize := range sz { + // initialize tpe with NumBlobTypes to indicate it's not set + indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)} + } + + hasDuplicates := false + // iterate over all blobs in index to generate packInfo + err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { + ip := indexPack[blob.PackID] + + // Set blob type if not yet set + if ip.tpe == restic.NumBlobTypes { + ip.tpe = blob.Type + } + + // mark mixed packs with "Invalid blob type" + if ip.tpe != blob.Type { + ip.tpe = restic.InvalidBlob + } + + bh := blob.BlobHandle + size := uint64(blob.Length) + dupCount, _ := usedBlobs.Get(bh) + switch { + case dupCount >= 2: + hasDuplicates = true + // mark as unused for now, we will later on select one copy + ip.unusedSize += size + ip.unusedBlobs++ + ip.duplicateBlobs++ + + // count as duplicate, will later on change one copy to be counted as used + stats.Size.Duplicate += size + stats.Blobs.Duplicate++ + case dupCount == 1: // used blob, not duplicate + ip.usedSize += size + ip.usedBlobs++ + + stats.Size.Used += size + stats.Blobs.Used++ + default: // unused blob + ip.unusedSize += size + ip.unusedBlobs++ + + stats.Size.Unused += size + stats.Blobs.Unused++ + } + if !blob.IsCompressed() { + ip.uncompressed = true + } + // update indexPack + indexPack[blob.PackID] = ip + }) + if err != nil { + return nil, nil, err + } + + // if duplicate blobs exist, those will be set to either "used" or "unused": + // - mark only one occurrence of duplicate blobs as used + // - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used" + // - if a pack only consists of duplicates (which by definition are used blobs), mark it as "used". This + // ensures that already rewritten packs are kept. + // - if there are no used blobs in a pack, possibly mark duplicates as "unused" + if hasDuplicates { + // iterate again over all blobs in index (this is pretty cheap, all in-mem) + err = idx.ListBlobs(ctx, func(blob restic.PackedBlob) { + bh := blob.BlobHandle + count, ok := usedBlobs.Get(bh) + // skip non-duplicate, aka. normal blobs + // count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining + if !ok || count == 1 { + return + } + + ip := indexPack[blob.PackID] + size := uint64(blob.Length) + switch { + case ip.usedBlobs > 0, (ip.duplicateBlobs == ip.unusedBlobs), count == 0: + // other used blobs in pack, only duplicate blobs or "last" occurrence -> transition to used + // a pack file created by an interrupted prune run will consist of only duplicate blobs + // thus select such already repacked pack files + ip.usedSize += size + ip.usedBlobs++ + ip.unusedSize -= size + ip.unusedBlobs-- + // same for the global statistics + stats.Size.Used += size + stats.Blobs.Used++ + stats.Size.Duplicate -= size + stats.Blobs.Duplicate-- + // let other occurrences remain marked as unused + usedBlobs.Set(bh, 1) + default: + // remain unused and decrease counter + count-- + if count == 1 { + // setting count to 1 would lead to forgetting that this blob had duplicates + // thus use the special value zero. This will select the last instance of the blob for keeping. + count = 0 + } + usedBlobs.Set(bh, count) + } + // update indexPack + indexPack[blob.PackID] = ip + }) + if err != nil { + return nil, nil, err + } + } + + // Sanity check. If no duplicates exist, all blobs have value 1. After handling + // duplicates, this also applies to duplicates. + usedBlobs.For(func(_ restic.BlobHandle, count uint8) { + if count != 1 { + panic("internal error during blob selection") + } + }) + + return usedBlobs, indexPack, nil +} + +func decidePackAction(ctx context.Context, opts PruneOptions, repo *Repository, indexPack map[restic.ID]packInfo, stats *PruneStats, printer progress.Printer) (PrunePlan, error) { + removePacksFirst := restic.NewIDSet() + removePacks := restic.NewIDSet() + repackPacks := restic.NewIDSet() + + var repackCandidates []packInfoWithID + var repackSmallCandidates []packInfoWithID + repoVersion := repo.Config().Version + // only repack very small files by default + targetPackSize := repo.packSize() / 25 + if opts.RepackSmall { + // consider files with at least 80% of the target size as large enough + targetPackSize = repo.packSize() / 5 * 4 + } + + // loop over all packs and decide what to do + bar := printer.NewCounter("packs processed") + bar.SetMax(uint64(len(indexPack))) + err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error { + p, ok := indexPack[id] + if !ok { + // Pack was not referenced in index and is not used => immediately remove! + printer.V("will remove pack %v as it is unused and not indexed\n", id.Str()) + removePacksFirst.Insert(id) + stats.Size.Unref += uint64(packSize) + return nil + } + + if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 { + // Pack size does not fit and pack is needed => error + // If the pack is not needed, this is no error, the pack can + // and will be simply removed, see below. + printer.E("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n", + id.Str(), p.unusedSize+p.usedSize, packSize) + return ErrSizeNotMatching + } + + // statistics + switch { + case p.usedBlobs == 0: + stats.Packs.Unused++ + case p.unusedBlobs == 0: + stats.Packs.Used++ + default: + stats.Packs.PartlyUsed++ + } + + if p.uncompressed { + stats.Size.Uncompressed += p.unusedSize + p.usedSize + } + mustCompress := false + if repoVersion >= 2 { + // repo v2: always repack tree blobs if uncompressed + // compress data blobs if requested + mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed + } + + // decide what to do + switch { + case p.usedBlobs == 0: + // All blobs in pack are no longer used => remove pack! + removePacks.Insert(id) + stats.Blobs.Remove += p.unusedBlobs + stats.Size.Remove += p.unusedSize + + case opts.RepackCacheableOnly && p.tpe == restic.DataBlob: + // if this is a data pack and --repack-cacheable-only is set => keep pack! + stats.Packs.Keep++ + + case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress: + if packSize >= int64(targetPackSize) { + // All blobs in pack are used and not mixed => keep pack! + stats.Packs.Keep++ + } else { + repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress}) + } + + default: + // all other packs are candidates for repacking + repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress}) + } + + delete(indexPack, id) + bar.Add(1) + return nil + }) + bar.Done() + if err != nil { + return PrunePlan{}, err + } + + // At this point indexPacks contains only missing packs! + + // missing packs that are not needed can be ignored + ignorePacks := restic.NewIDSet() + for id, p := range indexPack { + if p.usedBlobs == 0 { + ignorePacks.Insert(id) + stats.Blobs.Remove += p.unusedBlobs + stats.Size.Remove += p.unusedSize + delete(indexPack, id) + } + } + + if len(indexPack) != 0 { + printer.E("The index references %d needed pack files which are missing from the repository:\n", len(indexPack)) + for id := range indexPack { + printer.E(" %v\n", id) + } + return PrunePlan{}, ErrPacksMissing + } + if len(ignorePacks) != 0 { + printer.E("Missing but unneeded pack files are referenced in the index, will be repaired\n") + for id := range ignorePacks { + printer.E("will forget missing pack file %v\n", id) + } + } + + if len(repackSmallCandidates) < 10 { + // too few small files to be worth the trouble, this also prevents endlessly repacking + // if there is just a single pack file below the target size + stats.Packs.Keep += uint(len(repackSmallCandidates)) + } else { + repackCandidates = append(repackCandidates, repackSmallCandidates...) + } + + // Sort repackCandidates such that packs with highest ratio unused/used space are picked first. + // This is equivalent to sorting by unused / total space. + // Instead of unused[i] / used[i] > unused[j] / used[j] we use + // unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64 + // Moreover packs containing trees and too short packs are sorted to the beginning + sort.Slice(repackCandidates, func(i, j int) bool { + pi := repackCandidates[i].packInfo + pj := repackCandidates[j].packInfo + switch { + case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob: + return true + case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob: + return false + case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize): + return true + case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize): + return false + } + return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize + }) + + repack := func(id restic.ID, p packInfo) { + repackPacks.Insert(id) + stats.Blobs.Repack += p.unusedBlobs + p.usedBlobs + stats.Size.Repack += p.unusedSize + p.usedSize + stats.Blobs.Repackrm += p.unusedBlobs + stats.Size.Repackrm += p.unusedSize + if p.uncompressed { + stats.Size.Uncompressed -= p.unusedSize + p.usedSize + } + } + + // calculate limit for number of unused bytes in the repo after repacking + maxUnusedSizeAfter := opts.MaxUnusedBytes(stats.Size.Used) + + for _, p := range repackCandidates { + reachedUnusedSizeAfter := (stats.Size.Unused-stats.Size.Remove-stats.Size.Repackrm < maxUnusedSizeAfter) + reachedRepackSize := stats.Size.Repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes + packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize) + + switch { + case reachedRepackSize: + stats.Packs.Keep++ + + case p.tpe != restic.DataBlob, p.mustCompress: + // repacking non-data packs / uncompressed-trees is only limited by repackSize + repack(p.ID, p.packInfo) + + case reachedUnusedSizeAfter && packIsLargeEnough: + // for all other packs stop repacking if tolerated unused size is reached. + stats.Packs.Keep++ + + default: + repack(p.ID, p.packInfo) + } + } + + stats.Packs.Unref = uint(len(removePacksFirst)) + stats.Packs.Repack = uint(len(repackPacks)) + stats.Packs.Remove = uint(len(removePacks)) + + if repo.Config().Version < 2 { + // compression not supported for repository format version 1 + stats.Size.Uncompressed = 0 + } + + return PrunePlan{removePacksFirst: removePacksFirst, + removePacks: removePacks, + repackPacks: repackPacks, + ignorePacks: ignorePacks, + }, nil +} + +func (plan *PrunePlan) Stats() PruneStats { + return plan.stats +} + +// Execute does the actual pruning: +// - remove unreferenced packs first +// - repack given pack files while keeping the given blobs +// - rebuild the index while ignoring all files that will be deleted +// - delete the files +// plan.removePacks and plan.ignorePacks are modified in this function. +func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) error { + if plan.opts.DryRun { + printer.V("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n") + if len(plan.removePacksFirst) > 0 { + printer.V("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst) + } + printer.V("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks) + printer.V("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks) + // Always quit here if DryRun was set! + return nil + } + + repo := plan.repo + // make sure the plan can only be used once + plan.repo = nil + + // unreferenced packs can be safely deleted first + if len(plan.removePacksFirst) != 0 { + printer.P("deleting unreferenced packs\n") + _ = deleteFiles(ctx, true, repo, plan.removePacksFirst, restic.PackFile, printer) + // forget unused data + plan.removePacksFirst = nil + } + if ctx.Err() != nil { + return ctx.Err() + } + + if len(plan.repackPacks) != 0 { + printer.P("repacking packs\n") + bar := printer.NewCounter("packs repacked") + bar.SetMax(uint64(len(plan.repackPacks))) + _, err := Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar) + bar.Done() + if err != nil { + return errors.Fatal(err.Error()) + } + + // Also remove repacked packs + plan.removePacks.Merge(plan.repackPacks) + // forget unused data + plan.repackPacks = nil + + if plan.keepBlobs.Len() != 0 { + printer.E("%v was not repacked\n\n"+ + "Integrity check failed.\n"+ + "Please report this error (along with the output of the 'prune' run) at\n"+ + "https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs) + return errors.Fatal("internal error: blobs were not repacked") + } + + // allow GC of the blob set + plan.keepBlobs = nil + } + + if len(plan.ignorePacks) == 0 { + plan.ignorePacks = plan.removePacks + } else { + plan.ignorePacks.Merge(plan.removePacks) + } + + if plan.opts.UnsafeRecovery { + printer.P("deleting index files\n") + indexFiles := repo.idx.IDs() + err := deleteFiles(ctx, false, repo, indexFiles, restic.IndexFile, printer) + if err != nil { + return errors.Fatalf("%s", err) + } + } else if len(plan.ignorePacks) != 0 { + err := rewriteIndexFiles(ctx, repo, plan.ignorePacks, nil, nil, printer) + if err != nil { + return errors.Fatalf("%s", err) + } + } + + if len(plan.removePacks) != 0 { + printer.P("removing %d old packs\n", len(plan.removePacks)) + _ = deleteFiles(ctx, true, repo, plan.removePacks, restic.PackFile, printer) + } + if ctx.Err() != nil { + return ctx.Err() + } + + if plan.opts.UnsafeRecovery { + err := repo.idx.SaveFallback(ctx, repo, plan.ignorePacks, printer.NewCounter("packs processed")) + if err != nil { + return errors.Fatalf("%s", err) + } + } + + // drop outdated in-memory index + repo.clearIndex() + + printer.P("done\n") + return nil +} + +// deleteFiles deletes the given fileList of fileType in parallel +// if ignoreError=true, it will print a warning if there was an error, else it will abort. +func deleteFiles(ctx context.Context, ignoreError bool, repo restic.RemoverUnpacked, fileList restic.IDSet, fileType restic.FileType, printer progress.Printer) error { + bar := printer.NewCounter("files deleted") + defer bar.Done() + + return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error { + if err != nil { + printer.E("unable to remove %v/%v from the repository\n", fileType, id) + if !ignoreError { + return err + } + } + printer.VV("removed %v/%v\n", fileType, id) + return nil + }, bar) +} diff --git a/mover-restic/restic/internal/repository/prune_test.go b/mover-restic/restic/internal/repository/prune_test.go new file mode 100644 index 000000000..02eefc463 --- /dev/null +++ b/mover-restic/restic/internal/repository/prune_test.go @@ -0,0 +1,108 @@ +package repository_test + +import ( + "context" + "math" + "testing" + + "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" + "golang.org/x/sync/errgroup" +) + +func testPrune(t *testing.T, opts repository.PruneOptions, errOnUnused bool) { + repo, be := repository.TestRepositoryWithVersion(t, 0) + createRandomBlobs(t, repo, 4, 0.5, true) + createRandomBlobs(t, repo, 5, 0.5, true) + keep, _ := selectBlobs(t, repo, 0.5) + + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + // duplicate a few blobs to exercise those code paths + for blob := range keep { + buf, err := repo.LoadBlob(context.TODO(), blob.Type, blob.ID, nil) + rtest.OK(t, err) + _, _, _, err = repo.SaveBlob(context.TODO(), blob.Type, buf, blob.ID, true) + rtest.OK(t, err) + } + rtest.OK(t, repo.Flush(context.TODO())) + + plan, err := repository.PlanPrune(context.TODO(), opts, repo, func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error { + for blob := range keep { + usedBlobs.Insert(blob) + } + return nil + }, &progress.NoopPrinter{}) + rtest.OK(t, err) + + rtest.OK(t, plan.Execute(context.TODO(), &progress.NoopPrinter{})) + + repo = repository.TestOpenBackend(t, be) + checker.TestCheckRepo(t, repo, true) + + if errOnUnused { + existing := listBlobs(repo) + rtest.Assert(t, existing.Equals(keep), "unexpected blobs, wanted %v got %v", keep, existing) + } +} + +func TestPrune(t *testing.T) { + for _, test := range []struct { + name string + opts repository.PruneOptions + errOnUnused bool + }{ + { + name: "0", + opts: repository.PruneOptions{ + MaxRepackBytes: math.MaxUint64, + MaxUnusedBytes: func(used uint64) (unused uint64) { return 0 }, + }, + errOnUnused: true, + }, + { + name: "50", + opts: repository.PruneOptions{ + MaxRepackBytes: math.MaxUint64, + MaxUnusedBytes: func(used uint64) (unused uint64) { return used / 2 }, + }, + }, + { + name: "unlimited", + opts: repository.PruneOptions{ + MaxRepackBytes: math.MaxUint64, + MaxUnusedBytes: func(used uint64) (unused uint64) { return math.MaxUint64 }, + }, + }, + { + name: "cachableonly", + opts: repository.PruneOptions{ + MaxRepackBytes: math.MaxUint64, + MaxUnusedBytes: func(used uint64) (unused uint64) { return used / 20 }, + RepackCacheableOnly: true, + }, + }, + { + name: "small", + opts: repository.PruneOptions{ + MaxRepackBytes: math.MaxUint64, + MaxUnusedBytes: func(used uint64) (unused uint64) { return math.MaxUint64 }, + RepackSmall: true, + }, + errOnUnused: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + testPrune(t, test.opts, test.errOnUnused) + }) + t.Run(test.name+"-recovery", func(t *testing.T) { + opts := test.opts + opts.UnsafeRecovery = true + // unsafeNoSpaceRecovery does not repack partially used pack files + testPrune(t, opts, false) + }) + } +} diff --git a/mover-restic/restic/internal/repository/raw.go b/mover-restic/restic/internal/repository/raw.go new file mode 100644 index 000000000..31443b010 --- /dev/null +++ b/mover-restic/restic/internal/repository/raw.go @@ -0,0 +1,56 @@ +package repository + +import ( + "bytes" + "context" + "fmt" + "io" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/restic" +) + +// LoadRaw reads all data stored in the backend for the file with id and filetype t. +// If the backend returns data that does not match the id, then the buffer is returned +// along with an error that is a restic.ErrInvalidData error. +func (r *Repository) LoadRaw(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) { + h := backend.Handle{Type: t, Name: id.String()} + + buf, err = loadRaw(ctx, r.be, h) + + // retry loading damaged data only once. If a file fails to download correctly + // the second time, then it is likely corrupted at the backend. + if h.Type != backend.ConfigFile && id != restic.Hash(buf) { + if r.Cache != nil { + // Cleanup cache to make sure it's not the cached copy that is broken. + // Ignore error as there's not much we can do in that case. + _ = r.Cache.Forget(h) + } + + buf, err = loadRaw(ctx, r.be, h) + + if err == nil && id != restic.Hash(buf) { + // Return corrupted data to the caller if it is still broken the second time to + // let the caller decide what to do with the data. + return buf, fmt.Errorf("LoadRaw(%v): %w", h, restic.ErrInvalidData) + } + } + + if err != nil { + return nil, err + } + return buf, nil +} + +func loadRaw(ctx context.Context, be backend.Backend, h backend.Handle) (buf []byte, err error) { + err = be.Load(ctx, h, 0, 0, func(rd io.Reader) error { + wr := new(bytes.Buffer) + _, cerr := io.Copy(wr, rd) + if cerr != nil { + return cerr + } + buf = wr.Bytes() + return cerr + }) + return buf, err +} diff --git a/mover-restic/restic/internal/repository/raw_test.go b/mover-restic/restic/internal/repository/raw_test.go new file mode 100644 index 000000000..ac65a8dc8 --- /dev/null +++ b/mover-restic/restic/internal/repository/raw_test.go @@ -0,0 +1,108 @@ +package repository_test + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/cache" + "github.com/restic/restic/internal/backend/mem" + "github.com/restic/restic/internal/backend/mock" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +const KiB = 1 << 10 +const MiB = 1 << 20 + +func TestLoadRaw(t *testing.T) { + b := mem.New() + repo, err := repository.New(b, repository.Options{}) + rtest.OK(t, err) + + for i := 0; i < 5; i++ { + data := rtest.Random(23+i, 500*KiB) + + id := restic.Hash(data) + h := backend.Handle{Name: id.String(), Type: backend.PackFile} + err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher())) + rtest.OK(t, err) + + buf, err := repo.LoadRaw(context.TODO(), backend.PackFile, id) + rtest.OK(t, err) + + if len(buf) != len(data) { + t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf)) + continue + } + + if !bytes.Equal(buf, data) { + t.Errorf("wrong data returned") + continue + } + } +} + +func TestLoadRawBroken(t *testing.T) { + b := mock.NewBackend() + repo, err := repository.New(b, repository.Options{}) + rtest.OK(t, err) + + data := rtest.Random(23, 10*KiB) + id := restic.Hash(data) + // damage buffer + data[0] ^= 0xff + + b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(data)), nil + } + + // must detect but still return corrupt data + buf, err := repo.LoadRaw(context.TODO(), backend.PackFile, id) + rtest.Assert(t, bytes.Equal(buf, data), "wrong data returned") + rtest.Assert(t, errors.Is(err, restic.ErrInvalidData), "missing expected ErrInvalidData error, got %v", err) + + // cause the first access to fail, but repair the data for the second access + data[0] ^= 0xff + loadCtr := 0 + b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + data[0] ^= 0xff + loadCtr++ + return io.NopCloser(bytes.NewReader(data)), nil + } + + // must retry load of corrupted data + buf, err = repo.LoadRaw(context.TODO(), backend.PackFile, id) + rtest.OK(t, err) + rtest.Assert(t, bytes.Equal(buf, data), "wrong data returned") + rtest.Equals(t, 2, loadCtr, "missing retry on broken data") +} + +func TestLoadRawBrokenWithCache(t *testing.T) { + b := mock.NewBackend() + c := cache.TestNewCache(t) + repo, err := repository.New(b, repository.Options{}) + rtest.OK(t, err) + repo.UseCache(c) + + data := rtest.Random(23, 10*KiB) + id := restic.Hash(data) + + loadCtr := 0 + // cause the first access to fail, but repair the data for the second access + b.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + data[0] ^= 0xff + loadCtr++ + return io.NopCloser(bytes.NewReader(data)), nil + } + + // must retry load of corrupted data + buf, err := repo.LoadRaw(context.TODO(), backend.SnapshotFile, id) + rtest.OK(t, err) + rtest.Assert(t, bytes.Equal(buf, data), "wrong data returned") + rtest.Equals(t, 2, loadCtr, "missing retry on broken data") +} diff --git a/mover-restic/restic/internal/repository/repack.go b/mover-restic/restic/internal/repository/repack.go index c82e63f28..8c9ca28bb 100644 --- a/mover-restic/restic/internal/repository/repack.go +++ b/mover-restic/restic/internal/repository/repack.go @@ -54,7 +54,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito downloadQueue := make(chan restic.PackBlobs) wg.Go(func() error { defer close(downloadQueue) - for pbs := range repo.Index().ListPacks(wgCtx, packs) { + for pbs := range repo.ListPacksFromIndex(wgCtx, packs) { var packBlobs []restic.Blob keepMutex.Lock() // filter out unnecessary blobs @@ -72,20 +72,15 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito return wgCtx.Err() } } - return nil + return wgCtx.Err() }) worker := func() error { for t := range downloadQueue { - err := StreamPack(wgCtx, repo.Backend().Load, repo.Key(), t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err := repo.LoadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { - var ierr error - // check whether we can get a valid copy somewhere else - buf, ierr = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) - if ierr != nil { - // no luck, return the original error - return err - } + // a required blob couldn't be retrieved + return err } keepMutex.Lock() diff --git a/mover-restic/restic/internal/repository/repack_test.go b/mover-restic/restic/internal/repository/repack_test.go index 1ecbf3e1c..476e63b47 100644 --- a/mover-restic/restic/internal/repository/repack_test.go +++ b/mover-restic/restic/internal/repository/repack_test.go @@ -6,10 +6,11 @@ import ( "testing" "time" - "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -17,7 +18,7 @@ func randomSize(min, max int) int { return rand.Intn(max-min) + min } -func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32) { +func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32, smallBlobs bool) { var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) @@ -29,7 +30,11 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl if rand.Float32() < pData { tpe = restic.DataBlob - length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data + if smallBlobs { + length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB of data + } else { + length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data + } } else { tpe = restic.TreeBlob length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB @@ -61,7 +66,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl } } -func createRandomWrongBlob(t testing.TB, repo restic.Repository) { +func createRandomWrongBlob(t testing.TB, repo restic.Repository) restic.BlobHandle { length := randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data buf := make([]byte, length) rand.Read(buf) @@ -79,10 +84,11 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { if err := repo.Flush(context.Background()); err != nil { t.Fatalf("repo.Flush() returned error %v", err) } + return restic.BlobHandle{ID: id, Type: restic.DataBlob} } // selectBlobs splits the list of all blobs randomly into two lists. A blob -// will be contained in the firstone ith probability p. +// will be contained in the firstone with probability p. func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 restic.BlobSet) { list1 = restic.NewBlobSet() list2 = restic.NewBlobSet() @@ -118,9 +124,13 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 return list1, list2 } -func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { +func listPacks(t *testing.T, repo restic.Lister) restic.IDSet { + return listFiles(t, repo, restic.PackFile) +} + +func listFiles(t *testing.T, repo restic.Lister, tpe backend.FileType) restic.IDSet { list := restic.NewIDSet() - err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { + err := repo.List(context.TODO(), tpe, func(id restic.ID, size int64) error { list.Insert(id) return nil }) @@ -135,9 +145,8 @@ func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSet) restic.IDSet { packs := restic.NewIDSet() - idx := repo.Index() for h := range blobs { - list := idx.Lookup(h) + list := repo.LookupBlob(h.Type, h.ID) if len(list) == 0 { t.Fatal("Failed to find blob", h.ID.Str(), "with type", h.Type) } @@ -157,65 +166,19 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest } for id := range repackedBlobs { - err = repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}) + err = repo.RemoveUnpacked(context.TODO(), restic.PackFile, id) if err != nil { t.Fatal(err) } } } -func flush(t *testing.T, repo restic.Repository) { - if err := repo.Flush(context.TODO()); err != nil { - t.Fatalf("repo.SaveIndex() %v", err) - } -} - -func rebuildIndex(t *testing.T, repo restic.Repository) { - err := repo.SetIndex(index.NewMasterIndex()) - if err != nil { - t.Fatal(err) - } - - packs := make(map[restic.ID]int64) - err = repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { - packs[id] = size - return nil - }) - if err != nil { - t.Fatal(err) - } - - _, err = repo.(*repository.Repository).CreateIndexFromPacks(context.TODO(), packs, nil) - if err != nil { - t.Fatal(err) - } - - err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { - h := restic.Handle{ - Type: restic.IndexFile, - Name: id.String(), - } - return repo.Backend().Remove(context.TODO(), h) - }) - if err != nil { - t.Fatal(err) - } - - _, err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), nil, nil) - if err != nil { - t.Fatal(err) - } -} - -func reloadIndex(t *testing.T, repo restic.Repository) { - err := repo.SetIndex(index.NewMasterIndex()) - if err != nil { - t.Fatal(err) - } +func rebuildAndReloadIndex(t *testing.T, repo *repository.Repository) { + rtest.OK(t, repository.RepairIndex(context.TODO(), repo, repository.RepairIndexOptions{ + ReadAllPacks: true, + }, &progress.NoopPrinter{})) - if err := repo.LoadIndex(context.TODO(), nil); err != nil { - t.Fatalf("error loading new index: %v", err) - } + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } func TestRepack(t *testing.T) { @@ -223,13 +186,15 @@ func TestRepack(t *testing.T) { } func testRepack(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) seed := time.Now().UnixNano() rand.Seed(seed) t.Logf("rand seed is %v", seed) - createRandomBlobs(t, repo, 100, 0.7) + // add a small amount of blobs twice to create multiple pack files + createRandomBlobs(t, repo, 10, 0.7, false) + createRandomBlobs(t, repo, 10, 0.7, false) packsBefore := listPacks(t, repo) @@ -243,15 +208,12 @@ func testRepack(t *testing.T, version uint) { packsBefore, packsAfter) } - flush(t, repo) - removeBlobs, keepBlobs := selectBlobs(t, repo, 0.2) removePacks := findPacksForBlobs(t, repo, removeBlobs) repack(t, repo, removePacks, keepBlobs) - rebuildIndex(t, repo) - reloadIndex(t, repo) + rebuildAndReloadIndex(t, repo) packsAfter = listPacks(t, repo) for id := range removePacks { @@ -260,10 +222,8 @@ func testRepack(t *testing.T, version uint) { } } - idx := repo.Index() - for h := range keepBlobs { - list := idx.Lookup(h) + list := repo.LookupBlob(h.Type, h.ID) if len(list) == 0 { t.Errorf("unable to find blob %v in repo", h.ID.Str()) continue @@ -282,7 +242,7 @@ func testRepack(t *testing.T, version uint) { } for h := range removeBlobs { - if _, found := repo.LookupBlobSize(h.ID, h.Type); found { + if _, found := repo.LookupBlobSize(h.Type, h.ID); found { t.Errorf("blob %v still contained in the repo", h) } } @@ -301,8 +261,8 @@ func (r oneConnectionRepo) Connections() uint { } func testRepackCopy(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) - dstRepo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) + dstRepo, _ := repository.TestRepositoryWithVersion(t, version) // test with minimal possible connection count repoWrapped := &oneConnectionRepo{repo} @@ -312,8 +272,9 @@ func testRepackCopy(t *testing.T, version uint) { rand.Seed(seed) t.Logf("rand seed is %v", seed) - createRandomBlobs(t, repo, 100, 0.7) - flush(t, repo) + // add a small amount of blobs twice to create multiple pack files + createRandomBlobs(t, repo, 10, 0.7, false) + createRandomBlobs(t, repo, 10, 0.7, false) _, keepBlobs := selectBlobs(t, repo, 0.2) copyPacks := findPacksForBlobs(t, repo, keepBlobs) @@ -322,13 +283,10 @@ func testRepackCopy(t *testing.T, version uint) { if err != nil { t.Fatal(err) } - rebuildIndex(t, dstRepo) - reloadIndex(t, dstRepo) - - idx := dstRepo.Index() + rebuildAndReloadIndex(t, dstRepo) for h := range keepBlobs { - list := idx.Lookup(h) + list := dstRepo.LookupBlob(h.Type, h.ID) if len(list) == 0 { t.Errorf("unable to find blob %v in repo", h.ID.Str()) continue @@ -347,13 +305,13 @@ func TestRepackWrongBlob(t *testing.T) { func testRepackWrongBlob(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) + repo, _ := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) t.Logf("rand seed is %v", seed) - createRandomBlobs(t, repo, 5, 0.7) + createRandomBlobs(t, repo, 5, 0.7, false) createRandomWrongBlob(t, repo) // just keep all blobs, but also rewrite every pack @@ -373,7 +331,7 @@ func TestRepackBlobFallback(t *testing.T) { func testRepackBlobFallback(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) + repo, _ := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/mover-restic/restic/internal/repository/repair_index.go b/mover-restic/restic/internal/repository/repair_index.go new file mode 100644 index 000000000..770809254 --- /dev/null +++ b/mover-restic/restic/internal/repository/repair_index.go @@ -0,0 +1,139 @@ +package repository + +import ( + "context" + + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" +) + +type RepairIndexOptions struct { + ReadAllPacks bool +} + +func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions, printer progress.Printer) error { + var obsoleteIndexes restic.IDs + packSizeFromList := make(map[restic.ID]int64) + packSizeFromIndex := make(map[restic.ID]int64) + removePacks := restic.NewIDSet() + + if opts.ReadAllPacks { + // get list of old index files but start with empty index + err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error { + obsoleteIndexes = append(obsoleteIndexes, id) + return nil + }) + if err != nil { + return err + } + repo.clearIndex() + + } else { + printer.P("loading indexes...\n") + mi := index.NewMasterIndex() + err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error { + if err != nil { + printer.E("removing invalid index %v: %v\n", id, err) + obsoleteIndexes = append(obsoleteIndexes, id) + return nil + } + + mi.Insert(idx) + return nil + }) + if err != nil { + return err + } + + err = mi.MergeFinalIndexes() + if err != nil { + return err + } + + err = repo.SetIndex(mi) + if err != nil { + return err + } + packSizeFromIndex, err = pack.Size(ctx, repo, false) + if err != nil { + return err + } + } + + oldIndexes := repo.idx.IDs() + + printer.P("getting pack files to read...\n") + err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error { + size, ok := packSizeFromIndex[id] + if !ok || size != packSize { + // Pack was not referenced in index or size does not match + packSizeFromList[id] = packSize + removePacks.Insert(id) + } + if !ok { + printer.E("adding pack file to index %v\n", id) + } else if size != packSize { + printer.E("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size) + } + delete(packSizeFromIndex, id) + return nil + }) + if err != nil { + return err + } + for id := range packSizeFromIndex { + // forget pack files that are referenced in the index but do not exist + // when rebuilding the index + removePacks.Insert(id) + printer.E("removing not found pack file %v\n", id) + } + + if len(packSizeFromList) > 0 { + printer.P("reading pack files\n") + bar := printer.NewCounter("packs") + bar.SetMax(uint64(len(packSizeFromList))) + invalidFiles, err := repo.createIndexFromPacks(ctx, packSizeFromList, bar) + bar.Done() + if err != nil { + return err + } + + for _, id := range invalidFiles { + printer.V("skipped incomplete pack file: %v\n", id) + } + } + + if err := repo.Flush(ctx); err != nil { + return err + } + + err = rewriteIndexFiles(ctx, repo, removePacks, oldIndexes, obsoleteIndexes, printer) + if err != nil { + return err + } + + // drop outdated in-memory index + repo.clearIndex() + return nil +} + +func rewriteIndexFiles(ctx context.Context, repo *Repository, removePacks restic.IDSet, oldIndexes restic.IDSet, extraObsolete restic.IDs, printer progress.Printer) error { + printer.P("rebuilding index\n") + + bar := printer.NewCounter("indexes processed") + return repo.idx.Rewrite(ctx, repo, removePacks, oldIndexes, extraObsolete, index.MasterIndexRewriteOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return printer.NewCounter("old indexes deleted") + }, + DeleteReport: func(id restic.ID, err error) { + if err != nil { + printer.VV("failed to remove index %v: %v\n", id.String(), err) + } else { + printer.VV("removed index %v\n", id.String()) + } + }, + }) +} diff --git a/mover-restic/restic/internal/repository/repair_index_test.go b/mover-restic/restic/internal/repository/repair_index_test.go new file mode 100644 index 000000000..ac47d59ff --- /dev/null +++ b/mover-restic/restic/internal/repository/repair_index_test.go @@ -0,0 +1,75 @@ +package repository_test + +import ( + "context" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/checker" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" +) + +func listIndex(t *testing.T, repo restic.Lister) restic.IDSet { + return listFiles(t, repo, restic.IndexFile) +} + +func testRebuildIndex(t *testing.T, readAllPacks bool, damage func(t *testing.T, repo *repository.Repository, be backend.Backend)) { + repo, be := repository.TestRepositoryWithVersion(t, 0) + createRandomBlobs(t, repo, 4, 0.5, true) + createRandomBlobs(t, repo, 5, 0.5, true) + indexes := listIndex(t, repo) + t.Logf("old indexes %v", indexes) + + damage(t, repo, be) + + repo = repository.TestOpenBackend(t, be) + rtest.OK(t, repository.RepairIndex(context.TODO(), repo, repository.RepairIndexOptions{ + ReadAllPacks: readAllPacks, + }, &progress.NoopPrinter{})) + + checker.TestCheckRepo(t, repo, true) +} + +func TestRebuildIndex(t *testing.T) { + for _, test := range []struct { + name string + damage func(t *testing.T, repo *repository.Repository, be backend.Backend) + }{ + { + "valid index", + func(t *testing.T, repo *repository.Repository, be backend.Backend) {}, + }, + { + "damaged index", + func(t *testing.T, repo *repository.Repository, be backend.Backend) { + index := listIndex(t, repo).List()[0] + replaceFile(t, be, backend.Handle{Type: restic.IndexFile, Name: index.String()}, func(b []byte) []byte { + b[0] ^= 0xff + return b + }) + }, + }, + { + "missing index", + func(t *testing.T, repo *repository.Repository, be backend.Backend) { + index := listIndex(t, repo).List()[0] + rtest.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: index.String()})) + }, + }, + { + "missing pack", + func(t *testing.T, repo *repository.Repository, be backend.Backend) { + pack := listPacks(t, repo).List()[0] + rtest.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: pack.String()})) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + testRebuildIndex(t, false, test.damage) + testRebuildIndex(t, true, test.damage) + }) + } +} diff --git a/mover-restic/restic/internal/repository/repair_pack.go b/mover-restic/restic/internal/repository/repair_pack.go new file mode 100644 index 000000000..811388cc9 --- /dev/null +++ b/mover-restic/restic/internal/repository/repair_pack.go @@ -0,0 +1,72 @@ +package repository + +import ( + "context" + "errors" + "io" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" + "golang.org/x/sync/errgroup" +) + +func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printer progress.Printer) error { + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + + printer.P("salvaging intact data from specified pack files") + bar := printer.NewCounter("pack files") + bar.SetMax(uint64(len(ids))) + defer bar.Done() + + wg.Go(func() error { + // examine all data the indexes have for the pack file + for b := range repo.ListPacksFromIndex(wgCtx, ids) { + blobs := b.Blobs + if len(blobs) == 0 { + printer.E("no blobs found for pack %v", b.PackID) + bar.Add(1) + continue + } + + err := repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if err != nil { + printer.E("failed to load blob %v: %v", blob.ID, err) + return nil + } + id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) + if !id.Equal(blob.ID) { + panic("pack id mismatch during upload") + } + return err + }) + // ignore truncated file parts + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + return err + } + bar.Add(1) + } + return repo.Flush(wgCtx) + }) + + err := wg.Wait() + bar.Done() + if err != nil { + return err + } + + // remove salvaged packs from index + err = rewriteIndexFiles(ctx, repo, ids, nil, nil, printer) + if err != nil { + return err + } + + // cleanup + printer.P("removing salvaged pack files") + // if we fail to delete the damaged pack files, then prune will remove them later on + bar = printer.NewCounter("files deleted") + _ = restic.ParallelRemove(ctx, repo, ids, restic.PackFile, nil, bar) + bar.Done() + + return nil +} diff --git a/mover-restic/restic/internal/repository/repair_pack_test.go b/mover-restic/restic/internal/repository/repair_pack_test.go new file mode 100644 index 000000000..0d6d340f4 --- /dev/null +++ b/mover-restic/restic/internal/repository/repair_pack_test.go @@ -0,0 +1,130 @@ +package repository_test + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/restic/restic/internal/backend" + backendtest "github.com/restic/restic/internal/backend/test" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" +) + +func listBlobs(repo restic.Repository) restic.BlobSet { + blobs := restic.NewBlobSet() + _ = repo.ListBlobs(context.TODO(), func(pb restic.PackedBlob) { + blobs.Insert(pb.BlobHandle) + }) + return blobs +} + +func replaceFile(t *testing.T, be backend.Backend, h backend.Handle, damage func([]byte) []byte) { + buf, err := backendtest.LoadAll(context.TODO(), be, h) + test.OK(t, err) + buf = damage(buf) + test.OK(t, be.Remove(context.TODO(), h)) + test.OK(t, be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))) +} + +func TestRepairBrokenPack(t *testing.T) { + repository.TestAllVersions(t, testRepairBrokenPack) +} + +func testRepairBrokenPack(t *testing.T, version uint) { + tests := []struct { + name string + damage func(t *testing.T, repo *repository.Repository, be backend.Backend, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) + }{ + { + "valid pack", + func(t *testing.T, repo *repository.Repository, be backend.Backend, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + return packsBefore, restic.NewBlobSet() + }, + }, + { + "broken pack", + func(t *testing.T, repo *repository.Repository, be backend.Backend, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + wrongBlob := createRandomWrongBlob(t, repo) + damagedPacks := findPacksForBlobs(t, repo, restic.NewBlobSet(wrongBlob)) + return damagedPacks, restic.NewBlobSet(wrongBlob) + }, + }, + { + "partially broken pack", + func(t *testing.T, repo *repository.Repository, be backend.Backend, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, be, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf[0] ^= 0xff + return buf + }) + + // find blob that starts at offset 0 + var damagedBlob restic.BlobHandle + for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + } + } + } + + return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) + }, + }, { + "truncated pack", + func(t *testing.T, repo *repository.Repository, be backend.Backend, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, be, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf = buf[0:10] + return buf + }) + + // all blobs in the file are broken + damagedBlobs := restic.NewBlobSet() + for blobs := range repo.ListPacksFromIndex(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + damagedBlobs.Insert(blob.BlobHandle) + } + } + return restic.NewIDSet(damagedID), damagedBlobs + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // disable verification to allow adding corrupted blobs to the repository + repo, be := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) + + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("rand seed is %v", seed) + + createRandomBlobs(t, repo, 5, 0.7, true) + packsBefore := listPacks(t, repo) + blobsBefore := listBlobs(repo) + + toRepair, damagedBlobs := test.damage(t, repo, be, packsBefore) + + rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) + // reload index + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) + + packsAfter := listPacks(t, repo) + blobsAfter := listBlobs(repo) + + rtest.Assert(t, len(packsAfter.Intersect(toRepair)) == 0, "some damaged packs were not removed") + rtest.Assert(t, len(packsBefore.Sub(toRepair).Sub(packsAfter)) == 0, "not-damaged packs were removed") + rtest.Assert(t, blobsBefore.Sub(damagedBlobs).Equals(blobsAfter), "diverging blob lists") + }) + } +} diff --git a/mover-restic/restic/internal/repository/repository.go b/mover-restic/restic/internal/repository/repository.go index 40794508f..838858c38 100644 --- a/mover-restic/restic/internal/repository/repository.go +++ b/mover-restic/restic/internal/repository/repository.go @@ -1,42 +1,39 @@ package repository import ( - "bufio" "bytes" "context" "fmt" "io" + "math" "os" "runtime" "sort" "sync" - "github.com/cenkalti/backoff/v4" "github.com/klauspost/compress/zstd" "github.com/restic/chunker" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/backend/dryrun" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/index" - "github.com/restic/restic/internal/pack" + "github.com/restic/restic/internal/repository/index" + "github.com/restic/restic/internal/repository/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) -const MaxStreamBufferSize = 4 * 1024 * 1024 - const MinPackSize = 4 * 1024 * 1024 const DefaultPackSize = 16 * 1024 * 1024 const MaxPackSize = 128 * 1024 * 1024 // Repository is used to access a repository in a backend. type Repository struct { - be restic.Backend + be backend.Backend cfg restic.Config key *crypto.Key keyID restic.ID @@ -45,8 +42,6 @@ type Repository struct { opts Options - noAutoIndexUpdate bool - packerWg *errgroup.Group uploader *packerUploader treePM *packerManager @@ -110,7 +105,7 @@ func (c *CompressionMode) Type() string { } // New returns a new repository with backend be. -func New(be restic.Backend, opts Options) (*Repository, error) { +func New(be backend.Backend, opts Options) (*Repository, error) { if opts.Compression == CompressionInvalid { return nil, errors.New("invalid compression mode") } @@ -133,18 +128,9 @@ func New(be restic.Backend, opts Options) (*Repository, error) { return repo, nil } -// DisableAutoIndexUpdate deactives the automatic finalization and upload of new -// indexes once these are full -func (r *Repository) DisableAutoIndexUpdate() { - r.noAutoIndexUpdate = true -} - // setConfig assigns the given config and updates the repository parameters accordingly func (r *Repository) setConfig(cfg restic.Config) { r.cfg = cfg - if r.cfg.Version >= 2 { - r.idx.MarkCompressed() - } } // Config returns the repository configuration. @@ -152,8 +138,8 @@ func (r *Repository) Config() restic.Config { return r.cfg } -// PackSize return the target size of a pack file when uploading -func (r *Repository) PackSize() uint { +// packSize return the target size of a pack file when uploading +func (r *Repository) packSize() uint { return r.opts.PackSize } @@ -180,46 +166,11 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res id = restic.ID{} } - ctx, cancel := context.WithCancel(ctx) - - h := restic.Handle{Type: t, Name: id.String()} - retriedInvalidData := false - var dataErr error - wr := new(bytes.Buffer) - - err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error { - // make sure this call is idempotent, in case an error occurs - wr.Reset() - _, cerr := io.Copy(wr, rd) - if cerr != nil { - return cerr - } - - buf := wr.Bytes() - if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) { - debug.Log("retry loading broken blob %v", h) - if !retriedInvalidData { - retriedInvalidData = true - } else { - // with a canceled context there is not guarantee which error will - // be returned by `be.Load`. - dataErr = fmt.Errorf("load(%v): %w", h, restic.ErrInvalidData) - cancel() - } - return restic.ErrInvalidData - - } - return nil - }) - - if dataErr != nil { - return nil, dataErr - } + buf, err := r.LoadRaw(ctx, t, id) if err != nil { return nil, err } - buf := wr.Bytes() nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil) if err != nil { @@ -233,7 +184,7 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res } type haver interface { - Has(restic.Handle) bool + Has(backend.Handle) bool } // sortCachedPacksFirst moves all cached pack files to the front of blobs. @@ -251,7 +202,7 @@ func sortCachedPacksFirst(cache haver, blobs []restic.PackedBlob) { noncached := make([]restic.PackedBlob, 0, len(blobs)/2) for _, blob := range blobs { - if cache.Has(restic.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { + if cache.Has(backend.Handle{Type: restic.PackFile, Name: blob.PackID.String()}) { cached = append(cached, blob) continue } @@ -276,16 +227,27 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. // try cached pack files first sortCachedPacksFirst(r.Cache, blobs) - var lastError error - for _, blob := range blobs { - debug.Log("blob %v/%v found: %v", t, id, blob) - - if blob.Type != t { - debug.Log("blob %v has wrong block type, want %v", blob, t) + buf, err := r.loadBlob(ctx, blobs, buf) + if err != nil { + if r.Cache != nil { + for _, blob := range blobs { + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} + // ignore errors as there's not much we can do here + _ = r.Cache.Forget(h) + } } + buf, err = r.loadBlob(ctx, blobs, buf) + } + return buf, err +} + +func (r *Repository) loadBlob(ctx context.Context, blobs []restic.PackedBlob, buf []byte) ([]byte, error) { + var lastError error + for _, blob := range blobs { + debug.Log("blob %v found: %v", blob.BlobHandle, blob) // load blob from pack - h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), ContainedBlobType: t} + h := backend.Handle{Type: restic.PackFile, Name: blob.PackID.String(), IsMetadata: blob.Type.IsMetadata()} switch { case cap(buf) < int(blob.Length): @@ -294,42 +256,26 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. buf = buf[:blob.Length] } - n, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) + _, err := backend.ReadAt(ctx, r.be, h, int64(blob.Offset), buf) if err != nil { debug.Log("error loading blob %v: %v", blob, err) lastError = err continue } - if uint(n) != blob.Length { - lastError = errors.Errorf("error loading blob %v: wrong length returned, want %d, got %d", - id.Str(), blob.Length, uint(n)) - debug.Log("lastError: %v", lastError) - continue - } - - // decrypt - nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] - plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil) - if err != nil { - lastError = errors.Errorf("decrypting blob %v failed: %v", id, err) - continue - } + it := newPackBlobIterator(blob.PackID, newByteReader(buf), uint(blob.Offset), []restic.Blob{blob.Blob}, r.key, r.getZstdDecoder()) + pbv, err := it.Next() - if blob.IsCompressed() { - plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, make([]byte, 0, blob.DataLength())) - if err != nil { - lastError = errors.Errorf("decompressing blob %v failed: %v", id, err) - continue - } + if err == nil { + err = pbv.Err } - - // check hash - if !restic.Hash(plaintext).Equal(id) { - lastError = errors.Errorf("blob %v returned invalid hash", id) + if err != nil { + debug.Log("error decoding blob %v: %v", blob, err) + lastError = err continue } + plaintext := pbv.Plaintext if len(plaintext) > cap(buf) { return plaintext, nil } @@ -343,12 +289,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic. return nil, lastError } - return nil, errors.Errorf("loading blob %v from %v packs failed", id.Str(), len(blobs)) -} - -// LookupBlobSize returns the size of blob id. -func (r *Repository) LookupBlobSize(id restic.ID, tpe restic.BlobType) (uint, bool) { - return r.idx.LookupSize(restic.BlobHandle{ID: id, Type: tpe}) + return nil, errors.Errorf("loading %v from %v packs failed", blobs[0].BlobHandle, len(blobs)) } func (r *Repository) getZstdEncoder() *zstd.Encoder { @@ -531,9 +472,9 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } else { id = restic.Hash(ciphertext) } - h := restic.Handle{Type: t, Name: id.String()} + h := backend.Handle{Type: t, Name: id.String()} - err = r.be.Save(ctx, h, restic.NewByteReader(ciphertext, r.be.Hasher())) + err = r.be.Save(ctx, h, backend.NewByteReader(ciphertext, r.be.Hasher())) if err != nil { debug.Log("error saving blob %v: %v", h, err) return restic.ID{}, err @@ -566,16 +507,17 @@ func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []by return nil } +func (r *Repository) RemoveUnpacked(ctx context.Context, t restic.FileType, id restic.ID) error { + // TODO prevent everything except removing snapshots for non-repository code + return r.be.Remove(ctx, backend.Handle{Type: t, Name: id.String()}) +} + // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { if err := r.flushPacks(ctx); err != nil { return err } - // Save index after flushing only if noAutoIndexUpdate is not set - if r.noAutoIndexUpdate { - return nil - } return r.idx.SaveIndex(ctx, r) } @@ -587,8 +529,8 @@ func (r *Repository) StartPackUploader(ctx context.Context, wg *errgroup.Group) innerWg, ctx := errgroup.WithContext(ctx) r.packerWg = innerWg r.uploader = newPackerUploader(ctx, innerWg, r, r.be.Connections()) - r.treePM = newPackerManager(r.key, restic.TreeBlob, r.PackSize(), r.uploader.QueuePacker) - r.dataPM = newPackerManager(r.key, restic.DataBlob, r.PackSize(), r.uploader.QueuePacker) + r.treePM = newPackerManager(r.key, restic.TreeBlob, r.packSize(), r.uploader.QueuePacker) + r.dataPM = newPackerManager(r.key, restic.DataBlob, r.packSize(), r.uploader.QueuePacker) wg.Go(func() error { return innerWg.Wait() @@ -620,18 +562,27 @@ func (r *Repository) flushPacks(ctx context.Context) error { return err } -// Backend returns the backend for the repository. -func (r *Repository) Backend() restic.Backend { - return r.be -} - func (r *Repository) Connections() uint { return r.be.Connections() } -// Index returns the currently used MasterIndex. -func (r *Repository) Index() restic.MasterIndex { - return r.idx +func (r *Repository) LookupBlob(tpe restic.BlobType, id restic.ID) []restic.PackedBlob { + return r.idx.Lookup(restic.BlobHandle{Type: tpe, ID: id}) +} + +// LookupBlobSize returns the size of blob id. +func (r *Repository) LookupBlobSize(tpe restic.BlobType, id restic.ID) (uint, bool) { + return r.idx.LookupSize(restic.BlobHandle{Type: tpe, ID: id}) +} + +// ListBlobs runs fn on all blobs known to the index. When the context is cancelled, +// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. +func (r *Repository) ListBlobs(ctx context.Context, fn func(restic.PackedBlob)) error { + return r.idx.Each(ctx, fn) +} + +func (r *Repository) ListPacksFromIndex(ctx context.Context, packs restic.IDSet) <-chan restic.PackBlobs { + return r.idx.ListPacks(ctx, packs) } // SetIndex instructs the repository to use the given index. @@ -640,50 +591,18 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error { return r.prepareCache() } +func (r *Repository) clearIndex() { + r.idx = index.NewMasterIndex() +} + // LoadIndex loads all index files from the backend in parallel and stores them func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { debug.Log("Loading index") - indexList, err := backend.MemorizeList(ctx, r.Backend(), restic.IndexFile) - if err != nil { - return err - } - - if p != nil { - var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(fi restic.FileInfo) error { - _, err := restic.ParseID(fi.Name) - if err != nil { - debug.Log("unable to parse %v as an ID", fi.Name) - return nil - } - - numIndexFiles++ - return nil - }) - if err != nil { - return err - } - p.SetMax(numIndexFiles) - defer p.Done() - } - - err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { - if err != nil { - return err - } - r.idx.Insert(idx) - if p != nil { - p.Add(1) - } - return nil - }) - - if err != nil { - return err - } + // reset in-memory index before loading it from the repository + r.clearIndex() - err = r.idx.MergeFinalIndexes() + err := r.idx.Load(ctx, r, p, nil) if err != nil { return err } @@ -697,24 +616,30 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { defer cancel() invalidIndex := false - r.idx.Each(ctx, func(blob restic.PackedBlob) { + err := r.idx.Each(ctx, func(blob restic.PackedBlob) { if blob.IsCompressed() { invalidIndex = true } }) + if err != nil { + return err + } if invalidIndex { return errors.New("index uses feature not supported by repository version 1") } } + if ctx.Err() != nil { + return ctx.Err() + } // remove index files from the cache which have been removed in the repo return r.prepareCache() } -// CreateIndexFromPacks creates a new index by reading all given pack files (with sizes). +// createIndexFromPacks creates a new index by reading all given pack files (with sizes). // The index is added to the MasterIndex but not marked as finalized. // Returned is the list of pack files which could not be read. -func (r *Repository) CreateIndexFromPacks(ctx context.Context, packsize map[restic.ID]int64, p *progress.Counter) (invalid restic.IDs, err error) { +func (r *Repository) createIndexFromPacks(ctx context.Context, packsize map[restic.ID]int64, p *progress.Counter) (invalid restic.IDs, err error) { var m sync.Mutex debug.Log("Loading index from pack files") @@ -809,12 +734,19 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int return err } + oldKey := r.key + oldKeyID := r.keyID + r.key = key.master r.keyID = key.ID() cfg, err := restic.LoadConfig(ctx, r) - if err == crypto.ErrUnauthenticated { - return fmt.Errorf("config or key %v is damaged: %w", key.ID(), err) - } else if err != nil { + if err != nil { + r.key = oldKey + r.keyID = oldKeyID + + if err == crypto.ErrUnauthenticated { + return fmt.Errorf("config or key %v is damaged: %w", key.ID(), err) + } return fmt.Errorf("config cannot be loaded: %w", err) } @@ -833,13 +765,30 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch return fmt.Errorf("repository version %v too low", version) } - _, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile}) + _, err := r.be.Stat(ctx, backend.Handle{Type: restic.ConfigFile}) if err != nil && !r.be.IsNotExist(err) { return err } if err == nil { return errors.New("repository master key and config already initialized") } + // double check to make sure that a repository is not accidentally reinitialized + // if the backend somehow fails to stat the config file. An initialized repository + // must always contain at least one key file. + if err := r.List(ctx, restic.KeyFile, func(_ restic.ID, _ int64) error { + return errors.New("repository already contains keys") + }); err != nil { + return err + } + // Also check for snapshots to detect repositories with a misconfigured retention + // policy that deletes files older than x days. For such repositories usually the + // config and key files are removed first and therefore the check would not detect + // the old repository. + if err := r.List(ctx, restic.SnapshotFile, func(_ restic.ID, _ int64) error { + return errors.New("repository already contains snapshots") + }); err != nil { + return err + } cfg, err := restic.CreateConfig(version) if err != nil { @@ -878,7 +827,7 @@ func (r *Repository) KeyID() restic.ID { // List runs fn for all files of type t in the repo. func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { - return r.be.List(ctx, t, func(fi restic.FileInfo) error { + return r.be.List(ctx, t, func(fi backend.FileInfo) error { id, err := restic.ParseID(fi.Name) if err != nil { debug.Log("unable to parse %v as an ID", fi.Name) @@ -891,9 +840,19 @@ func (r *Repository) List(ctx context.Context, t restic.FileType, fn func(restic // ListPack returns the list of blobs saved in the pack id and the length of // the pack header. func (r *Repository) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, uint32, error) { - h := restic.Handle{Type: restic.PackFile, Name: id.String()} + h := backend.Handle{Type: restic.PackFile, Name: id.String()} - return pack.List(r.Key(), backend.ReaderAt(ctx, r.Backend(), h), size) + entries, hdrSize, err := pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) + if err != nil { + if r.Cache != nil { + // ignore error as there is not much we can do here + _ = r.Cache.Forget(h) + } + + // retry on error + entries, hdrSize, err = pack.List(r.Key(), backend.ReaderAt(ctx, r.be, h), size) + } + return entries, hdrSize, err } // Delete calls backend.Delete() if implemented, and returns an error @@ -916,6 +875,10 @@ func (r *Repository) Close() error { // occupies in the repo (compressed or not, including encryption overhead). func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) { + if int64(len(buf)) > math.MaxUint32 { + return restic.ID{}, false, 0, fmt.Errorf("blob is larger than 4GB") + } + // compute plaintext hash if not already set if id.IsNull() { // Special case the hash calculation for all zero chunks. This is especially @@ -941,16 +904,22 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte return newID, known, size, err } -type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error +type backendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error +type loadBlobFn func(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) -// Skip sections with more than 4MB unused blobs -const maxUnusedRange = 4 * 1024 * 1024 +// Skip sections with more than 1MB unused blobs +const maxUnusedRange = 1 * 1024 * 1024 -// StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to +// LoadBlobsFromPack loads the listed blobs from the specified pack file. The plaintext blob is passed to // the handleBlobFn callback or an error if decryption failed or the blob hash does not match. -// handleBlobFn is never called multiple times for the same blob. If the callback returns an error, -// then StreamPack will abort and not retry it. -func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +// handleBlobFn is called at most once for each blob. If the callback returns an error, +// then LoadBlobsFromPack will abort and not retry it. The buf passed to the callback is only valid within +// this specific call. The callback must not keep a reference to buf. +func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + return streamPack(ctx, r.be.Load, r.LoadBlob, r.getZstdDecoder(), r.key, packID, blobs, handleBlobFn) +} + +func streamPack(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do return nil @@ -962,14 +931,29 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack lowerIdx := 0 lastPos := blobs[0].Offset + const maxChunkSize = 2 * DefaultPackSize + for i := 0; i < len(blobs); i++ { if blobs[i].Offset < lastPos { // don't wait for streamPackPart to fail return errors.Errorf("overlapping blobs in pack %v", packID) } + + chunkSizeAfter := (blobs[i].Offset + blobs[i].Length) - blobs[lowerIdx].Offset + split := false + // split if the chunk would become larger than maxChunkSize. Oversized chunks are + // handled by the requirement that the chunk contains at least one blob (i > lowerIdx) + if i > lowerIdx && chunkSizeAfter >= maxChunkSize { + split = true + } + // skip too large gaps as a new request is typically much cheaper than data transfers if blobs[i].Offset-lastPos > maxUnusedRange { + split = true + } + + if split { // load everything up to the skipped file section - err := streamPackPart(ctx, beLoad, key, packID, blobs[lowerIdx:i], handleBlobFn) + err := streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:i], handleBlobFn) if err != nil { return err } @@ -978,110 +962,204 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack lastPos = blobs[i].Offset + blobs[i].Length } // load remainder - return streamPackPart(ctx, beLoad, key, packID, blobs[lowerIdx:], handleBlobFn) + return streamPackPart(ctx, beLoad, loadBlobFn, dec, key, packID, blobs[lowerIdx:], handleBlobFn) } -func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { - h := restic.Handle{Type: restic.PackFile, Name: packID.String(), ContainedBlobType: restic.DataBlob} +func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBlobFn, dec *zstd.Decoder, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: blobs[0].Type.IsMetadata()} dataStart := blobs[0].Offset dataEnd := blobs[len(blobs)-1].Offset + blobs[len(blobs)-1].Length debug.Log("streaming pack %v (%d to %d bytes), blobs: %v", packID, dataStart, dataEnd, len(blobs)) - dec, err := zstd.NewReader(nil) + data := make([]byte, int(dataEnd-dataStart)) + err := beLoad(ctx, h, int(dataEnd-dataStart), int64(dataStart), func(rd io.Reader) error { + _, cerr := io.ReadFull(rd, data) + return cerr + }) + // prevent callbacks after cancellation + if ctx.Err() != nil { + return ctx.Err() + } if err != nil { - panic(dec) + // the context is only still valid if handleBlobFn never returned an error + if loadBlobFn != nil { + // check whether we can get the remaining blobs somewhere else + for _, entry := range blobs { + buf, ierr := loadBlobFn(ctx, entry.Type, entry.ID, nil) + err = handleBlobFn(entry.BlobHandle, buf, ierr) + if err != nil { + break + } + } + } + return errors.Wrap(err, "StreamPack") } - defer dec.Close() - ctx, cancel := context.WithCancel(ctx) - // stream blobs in pack - err = beLoad(ctx, h, int(dataEnd-dataStart), int64(dataStart), func(rd io.Reader) error { - // prevent callbacks after cancelation - if ctx.Err() != nil { - return ctx.Err() - } - bufferSize := int(dataEnd - dataStart) - if bufferSize > MaxStreamBufferSize { - bufferSize = MaxStreamBufferSize + it := newPackBlobIterator(packID, newByteReader(data), dataStart, blobs, key, dec) + + for { + val, err := it.Next() + if err == errPackEOF { + break + } else if err != nil { + return err } - // create reader here to allow reusing the buffered reader from checker.checkData - bufRd := bufio.NewReaderSize(rd, bufferSize) - currentBlobEnd := dataStart - var buf []byte - var decode []byte - for len(blobs) > 0 { - entry := blobs[0] - - skipBytes := int(entry.Offset - currentBlobEnd) - if skipBytes < 0 { - return errors.Errorf("overlapping blobs in pack %v", packID) - } - _, err := bufRd.Discard(skipBytes) - if err != nil { - return err + if val.Err != nil && loadBlobFn != nil { + var ierr error + // check whether we can get a valid copy somewhere else + buf, ierr := loadBlobFn(ctx, val.Handle.Type, val.Handle.ID, nil) + if ierr == nil { + // success + val.Plaintext = buf + val.Err = nil } + } - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} - debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) + err = handleBlobFn(val.Handle, val.Plaintext, val.Err) + if err != nil { + return err + } + // ensure that each blob is only passed once to handleBlobFn + blobs = blobs[1:] + } - if uint(cap(buf)) < entry.Length { - buf = make([]byte, entry.Length) - } - buf = buf[:entry.Length] + return errors.Wrap(err, "StreamPack") +} - n, err := io.ReadFull(bufRd, buf) - if err != nil { - debug.Log(" read error %v", err) - return errors.Wrap(err, "ReadFull") - } +// discardReader allows the PackBlobIterator to perform zero copy +// reads if the underlying data source is a byte slice. +type discardReader interface { + Discard(n int) (discarded int, err error) + // ReadFull reads the next n bytes into a byte slice. The caller must not + // retain a reference to the byte. Modifications are only allowed within + // the boundaries of the returned slice. + ReadFull(n int) (buf []byte, err error) +} - if n != len(buf) { - return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", - h, packID.Str(), len(buf), n) - } - currentBlobEnd = entry.Offset + entry.Length +type byteReader struct { + buf []byte +} - if int(entry.Length) <= key.NonceSize() { - debug.Log("%v", blobs) - return errors.Errorf("invalid blob length %v", entry) - } +func newByteReader(buf []byte) *byteReader { + return &byteReader{ + buf: buf, + } +} - // decryption errors are likely permanent, give the caller a chance to skip them - nonce, ciphertext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(ciphertext[:0], nonce, ciphertext, nil) - if err == nil && entry.IsCompressed() { - // DecodeAll will allocate a slice if it is not large enough since it - // knows the decompressed size (because we're using EncodeAll) - decode, err = dec.DecodeAll(plaintext, decode[:0]) - plaintext = decode - if err != nil { - err = errors.Errorf("decompressing blob %v failed: %v", h, err) - } - } - if err == nil { - id := restic.Hash(plaintext) - if !id.Equal(entry.ID) { - debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", - h.Type, h.ID, packID.Str(), id) - err = errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", - h, packID.Str(), id) - } - } +func (b *byteReader) Discard(n int) (discarded int, err error) { + if len(b.buf) < n { + return 0, io.ErrUnexpectedEOF + } + b.buf = b.buf[n:] + return n, nil +} - err = handleBlobFn(entry.BlobHandle, plaintext, err) - if err != nil { - cancel() - return backoff.Permanent(err) - } - // ensure that each blob is only passed once to handleBlobFn - blobs = blobs[1:] +func (b *byteReader) ReadFull(n int) (buf []byte, err error) { + if len(b.buf) < n { + return nil, io.ErrUnexpectedEOF + } + buf = b.buf[:n] + b.buf = b.buf[n:] + return buf, nil +} + +type packBlobIterator struct { + packID restic.ID + rd discardReader + currentOffset uint + + blobs []restic.Blob + key *crypto.Key + dec *zstd.Decoder + + decode []byte +} + +type packBlobValue struct { + Handle restic.BlobHandle + Plaintext []byte + Err error +} + +var errPackEOF = errors.New("reached EOF of pack file") + +func newPackBlobIterator(packID restic.ID, rd discardReader, currentOffset uint, + blobs []restic.Blob, key *crypto.Key, dec *zstd.Decoder) *packBlobIterator { + return &packBlobIterator{ + packID: packID, + rd: rd, + currentOffset: currentOffset, + blobs: blobs, + key: key, + dec: dec, + } +} + +// Next returns the next blob, an error or ErrPackEOF if all blobs were read +func (b *packBlobIterator) Next() (packBlobValue, error) { + if len(b.blobs) == 0 { + return packBlobValue{}, errPackEOF + } + + entry := b.blobs[0] + b.blobs = b.blobs[1:] + + skipBytes := int(entry.Offset - b.currentOffset) + if skipBytes < 0 { + return packBlobValue{}, fmt.Errorf("overlapping blobs in pack %v", b.packID) + } + + _, err := b.rd.Discard(skipBytes) + if err != nil { + return packBlobValue{}, err + } + b.currentOffset = entry.Offset + + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) + + buf, err := b.rd.ReadFull(int(entry.Length)) + if err != nil { + debug.Log(" read error %v", err) + return packBlobValue{}, fmt.Errorf("readFull: %w", err) + } + + b.currentOffset = entry.Offset + entry.Length + + if int(entry.Length) <= b.key.NonceSize() { + debug.Log("%v", b.blobs) + return packBlobValue{}, fmt.Errorf("invalid blob length %v", entry) + } + + // decryption errors are likely permanent, give the caller a chance to skip them + nonce, ciphertext := buf[:b.key.NonceSize()], buf[b.key.NonceSize():] + plaintext, err := b.key.Open(ciphertext[:0], nonce, ciphertext, nil) + if err != nil { + err = fmt.Errorf("decrypting blob %v from %v failed: %w", h, b.packID.Str(), err) + } + if err == nil && entry.IsCompressed() { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + b.decode, err = b.dec.DecodeAll(plaintext, b.decode[:0]) + plaintext = b.decode + if err != nil { + err = fmt.Errorf("decompressing blob %v from %v failed: %w", h, b.packID.Str(), err) } - return nil - }) - return errors.Wrap(err, "StreamPack") + } + if err == nil { + id := restic.Hash(plaintext) + if !id.Equal(entry.ID) { + debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", + h.Type, h.ID, b.packID.Str(), id) + err = fmt.Errorf("read blob %v from %v: wrong data returned, hash is %v", + h, b.packID.Str(), id) + } + } + + return packBlobValue{entry.BlobHandle, plaintext, err}, nil } var zeroChunkOnce sync.Once diff --git a/mover-restic/restic/internal/repository/repository_internal_test.go b/mover-restic/restic/internal/repository/repository_internal_test.go index 2a9976ace..35082774c 100644 --- a/mover-restic/restic/internal/repository/repository_internal_test.go +++ b/mover-restic/restic/internal/repository/repository_internal_test.go @@ -1,19 +1,28 @@ package repository import ( + "bytes" + "context" + "encoding/json" + "io" "math/rand" "sort" "strings" "testing" + "github.com/cenkalti/backoff/v4" + "github.com/google/go-cmp/cmp" + "github.com/klauspost/compress/zstd" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) -type mapcache map[restic.Handle]bool +type mapcache map[backend.Handle]bool -func (c mapcache) Has(h restic.Handle) bool { return c[h] } +func (c mapcache) Has(h backend.Handle) bool { return c[h] } func TestSortCachedPacksFirst(t *testing.T) { var ( @@ -29,15 +38,15 @@ func TestSortCachedPacksFirst(t *testing.T) { blobs[i] = restic.PackedBlob{PackID: id} if i%3 == 0 { - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} cache[h] = true } } copy(sorted[:], blobs[:]) sort.SliceStable(sorted[:], func(i, j int) bool { - hi := restic.Handle{Type: restic.PackFile, Name: sorted[i].PackID.String()} - hj := restic.Handle{Type: restic.PackFile, Name: sorted[j].PackID.String()} + hi := backend.Handle{Type: backend.PackFile, Name: sorted[i].PackID.String()} + hj := backend.Handle{Type: backend.PackFile, Name: sorted[j].PackID.String()} return cache.Has(hi) && !cache.Has(hj) }) @@ -60,7 +69,7 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { blobs[i] = restic.PackedBlob{PackID: id} if i%3 == 0 { - h := restic.Handle{Name: id.String(), Type: restic.PackFile} + h := backend.Handle{Name: id.String(), Type: backend.PackFile} cache[h] = true } } @@ -75,8 +84,276 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { } } +// buildPackfileWithoutHeader returns a manually built pack file without a header. +func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { + opts := []zstd.EOption{ + // Set the compression level configured. + zstd.WithEncoderLevel(zstd.SpeedDefault), + // Disable CRC, we have enough checks in place, makes the + // compressed data four bytes shorter. + zstd.WithEncoderCRC(false), + // Set a window of 512kbyte, so we have good lookbehind for usual + // blob sizes. + zstd.WithWindowSize(512 * 1024), + } + enc, err := zstd.NewWriter(nil, opts...) + if err != nil { + panic(err) + } + + var offset uint + for i, size := range blobSizes { + plaintext := rtest.Random(800+i, size) + id := restic.Hash(plaintext) + uncompressedLength := uint(0) + if compress { + uncompressedLength = uint(len(plaintext)) + plaintext = enc.EncodeAll(plaintext, nil) + } + + // we use a deterministic nonce here so the whole process is + // deterministic, last byte is the blob index + var nonce = []byte{ + 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, + 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), + } + + before := len(packfile) + packfile = append(packfile, nonce...) + packfile = key.Seal(packfile, nonce, plaintext, nil) + after := len(packfile) + + ciphertextLength := after - before + + blobs = append(blobs, restic.Blob{ + BlobHandle: restic.BlobHandle{ + Type: restic.DataBlob, + ID: id, + }, + Length: uint(ciphertextLength), + UncompressedLength: uncompressedLength, + Offset: offset, + }) + + offset = uint(len(packfile)) + } + + return blobs, packfile +} + +func TestStreamPack(t *testing.T) { + TestAllVersions(t, testStreamPack) +} + +func testStreamPack(t *testing.T, version uint) { + dec, err := zstd.NewReader(nil) + if err != nil { + panic(dec) + } + defer dec.Close() + + // always use the same key for deterministic output + key := testKey(t) + + blobSizes := []int{ + 5522811, + 10, + 5231, + 18812, + 123123, + 13522811, + 12301, + 892242, + 28616, + 13351, + 252287, + 188883, + 3522811, + 18883, + } + + var compress bool + switch version { + case 1: + compress = false + case 2: + compress = true + default: + t.Fatal("test does not support repository version", version) + } + + packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) + + loadCalls := 0 + shortFirstLoad := false + + loadBytes := func(length int, offset int64) []byte { + data := packfile + + if offset > int64(len(data)) { + offset = 0 + length = 0 + } + data = data[offset:] + + if length > len(data) { + length = len(data) + } + if shortFirstLoad { + length /= 2 + shortFirstLoad = false + } + + return data[:length] + } + + load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + data := loadBytes(length, offset) + if shortFirstLoad { + data = data[:len(data)/2] + shortFirstLoad = false + } + + loadCalls++ + + err := fn(bytes.NewReader(data)) + if err == nil { + return nil + } + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return err + } + + // retry loading once + return fn(bytes.NewReader(loadBytes(length, offset))) + } + + // first, test regular usage + t.Run("regular", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + calls int + shortFirstLoad bool + }{ + {packfileBlobs[1:2], 1, false}, + {packfileBlobs[2:5], 1, false}, + {packfileBlobs[2:8], 1, false}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[4], + packfileBlobs[2], + }, 1, false}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[len(packfileBlobs)-1], + }, 2, false}, + {packfileBlobs[:], 1, true}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gotBlobs := make(map[restic.ID]int) + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + gotBlobs[blob.ID]++ + + id := restic.Hash(buf) + if !id.Equal(blob.ID) { + t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) + } + + return err + } + + wantBlobs := make(map[restic.ID]int) + for _, blob := range test.blobs { + wantBlobs[blob.ID] = 1 + } + + loadCalls = 0 + shortFirstLoad = test.shortFirstLoad + err := streamPack(ctx, load, nil, dec, &key, restic.ID{}, test.blobs, handleBlob) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(wantBlobs, gotBlobs) { + t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) + } + rtest.Equals(t, test.calls, loadCalls) + }) + } + }) + shortFirstLoad = false + + // next, test invalid uses, which should return an error + t.Run("invalid", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + err string + }{ + { + // pass one blob several times + blobs: []restic.Blob{ + packfileBlobs[3], + packfileBlobs[8], + packfileBlobs[3], + packfileBlobs[4], + }, + err: "overlapping blobs in pack", + }, + + { + // pass something that's not a valid blob in the current pack file + blobs: []restic.Blob{ + { + Offset: 123, + Length: 20000, + }, + }, + err: "ciphertext verification failed", + }, + + { + // pass a blob that's too small + blobs: []restic.Blob{ + { + Offset: 123, + Length: 10, + }, + }, + err: "invalid blob length", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + return err + } + + err := streamPack(ctx, load, nil, dec, &key, restic.ID{}, test.blobs, handleBlob) + if err == nil { + t.Fatalf("wanted error %v, got nil", test.err) + } + + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) + } + }) + } + }) +} + func TestBlobVerification(t *testing.T) { - repo := TestRepository(t).(*Repository) + repo := TestRepository(t) type DamageType string const ( @@ -125,7 +402,7 @@ func TestBlobVerification(t *testing.T) { } func TestUnpackedVerification(t *testing.T) { - repo := TestRepository(t).(*Repository) + repo := TestRepository(t) type DamageType string const ( @@ -172,3 +449,83 @@ func TestUnpackedVerification(t *testing.T) { } } } + +func testKey(t *testing.T) crypto.Key { + const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` + + var key crypto.Key + err := json.Unmarshal([]byte(jsonKey), &key) + if err != nil { + t.Fatal(err) + } + return key +} + +func TestStreamPackFallback(t *testing.T) { + dec, err := zstd.NewReader(nil) + if err != nil { + panic(dec) + } + defer dec.Close() + + test := func(t *testing.T, failLoad bool) { + key := testKey(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + plaintext := rtest.Random(800, 42) + blobID := restic.Hash(plaintext) + blobs := []restic.Blob{ + { + Length: uint(crypto.CiphertextLength(len(plaintext))), + Offset: 0, + BlobHandle: restic.BlobHandle{ + ID: blobID, + Type: restic.DataBlob, + }, + }, + } + + var loadPack backendLoadFn + if failLoad { + loadPack = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + return errors.New("load error") + } + } else { + loadPack = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + // just return an empty array to provoke an error + data := make([]byte, length) + return fn(bytes.NewReader(data)) + } + } + + loadBlob := func(ctx context.Context, t restic.BlobType, id restic.ID, buf []byte) ([]byte, error) { + if id == blobID { + return plaintext, nil + } + return nil, errors.New("unknown blob") + } + + blobOK := false + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + rtest.OK(t, err) + rtest.Equals(t, blobID, blob.ID) + rtest.Equals(t, plaintext, buf) + blobOK = true + return err + } + + err := streamPack(ctx, loadPack, loadBlob, dec, &key, restic.ID{}, blobs, handleBlob) + rtest.OK(t, err) + rtest.Assert(t, blobOK, "blob failed to load") + } + + t.Run("corrupted blob", func(t *testing.T) { + test(t, false) + }) + + // test fallback for failed pack loading + t.Run("failed load", func(t *testing.T) { + test(t, true) + }) +} diff --git a/mover-restic/restic/internal/repository/repository_test.go b/mover-restic/restic/internal/repository/repository_test.go index c4550d77d..ea21ea3f3 100644 --- a/mover-restic/restic/internal/repository/repository_test.go +++ b/mover-restic/restic/internal/repository/repository_test.go @@ -4,25 +4,26 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/json" - "errors" "fmt" "io" "math/rand" "os" "path/filepath" "strings" + "sync" "testing" "time" - "github.com/cenkalti/backoff/v4" - "github.com/google/go-cmp/cmp" - "github.com/klauspost/compress/zstd" + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/cache" "github.com/restic/restic/internal/backend/local" + "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/crypto" - "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/repository/index" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) @@ -32,52 +33,20 @@ var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) func TestSave(t *testing.T) { - repository.TestAllVersions(t, testSave) + repository.TestAllVersions(t, testSavePassID) + repository.TestAllVersions(t, testSaveCalculateID) } -func testSave(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) - - for _, size := range testSizes { - data := make([]byte, size) - _, err := io.ReadFull(rnd, data) - rtest.OK(t, err) - - id := restic.Hash(data) - - var wg errgroup.Group - repo.StartPackUploader(context.TODO(), &wg) - - // save - sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) - rtest.OK(t, err) - - rtest.Equals(t, id, sid) - - rtest.OK(t, repo.Flush(context.Background())) - // rtest.OK(t, repo.SaveIndex()) - - // read back - buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil) - rtest.OK(t, err) - rtest.Equals(t, size, len(buf)) - - rtest.Assert(t, len(buf) == len(data), - "number of bytes read back does not match: expected %d, got %d", - len(data), len(buf)) - - rtest.Assert(t, bytes.Equal(buf, data), - "data does not match: expected %02x, got %02x", - data, buf) - } +func testSavePassID(t *testing.T, version uint) { + testSave(t, version, false) } -func TestSaveFrom(t *testing.T) { - repository.TestAllVersions(t, testSaveFrom) +func testSaveCalculateID(t *testing.T, version uint) { + testSave(t, version, true) } -func testSaveFrom(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) +func testSave(t *testing.T, version uint, calculateID bool) { + repo, _ := repository.TestRepositoryWithVersion(t, version) for _, size := range testSizes { data := make([]byte, size) @@ -90,9 +59,13 @@ func testSaveFrom(t *testing.T, version uint) { repo.StartPackUploader(context.TODO(), &wg) // save - id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) + inputID := restic.ID{} + if !calculateID { + inputID = id + } + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, inputID, false) rtest.OK(t, err) - rtest.Equals(t, id, id2) + rtest.Equals(t, id, sid) rtest.OK(t, repo.Flush(context.Background())) @@ -116,7 +89,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) { } func benchmarkSaveAndEncrypt(t *testing.B, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) size := 4 << 20 // 4MiB data := make([]byte, size) @@ -142,7 +115,7 @@ func TestLoadBlob(t *testing.T) { } func testLoadBlob(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) length := 1000000 buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) @@ -171,12 +144,34 @@ func testLoadBlob(t *testing.T, version uint) { } } +func TestLoadBlobBroken(t *testing.T) { + be := mem.New() + repo, _ := repository.TestRepositoryWithBackend(t, &damageOnceBackend{Backend: be}, restic.StableRepoVersion, repository.Options{}) + buf := test.Random(42, 1000) + + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + id, _, _, err := repo.SaveBlob(context.TODO(), restic.TreeBlob, buf, restic.ID{}, false) + rtest.OK(t, err) + rtest.OK(t, repo.Flush(context.Background())) + + // setup cache after saving the blob to make sure that the damageOnceBackend damages the cached data + c := cache.TestNewCache(t) + repo.UseCache(c) + + data, err := repo.LoadBlob(context.TODO(), restic.TreeBlob, id, nil) + rtest.OK(t, err) + rtest.Assert(t, bytes.Equal(buf, data), "data mismatch") + pack := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + rtest.Assert(t, c.Has(backend.Handle{Type: restic.PackFile, Name: pack.String()}), "expected tree pack to be cached") +} + func BenchmarkLoadBlob(b *testing.B) { repository.BenchmarkAllVersions(b, benchmarkLoadBlob) } func benchmarkLoadBlob(b *testing.B, version uint) { - repo := repository.TestRepositoryWithVersion(b, version) + repo, _ := repository.TestRepositoryWithVersion(b, version) length := 1000000 buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) @@ -217,7 +212,7 @@ func BenchmarkLoadUnpacked(b *testing.B) { } func benchmarkLoadUnpacked(b *testing.B, version uint) { - repo := repository.TestRepositoryWithVersion(b, version) + repo, _ := repository.TestRepositoryWithVersion(b, version) length := 1000000 buf := crypto.NewBlobBuffer(length) _, err := io.ReadFull(rnd, buf) @@ -253,15 +248,14 @@ func benchmarkLoadUnpacked(b *testing.B, version uint) { var repoFixture = filepath.Join("testdata", "test-repo.tar.gz") func TestRepositoryLoadIndex(t *testing.T) { - repodir, cleanup := rtest.Env(t, repoFixture) + repo, _, cleanup := repository.TestFromFixture(t, repoFixture) defer cleanup() - repo := repository.TestOpenLocal(t, repodir) rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } // loadIndex loads the index id from backend and returns it. -func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) { +func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*index.Index, error) { buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id) if err != nil { return nil, err @@ -275,44 +269,40 @@ func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*inde } func TestRepositoryLoadUnpackedBroken(t *testing.T) { - repodir, cleanup := rtest.Env(t, repoFixture) - defer cleanup() + repo, be := repository.TestRepositoryWithVersion(t, 0) data := rtest.Random(23, 12345) id := restic.Hash(data) - h := restic.Handle{Type: restic.IndexFile, Name: id.String()} + h := backend.Handle{Type: restic.IndexFile, Name: id.String()} // damage buffer data[0] ^= 0xff - repo := repository.TestOpenLocal(t, repodir) // store broken file - err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil)) + err := be.Save(context.TODO(), h, backend.NewByteReader(data, be.Hasher())) rtest.OK(t, err) - // without a retry backend this will just return an error that the file is broken _, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id) - if err == nil { - t.Fatal("missing expected error") - } - rtest.Assert(t, strings.Contains(err.Error(), "invalid data returned"), "unexpected error: %v", err) + rtest.Assert(t, errors.Is(err, restic.ErrInvalidData), "unexpected error: %v", err) } type damageOnceBackend struct { - restic.Backend + backend.Backend + m sync.Map } -func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *damageOnceBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { // don't break the config file as we can't retry it if h.Type == restic.ConfigFile { return be.Backend.Load(ctx, h, length, offset, fn) } - // return broken data on the first try - err := be.Backend.Load(ctx, h, length+1, offset, fn) - if err != nil { - // retry - err = be.Backend.Load(ctx, h, length, offset, fn) + + h.IsMetadata = false + _, isRetry := be.m.LoadOrStore(h, true) + if !isRetry { + // return broken data on the first try + offset++ } - return err + return be.Backend.Load(ctx, h, length, offset, fn) } func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { @@ -321,10 +311,7 @@ func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) { be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2}) rtest.OK(t, err) - repo, err := repository.New(&damageOnceBackend{Backend: be}, repository.Options{}) - rtest.OK(t, err) - err = repo.SearchKey(context.TODO(), rtest.TestPassword, 10, "") - rtest.OK(t, err) + repo := repository.TestOpenBackend(t, &damageOnceBackend{Backend: be}) rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) } @@ -336,7 +323,7 @@ func BenchmarkLoadIndex(b *testing.B) { func benchmarkLoadIndex(b *testing.B, version uint) { repository.TestUseLowSecurityKDFParameters(b) - repo := repository.TestRepositoryWithVersion(b, version) + repo, be := repository.TestRepositoryWithVersion(b, version) idx := index.NewIndex() for i := 0; i < 5000; i++ { @@ -350,11 +337,11 @@ func benchmarkLoadIndex(b *testing.B, version uint) { } idx.Finalize() - id, err := index.SaveIndex(context.TODO(), repo, idx) + id, err := idx.SaveIndex(context.TODO(), repo) rtest.OK(b, err) b.Logf("index saved as %v", id.Str()) - fi, err := repo.Backend().Stat(context.TODO(), restic.Handle{Type: restic.IndexFile, Name: id.String()}) + fi, err := be.Stat(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: id.String()}) rtest.OK(b, err) b.Logf("filesize is %v", fi.Size) @@ -388,9 +375,9 @@ func TestRepositoryIncrementalIndex(t *testing.T) { } func testRepositoryIncrementalIndex(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version).(*repository.Repository) + repo, _ := repository.TestRepositoryWithVersion(t, version) - index.IndexFull = func(*index.Index, bool) bool { return true } + index.IndexFull = func(*index.Index) bool { return true } // add a few rounds of packs for j := 0; j < 5; j++ { @@ -408,13 +395,13 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { idx, err := loadIndex(context.TODO(), repo, id) rtest.OK(t, err) - idx.Each(context.TODO(), func(pb restic.PackedBlob) { + rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) { if _, ok := packEntries[pb.PackID]; !ok { packEntries[pb.PackID] = make(map[restic.ID]struct{}) } packEntries[pb.PackID][id] = struct{}{} - }) + })) return nil }) if err != nil { @@ -429,278 +416,71 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { } -// buildPackfileWithoutHeader returns a manually built pack file without a header. -func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { - opts := []zstd.EOption{ - // Set the compression level configured. - zstd.WithEncoderLevel(zstd.SpeedDefault), - // Disable CRC, we have enough checks in place, makes the - // compressed data four bytes shorter. - zstd.WithEncoderCRC(false), - // Set a window of 512kbyte, so we have good lookbehind for usual - // blob sizes. - zstd.WithWindowSize(512 * 1024), - } - enc, err := zstd.NewWriter(nil, opts...) - if err != nil { - panic(err) - } - - var offset uint - for i, size := range blobSizes { - plaintext := rtest.Random(800+i, size) - id := restic.Hash(plaintext) - uncompressedLength := uint(0) - if compress { - uncompressedLength = uint(len(plaintext)) - plaintext = enc.EncodeAll(plaintext, nil) - } - - // we use a deterministic nonce here so the whole process is - // deterministic, last byte is the blob index - var nonce = []byte{ - 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, - 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), - } - - before := len(packfile) - packfile = append(packfile, nonce...) - packfile = key.Seal(packfile, nonce, plaintext, nil) - after := len(packfile) - - ciphertextLength := after - before - - blobs = append(blobs, restic.Blob{ - BlobHandle: restic.BlobHandle{ - Type: restic.DataBlob, - ID: id, - }, - Length: uint(ciphertextLength), - UncompressedLength: uncompressedLength, - Offset: offset, - }) - - offset = uint(len(packfile)) - } - - return blobs, packfile -} - -func TestStreamPack(t *testing.T) { - repository.TestAllVersions(t, testStreamPack) +func TestInvalidCompression(t *testing.T) { + var comp repository.CompressionMode + err := comp.Set("nope") + rtest.Assert(t, err != nil, "missing error") + _, err = repository.New(nil, repository.Options{Compression: comp}) + rtest.Assert(t, err != nil, "missing error") } -func testStreamPack(t *testing.T, version uint) { - // always use the same key for deterministic output - const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` - - var key crypto.Key - err := json.Unmarshal([]byte(jsonKey), &key) - if err != nil { - t.Fatal(err) - } - - blobSizes := []int{ - 5522811, - 10, - 5231, - 18812, - 123123, - 13522811, - 12301, - 892242, - 28616, - 13351, - 252287, - 188883, - 3522811, - 18883, - } - - var compress bool - switch version { - case 1: - compress = false - case 2: - compress = true - default: - t.Fatal("test does not suport repository version", version) - } - - packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) - - loadCalls := 0 - shortFirstLoad := false - - loadBytes := func(length int, offset int64) []byte { - data := packfile - - if offset > int64(len(data)) { - offset = 0 - length = 0 - } - data = data[offset:] - - if length > len(data) { - length = len(data) - } - if shortFirstLoad { - length /= 2 - shortFirstLoad = false - } - - return data[:length] - } - - load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - data := loadBytes(length, offset) - if shortFirstLoad { - data = data[:len(data)/2] - shortFirstLoad = false - } - - loadCalls++ +func TestListPack(t *testing.T) { + be := mem.New() + repo, _ := repository.TestRepositoryWithBackend(t, &damageOnceBackend{Backend: be}, restic.StableRepoVersion, repository.Options{}) + buf := test.Random(42, 1000) - err := fn(bytes.NewReader(data)) - if err == nil { - return nil - } - var permanent *backoff.PermanentError - if errors.As(err, &permanent) { - return err - } + var wg errgroup.Group + repo.StartPackUploader(context.TODO(), &wg) + id, _, _, err := repo.SaveBlob(context.TODO(), restic.TreeBlob, buf, restic.ID{}, false) + rtest.OK(t, err) + rtest.OK(t, repo.Flush(context.Background())) - // retry loading once - return fn(bytes.NewReader(loadBytes(length, offset))) - } + // setup cache after saving the blob to make sure that the damageOnceBackend damages the cached data + c := cache.TestNewCache(t) + repo.UseCache(c) - // first, test regular usage - t.Run("regular", func(t *testing.T) { - tests := []struct { - blobs []restic.Blob - calls int - shortFirstLoad bool - }{ - {packfileBlobs[1:2], 1, false}, - {packfileBlobs[2:5], 1, false}, - {packfileBlobs[2:8], 1, false}, - {[]restic.Blob{ - packfileBlobs[0], - packfileBlobs[4], - packfileBlobs[2], - }, 1, false}, - {[]restic.Blob{ - packfileBlobs[0], - packfileBlobs[len(packfileBlobs)-1], - }, 2, false}, - {packfileBlobs[:], 1, true}, - } + // Forcibly cache pack file + packID := repo.LookupBlob(restic.TreeBlob, id)[0].PackID + rtest.OK(t, be.Load(context.TODO(), backend.Handle{Type: restic.PackFile, IsMetadata: true, Name: packID.String()}, 0, 0, func(rd io.Reader) error { return nil })) - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - gotBlobs := make(map[restic.ID]int) - - handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { - gotBlobs[blob.ID]++ - - id := restic.Hash(buf) - if !id.Equal(blob.ID) { - t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) - } - - return err - } - - wantBlobs := make(map[restic.ID]int) - for _, blob := range test.blobs { - wantBlobs[blob.ID] = 1 - } - - loadCalls = 0 - shortFirstLoad = test.shortFirstLoad - err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) - if err != nil { - t.Fatal(err) - } - - if !cmp.Equal(wantBlobs, gotBlobs) { - t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) - } - rtest.Equals(t, test.calls, loadCalls) - }) - } - }) - shortFirstLoad = false - - // next, test invalid uses, which should return an error - t.Run("invalid", func(t *testing.T) { - tests := []struct { - blobs []restic.Blob - err string - }{ - { - // pass one blob several times - blobs: []restic.Blob{ - packfileBlobs[3], - packfileBlobs[8], - packfileBlobs[3], - packfileBlobs[4], - }, - err: "overlapping blobs in pack", - }, - - { - // pass something that's not a valid blob in the current pack file - blobs: []restic.Blob{ - { - Offset: 123, - Length: 20000, - }, - }, - err: "ciphertext verification failed", - }, - - { - // pass a blob that's too small - blobs: []restic.Blob{ - { - Offset: 123, - Length: 10, - }, - }, - err: "invalid blob length", - }, + // Get size to list pack + var size int64 + rtest.OK(t, repo.List(context.TODO(), restic.PackFile, func(id restic.ID, sz int64) error { + if id == packID { + size = sz } + return nil + })) - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + blobs, _, err := repo.ListPack(context.TODO(), packID, size) + rtest.OK(t, err) + rtest.Assert(t, len(blobs) == 1 && blobs[0].ID == id, "unexpected blobs in pack: %v", blobs) - handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { - return err - } + rtest.Assert(t, !c.Has(backend.Handle{Type: restic.PackFile, Name: packID.String()}), "tree pack should no longer be cached as ListPack does not set IsMetadata in the backend.Handle") +} - err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) - if err == nil { - t.Fatalf("wanted error %v, got nil", test.err) - } +func TestNoDoubleInit(t *testing.T) { + r, be := repository.TestRepositoryWithVersion(t, restic.StableRepoVersion) - if !strings.Contains(err.Error(), test.err) { - t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) - } - }) - } - }) -} + repo, err := repository.New(be, repository.Options{}) + rtest.OK(t, err) -func TestInvalidCompression(t *testing.T) { - var comp repository.CompressionMode - err := comp.Set("nope") - rtest.Assert(t, err != nil, "missing error") - _, err = repository.New(nil, repository.Options{Compression: comp}) - rtest.Assert(t, err != nil, "missing error") + pol := r.Config().ChunkerPolynomial + err = repo.Init(context.TODO(), r.Config().Version, test.TestPassword, &pol) + rtest.Assert(t, strings.Contains(err.Error(), "repository master key and config already initialized"), "expected config exist error, got %q", err) + + // must also prevent init if only keys exist + rtest.OK(t, be.Remove(context.TODO(), backend.Handle{Type: backend.ConfigFile})) + err = repo.Init(context.TODO(), r.Config().Version, test.TestPassword, &pol) + rtest.Assert(t, strings.Contains(err.Error(), "repository already contains keys"), "expected already contains keys error, got %q", err) + + // must also prevent init if a snapshot exists and keys were deleted + var data [32]byte + hash := restic.Hash(data[:]) + rtest.OK(t, be.Save(context.TODO(), backend.Handle{Type: backend.SnapshotFile, Name: hash.String()}, backend.NewByteReader(data[:], be.Hasher()))) + rtest.OK(t, be.List(context.TODO(), restic.KeyFile, func(fi backend.FileInfo) error { + return be.Remove(context.TODO(), backend.Handle{Type: restic.KeyFile, Name: fi.Name}) + })) + err = repo.Init(context.TODO(), r.Config().Version, test.TestPassword, &pol) + rtest.Assert(t, strings.Contains(err.Error(), "repository already contains snapshots"), "expected already contains snapshots error, got %q", err) } diff --git a/mover-restic/restic/internal/repository/s3_backend.go b/mover-restic/restic/internal/repository/s3_backend.go new file mode 100644 index 000000000..4c77c69a2 --- /dev/null +++ b/mover-restic/restic/internal/repository/s3_backend.go @@ -0,0 +1,12 @@ +package repository + +import ( + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/backend/s3" +) + +// AsS3Backend extracts the S3 backend from a repository +// TODO remove me once restic 0.17 was released +func AsS3Backend(repo *Repository) *s3.Backend { + return backend.AsBackend[*s3.Backend](repo.be) +} diff --git a/mover-restic/restic/internal/repository/testing.go b/mover-restic/restic/internal/repository/testing.go index 9bdd65901..2155cad16 100644 --- a/mover-restic/restic/internal/repository/testing.go +++ b/mover-restic/restic/internal/repository/testing.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "os" + "sync" "testing" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/backend/retry" @@ -16,34 +18,35 @@ import ( "github.com/restic/chunker" ) -// testKDFParams are the parameters for the KDF to be used during testing. -var testKDFParams = crypto.Params{ - N: 128, - R: 1, - P: 1, -} - type logger interface { Logf(format string, args ...interface{}) } +var paramsOnce sync.Once + // TestUseLowSecurityKDFParameters configures low-security KDF parameters for testing. func TestUseLowSecurityKDFParameters(t logger) { t.Logf("using low-security KDF parameters for test") - Params = &testKDFParams + paramsOnce.Do(func() { + params = &crypto.Params{ + N: 128, + R: 1, + P: 1, + } + }) } // TestBackend returns a fully configured in-memory backend. -func TestBackend(_ testing.TB) restic.Backend { +func TestBackend(_ testing.TB) backend.Backend { return mem.New() } -const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) +const testChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, opts Options) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, opts Options) (*Repository, backend.Backend) { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -57,25 +60,29 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, op t.Fatalf("TestRepository(): new repo failed: %v", err) } - cfg := restic.TestCreateConfig(t, TestChunkerPol, version) - err = repo.init(context.TODO(), test.TestPassword, cfg) + if version == 0 { + version = restic.StableRepoVersion + } + pol := testChunkerPol + err = repo.Init(context.TODO(), version, test.TestPassword, &pol) if err != nil { t.Fatalf("TestRepository(): initialize repo failed: %v", err) } - return repo + return repo, be } // TestRepository returns a repository initialized with a test password on an // in-memory backend. When the environment variable RESTIC_TEST_REPO is set to // a non-existing directory, a local backend is created there and this is used // instead. The directory is not removed, but left there for inspection. -func TestRepository(t testing.TB) restic.Repository { +func TestRepository(t testing.TB) *Repository { t.Helper() - return TestRepositoryWithVersion(t, 0) + repo, _ := TestRepositoryWithVersion(t, 0) + return repo } -func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { +func TestRepositoryWithVersion(t testing.TB, version uint) (*Repository, backend.Backend) { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") opts := Options{} @@ -97,9 +104,16 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { return TestRepositoryWithBackend(t, nil, version, opts) } +func TestFromFixture(t testing.TB, repoFixture string) (*Repository, backend.Backend, func()) { + repodir, cleanup := test.Env(t, repoFixture) + repo, be := TestOpenLocal(t, repodir) + + return repo, be, cleanup +} + // TestOpenLocal opens a local repository. -func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { - var be restic.Backend +func TestOpenLocal(t testing.TB, dir string) (*Repository, backend.Backend) { + var be backend.Backend be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2}) if err != nil { t.Fatal(err) @@ -107,6 +121,10 @@ func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) { be = retry.New(be, 3, nil, nil) + return TestOpenBackend(t, be), be +} + +func TestOpenBackend(t testing.TB, be backend.Backend) *Repository { repo, err := New(be, Options{}) if err != nil { t.Fatal(err) diff --git a/mover-restic/restic/internal/repository/upgrade_repo.go b/mover-restic/restic/internal/repository/upgrade_repo.go new file mode 100644 index 000000000..ea3ae2c0e --- /dev/null +++ b/mover-restic/restic/internal/repository/upgrade_repo.go @@ -0,0 +1,103 @@ +package repository + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/restic" +) + +type upgradeRepoV2Error struct { + UploadNewConfigError error + ReuploadOldConfigError error + + BackupFilePath string +} + +func (err *upgradeRepoV2Error) Error() string { + if err.ReuploadOldConfigError != nil { + return fmt.Sprintf("error uploading config (%v), re-uploading old config filed failed as well (%v), but there is a backup of the config file in %v", err.UploadNewConfigError, err.ReuploadOldConfigError, err.BackupFilePath) + } + + return fmt.Sprintf("error uploading config (%v), re-uploaded old config was successful, there is a backup of the config file in %v", err.UploadNewConfigError, err.BackupFilePath) +} + +func (err *upgradeRepoV2Error) Unwrap() error { + // consider the original upload error as the primary cause + return err.UploadNewConfigError +} + +func upgradeRepository(ctx context.Context, repo *Repository) error { + h := backend.Handle{Type: backend.ConfigFile} + + if !repo.be.HasAtomicReplace() { + // remove the original file for backends which do not support atomic overwriting + err := repo.be.Remove(ctx, h) + if err != nil { + return fmt.Errorf("remove config failed: %w", err) + } + } + + // upgrade config + cfg := repo.Config() + cfg.Version = 2 + + err := restic.SaveConfig(ctx, repo, cfg) + if err != nil { + return fmt.Errorf("save new config file failed: %w", err) + } + + return nil +} + +func UpgradeRepo(ctx context.Context, repo *Repository) error { + if repo.Config().Version != 1 { + return fmt.Errorf("repository has version %v, only upgrades from version 1 are supported", repo.Config().Version) + } + + tempdir, err := os.MkdirTemp("", "restic-migrate-upgrade-repo-v2-") + if err != nil { + return fmt.Errorf("create temp dir failed: %w", err) + } + + h := backend.Handle{Type: restic.ConfigFile} + + // read raw config file and save it to a temp dir, just in case + rawConfigFile, err := repo.LoadRaw(ctx, restic.ConfigFile, restic.ID{}) + if err != nil { + return fmt.Errorf("load config file failed: %w", err) + } + + backupFileName := filepath.Join(tempdir, "config") + err = os.WriteFile(backupFileName, rawConfigFile, 0600) + if err != nil { + return fmt.Errorf("write config file backup to %v failed: %w", tempdir, err) + } + + // run the upgrade + err = upgradeRepository(ctx, repo) + if err != nil { + + // build an error we can return to the caller + repoError := &upgradeRepoV2Error{ + UploadNewConfigError: err, + BackupFilePath: backupFileName, + } + + // try contingency methods, reupload the original file + _ = repo.be.Remove(ctx, h) + err = repo.be.Save(ctx, h, backend.NewByteReader(rawConfigFile, nil)) + if err != nil { + repoError.ReuploadOldConfigError = err + } + + return repoError + } + + _ = os.Remove(backupFileName) + _ = os.Remove(tempdir) + return nil +} diff --git a/mover-restic/restic/internal/repository/upgrade_repo_test.go b/mover-restic/restic/internal/repository/upgrade_repo_test.go new file mode 100644 index 000000000..61ca6ef95 --- /dev/null +++ b/mover-restic/restic/internal/repository/upgrade_repo_test.go @@ -0,0 +1,82 @@ +package repository + +import ( + "context" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/errors" + rtest "github.com/restic/restic/internal/test" +) + +func TestUpgradeRepoV2(t *testing.T) { + repo, _ := TestRepositoryWithVersion(t, 1) + if repo.Config().Version != 1 { + t.Fatal("test repo has wrong version") + } + + err := UpgradeRepo(context.Background(), repo) + rtest.OK(t, err) +} + +type failBackend struct { + backend.Backend + + mu sync.Mutex + ConfigFileSavesUntilError uint +} + +func (be *failBackend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { + if h.Type != backend.ConfigFile { + return be.Backend.Save(ctx, h, rd) + } + + be.mu.Lock() + if be.ConfigFileSavesUntilError == 0 { + be.mu.Unlock() + return errors.New("failure induced for testing") + } + + be.ConfigFileSavesUntilError-- + be.mu.Unlock() + + return be.Backend.Save(ctx, h, rd) +} + +func TestUpgradeRepoV2Failure(t *testing.T) { + be := TestBackend(t) + + // wrap backend so that it fails upgrading the config after the initial write + be = &failBackend{ + ConfigFileSavesUntilError: 1, + Backend: be, + } + + repo, _ := TestRepositoryWithBackend(t, be, 1, Options{}) + if repo.Config().Version != 1 { + t.Fatal("test repo has wrong version") + } + + err := UpgradeRepo(context.Background(), repo) + if err == nil { + t.Fatal("expected error returned from Apply(), got nil") + } + + upgradeErr := err.(*upgradeRepoV2Error) + if upgradeErr.UploadNewConfigError == nil { + t.Fatal("expected upload error, got nil") + } + + if upgradeErr.ReuploadOldConfigError == nil { + t.Fatal("expected reupload error, got nil") + } + + if upgradeErr.BackupFilePath == "" { + t.Fatal("no backup file path found") + } + rtest.OK(t, os.Remove(upgradeErr.BackupFilePath)) + rtest.OK(t, os.Remove(filepath.Dir(upgradeErr.BackupFilePath))) +} diff --git a/mover-restic/restic/internal/restic/backend_find.go b/mover-restic/restic/internal/restic/backend_find.go index 7c78b3355..2f00595c4 100644 --- a/mover-restic/restic/internal/restic/backend_find.go +++ b/mover-restic/restic/internal/restic/backend_find.go @@ -3,8 +3,6 @@ package restic import ( "context" "fmt" - - "github.com/restic/restic/internal/debug" ) // A MultipleIDMatchesError is returned by Find() when multiple IDs with a @@ -32,15 +30,9 @@ func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) ctx, cancel := context.WithCancel(ctx) defer cancel() - err := be.List(ctx, t, func(fi FileInfo) error { - // ignore filename which are not an id - id, err := ParseID(fi.Name) - if err != nil { - debug.Log("unable to parse %v as an ID", fi.Name) - return nil - } - - if len(fi.Name) >= len(prefix) && prefix == fi.Name[:len(prefix)] { + err := be.List(ctx, t, func(id ID, _ int64) error { + name := id.String() + if len(name) >= len(prefix) && prefix == name[:len(prefix)] { if match.IsNull() { match = id } else { diff --git a/mover-restic/restic/internal/restic/backend_find_test.go b/mover-restic/restic/internal/restic/backend_find_test.go index cbd5e7f48..5f020fda6 100644 --- a/mover-restic/restic/internal/restic/backend_find_test.go +++ b/mover-restic/restic/internal/restic/backend_find_test.go @@ -1,37 +1,31 @@ -package restic +package restic_test import ( "context" "strings" "testing" -) - -type mockBackend struct { - list func(context.Context, FileType, func(FileInfo) error) error -} -func (m mockBackend) List(ctx context.Context, t FileType, fn func(FileInfo) error) error { - return m.list(ctx, t, fn) -} + "github.com/restic/restic/internal/restic" +) -var samples = IDs{ - TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), - TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), - TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), - TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), - TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), - TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), - TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), - TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), +var samples = restic.IDs{ + restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), + restic.TestParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), + restic.TestParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), + restic.TestParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), + restic.TestParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), + restic.TestParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), + restic.TestParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), + restic.TestParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), } func TestFind(t *testing.T) { list := samples - m := mockBackend{} - m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error { + m := &ListHelper{} + m.ListFn = func(ctx context.Context, t restic.FileType, fn func(id restic.ID, size int64) error) error { for _, id := range list { - err := fn(FileInfo{Name: id.String()}) + err := fn(id, 0) if err != nil { return err } @@ -39,17 +33,17 @@ func TestFind(t *testing.T) { return nil } - f, err := Find(context.TODO(), m, SnapshotFile, "20bdc1402a6fc9b633aa") + f, err := restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc1402a6fc9b633aa") if err != nil { t.Error(err) } - expectedMatch := TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff") + expectedMatch := restic.TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff") if f != expectedMatch { t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f) } - f, err = Find(context.TODO(), m, SnapshotFile, "NotAPrefix") - if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") { + f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "NotAPrefix") + if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") { t.Error("Expected no snapshots to be found.") } if !f.IsNull() { @@ -58,8 +52,8 @@ func TestFind(t *testing.T) { // Try to match with a prefix longer than any ID. extraLengthID := samples[0].String() + "f" - f, err = Find(context.TODO(), m, SnapshotFile, extraLengthID) - if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) { + f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, extraLengthID) + if _, ok := err.(*restic.NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) { t.Errorf("Wrong error %v for no snapshots matched", err) } if !f.IsNull() { @@ -67,8 +61,8 @@ func TestFind(t *testing.T) { } // Use a prefix that will match the prefix of multiple Ids in `samples`. - f, err = Find(context.TODO(), m, SnapshotFile, "20bdc140") - if _, ok := err.(*MultipleIDMatchesError); !ok { + f, err = restic.Find(context.TODO(), m, restic.SnapshotFile, "20bdc140") + if _, ok := err.(*restic.MultipleIDMatchesError); !ok { t.Errorf("Wrong error %v for multiple snapshots", err) } if !f.IsNull() { diff --git a/mover-restic/restic/internal/restic/backend_test.go b/mover-restic/restic/internal/restic/backend_test.go deleted file mode 100644 index a970eb5b3..000000000 --- a/mover-restic/restic/internal/restic/backend_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package restic_test - -import ( - "testing" - - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" -) - -type testBackend struct { - restic.Backend -} - -func (t *testBackend) Unwrap() restic.Backend { - return nil -} - -type otherTestBackend struct { - restic.Backend -} - -func (t *otherTestBackend) Unwrap() restic.Backend { - return t.Backend -} - -func TestAsBackend(t *testing.T) { - other := otherTestBackend{} - test.Assert(t, restic.AsBackend[*testBackend](other) == nil, "otherTestBackend is not a testBackend backend") - - testBe := &testBackend{} - test.Assert(t, restic.AsBackend[*testBackend](testBe) == testBe, "testBackend was not returned") - - wrapper := &otherTestBackend{Backend: testBe} - test.Assert(t, restic.AsBackend[*testBackend](wrapper) == testBe, "failed to unwrap testBackend backend") - - wrapper.Backend = other - test.Assert(t, restic.AsBackend[*testBackend](wrapper) == nil, "a wrapped otherTestBackend is not a testBackend") -} diff --git a/mover-restic/restic/internal/restic/blob.go b/mover-restic/restic/internal/restic/blob.go index 260f40fde..3a6872af3 100644 --- a/mover-restic/restic/internal/restic/blob.go +++ b/mover-restic/restic/internal/restic/blob.go @@ -75,6 +75,15 @@ func (t BlobType) String() string { return fmt.Sprintf("", t) } +func (t BlobType) IsMetadata() bool { + switch t { + case TreeBlob: + return true + default: + return false + } +} + // MarshalJSON encodes the BlobType into JSON. func (t BlobType) MarshalJSON() ([]byte, error) { switch t { diff --git a/mover-restic/restic/internal/restic/config.go b/mover-restic/restic/internal/restic/config.go index 67ee190bc..3fb61cc13 100644 --- a/mover-restic/restic/internal/restic/config.go +++ b/mover-restic/restic/internal/restic/config.go @@ -2,6 +2,7 @@ package restic import ( "context" + "sync" "testing" "github.com/restic/restic/internal/errors" @@ -50,29 +51,16 @@ func CreateConfig(version uint) (Config, error) { return cfg, nil } -// TestCreateConfig creates a config for use within tests. -func TestCreateConfig(t testing.TB, pol chunker.Pol, version uint) (cfg Config) { - cfg.ChunkerPolynomial = pol - - cfg.ID = NewRandomID().String() - if version == 0 { - version = StableRepoVersion - } - if version < MinRepoVersion || version > MaxRepoVersion { - t.Fatalf("version %d is out of range", version) - } - cfg.Version = version - - return cfg -} - var checkPolynomial = true +var checkPolynomialOnce sync.Once // TestDisableCheckPolynomial disables the check that the polynomial used for // the chunker. func TestDisableCheckPolynomial(t testing.TB) { t.Logf("disabling check of the chunker polynomial") - checkPolynomial = false + checkPolynomialOnce.Do(func() { + checkPolynomial = false + }) } // LoadConfig returns loads, checks and returns the config for a repository. diff --git a/mover-restic/restic/internal/restic/counted_blob_set.go b/mover-restic/restic/internal/restic/counted_blob_set.go deleted file mode 100644 index f965d3129..000000000 --- a/mover-restic/restic/internal/restic/counted_blob_set.go +++ /dev/null @@ -1,68 +0,0 @@ -package restic - -import "sort" - -// CountedBlobSet is a set of blobs. For each blob it also stores a uint8 value -// which can be used to track some information. The CountedBlobSet does not use -// that value in any way. New entries are created with value 0. -type CountedBlobSet map[BlobHandle]uint8 - -// NewCountedBlobSet returns a new CountedBlobSet, populated with ids. -func NewCountedBlobSet(handles ...BlobHandle) CountedBlobSet { - m := make(CountedBlobSet) - for _, h := range handles { - m[h] = 0 - } - - return m -} - -// Has returns true iff id is contained in the set. -func (s CountedBlobSet) Has(h BlobHandle) bool { - _, ok := s[h] - return ok -} - -// Insert adds id to the set. -func (s CountedBlobSet) Insert(h BlobHandle) { - s[h] = 0 -} - -// Delete removes id from the set. -func (s CountedBlobSet) Delete(h BlobHandle) { - delete(s, h) -} - -func (s CountedBlobSet) Len() int { - return len(s) -} - -// List returns a sorted slice of all BlobHandle in the set. -func (s CountedBlobSet) List() BlobHandles { - list := make(BlobHandles, 0, len(s)) - for h := range s { - list = append(list, h) - } - - sort.Sort(list) - - return list -} - -func (s CountedBlobSet) String() string { - str := s.List().String() - if len(str) < 2 { - return "{}" - } - - return "{" + str[1:len(str)-1] + "}" -} - -// Copy returns a copy of the CountedBlobSet. -func (s CountedBlobSet) Copy() CountedBlobSet { - cp := make(CountedBlobSet, len(s)) - for k, v := range s { - cp[k] = v - } - return cp -} diff --git a/mover-restic/restic/internal/restic/counted_blob_set_test.go b/mover-restic/restic/internal/restic/counted_blob_set_test.go deleted file mode 100644 index 681751e91..000000000 --- a/mover-restic/restic/internal/restic/counted_blob_set_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package restic_test - -import ( - "testing" - - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/test" -) - -func TestCountedBlobSet(t *testing.T) { - bs := restic.NewCountedBlobSet() - test.Equals(t, bs.Len(), 0) - test.Equals(t, bs.List(), restic.BlobHandles{}) - - bh := restic.NewRandomBlobHandle() - // check non existant - test.Equals(t, bs.Has(bh), false) - - // test insert - bs.Insert(bh) - test.Equals(t, bs.Has(bh), true) - test.Equals(t, bs.Len(), 1) - test.Equals(t, bs.List(), restic.BlobHandles{bh}) - - // test remove - bs.Delete(bh) - test.Equals(t, bs.Len(), 0) - test.Equals(t, bs.Has(bh), false) - test.Equals(t, bs.List(), restic.BlobHandles{}) - - bs = restic.NewCountedBlobSet(bh) - test.Equals(t, bs.Len(), 1) - test.Equals(t, bs.List(), restic.BlobHandles{bh}) - - s := bs.String() - test.Assert(t, len(s) > 10, "invalid string: %v", s) -} - -func TestCountedBlobSetCopy(t *testing.T) { - bs := restic.NewCountedBlobSet(restic.NewRandomBlobHandle(), restic.NewRandomBlobHandle(), restic.NewRandomBlobHandle()) - test.Equals(t, bs.Len(), 3) - cp := bs.Copy() - test.Equals(t, cp.Len(), 3) - test.Equals(t, bs.List(), cp.List()) -} diff --git a/mover-restic/restic/internal/restic/find.go b/mover-restic/restic/internal/restic/find.go index 08670a49f..d7b032bf8 100644 --- a/mover-restic/restic/internal/restic/find.go +++ b/mover-restic/restic/internal/restic/find.go @@ -11,18 +11,18 @@ import ( // Loader loads a blob from a repository. type Loader interface { LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) - LookupBlobSize(id ID, tpe BlobType) (uint, bool) + LookupBlobSize(tpe BlobType, id ID) (uint, bool) Connections() uint } -type findBlobSet interface { +type FindBlobSet interface { Has(bh BlobHandle) bool Insert(bh BlobHandle) } // FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data // blobs) to the set blobs. Already seen tree blobs will not be visited again. -func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs findBlobSet, p *progress.Counter) error { +func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs FindBlobSet, p *progress.Counter) error { var lock sync.Mutex wg, ctx := errgroup.WithContext(ctx) diff --git a/mover-restic/restic/internal/restic/find_test.go b/mover-restic/restic/internal/restic/find_test.go index 1ae30ded9..9b8315ad4 100644 --- a/mover-restic/restic/internal/restic/find_test.go +++ b/mover-restic/restic/internal/restic/find_test.go @@ -166,7 +166,7 @@ func (r ForbiddenRepo) LoadBlob(context.Context, restic.BlobType, restic.ID, []b return nil, errors.New("should not be called") } -func (r ForbiddenRepo) LookupBlobSize(_ restic.ID, _ restic.BlobType) (uint, bool) { +func (r ForbiddenRepo) LookupBlobSize(_ restic.BlobType, _ restic.ID) (uint, bool) { return 0, false } diff --git a/mover-restic/restic/internal/restic/idset.go b/mover-restic/restic/internal/restic/idset.go index 1b12a6398..9e6e3c6fd 100644 --- a/mover-restic/restic/internal/restic/idset.go +++ b/mover-restic/restic/internal/restic/idset.go @@ -105,3 +105,9 @@ func (s IDSet) String() string { str := s.List().String() return "{" + str[1:len(str)-1] + "}" } + +func (s IDSet) Clone() IDSet { + c := NewIDSet() + c.Merge(s) + return c +} diff --git a/mover-restic/restic/internal/restic/idset_test.go b/mover-restic/restic/internal/restic/idset_test.go index 734b31237..14c88b314 100644 --- a/mover-restic/restic/internal/restic/idset_test.go +++ b/mover-restic/restic/internal/restic/idset_test.go @@ -35,4 +35,7 @@ func TestIDSet(t *testing.T) { } rtest.Equals(t, "{1285b303 7bb086db f658198b}", set.String()) + + copied := set.Clone() + rtest.Equals(t, "{1285b303 7bb086db f658198b}", copied.String()) } diff --git a/mover-restic/restic/internal/restic/lister.go b/mover-restic/restic/internal/restic/lister.go new file mode 100644 index 000000000..23da30b7d --- /dev/null +++ b/mover-restic/restic/internal/restic/lister.go @@ -0,0 +1,52 @@ +package restic + +import ( + "context" + "fmt" +) + +type fileInfo struct { + id ID + size int64 +} + +type memorizedLister struct { + fileInfos []fileInfo + tpe FileType +} + +func (m *memorizedLister) List(ctx context.Context, t FileType, fn func(ID, int64) error) error { + if t != m.tpe { + return fmt.Errorf("filetype mismatch, expected %s got %s", m.tpe, t) + } + for _, fi := range m.fileInfos { + if ctx.Err() != nil { + break + } + err := fn(fi.id, fi.size) + if err != nil { + return err + } + } + return ctx.Err() +} + +func MemorizeList(ctx context.Context, be Lister, t FileType) (Lister, error) { + if _, ok := be.(*memorizedLister); ok { + return be, nil + } + + var fileInfos []fileInfo + err := be.List(ctx, t, func(id ID, size int64) error { + fileInfos = append(fileInfos, fileInfo{id, size}) + return nil + }) + if err != nil { + return nil, err + } + + return &memorizedLister{ + fileInfos: fileInfos, + tpe: t, + }, nil +} diff --git a/mover-restic/restic/internal/restic/lister_test.go b/mover-restic/restic/internal/restic/lister_test.go new file mode 100644 index 000000000..245a5d3da --- /dev/null +++ b/mover-restic/restic/internal/restic/lister_test.go @@ -0,0 +1,68 @@ +package restic_test + +import ( + "context" + "fmt" + "testing" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" +) + +type ListHelper struct { + ListFn func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error +} + +func (l *ListHelper) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { + return l.ListFn(ctx, t, fn) +} + +func TestMemoizeList(t *testing.T) { + // setup backend to serve as data source for memoized list + be := &ListHelper{} + + type FileInfo struct { + ID restic.ID + Size int64 + } + files := []FileInfo{ + {ID: restic.NewRandomID(), Size: 42}, + {ID: restic.NewRandomID(), Size: 45}, + } + be.ListFn = func(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error { + for _, fi := range files { + if err := fn(fi.ID, fi.Size); err != nil { + return err + } + } + return nil + } + + mem, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile) + rtest.OK(t, err) + + err = mem.List(context.TODO(), backend.IndexFile, func(id restic.ID, size int64) error { + t.Fatal("file type mismatch") + return nil // the memoized lister must return an error by itself + }) + rtest.Assert(t, err != nil, "missing error on file typ mismatch") + + var memFiles []FileInfo + err = mem.List(context.TODO(), backend.SnapshotFile, func(id restic.ID, size int64) error { + memFiles = append(memFiles, FileInfo{ID: id, Size: size}) + return nil + }) + rtest.OK(t, err) + rtest.Equals(t, files, memFiles) +} + +func TestMemoizeListError(t *testing.T) { + // setup backend to serve as data source for memoized list + be := &ListHelper{} + be.ListFn = func(ctx context.Context, t backend.FileType, fn func(restic.ID, int64) error) error { + return fmt.Errorf("list error") + } + _, err := restic.MemorizeList(context.TODO(), be, backend.SnapshotFile) + rtest.Assert(t, err != nil, "missing error on list error") +} diff --git a/mover-restic/restic/internal/restic/lock.go b/mover-restic/restic/internal/restic/lock.go index a65ed6b5c..49c7cedf2 100644 --- a/mover-restic/restic/internal/restic/lock.go +++ b/mover-restic/restic/internal/restic/lock.go @@ -17,6 +17,10 @@ import ( "github.com/restic/restic/internal/debug" ) +// UnlockCancelDelay bounds the duration how long lock cleanup operations will wait +// if the passed in context was canceled. +const UnlockCancelDelay time.Duration = 1 * time.Minute + // Lock represents a process locking the repository for an operation. // // There are two types of locks: exclusive and non-exclusive. There may be many @@ -35,7 +39,7 @@ type Lock struct { UID uint32 `json:"uid,omitempty"` GID uint32 `json:"gid,omitempty"` - repo Repository + repo Unpacked lockID *ID } @@ -86,14 +90,14 @@ var ErrRemovedLock = errors.New("lock file was removed in the meantime") // NewLock returns a new, non-exclusive lock for the repository. If an // exclusive lock is already held by another process, it returns an error // that satisfies IsAlreadyLocked. -func NewLock(ctx context.Context, repo Repository) (*Lock, error) { +func NewLock(ctx context.Context, repo Unpacked) (*Lock, error) { return newLock(ctx, repo, false) } // NewExclusiveLock returns a new, exclusive lock for the repository. If // another lock (normal and exclusive) is already held by another process, // it returns an error that satisfies IsAlreadyLocked. -func NewExclusiveLock(ctx context.Context, repo Repository) (*Lock, error) { +func NewExclusiveLock(ctx context.Context, repo Unpacked) (*Lock, error) { return newLock(ctx, repo, true) } @@ -105,7 +109,7 @@ func TestSetLockTimeout(t testing.TB, d time.Duration) { waitBeforeLockCheck = d } -func newLock(ctx context.Context, repo Repository, excl bool) (*Lock, error) { +func newLock(ctx context.Context, repo Unpacked, excl bool) (*Lock, error) { lock := &Lock{ Time: time.Now(), PID: os.Getpid(), @@ -136,7 +140,7 @@ func newLock(ctx context.Context, repo Repository, excl bool) (*Lock, error) { time.Sleep(waitBeforeLockCheck) if err = lock.checkForOtherLocks(ctx); err != nil { - _ = lock.Unlock() + _ = lock.Unlock(ctx) return nil, err } @@ -162,9 +166,16 @@ func (l *Lock) fillUserInfo() error { // exclusive lock is found. func (l *Lock) checkForOtherLocks(ctx context.Context) error { var err error + checkedIDs := NewIDSet() + if l.lockID != nil { + checkedIDs.Insert(*l.lockID) + } // retry locking a few times for i := 0; i < 3; i++ { - err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error { + // Store updates in new IDSet to prevent data races + var m sync.Mutex + newCheckedIDs := NewIDSet(checkedIDs.List()...) + err = ForAllLocks(ctx, l.repo, checkedIDs, func(id ID, lock *Lock, err error) error { if err != nil { // if we cannot load a lock then it is unclear whether it can be ignored // it could either be invalid or just unreadable due to network/permission problems @@ -180,8 +191,13 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { return &alreadyLockedError{otherLock: lock} } + // valid locks will remain valid + m.Lock() + newCheckedIDs.Insert(id) + m.Unlock() return nil }) + checkedIDs = newCheckedIDs // no lock detected if err == nil { return nil @@ -208,12 +224,15 @@ func (l *Lock) createLock(ctx context.Context) (ID, error) { } // Unlock removes the lock from the repository. -func (l *Lock) Unlock() error { +func (l *Lock) Unlock(ctx context.Context) error { if l == nil || l.lockID == nil { return nil } - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()}) + ctx, cancel := delayedCancelContext(ctx, UnlockCancelDelay) + defer cancel() + + return l.repo.RemoveUnpacked(ctx, LockFile, *l.lockID) } var StaleLockTimeout = 30 * time.Minute @@ -254,6 +273,23 @@ func (l *Lock) Stale() bool { return false } +func delayedCancelContext(parentCtx context.Context, delay time.Duration) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + select { + case <-parentCtx.Done(): + case <-ctx.Done(): + return + } + + time.Sleep(delay) + cancel() + }() + + return ctx, cancel +} + // Refresh refreshes the lock by creating a new file in the backend with a new // timestamp. Afterwards the old lock is removed. func (l *Lock) Refresh(ctx context.Context) error { @@ -273,7 +309,10 @@ func (l *Lock) Refresh(ctx context.Context) error { oldLockID := l.lockID l.lockID = &id - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) + ctx, cancel := delayedCancelContext(ctx, UnlockCancelDelay) + defer cancel() + + return l.repo.RemoveUnpacked(ctx, LockFile, *oldLockID) } // RefreshStaleLock is an extended variant of Refresh that can also refresh stale lock files. @@ -300,15 +339,19 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error { time.Sleep(waitBeforeLockCheck) exists, err = l.checkExistence(ctx) + + ctx, cancel := delayedCancelContext(ctx, UnlockCancelDelay) + defer cancel() + if err != nil { // cleanup replacement lock - _ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) + _ = l.repo.RemoveUnpacked(ctx, LockFile, id) return err } if !exists { // cleanup replacement lock - _ = l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: id.String()}) + _ = l.repo.RemoveUnpacked(ctx, LockFile, id) return ErrRemovedLock } @@ -319,7 +362,7 @@ func (l *Lock) RefreshStaleLock(ctx context.Context) error { oldLockID := l.lockID l.lockID = &id - return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()}) + return l.repo.RemoveUnpacked(ctx, LockFile, *oldLockID) } func (l *Lock) checkExistence(ctx context.Context) (bool, error) { @@ -328,8 +371,8 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) { exists := false - err := l.repo.Backend().List(ctx, LockFile, func(fi FileInfo) error { - if fi.Name == l.lockID.String() { + err := l.repo.List(ctx, LockFile, func(id ID, _ int64) error { + if id.Equal(*l.lockID) { exists = true } return nil @@ -366,7 +409,7 @@ func init() { } // LoadLock loads and unserializes a lock from a repository. -func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) { +func LoadLock(ctx context.Context, repo LoaderUnpacked, id ID) (*Lock, error) { lock := &Lock{} if err := LoadJSONUnpacked(ctx, repo, LockFile, id, lock); err != nil { return nil, err @@ -377,7 +420,7 @@ func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) { } // RemoveStaleLocks deletes all locks detected as stale from the repository. -func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { +func RemoveStaleLocks(ctx context.Context, repo Unpacked) (uint, error) { var processed uint err := ForAllLocks(ctx, repo, nil, func(id ID, lock *Lock, err error) error { if err != nil { @@ -387,7 +430,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { } if lock.Stale() { - err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) + err = repo.RemoveUnpacked(ctx, LockFile, id) if err == nil { processed++ } @@ -400,10 +443,10 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { } // RemoveAllLocks removes all locks forcefully. -func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { +func RemoveAllLocks(ctx context.Context, repo Unpacked) (uint, error) { var processed uint32 - err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { - err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()}) + err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, _ int64) error { + err := repo.RemoveUnpacked(ctx, LockFile, id) if err == nil { atomic.AddUint32(&processed, 1) } @@ -416,12 +459,12 @@ func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { // It is guaranteed that the function is not run concurrently. If the // callback returns an error, this function is cancelled and also returns that error. // If a lock ID is passed via excludeID, it will be ignored. -func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID, *Lock, error) error) error { +func ForAllLocks(ctx context.Context, repo ListerLoaderUnpacked, excludeIDs IDSet, fn func(ID, *Lock, error) error) error { var m sync.Mutex // For locks decoding is nearly for free, thus just assume were only limited by IO - return ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { - if excludeID != nil && id.Equal(*excludeID) { + return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { + if excludeIDs.Has(id) { return nil } if size == 0 { diff --git a/mover-restic/restic/internal/restic/lock_test.go b/mover-restic/restic/internal/restic/lock_test.go index 2eb22be1b..606ed210d 100644 --- a/mover-restic/restic/internal/restic/lock_test.go +++ b/mover-restic/restic/internal/restic/lock_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -21,7 +22,7 @@ func TestLock(t *testing.T) { lock, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) - rtest.OK(t, lock.Unlock()) + rtest.OK(t, lock.Unlock(context.TODO())) } func TestDoubleUnlock(t *testing.T) { @@ -31,9 +32,9 @@ func TestDoubleUnlock(t *testing.T) { lock, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) - rtest.OK(t, lock.Unlock()) + rtest.OK(t, lock.Unlock(context.TODO())) - err = lock.Unlock() + err = lock.Unlock(context.TODO()) rtest.Assert(t, err != nil, "double unlock didn't return an error, got %v", err) } @@ -48,15 +49,15 @@ func TestMultipleLock(t *testing.T) { lock2, err := restic.NewLock(context.TODO(), repo) rtest.OK(t, err) - rtest.OK(t, lock1.Unlock()) - rtest.OK(t, lock2.Unlock()) + rtest.OK(t, lock1.Unlock(context.TODO())) + rtest.OK(t, lock2.Unlock(context.TODO())) } type failLockLoadingBackend struct { - restic.Backend + backend.Backend } -func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { +func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { if h.Type == restic.LockFile { return fmt.Errorf("error loading lock") } @@ -65,7 +66,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} - repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) + repo, _ := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) @@ -74,7 +75,7 @@ func TestMultipleLockFailure(t *testing.T) { _, err = restic.NewLock(context.TODO(), repo) rtest.Assert(t, err != nil, "unreadable lock file did not result in an error") - rtest.OK(t, lock1.Unlock()) + rtest.OK(t, lock1.Unlock(context.TODO())) } func TestLockExclusive(t *testing.T) { @@ -82,7 +83,7 @@ func TestLockExclusive(t *testing.T) { elock, err := restic.NewExclusiveLock(context.TODO(), repo) rtest.OK(t, err) - rtest.OK(t, elock.Unlock()) + rtest.OK(t, elock.Unlock(context.TODO())) } func TestLockOnExclusiveLockedRepo(t *testing.T) { @@ -98,8 +99,8 @@ func TestLockOnExclusiveLockedRepo(t *testing.T) { rtest.Assert(t, restic.IsAlreadyLocked(err), "create normal lock with exclusively locked repo didn't return the correct error") - rtest.OK(t, lock.Unlock()) - rtest.OK(t, elock.Unlock()) + rtest.OK(t, lock.Unlock(context.TODO())) + rtest.OK(t, elock.Unlock(context.TODO())) } func TestExclusiveLockOnLockedRepo(t *testing.T) { @@ -115,11 +116,11 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) { rtest.Assert(t, restic.IsAlreadyLocked(err), "create normal lock with exclusively locked repo didn't return the correct error") - rtest.OK(t, lock.Unlock()) - rtest.OK(t, elock.Unlock()) + rtest.OK(t, lock.Unlock(context.TODO())) + rtest.OK(t, elock.Unlock(context.TODO())) } -func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, error) { +func createFakeLock(repo restic.SaverUnpacked, t time.Time, pid int) (restic.ID, error) { hostname, err := os.Hostname() if err != nil { return restic.ID{}, err @@ -129,9 +130,8 @@ func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, er return restic.SaveJSONUnpacked(context.TODO(), repo, restic.LockFile, &newLock) } -func removeLock(repo restic.Repository, id restic.ID) error { - h := restic.Handle{Type: restic.LockFile, Name: id.String()} - return repo.Backend().Remove(context.TODO(), h) +func removeLock(repo restic.RemoverUnpacked, id restic.ID) error { + return repo.RemoveUnpacked(context.TODO(), restic.LockFile, id) } var staleLockTests = []struct { @@ -190,13 +190,16 @@ func TestLockStale(t *testing.T) { } } -func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool { - h := restic.Handle{Type: restic.LockFile, Name: id.String()} - _, err := repo.Backend().Stat(context.TODO(), h) - if err != nil && !repo.Backend().IsNotExist(err) { - t.Fatal(err) - } - return err == nil +func lockExists(repo restic.Lister, t testing.TB, lockID restic.ID) bool { + var exists bool + rtest.OK(t, repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { + if id == lockID { + exists = true + } + return nil + })) + + return exists } func TestLockWithStaleLock(t *testing.T) { @@ -253,7 +256,7 @@ func TestRemoveAllLocks(t *testing.T) { 3, processed) } -func checkSingleLock(t *testing.T, repo restic.Repository) restic.ID { +func checkSingleLock(t *testing.T, repo restic.Lister) restic.ID { t.Helper() var lockID *restic.ID err := repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { @@ -293,7 +296,7 @@ func testLockRefresh(t *testing.T, refresh func(lock *restic.Lock) error) { rtest.OK(t, err) rtest.Assert(t, lock2.Time.After(time0), "expected a later timestamp after lock refresh") - rtest.OK(t, lock.Unlock()) + rtest.OK(t, lock.Unlock(context.TODO())) } func TestLockRefresh(t *testing.T) { @@ -309,7 +312,7 @@ func TestLockRefreshStale(t *testing.T) { } func TestLockRefreshStaleMissing(t *testing.T) { - repo := repository.TestRepository(t) + repo, be := repository.TestRepositoryWithVersion(t, 0) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock, err := restic.NewLock(context.TODO(), repo) @@ -317,7 +320,7 @@ func TestLockRefreshStaleMissing(t *testing.T) { lockID := checkSingleLock(t, repo) // refresh must fail if lock was removed - rtest.OK(t, repo.Backend().Remove(context.TODO(), restic.Handle{Type: restic.LockFile, Name: lockID.String()})) + rtest.OK(t, be.Remove(context.TODO(), backend.Handle{Type: restic.LockFile, Name: lockID.String()})) time.Sleep(time.Millisecond) err = lock.RefreshStaleLock(context.TODO()) rtest.Assert(t, err == restic.ErrRemovedLock, "unexpected error, expected %v, got %v", restic.ErrRemovedLock, err) diff --git a/mover-restic/restic/internal/restic/node.go b/mover-restic/restic/internal/restic/node.go index 7edc41ce8..51c6071b7 100644 --- a/mover-restic/restic/internal/restic/node.go +++ b/mover-restic/restic/internal/restic/node.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "os/user" + "reflect" "strconv" + "strings" "sync" "syscall" "time" @@ -20,12 +22,55 @@ import ( "github.com/restic/restic/internal/fs" ) -// ExtendedAttribute is a tuple storing the xattr name and value. +// ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. type ExtendedAttribute struct { Name string `json:"name"` Value []byte `json:"value"` } +// GenericAttributeType can be used for OS specific functionalities by defining specific types +// in node.go to be used by the specific node_xx files. +// OS specific attribute types should follow the convention Attributes. +// GenericAttributeTypes should follow the convention . +// The attributes in OS specific attribute types must be pointers as we want to distinguish nil values +// and not create GenericAttributes for them. +type GenericAttributeType string + +// OSType is the type created to represent each specific OS +type OSType string + +const ( + // When new GenericAttributeType are defined, they must be added in the init function as well. + + // Below are windows specific attributes. + + // TypeCreationTime is the GenericAttributeType used for storing creation time for windows files within the generic attributes map. + TypeCreationTime GenericAttributeType = "windows.creation_time" + // TypeFileAttributes is the GenericAttributeType used for storing file attributes for windows files within the generic attributes map. + TypeFileAttributes GenericAttributeType = "windows.file_attributes" + // TypeSecurityDescriptor is the GenericAttributeType used for storing security descriptors including owner, group, discretionary access control list (DACL), system access control list (SACL)) for windows files within the generic attributes map. + TypeSecurityDescriptor GenericAttributeType = "windows.security_descriptor" + + // Generic Attributes for other OS types should be defined here. +) + +// init is called when the package is initialized. Any new GenericAttributeTypes being created must be added here as well. +func init() { + storeGenericAttributeType(TypeCreationTime, TypeFileAttributes, TypeSecurityDescriptor) +} + +// genericAttributesForOS maintains a map of known genericAttributesForOS to the OSType +var genericAttributesForOS = map[GenericAttributeType]OSType{} + +// storeGenericAttributeType adds and entry in genericAttributesForOS map +func storeGenericAttributeType(attributeTypes ...GenericAttributeType) { + for _, attributeType := range attributeTypes { + // Get the OS attribute type from the GenericAttributeType + osAttributeName := strings.Split(string(attributeType), ".")[0] + genericAttributesForOS[attributeType] = OSType(osAttributeName) + } +} + // Node is a file, directory or other item in a backup. type Node struct { Name string `json:"name"` @@ -39,7 +84,7 @@ type Node struct { User string `json:"user,omitempty"` Group string `json:"group,omitempty"` Inode uint64 `json:"inode,omitempty"` - DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev + DeviceID uint64 `json:"device_id,omitempty"` // device id of the file, stat.st_dev, only stored for hardlinks Size uint64 `json:"size,omitempty"` Links uint64 `json:"links,omitempty"` LinkTarget string `json:"linktarget,omitempty"` @@ -47,11 +92,12 @@ type Node struct { // This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems, // as LinkTarget without breaking backwards-compatibility. // Must only be set of the linktarget cannot be encoded as valid utf8. - LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` - ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` - Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev - Content IDs `json:"content"` - Subtree *ID `json:"subtree,omitempty"` + LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` + ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` + GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"` + Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev + Content IDs `json:"content"` + Subtree *ID `json:"subtree,omitempty"` Error string `json:"error,omitempty"` @@ -90,7 +136,7 @@ func (node Node) String() string { // NodeFromFileInfo returns a new node from the given path and FileInfo. It // returns the first error that is encountered, together with a node. -func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { +func NodeFromFileInfo(path string, fi os.FileInfo, ignoreXattrListError bool) (*Node, error) { mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky node := &Node{ Path: path, @@ -104,7 +150,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { node.Size = uint64(fi.Size()) } - err := node.fillExtra(path, fi) + err := node.fillExtra(path, fi, ignoreXattrListError) return node, err } @@ -142,7 +188,7 @@ func (node Node) GetExtendedAttribute(a string) []byte { } // CreateAt creates the node at the given path but does NOT restore node meta data. -func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) error { +func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error { debug.Log("create node %v at %v", node.Name, path) switch node.Type { @@ -180,8 +226,8 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) er } // RestoreMetadata restores node metadata -func (node Node) RestoreMetadata(path string) error { - err := node.restoreMetadata(path) +func (node Node) RestoreMetadata(path string, warn func(msg string)) error { + err := node.restoreMetadata(path, warn) if err != nil { debug.Log("restoreMetadata(%s) error %v", path, err) } @@ -189,7 +235,7 @@ func (node Node) RestoreMetadata(path string) error { return err } -func (node Node) restoreMetadata(path string) error { +func (node Node) restoreMetadata(path string, warn func(msg string)) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { @@ -203,14 +249,6 @@ func (node Node) restoreMetadata(path string) error { } } - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr != nil { - firsterr = errors.WithStack(err) - } - } - } - if err := node.RestoreTimestamps(path); err != nil { debug.Log("error restoring timestamps for dir %v: %v", path, err) if firsterr != nil { @@ -225,17 +263,25 @@ func (node Node) restoreMetadata(path string) error { } } - return firsterr -} + if err := node.restoreGenericAttributes(path, warn); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr != nil { + firsterr = err + } + } -func (node Node) restoreExtendedAttributes(path string) error { - for _, attr := range node.ExtendedAttributes { - err := Setxattr(path, attr.Name, attr.Value) - if err != nil { - return err + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := fs.Chmod(path, node.Mode); err != nil { + if firsterr != nil { + firsterr = errors.WithStack(err) + } } } - return nil + + return firsterr } func (node Node) RestoreTimestamps(path string) error { @@ -264,7 +310,7 @@ func (node Node) createDirAt(path string) error { return nil } -func (node Node) createFileAt(ctx context.Context, path string, repo Repository) error { +func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error { f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { return errors.WithStack(err) @@ -284,7 +330,7 @@ func (node Node) createFileAt(ctx context.Context, path string, repo Repository) return nil } -func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.File) error { +func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error { var buf []byte for _, id := range node.Content { buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) @@ -302,11 +348,6 @@ func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.Fi } func (node Node) createSymlinkAt(path string) error { - - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return errors.Wrap(err, "Symlink") - } - if err := fs.Symlink(node.LinkTarget, path); err != nil { return errors.WithStack(err) } @@ -438,6 +479,9 @@ func (node Node) Equals(other Node) bool { if !node.sameExtendedAttributes(other) { return false } + if !node.sameGenericAttributes(other) { + return false + } if node.Subtree != nil { if other.Subtree == nil { return false @@ -480,8 +524,13 @@ func (node Node) sameContent(other Node) bool { } func (node Node) sameExtendedAttributes(other Node) bool { - if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) { + ln := len(node.ExtendedAttributes) + lo := len(other.ExtendedAttributes) + if ln != lo { return false + } else if ln == 0 { + // This means lo is also of length 0 + return true } // build a set of all attributes that node has @@ -525,6 +574,33 @@ func (node Node) sameExtendedAttributes(other Node) bool { return true } +func (node Node) sameGenericAttributes(other Node) bool { + return deepEqual(node.GenericAttributes, other.GenericAttributes) +} + +func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { + // Check if the maps have the same number of keys + if len(map1) != len(map2) { + return false + } + + // Iterate over each key-value pair in map1 + for key, value1 := range map1 { + // Check if the key exists in map2 + value2, ok := map2[key] + if !ok { + return false + } + + // Check if the JSON.RawMessage values are equal byte by byte + if !bytes.Equal(value1, value2) { + return false + } + } + + return true +} + func (node *Node) fillUser(stat *statT) { uid, gid := stat.uid(), stat.gid() node.UID, node.GID = uid, gid @@ -586,7 +662,7 @@ func lookupGroup(gid uint32) string { return group } -func (node *Node) fillExtra(path string, fi os.FileInfo) error { +func (node *Node) fillExtra(path string, fi os.FileInfo, ignoreXattrListError bool) error { stat, ok := toStatT(fi.Sys()) if !ok { // fill minimal info with current values for uid, gid @@ -627,32 +703,12 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { return errors.Errorf("unsupported file type %q", node.Type) } - return node.fillExtendedAttributes(path) -} - -func (node *Node) fillExtendedAttributes(path string) error { - xattrs, err := Listxattr(path) - debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) - if err != nil { - return err - } - - node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) - for _, attr := range xattrs { - attrVal, err := Getxattr(path, attr) - if err != nil { - fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) - continue - } - attr := ExtendedAttribute{ - Name: attr, - Value: attrVal, - } - - node.ExtendedAttributes = append(node.ExtendedAttributes, attr) + allowExtended, err := node.fillGenericAttributes(path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + err = errors.CombineErrors(err, node.fillExtendedAttributes(path, ignoreXattrListError)) } - - return nil + return err } func mkfifo(path string, mode uint32) (err error) { @@ -665,3 +721,119 @@ func (node *Node) fillTimes(stat *statT) { node.ChangeTime = time.Unix(ctim.Unix()) node.AccessTime = time.Unix(atim.Unix()) } + +// HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { + for _, unknownAttrib := range unknownAttribs { + handleUnknownGenericAttributeFound(unknownAttrib, warn) + } +} + +// handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType, warn func(msg string)) { + if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) { + // Print the unique error only once for a given execution + os, exists := genericAttributesForOS[genericAttributeType] + + if exists { + // If genericAttributesForOS contains an entry but we still got here, it means the specific node_xx.go for the current OS did not handle it and the repository may have been originally created on a different OS. + // The fact that node.go knows about the attribute, means it is not a new attribute. This may be a common situation if a repo is used across OSs. + debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os) + } else { + // If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType. + warn(fmt.Sprintf("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType)) + } + } +} + +// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. +// This is not used on windows currently because windows has handling for generic attributes. +// nolint:unused +func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error { + for name := range node.GenericAttributes { + handleUnknownGenericAttributeFound(name, warn) + } + return nil +} + +var unknownGenericAttributesHandlingHistory sync.Map + +// checkGenericAttributeNameNotHandledAndPut checks if the GenericAttributeType name entry +// already exists and puts it in the map if not. +func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool { + // If Key doesn't exist, put the value and return true because it is not already handled + _, exists := unknownGenericAttributesHandlingHistory.LoadOrStore(value, "") + // Key exists, then it is already handled so return false + return !exists +} + +// The functions below are common helper functions which can be used for generic attributes support +// across different OS. + +// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection +// nolint:unused +func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { + attributeValue := *attributeValuePtr + + for key, rawMsg := range attrs { + found := false + for i := 0; i < attributeType.NumField(); i++ { + if getFQKeyByIndex(attributeType, i, keyPrefix) == key { + found = true + fieldValue := attributeValue.Field(i) + // For directly supported types, use json.Unmarshal directly + if err := json.Unmarshal(rawMsg, fieldValue.Addr().Interface()); err != nil { + return unknownAttribs, errors.Wrap(err, "Unmarshal") + } + break + } + } + if !found { + unknownAttribs = append(unknownAttribs, key) + } + } + return unknownAttribs, nil +} + +// getFQKey gets the fully qualified key for the field +// nolint:unused +func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType { + return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic"))) +} + +// getFQKeyByIndex gets the fully qualified key for the field index +// nolint:unused +func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType { + return getFQKey(attributeType.Field(index), keyPrefix) +} + +// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection +// nolint:unused +func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { + attributeValue := *attributeValuePtr + attrs = make(map[GenericAttributeType]json.RawMessage) + + // Iterate over the fields of the struct + for i := 0; i < attributeType.NumField(); i++ { + field := attributeType.Field(i) + + // Get the field value using reflection + fieldValue := attributeValue.FieldByName(field.Name) + + // Check if the field is nil + if fieldValue.IsNil() { + // If it's nil, skip this field + continue + } + + // Marshal the field value into a json.RawMessage + var fieldBytes []byte + if fieldBytes, err = json.Marshal(fieldValue.Interface()); err != nil { + return attrs, errors.Wrap(err, "Marshal") + } + + // Insert the field into the map + attrs[getFQKey(field, keyPrefix)] = json.RawMessage(fieldBytes) + } + return attrs, nil +} diff --git a/mover-restic/restic/internal/restic/node_aix.go b/mover-restic/restic/internal/restic/node_aix.go index 572e33a65..32f63af15 100644 --- a/mover-restic/restic/internal/restic/node_aix.go +++ b/mover-restic/restic/internal/restic/node_aix.go @@ -3,9 +3,12 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -20,17 +23,27 @@ func (s statT) atim() syscall.Timespec { return toTimespec(s.Atim) } func (s statT) mtim() syscall.Timespec { return toTimespec(s.Mtim) } func (s statT) ctim() syscall.Timespec { return toTimespec(s.Ctim) } -// Getxattr is a no-op on AIX. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restoreExtendedAttributes is a no-op on AIX. +func (node Node) restoreExtendedAttributes(_ string) error { + return nil } -// Listxattr is a no-op on AIX. -func Listxattr(path string) ([]string, error) { - return nil, nil +// fillExtendedAttributes is a no-op on AIX. +func (node *Node) fillExtendedAttributes(_ string, _ bool) error { + return nil } -// Setxattr is a no-op on AIX. -func Setxattr(path, name string, data []byte) error { - return nil +// IsListxattrPermissionError is a no-op on AIX. +func IsListxattrPermissionError(_ error) bool { + return false +} + +// restoreGenericAttributes is no-op on AIX. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op on AIX. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil } diff --git a/mover-restic/restic/internal/restic/node_netbsd.go b/mover-restic/restic/internal/restic/node_netbsd.go index 0eade2f37..0fe46a3f2 100644 --- a/mover-restic/restic/internal/restic/node_netbsd.go +++ b/mover-restic/restic/internal/restic/node_netbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec } func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } -// Getxattr retrieves extended attribute data associated with path. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restoreExtendedAttributes is a no-op on netbsd. +func (node Node) restoreExtendedAttributes(_ string) error { + return nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. -func Listxattr(path string) ([]string, error) { - return nil, nil +// fillExtendedAttributes is a no-op on netbsd. +func (node *Node) fillExtendedAttributes(_ string, _ bool) error { + return nil } -// Setxattr associates name and data together as an attribute of path. -func Setxattr(path, name string, data []byte) error { - return nil +// IsListxattrPermissionError is a no-op on netbsd. +func IsListxattrPermissionError(_ error) bool { + return false +} + +// restoreGenericAttributes is no-op on netbsd. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op on netbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil } diff --git a/mover-restic/restic/internal/restic/node_openbsd.go b/mover-restic/restic/internal/restic/node_openbsd.go index a4ccc7211..71841f59f 100644 --- a/mover-restic/restic/internal/restic/node_openbsd.go +++ b/mover-restic/restic/internal/restic/node_openbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atim } func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } -// Getxattr retrieves extended attribute data associated with path. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restoreExtendedAttributes is a no-op on openbsd. +func (node Node) restoreExtendedAttributes(_ string) error { + return nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. -func Listxattr(path string) ([]string, error) { - return nil, nil +// fillExtendedAttributes is a no-op on openbsd. +func (node *Node) fillExtendedAttributes(_ string, _ bool) error { + return nil } -// Setxattr associates name and data together as an attribute of path. -func Setxattr(path, name string, data []byte) error { - return nil +// IsListxattrPermissionError is a no-op on openbsd. +func IsListxattrPermissionError(_ error) bool { + return false +} + +// restoreGenericAttributes is no-op on openbsd. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op on openbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil } diff --git a/mover-restic/restic/internal/restic/node_test.go b/mover-restic/restic/internal/restic/node_test.go index aae010421..6e0f31e21 100644 --- a/mover-restic/restic/internal/restic/node_test.go +++ b/mover-restic/restic/internal/restic/node_test.go @@ -1,4 +1,4 @@ -package restic_test +package restic import ( "context" @@ -8,10 +8,11 @@ import ( "path/filepath" "reflect" "runtime" + "strings" "testing" "time" - "github.com/restic/restic/internal/restic" + "github.com/google/go-cmp/cmp" "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -32,7 +33,7 @@ func BenchmarkNodeFillUser(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi, false) rtest.OK(t, err) } @@ -56,7 +57,7 @@ func BenchmarkNodeFromFileInfo(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi, false) if err != nil { t.Fatal(err) } @@ -75,11 +76,11 @@ func parseTime(s string) time.Time { return t.Local() } -var nodeTests = []restic.Node{ +var nodeTests = []Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -90,7 +91,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetuid, @@ -101,7 +102,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile2", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetgid, @@ -112,7 +113,7 @@ var nodeTests = []restic.Node{ { Name: "testSticky", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSticky, @@ -148,7 +149,7 @@ var nodeTests = []restic.Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -170,14 +171,14 @@ var nodeTests = []restic.Node{ { Name: "testXattrFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -191,7 +192,7 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -205,8 +206,14 @@ func TestNodeRestoreAt(t *testing.T) { var nodePath string if test.ExtendedAttributes != nil { if runtime.GOOS == "windows" { - // restic does not support xattrs on windows - return + // In windows extended attributes are case insensitive and windows returns + // the extended attributes in UPPER case. + // Update the tests to use UPPER case xattr names for windows. + extAttrArr := test.ExtendedAttributes + // Iterate through the array using pointers + for i := 0; i < len(extAttrArr); i++ { + extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) + } } // tempdir might be backed by a filesystem that does not support @@ -219,7 +226,7 @@ func TestNodeRestoreAt(t *testing.T) { nodePath = filepath.Join(tempdir, test.Name) } rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath)) + rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) if test.Type == "dir" { rtest.OK(t, test.RestoreTimestamps(nodePath)) @@ -228,8 +235,11 @@ func TestNodeRestoreAt(t *testing.T) { fi, err := os.Lstat(nodePath) rtest.OK(t, err) - n2, err := restic.NodeFromFileInfo(nodePath, fi) + n2, err := NodeFromFileInfo(nodePath, fi, false) + rtest.OK(t, err) + n3, err := NodeFromFileInfo(nodePath, fi, true) rtest.OK(t, err) + rtest.Assert(t, n2.Equals(*n3), "unexpected node info mismatch %v", cmp.Diff(n2, n3)) rtest.Assert(t, test.Name == n2.Name, "%v: name doesn't match (%v != %v)", test.Type, test.Name, n2.Name) @@ -330,7 +340,7 @@ func TestFixTime(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - res := restic.FixTime(test.src) + res := FixTime(test.src) if !res.Equal(test.want) { t.Fatalf("wrong result for %v, want:\n %v\ngot:\n %v", test.src, test.want, res) } @@ -343,12 +353,12 @@ func TestSymlinkSerialization(t *testing.T) { "válîd \t Üñi¢òde \n śẗŕinǵ", string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc}), } { - n := restic.Node{ + n := Node{ LinkTarget: link, } ser, err := json.Marshal(n) test.OK(t, err) - var n2 restic.Node + var n2 Node err = json.Unmarshal(ser, &n2) test.OK(t, err) fmt.Println(string(ser)) @@ -365,7 +375,7 @@ func TestSymlinkSerializationFormat(t *testing.T) { {`{"linktarget":"test"}`, "test"}, {`{"linktarget":"\u0000\u0001\u0002\ufffd\ufffd\ufffd","linktarget_raw":"AAEC+vv8"}`, string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc})}, } { - var n2 restic.Node + var n2 Node err := json.Unmarshal([]byte(d.ser), &n2) test.OK(t, err) test.Equals(t, d.linkTarget, n2.LinkTarget) diff --git a/mover-restic/restic/internal/restic/node_unix_test.go b/mover-restic/restic/internal/restic/node_unix_test.go index 374326bf7..9ea7b1725 100644 --- a/mover-restic/restic/internal/restic/node_unix_test.go +++ b/mover-restic/restic/internal/restic/node_unix_test.go @@ -128,7 +128,7 @@ func TestNodeFromFileInfo(t *testing.T) { return } - node, err := NodeFromFileInfo(test.filename, fi) + node, err := NodeFromFileInfo(test.filename, fi, false) if err != nil { t.Fatal(err) } diff --git a/mover-restic/restic/internal/restic/node_windows.go b/mover-restic/restic/internal/restic/node_windows.go index fc6439b40..2785e0412 100644 --- a/mover-restic/restic/internal/restic/node_windows.go +++ b/mover-restic/restic/internal/restic/node_windows.go @@ -1,21 +1,50 @@ package restic import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" "syscall" + "unsafe" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "golang.org/x/sys/windows" +) + +// WindowsAttributes are the genericAttributes for Windows OS +type WindowsAttributes struct { + // CreationTime is used for storing creation time for windows files. + CreationTime *syscall.Filetime `generic:"creation_time"` + // FileAttributes is used for storing file attributes for windows files. + FileAttributes *uint32 `generic:"file_attributes"` + // SecurityDescriptor is used for storing security descriptors which includes + // owner, group, discretionary access control list (DACL), system access control list (SACL) + SecurityDescriptor *[]byte `generic:"security_descriptor"` +} + +var ( + modAdvapi32 = syscall.NewLazyDLL("advapi32.dll") + procEncryptFile = modAdvapi32.NewProc("EncryptFileW") + procDecryptFile = modAdvapi32.NewProc("DecryptFileW") ) // mknod is not supported on Windows. -func mknod(path string, mode uint32, dev uint64) (err error) { +func mknod(_ string, _ uint32, _ uint64) (err error) { return errors.New("device nodes cannot be created on windows") } // Windows doesn't need lchown -func lchown(path string, uid int, gid int) (err error) { +func lchown(_ string, _ int, _ int) (err error) { return nil } +// restoreSymlinkTimestamps restores timestamps for symlinks func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(path) @@ -28,25 +57,110 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe if e != nil { return e } - defer syscall.Close(h) + + defer func() { + err := syscall.Close(h) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + a := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[0])) w := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[1])) return syscall.SetFileTime(h, nil, &a, &w) } -// Getxattr retrieves extended attribute data associated with path. -func Getxattr(path, name string) ([]byte, error) { - return nil, nil +// restore extended attributes for windows +func (node Node) restoreExtendedAttributes(path string) (err error) { + count := len(node.ExtendedAttributes) + if count > 0 { + eas := make([]fs.ExtendedAttribute, count) + for i, attr := range node.ExtendedAttributes { + eas[i] = fs.ExtendedAttribute{Name: attr.Name, Value: attr.Value} + } + if errExt := restoreExtendedAttributes(node.Type, path, eas); errExt != nil { + return errExt + } + } + return nil +} + +// fill extended attributes in the node. This also includes the Generic attributes for windows. +func (node *Node) fillExtendedAttributes(path string, _ bool) (err error) { + var fileHandle windows.Handle + if fileHandle, err = fs.OpenHandleForEA(node.Type, path, false); fileHandle == 0 { + return nil + } + if err != nil { + return errors.Errorf("get EA failed while opening file handle for path %v, with: %v", path, err) + } + defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call + //Get the windows Extended Attributes using the file handle + var extAtts []fs.ExtendedAttribute + extAtts, err = fs.GetFileEA(fileHandle) + debug.Log("fillExtendedAttributes(%v) %v", path, extAtts) + if err != nil { + return errors.Errorf("get EA failed for path %v, with: %v", path, err) + } + if len(extAtts) == 0 { + return nil + } + + //Fill the ExtendedAttributes in the node using the name/value pairs in the windows EA + for _, attr := range extAtts { + extendedAttr := ExtendedAttribute{ + Name: attr.Name, + Value: attr.Value, + } + + node.ExtendedAttributes = append(node.ExtendedAttributes, extendedAttr) + } + return nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. -func Listxattr(path string) ([]string, error) { - return nil, nil +// closeFileHandle safely closes a file handle and logs any errors. +func closeFileHandle(fileHandle windows.Handle, path string) { + err := windows.CloseHandle(fileHandle) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } } -// Setxattr associates name and data together as an attribute of path. -func Setxattr(path, name string, data []byte) error { +// restoreExtendedAttributes handles restore of the Windows Extended Attributes to the specified path. +// The Windows API requires setting of all the Extended Attributes in one call. +func restoreExtendedAttributes(nodeType, path string, eas []fs.ExtendedAttribute) (err error) { + var fileHandle windows.Handle + if fileHandle, err = fs.OpenHandleForEA(nodeType, path, true); fileHandle == 0 { + return nil + } + if err != nil { + return errors.Errorf("set EA failed while opening file handle for path %v, with: %v", path, err) + } + defer closeFileHandle(fileHandle, path) // Replaced inline defer with named function call + + // clear old unexpected xattrs by setting them to an empty value + oldEAs, err := fs.GetFileEA(fileHandle) + if err != nil { + return err + } + + for _, oldEA := range oldEAs { + found := false + for _, ea := range eas { + if strings.EqualFold(ea.Name, oldEA.Name) { + found = true + break + } + } + + if !found { + eas = append(eas, fs.ExtendedAttribute{Name: oldEA.Name, Value: nil}) + } + } + + if err = fs.SetFileEA(fileHandle, eas); err != nil { + return errors.Errorf("set EA failed for path %v, with: %v", path, err) + } return nil } @@ -81,5 +195,208 @@ func (s statT) mtim() syscall.Timespec { func (s statT) ctim() syscall.Timespec { // Windows does not have the concept of a "change time" in the sense Unix uses it, so we're using the LastWriteTime here. - return syscall.NsecToTimespec(s.LastWriteTime.Nanoseconds()) + return s.mtim() +} + +// restoreGenericAttributes restores generic attributes for Windows +func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) { + if len(node.GenericAttributes) == 0 { + return nil + } + var errs []error + windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + if err != nil { + return fmt.Errorf("error parsing generic attribute for: %s : %v", path, err) + } + if windowsAttributes.CreationTime != nil { + if err := restoreCreationTime(path, windowsAttributes.CreationTime); err != nil { + errs = append(errs, fmt.Errorf("error restoring creation time for: %s : %v", path, err)) + } + } + if windowsAttributes.FileAttributes != nil { + if err := restoreFileAttributes(path, windowsAttributes.FileAttributes); err != nil { + errs = append(errs, fmt.Errorf("error restoring file attributes for: %s : %v", path, err)) + } + } + if windowsAttributes.SecurityDescriptor != nil { + if err := fs.SetSecurityDescriptor(path, windowsAttributes.SecurityDescriptor); err != nil { + errs = append(errs, fmt.Errorf("error restoring security descriptor for: %s : %v", path, err)) + } + } + + HandleUnknownGenericAttributesFound(unknownAttribs, warn) + return errors.CombineErrors(errs...) +} + +// genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unknown attributes that it could not convert. +func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { + waValue := reflect.ValueOf(&windowsAttributes).Elem() + unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + return windowsAttributes, unknownAttribs, err +} + +// restoreCreationTime gets the creation time from the data and sets it to the file/folder at +// the specified path. +func restoreCreationTime(path string, creationTime *syscall.Filetime) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + handle, err := syscall.CreateFile(pathPointer, + syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil, + syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return err + } + defer func() { + if err := syscall.Close(handle); err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + return syscall.SetFileTime(handle, creationTime, nil, nil) +} + +// restoreFileAttributes gets the File Attributes from the data and sets them to the file/folder +// at the specified path. +func restoreFileAttributes(path string, fileAttributes *uint32) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + err = fixEncryptionAttribute(path, fileAttributes, pathPointer) + if err != nil { + debug.Log("Could not change encryption attribute for path: %s: %v", path, err) + } + return syscall.SetFileAttributes(pathPointer, *fileAttributes) +} + +// fixEncryptionAttribute checks if a file needs to be marked encrypted and is not already encrypted, it sets +// the FILE_ATTRIBUTE_ENCRYPTED. Conversely, if the file needs to be marked unencrypted and it is already +// marked encrypted, it removes the FILE_ATTRIBUTE_ENCRYPTED. +func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (err error) { + if *attrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should be encrypted. + err = encryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + // If existing file already has readonly or system flag, encrypt file call fails. + // The readonly and system flags will be set again at the end of this func if they are needed. + err = fs.ResetPermissions(path) + if err != nil { + return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) + } + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) + } + err = encryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed retry to encrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } + } else { + existingAttrs, err := windows.GetFileAttributes(pathPointer) + if err != nil { + return fmt.Errorf("failed to get file attributes for existing file: %s : %v", path, err) + } + if existingAttrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should not be encrypted, but its already encrypted. Decrypt it. + err = decryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) || errors.Is(err, windows.ERROR_FILE_READ_ONLY) { + // If existing file already has readonly or system flag, decrypt file call fails. + // The readonly and system flags will be set again after this func if they are needed. + err = fs.ResetPermissions(path) + if err != nil { + return fmt.Errorf("failed to encrypt file: failed to reset permissions: %s : %v", path, err) + } + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) + } + err = decryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed retry to decrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } + } + } + return err +} + +// encryptFile set the encrypted flag on the file. +func encryptFile(pathPointer *uint16) error { + // Call EncryptFile function + ret, _, err := procEncryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// decryptFile removes the encrypted flag from the file. +func decryptFile(pathPointer *uint16) error { + // Call DecryptFile function + ret, _, err := procDecryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// fillGenericAttributes fills in the generic attributes for windows like File Attributes, +// Created time etc. +func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { + if strings.Contains(filepath.Base(path), ":") { + //Do not process for Alternate Data Streams in Windows + // Also do not allow processing of extended attributes for ADS. + return false, nil + } + if !strings.HasSuffix(filepath.Clean(path), `\`) { + // Do not process file attributes and created time for windows directories like + // C:, D: + // Filepath.Clean(path) ends with '\' for Windows root drives only. + var sd *[]byte + if node.Type == "file" || node.Type == "dir" { + if sd, err = fs.GetSecurityDescriptor(path); err != nil { + return true, err + } + } + + // Add Windows attributes + node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{ + CreationTime: getCreationTime(fi, path), + FileAttributes: &stat.FileAttributes, + SecurityDescriptor: sd, + }) + } + return true, err +} + +// windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { + // Get the value of the WindowsAttributes + windowsAttributesValue := reflect.ValueOf(windowsAttributes) + return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) +} + +// getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. +// The value is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (UTC) +// split into two 32-bit parts: the low-order DWORD and the high-order DWORD for efficiency and interoperability. +// The low-order DWORD represents the number of 100-nanosecond intervals elapsed since January 1, 1601, modulo +// 2^32. The high-order DWORD represents the number of times the low-order DWORD has overflowed. +func getCreationTime(fi os.FileInfo, path string) (creationTimeAttribute *syscall.Filetime) { + attrib, success := fi.Sys().(*syscall.Win32FileAttributeData) + if success && attrib != nil { + return &attrib.CreationTime + } else { + debug.Log("Could not get create time for path: %s", path) + return nil + } } diff --git a/mover-restic/restic/internal/restic/node_windows_test.go b/mover-restic/restic/internal/restic/node_windows_test.go new file mode 100644 index 000000000..4fd57bbb7 --- /dev/null +++ b/mover-restic/restic/internal/restic/node_windows_test.go @@ -0,0 +1,331 @@ +//go:build windows +// +build windows + +package restic + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "github.com/restic/restic/internal/test" + "golang.org/x/sys/windows" +) + +func TestRestoreSecurityDescriptors(t *testing.T) { + t.Parallel() + tempDir := t.TempDir() + for i, sd := range fs.TestFileSDs { + testRestoreSecurityDescriptor(t, sd, tempDir, "file", fmt.Sprintf("testfile%d", i)) + } + for i, sd := range fs.TestDirSDs { + testRestoreSecurityDescriptor(t, sd, tempDir, "dir", fmt.Sprintf("testdir%d", i)) + } +} + +func testRestoreSecurityDescriptor(t *testing.T, sd string, tempDir, fileType, fileName string) { + // Decode the encoded string SD to get the security descriptor input in bytes. + sdInputBytes, err := base64.StdEncoding.DecodeString(sd) + test.OK(t, errors.Wrapf(err, "Error decoding SD for: %s", fileName)) + // Wrap the security descriptor bytes in windows attributes and convert to generic attributes. + genericAttributes, err := WindowsAttrsToGenericAttributes(WindowsAttributes{CreationTime: nil, FileAttributes: nil, SecurityDescriptor: &sdInputBytes}) + test.OK(t, errors.Wrapf(err, "Error constructing windows attributes for: %s", fileName)) + // Construct a Node with the generic attributes. + expectedNode := getNode(fileName, fileType, genericAttributes) + + // Restore the file/dir and restore the meta data including the security descriptors. + testPath, node := restoreAndGetNode(t, tempDir, expectedNode, false) + // Get the security descriptor from the node constructed from the file info of the restored path. + sdByteFromRestoredNode := getWindowsAttr(t, testPath, node).SecurityDescriptor + + // Get the security descriptor for the test path after the restore. + sdBytesFromRestoredPath, err := fs.GetSecurityDescriptor(testPath) + test.OK(t, errors.Wrapf(err, "Error while getting the security descriptor for: %s", testPath)) + + // Compare the input SD and the SD got from the restored file. + fs.CompareSecurityDescriptors(t, testPath, sdInputBytes, *sdBytesFromRestoredPath) + // Compare the SD got from node constructed from the restored file info and the SD got directly from the restored file. + fs.CompareSecurityDescriptors(t, testPath, *sdByteFromRestoredNode, *sdBytesFromRestoredPath) +} + +func getNode(name string, fileType string, genericAttributes map[GenericAttributeType]json.RawMessage) Node { + return Node{ + Name: name, + Type: fileType, + Mode: 0644, + ModTime: parseTime("2024-02-21 6:30:01.111"), + AccessTime: parseTime("2024-02-22 7:31:02.222"), + ChangeTime: parseTime("2024-02-23 8:32:03.333"), + GenericAttributes: genericAttributes, + } +} + +func getWindowsAttr(t *testing.T, testPath string, node *Node) WindowsAttributes { + windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + test.OK(t, errors.Wrapf(err, "Error getting windows attr from generic attr: %s", testPath)) + test.Assert(t, len(unknownAttribs) == 0, "Unknown attribs found: %s for: %s", unknownAttribs, testPath) + return windowsAttributes +} + +func TestRestoreCreationTime(t *testing.T) { + t.Parallel() + path := t.TempDir() + fi, err := os.Lstat(path) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", path)) + creationTimeAttribute := getCreationTime(fi, path) + test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) + //Using the temp dir creation time as the test creation time for the test file and folder + runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) +} + +func TestRestoreFileAttributes(t *testing.T) { + t.Parallel() + genericAttributeName := TypeFileAttributes + tempDir := t.TempDir() + normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) + hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) + system := uint32(syscall.FILE_ATTRIBUTE_SYSTEM) + archive := uint32(syscall.FILE_ATTRIBUTE_ARCHIVE) + encrypted := uint32(windows.FILE_ATTRIBUTE_ENCRYPTED) + fileAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, fileAttr := range fileAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testfile%d", i), + Type: "file", + Mode: 0655, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, fileAttr, false) + } + normal = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY) + hidden = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | syscall.FILE_ATTRIBUTE_HIDDEN) + system = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_SYSTEM) + archive = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ARCHIVE) + encrypted = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ENCRYPTED) + folderAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, folderAttr := range folderAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testdirectory%d", i), + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, folderAttr, false) + } +} + +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) +} +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected) + rawMessage := node.GenericAttributes[genericAttr] + genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + rawMessageExpected := genericAttrsExpected[genericAttr] + test.Equals(t, rawMessageExpected, rawMessage, "Generic attribute: %s got from NodeFromFileInfo not equal for path: %s", string(genericAttr), testPath) + } +} + +func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) { + testPath := filepath.Join(tempDir, "001", testNode.Name) + err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) + + if testNode.Type == "file" { + + testFile, err := os.Create(testPath) + test.OK(t, errors.Wrapf(err, "Failed to create test file: %s", testPath)) + testFile.Close() + } else if testNode.Type == "dir" { + + err := os.Mkdir(testPath, testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) + } + + err = testNode.RestoreMetadata(testPath, func(msg string) { + if warningExpected { + test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg) + } else { + // If warning is not expected, this code should not get triggered. + test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg)) + } + }) + test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath)) + + fi, err := os.Lstat(testPath) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath)) + + nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi, false) + test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath)) + + return testPath, nodeFromFileInfo +} + +const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" + +func TestNewGenericAttributeType(t *testing.T) { + t.Parallel() + + newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") + + tempDir := t.TempDir() + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + } + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, true) + _, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + test.OK(t, err) + // Since this GenericAttribute is unknown to this version of the software, it will not get set on the file. + test.Assert(t, len(ua) == 0, "Unknown attributes: %s found for path: %s", ua, testPath) + } +} + +func TestRestoreExtendedAttributes(t *testing.T) { + t.Parallel() + tempDir := t.TempDir() + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []ExtendedAttribute{ + {"user.foo", []byte("bar")}, + }, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []ExtendedAttribute{ + {"user.foo", []byte("bar")}, + }, + }, + } + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, false) + + var handle windows.Handle + var err error + utf16Path := windows.StringToUTF16Ptr(testPath) + if node.Type == "file" { + handle, err = windows.CreateFile(utf16Path, windows.FILE_READ_EA, 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0) + } else if node.Type == "dir" { + handle, err = windows.CreateFile(utf16Path, windows.FILE_READ_EA, 0, nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_BACKUP_SEMANTICS, 0) + } + test.OK(t, errors.Wrapf(err, "Error opening file/directory for: %s", testPath)) + defer func() { + err := windows.Close(handle) + test.OK(t, errors.Wrapf(err, "Error closing file for: %s", testPath)) + }() + + extAttr, err := fs.GetFileEA(handle) + test.OK(t, errors.Wrapf(err, "Error getting extended attributes for: %s", testPath)) + test.Equals(t, len(node.ExtendedAttributes), len(extAttr)) + + for _, expectedExtAttr := range node.ExtendedAttributes { + var foundExtAttr *fs.ExtendedAttribute + for _, ea := range extAttr { + if strings.EqualFold(ea.Name, expectedExtAttr.Name) { + foundExtAttr = &ea + break + + } + } + test.Assert(t, foundExtAttr != nil, "Expected extended attribute not found") + test.Equals(t, expectedExtAttr.Value, foundExtAttr.Value) + } + } +} diff --git a/mover-restic/restic/internal/restic/node_xattr.go b/mover-restic/restic/internal/restic/node_xattr.go index ea9eafe94..5a5a253d9 100644 --- a/mover-restic/restic/internal/restic/node_xattr.go +++ b/mover-restic/restic/internal/restic/node_xattr.go @@ -4,31 +4,47 @@ package restic import ( + "fmt" + "os" "syscall" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/pkg/xattr" ) -// Getxattr retrieves extended attribute data associated with path. -func Getxattr(path, name string) ([]byte, error) { +// getxattr retrieves extended attribute data associated with path. +func getxattr(path, name string) ([]byte, error) { b, err := xattr.LGet(path, name) return b, handleXattrErr(err) } -// Listxattr retrieves a list of names of extended attributes associated with the +// listxattr retrieves a list of names of extended attributes associated with the // given path in the file system. -func Listxattr(path string) ([]string, error) { +func listxattr(path string) ([]string, error) { l, err := xattr.LList(path) return l, handleXattrErr(err) } -// Setxattr associates name and data together as an attribute of path. -func Setxattr(path, name string, data []byte) error { +func IsListxattrPermissionError(err error) bool { + var xerr *xattr.Error + if errors.As(err, &xerr) { + return xerr.Op == "xattr.list" && errors.Is(xerr.Err, os.ErrPermission) + } + return false +} + +// setxattr associates name and data together as an attribute of path. +func setxattr(path, name string, data []byte) error { return handleXattrErr(xattr.LSet(path, name, data)) } +// removexattr removes the attribute name from path. +func removexattr(path, name string) error { + return handleXattrErr(xattr.LRemove(path, name)) +} + func handleXattrErr(err error) error { switch e := err.(type) { case nil: @@ -47,3 +63,68 @@ func handleXattrErr(err error) error { return errors.WithStack(e) } } + +// restoreGenericAttributes is no-op. +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) +} + +// fillGenericAttributes is a no-op. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} + +func (node Node) restoreExtendedAttributes(path string) error { + expectedAttrs := map[string]struct{}{} + for _, attr := range node.ExtendedAttributes { + err := setxattr(path, attr.Name, attr.Value) + if err != nil { + return err + } + expectedAttrs[attr.Name] = struct{}{} + } + + // remove unexpected xattrs + xattrs, err := listxattr(path) + if err != nil { + return err + } + for _, name := range xattrs { + if _, ok := expectedAttrs[name]; ok { + continue + } + if err := removexattr(path, name); err != nil { + return err + } + } + + return nil +} + +func (node *Node) fillExtendedAttributes(path string, ignoreListError bool) error { + xattrs, err := listxattr(path) + debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err) + if err != nil { + if ignoreListError && IsListxattrPermissionError(err) { + return nil + } + return err + } + + node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs)) + for _, attr := range xattrs { + attrVal, err := getxattr(path, attr) + if err != nil { + fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path) + continue + } + attr := ExtendedAttribute{ + Name: attr, + Value: attrVal, + } + + node.ExtendedAttributes = append(node.ExtendedAttributes, attr) + } + + return nil +} diff --git a/mover-restic/restic/internal/restic/node_xattr_all_test.go b/mover-restic/restic/internal/restic/node_xattr_all_test.go new file mode 100644 index 000000000..56ce5e286 --- /dev/null +++ b/mover-restic/restic/internal/restic/node_xattr_all_test.go @@ -0,0 +1,56 @@ +//go:build darwin || freebsd || linux || solaris || windows +// +build darwin freebsd linux solaris windows + +package restic + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + rtest "github.com/restic/restic/internal/test" +) + +func setAndVerifyXattr(t *testing.T, file string, attrs []ExtendedAttribute) { + if runtime.GOOS == "windows" { + // windows seems to convert the xattr name to upper case + for i := range attrs { + attrs[i].Name = strings.ToUpper(attrs[i].Name) + } + } + + node := Node{ + Type: "file", + ExtendedAttributes: attrs, + } + rtest.OK(t, node.restoreExtendedAttributes(file)) + + nodeActual := Node{ + Type: "file", + } + rtest.OK(t, nodeActual.fillExtendedAttributes(file, false)) + + rtest.Assert(t, nodeActual.sameExtendedAttributes(node), "xattr mismatch got %v expected %v", nodeActual.ExtendedAttributes, node.ExtendedAttributes) +} + +func TestOverwriteXattr(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "file") + rtest.OK(t, os.WriteFile(file, []byte("hello world"), 0o600)) + + setAndVerifyXattr(t, file, []ExtendedAttribute{ + { + Name: "user.foo", + Value: []byte("bar"), + }, + }) + + setAndVerifyXattr(t, file, []ExtendedAttribute{ + { + Name: "user.other", + Value: []byte("some"), + }, + }) +} diff --git a/mover-restic/restic/internal/restic/node_xattr_test.go b/mover-restic/restic/internal/restic/node_xattr_test.go new file mode 100644 index 000000000..5ce77bd28 --- /dev/null +++ b/mover-restic/restic/internal/restic/node_xattr_test.go @@ -0,0 +1,28 @@ +//go:build darwin || freebsd || linux || solaris +// +build darwin freebsd linux solaris + +package restic + +import ( + "os" + "testing" + + "github.com/pkg/xattr" + rtest "github.com/restic/restic/internal/test" +) + +func TestIsListxattrPermissionError(t *testing.T) { + xerr := &xattr.Error{ + Op: "xattr.list", + Name: "test", + Err: os.ErrPermission, + } + err := handleXattrErr(xerr) + rtest.Assert(t, err != nil, "missing error") + rtest.Assert(t, IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return true for %v", err) + + xerr.Err = os.ErrNotExist + err = handleXattrErr(xerr) + rtest.Assert(t, err != nil, "missing error") + rtest.Assert(t, !IsListxattrPermissionError(err), "expected IsListxattrPermissionError to return false for %v", err) +} diff --git a/mover-restic/restic/internal/restic/parallel.go b/mover-restic/restic/internal/restic/parallel.go index 34a2a019c..0c2215325 100644 --- a/mover-restic/restic/internal/restic/parallel.go +++ b/mover-restic/restic/internal/restic/parallel.go @@ -4,11 +4,11 @@ import ( "context" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error { - type FileIDInfo struct { ID Size int64 @@ -22,17 +22,11 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f // send list of index files through ch, which is closed afterwards wg.Go(func() error { defer close(ch) - return r.List(ctx, t, func(fi FileInfo) error { - id, err := ParseID(fi.Name) - if err != nil { - debug.Log("unable to parse %v as an ID", fi.Name) - return nil - } - + return r.List(ctx, t, func(id ID, size int64) error { select { case <-ctx.Done(): return nil - case ch <- FileIDInfo{id, fi.Size}: + case ch <- FileIDInfo{id, size}: } return nil }) @@ -57,3 +51,42 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f return wg.Wait() } + +// ParallelRemove deletes the given fileList of fileType in parallel +// if callback returns an error, then it will abort. +func ParallelRemove(ctx context.Context, repo RemoverUnpacked, fileList IDSet, fileType FileType, report func(id ID, err error) error, bar *progress.Counter) error { + fileChan := make(chan ID) + wg, ctx := errgroup.WithContext(ctx) + wg.Go(func() error { + defer close(fileChan) + for id := range fileList { + select { + case fileChan <- id: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + + bar.SetMax(uint64(len(fileList))) + + // deleting files is IO-bound + workerCount := repo.Connections() + for i := 0; i < int(workerCount); i++ { + wg.Go(func() error { + for id := range fileChan { + err := repo.RemoveUnpacked(ctx, fileType, id) + if report != nil { + err = report(id, err) + } + if err != nil { + return err + } + bar.Add(1) + } + return nil + }) + } + return wg.Wait() +} diff --git a/mover-restic/restic/internal/restic/repository.go b/mover-restic/restic/internal/restic/repository.go index 60e57f38a..b18b036a7 100644 --- a/mover-restic/restic/internal/restic/repository.go +++ b/mover-restic/restic/internal/restic/repository.go @@ -3,6 +3,7 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/ui/progress" @@ -15,51 +16,63 @@ var ErrInvalidData = errors.New("invalid data returned") // Repository stores data in a backend. It provides high-level functions and // transparently encrypts/decrypts data. type Repository interface { - - // Backend returns the backend used by the repository - Backend() Backend // Connections returns the maximum number of concurrent backend operations Connections() uint - + Config() Config Key() *crypto.Key - Index() MasterIndex - LoadIndex(context.Context, *progress.Counter) error - SetIndex(MasterIndex) error - LookupBlobSize(ID, BlobType) (uint, bool) + LoadIndex(ctx context.Context, p *progress.Counter) error + SetIndex(mi MasterIndex) error - Config() Config - PackSize() uint - - // List calls the function fn for each file of type t in the repository. - // When an error is returned by fn, processing stops and List() returns the - // error. - // - // The function fn is called in the same Goroutine List() was called from. - List(ctx context.Context, t FileType, fn func(ID, int64) error) error + LookupBlob(t BlobType, id ID) []PackedBlob + LookupBlobSize(t BlobType, id ID) (size uint, exists bool) + // ListBlobs runs fn on all blobs known to the index. When the context is cancelled, + // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. + ListBlobs(ctx context.Context, fn func(PackedBlob)) error + ListPacksFromIndex(ctx context.Context, packs IDSet) <-chan PackBlobs // ListPack returns the list of blobs saved in the pack id and the length of // the pack header. - ListPack(context.Context, ID, int64) ([]Blob, uint32, error) + ListPack(ctx context.Context, id ID, packSize int64) (entries []Blob, hdrSize uint32, err error) - LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) - SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) + LoadBlob(ctx context.Context, t BlobType, id ID, buf []byte) ([]byte, error) + LoadBlobsFromPack(ctx context.Context, packID ID, blobs []Blob, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error // StartPackUploader start goroutines to upload new pack files. The errgroup // is used to immediately notify about an upload error. Flush() will also return // that error. StartPackUploader(ctx context.Context, wg *errgroup.Group) - Flush(context.Context) error + SaveBlob(ctx context.Context, t BlobType, buf []byte, id ID, storeDuplicate bool) (newID ID, known bool, size int, err error) + Flush(ctx context.Context) error + // List calls the function fn for each file of type t in the repository. + // When an error is returned by fn, processing stops and List() returns the + // error. + // + // The function fn is called in the same Goroutine List() was called from. + List(ctx context.Context, t FileType, fn func(ID, int64) error) error + // LoadRaw reads all data stored in the backend for the file with id and filetype t. + // If the backend returns data that does not match the id, then the buffer is returned + // along with an error that is a restic.ErrInvalidData error. + LoadRaw(ctx context.Context, t FileType, id ID) (data []byte, err error) // LoadUnpacked loads and decrypts the file with the given type and ID. LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error) - SaveUnpacked(context.Context, FileType, []byte) (ID, error) + SaveUnpacked(ctx context.Context, t FileType, buf []byte) (ID, error) + // RemoveUnpacked removes a file from the repository. This will eventually be restricted to deleting only snapshots. + RemoveUnpacked(ctx context.Context, t FileType, id ID) error } -// Lister allows listing files in a backend. -type Lister interface { - List(context.Context, FileType, func(FileInfo) error) error -} +type FileType = backend.FileType + +// These are the different data types a backend can store. +const ( + PackFile FileType = backend.PackFile + KeyFile FileType = backend.KeyFile + LockFile FileType = backend.LockFile + SnapshotFile FileType = backend.SnapshotFile + IndexFile FileType = backend.IndexFile + ConfigFile FileType = backend.ConfigFile +) // LoaderUnpacked allows loading a blob not stored in a pack file type LoaderUnpacked interface { @@ -72,7 +85,19 @@ type LoaderUnpacked interface { type SaverUnpacked interface { // Connections returns the maximum number of concurrent backend operations Connections() uint - SaveUnpacked(context.Context, FileType, []byte) (ID, error) + SaveUnpacked(ctx context.Context, t FileType, buf []byte) (ID, error) +} + +// RemoverUnpacked allows removing an unpacked blob +type RemoverUnpacked interface { + // Connections returns the maximum number of concurrent backend operations + Connections() uint + RemoveUnpacked(ctx context.Context, t FileType, id ID) error +} + +type SaverRemoverUnpacked interface { + SaverUnpacked + RemoverUnpacked } type PackBlobs struct { @@ -82,13 +107,31 @@ type PackBlobs struct { // MasterIndex keeps track of the blobs are stored within files. type MasterIndex interface { - Has(BlobHandle) bool - Lookup(BlobHandle) []PackedBlob + Has(bh BlobHandle) bool + Lookup(bh BlobHandle) []PackedBlob // Each runs fn on all blobs known to the index. When the context is cancelled, - // the index iteration return immediately. This blocks any modification of the index. - Each(ctx context.Context, fn func(PackedBlob)) + // the index iteration returns immediately with ctx.Err(). This blocks any modification of the index. + Each(ctx context.Context, fn func(PackedBlob)) error ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs +} + +// Lister allows listing files in a backend. +type Lister interface { + List(ctx context.Context, t FileType, fn func(ID, int64) error) error +} + +type ListerLoaderUnpacked interface { + Lister + LoaderUnpacked +} + +type Unpacked interface { + ListerLoaderUnpacked + SaverUnpacked + RemoverUnpacked +} - Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) +type ListBlobser interface { + ListBlobs(ctx context.Context, fn func(PackedBlob)) error } diff --git a/mover-restic/restic/internal/restic/snapshot.go b/mover-restic/restic/internal/restic/snapshot.go index 13e795ec8..39ed80627 100644 --- a/mover-restic/restic/internal/restic/snapshot.go +++ b/mover-restic/restic/internal/restic/snapshot.go @@ -25,11 +25,31 @@ type Snapshot struct { Tags []string `json:"tags,omitempty"` Original *ID `json:"original,omitempty"` - ProgramVersion string `json:"program_version,omitempty"` + ProgramVersion string `json:"program_version,omitempty"` + Summary *SnapshotSummary `json:"summary,omitempty"` id *ID // plaintext ID, used during restore } +type SnapshotSummary struct { + BackupStart time.Time `json:"backup_start"` + BackupEnd time.Time `json:"backup_end"` + + // statistics from the backup json output + FilesNew uint `json:"files_new"` + FilesChanged uint `json:"files_changed"` + FilesUnmodified uint `json:"files_unmodified"` + DirsNew uint `json:"dirs_new"` + DirsChanged uint `json:"dirs_changed"` + DirsUnmodified uint `json:"dirs_unmodified"` + DataBlobs int `json:"data_blobs"` + TreeBlobs int `json:"tree_blobs"` + DataAdded uint64 `json:"data_added"` + DataAddedPacked uint64 `json:"data_added_packed"` + TotalFilesProcessed uint `json:"total_files_processed"` + TotalBytesProcessed uint64 `json:"total_bytes_processed"` +} + // NewSnapshot returns an initialized snapshot struct for the current user and // time. func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) { @@ -83,7 +103,7 @@ func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excl var m sync.Mutex // For most snapshots decoding is nearly for free, thus just assume were only limited by IO - return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, size int64) error { + return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, _ int64) error { if excludeIDs.Has(id) { return nil } @@ -96,7 +116,7 @@ func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excl } func (sn Snapshot) String() string { - return fmt.Sprintf("", + return fmt.Sprintf("snapshot %s of %v at %s by %s@%s", sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) } diff --git a/mover-restic/restic/internal/restic/snapshot_find.go b/mover-restic/restic/internal/restic/snapshot_find.go index cb761aee3..6d1ab9a7a 100644 --- a/mover-restic/restic/internal/restic/snapshot_find.go +++ b/mover-restic/restic/internal/restic/snapshot_find.go @@ -24,7 +24,7 @@ type SnapshotFilter struct { TimestampLimit time.Time } -func (f *SnapshotFilter) empty() bool { +func (f *SnapshotFilter) Empty() bool { return len(f.Hosts)+len(f.Tags)+len(f.Paths) == 0 } @@ -173,7 +173,7 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn } // Give the user some indication their filters are not used. - if !usedFilter && !f.empty() { + if !usedFilter && !f.Empty() { return fn("filters", nil, errors.Errorf("explicit snapshot ids are given")) } return nil diff --git a/mover-restic/restic/internal/restic/snapshot_find_test.go b/mover-restic/restic/internal/restic/snapshot_find_test.go index 2f16dcb2f..84bffd694 100644 --- a/mover-restic/restic/internal/restic/snapshot_find_test.go +++ b/mover-restic/restic/internal/restic/snapshot_find_test.go @@ -16,7 +16,7 @@ func TestFindLatestSnapshot(t *testing.T) { latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1) f := restic.SnapshotFilter{Hosts: []string{"foo"}} - sn, _, err := f.FindLatest(context.TODO(), repo.Backend(), repo, "latest") + sn, _, err := f.FindLatest(context.TODO(), repo, repo, "latest") if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -35,7 +35,7 @@ func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) { sn, _, err := (&restic.SnapshotFilter{ Hosts: []string{"foo"}, TimestampLimit: parseTimeUTC("2018-08-08 08:08:08"), - }).FindLatest(context.TODO(), repo.Backend(), repo, "latest") + }).FindLatest(context.TODO(), repo, repo, "latest") if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -62,7 +62,7 @@ func TestFindLatestWithSubpath(t *testing.T) { {desiredSnapshot.ID().String() + ":subfolder", "subfolder"}, } { t.Run("", func(t *testing.T) { - sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo.Backend(), repo, exp.query) + sn, subfolder, err := (&restic.SnapshotFilter{}).FindLatest(context.TODO(), repo, repo, exp.query) if err != nil { t.Fatalf("FindLatest returned error: %v", err) } @@ -78,7 +78,7 @@ func TestFindAllSubpathError(t *testing.T) { desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1) count := 0 - test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo.Backend(), repo, + test.OK(t, (&restic.SnapshotFilter{}).FindAll(context.TODO(), repo, repo, []string{"latest:subfolder", desiredSnapshot.ID().Str() + ":subfolder"}, func(id string, sn *restic.Snapshot, err error) error { if err == restic.ErrInvalidSnapshotSyntax { diff --git a/mover-restic/restic/internal/restic/snapshot_group.go b/mover-restic/restic/internal/restic/snapshot_group.go index 964a230b3..f4e1ed384 100644 --- a/mover-restic/restic/internal/restic/snapshot_group.go +++ b/mover-restic/restic/internal/restic/snapshot_group.go @@ -66,6 +66,20 @@ type SnapshotGroupKey struct { Tags []string `json:"tags"` } +func (s *SnapshotGroupKey) String() string { + var parts []string + if s.Hostname != "" { + parts = append(parts, fmt.Sprintf("host %v", s.Hostname)) + } + if len(s.Paths) != 0 { + parts = append(parts, fmt.Sprintf("path %v", s.Paths)) + } + if len(s.Tags) != 0 { + parts = append(parts, fmt.Sprintf("tags %v", s.Tags)) + } + return strings.Join(parts, ", ") +} + // GroupSnapshots takes a list of snapshots and a grouping criteria and creates // a grouped list of snapshots. func GroupSnapshots(snapshots Snapshots, groupBy SnapshotGroupByOptions) (map[string]Snapshots, bool, error) { diff --git a/mover-restic/restic/internal/restic/snapshot_group_test.go b/mover-restic/restic/internal/restic/snapshot_group_test.go index 78ac99ab1..f9d6ff460 100644 --- a/mover-restic/restic/internal/restic/snapshot_group_test.go +++ b/mover-restic/restic/internal/restic/snapshot_group_test.go @@ -38,7 +38,7 @@ func TestGroupByOptions(t *testing.T) { var opts restic.SnapshotGroupByOptions test.OK(t, opts.Set(exp.from)) if !cmp.Equal(opts, exp.opts) { - t.Errorf("unexpeted opts %s", cmp.Diff(opts, exp.opts)) + t.Errorf("unexpected opts %s", cmp.Diff(opts, exp.opts)) } test.Equals(t, opts.String(), exp.normalized) } diff --git a/mover-restic/restic/internal/restic/snapshot_policy.go b/mover-restic/restic/internal/restic/snapshot_policy.go index 0ff0c5ec8..950c26c91 100644 --- a/mover-restic/restic/internal/restic/snapshot_policy.go +++ b/mover-restic/restic/internal/restic/snapshot_policy.go @@ -94,7 +94,11 @@ func (e ExpirePolicy) String() (s string) { s += fmt.Sprintf("all snapshots within %s of the newest", e.Within) } - s = "keep " + s + if s == "" { + s = "remove" + } else { + s = "keep " + s + } return s } @@ -186,16 +190,6 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots, reason // sort newest snapshots first sort.Stable(list) - if p.Empty() { - for _, sn := range list { - reasons = append(reasons, KeepReason{ - Snapshot: sn, - Matches: []string{"policy is empty"}, - }) - } - return list, remove, reasons - } - if len(list) == 0 { return list, nil, nil } diff --git a/mover-restic/restic/internal/restic/snapshot_test.go b/mover-restic/restic/internal/restic/snapshot_test.go index b32c771d4..9099c8b5f 100644 --- a/mover-restic/restic/internal/restic/snapshot_test.go +++ b/mover-restic/restic/internal/restic/snapshot_test.go @@ -32,7 +32,7 @@ func TestLoadJSONUnpacked(t *testing.T) { } func testLoadJSONUnpacked(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) // archive a snapshot sn := restic.Snapshot{} diff --git a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_0 b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_0 index 11ca587c8..96cc25cc7 100644 --- a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_0 +++ b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_0 @@ -1,1782 +1,3 @@ { - "keep": [ - { - "time": "2016-01-18T12:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-12T21:08:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-12T21:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-09T21:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-08T20:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-07T10:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-06T08:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-05T09:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T16:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:30:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:28:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:24:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T12:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T11:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-04T10:23:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-03T07:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-01T07:08:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-01T01:03:03Z", - "tree": null, - "paths": null - }, - { - "time": "2016-01-01T01:02:03Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-21T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-18T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-15T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-13T10:20:30.1Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-13T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-12T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-10T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-11-08T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo", - "bar" - ] - }, - { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo", - "bar" - ] - }, - { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] - }, - { - "time": "2015-10-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-11T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-10T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-09T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-08T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-06T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-05T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-02T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-10-01T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-11T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-10T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-09T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-08T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-06T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-05T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-02T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-09-01T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-21T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-18T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-15T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-13T10:20:30.1Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-13T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-12T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-10T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2015-08-08T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-11-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-11-21T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-11-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-11-18T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-11-15T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo", - "bar" - ] - }, - { - "time": "2014-11-13T10:20:30.1Z", - "tree": null, - "paths": null, - "tags": [ - "bar" - ] - }, - { - "time": "2014-11-13T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-11-12T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-11-10T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-11-08T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-20T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-11T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-10T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-09T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-08T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-06T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-05T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-02T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-10-01T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - { - "time": "2014-09-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-11T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-10T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-09T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-08T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-06T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-05T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-02T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-09-01T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-22T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-21T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-20T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-18T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-15T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-13T10:20:30.1Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-13T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-12T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-10T10:20:30Z", - "tree": null, - "paths": null - }, - { - "time": "2014-08-08T10:20:30Z", - "tree": null, - "paths": null - } - ], - "reasons": [ - { - "snapshot": { - "time": "2016-01-18T12:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-12T21:08:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-12T21:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-09T21:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-08T20:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-07T10:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-06T08:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-05T09:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T16:23:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T12:30:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T12:28:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T12:24:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T12:23:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T11:23:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-04T10:23:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-03T07:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-01T07:08:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-01T01:03:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2016-01-01T01:02:03Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-21T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-18T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-15T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-13T10:20:30.1Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-13T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-12T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-10T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-11-08T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo", - "bar" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo", - "bar" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-22T10:20:30Z", - "tree": null, - "paths": [ - "path1", - "path2" - ], - "tags": [ - "foo", - "bar" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-11T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-10T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-09T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-08T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-06T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-05T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-02T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-10-01T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-11T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-10T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-09T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-08T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-06T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-05T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-02T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-09-01T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-21T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-18T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-15T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-13T10:20:30.1Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-13T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-12T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-10T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2015-08-08T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-21T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-18T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-15T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo", - "bar" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-13T10:20:30.1Z", - "tree": null, - "paths": null, - "tags": [ - "bar" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-13T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-12T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-10T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-11-08T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-22T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-20T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-11T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-10T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-09T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-08T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-06T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-05T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-02T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-10-01T10:20:30Z", - "tree": null, - "paths": null, - "tags": [ - "foo" - ] - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-11T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-10T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-09T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-08T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-06T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-05T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-02T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-09-01T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-22T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-21T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-20T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-18T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-15T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-13T10:20:30.1Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-13T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-12T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-10T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - }, - { - "snapshot": { - "time": "2014-08-08T10:20:30Z", - "tree": null, - "paths": null - }, - "matches": [ - "policy is empty" - ], - "counters": {} - } - ] + "keep": null } \ No newline at end of file diff --git a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_36 b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_36 index 75a3a5b46..cce4cf537 100644 --- a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_36 +++ b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_36 @@ -590,7 +590,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -601,7 +603,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -612,7 +616,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -623,7 +629,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -634,7 +642,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -645,7 +655,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -656,7 +668,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -667,7 +681,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -678,7 +694,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -689,7 +707,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -700,7 +720,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -711,7 +733,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -722,7 +746,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -733,7 +759,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -744,7 +772,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -755,7 +785,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -766,7 +798,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -777,7 +811,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -788,7 +824,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -799,7 +837,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -810,7 +850,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -821,7 +863,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -832,7 +876,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -843,7 +889,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -854,7 +902,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -865,7 +915,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -876,7 +928,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -887,7 +941,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -898,7 +954,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -909,7 +967,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -920,7 +980,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -935,7 +997,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -950,7 +1014,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -968,7 +1034,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -979,7 +1047,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -990,7 +1060,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1001,7 +1073,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1012,7 +1086,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1023,7 +1099,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1034,7 +1112,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1045,7 +1125,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1056,7 +1138,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1067,7 +1151,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1078,7 +1164,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1089,7 +1177,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1100,7 +1190,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1111,7 +1203,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1122,7 +1216,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1133,7 +1229,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1144,7 +1242,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1155,7 +1255,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1166,7 +1268,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1177,7 +1281,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1188,7 +1294,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1199,7 +1307,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1210,7 +1320,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1221,7 +1333,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1232,7 +1346,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1243,7 +1359,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1254,7 +1372,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1265,7 +1385,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1276,7 +1398,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1287,7 +1411,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1298,7 +1424,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1309,7 +1437,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1320,7 +1450,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1331,7 +1463,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1346,7 +1480,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1360,7 +1496,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1374,7 +1512,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1388,7 +1528,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1402,7 +1544,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1416,7 +1560,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1430,7 +1576,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1444,7 +1592,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1458,7 +1608,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1472,7 +1624,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1486,7 +1640,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1500,7 +1656,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1514,7 +1672,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1528,7 +1688,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1542,7 +1704,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1556,7 +1720,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1567,7 +1733,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1578,7 +1746,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1589,7 +1759,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1600,7 +1772,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1611,7 +1785,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1622,7 +1798,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1633,7 +1811,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1644,7 +1824,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1655,7 +1837,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1666,7 +1850,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1677,7 +1863,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1688,7 +1876,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1699,7 +1889,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1710,7 +1902,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1721,7 +1915,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1732,7 +1928,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1743,7 +1941,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1754,7 +1954,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1765,7 +1967,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } }, { "snapshot": { @@ -1776,7 +1980,9 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1} + "counters": { + "last": -1 + } } ] } \ No newline at end of file diff --git a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_37 b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_37 index f6ffa40ea..9856a83d6 100644 --- a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_37 +++ b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_37 @@ -591,7 +591,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -603,7 +606,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -614,7 +620,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -626,7 +635,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -638,7 +650,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -650,7 +665,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -662,7 +680,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -674,7 +695,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -686,7 +710,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -698,7 +725,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -709,7 +739,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -720,7 +753,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -731,7 +767,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -743,7 +782,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -755,7 +797,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -767,7 +812,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -779,7 +827,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -791,7 +842,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -802,7 +856,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -814,7 +871,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -826,7 +886,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -838,7 +901,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -850,7 +916,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -862,7 +931,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -874,7 +946,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -885,7 +960,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -897,7 +975,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -909,7 +990,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -921,7 +1005,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -933,7 +1020,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -944,7 +1034,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -959,7 +1052,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -974,7 +1070,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -992,7 +1091,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1004,7 +1106,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1016,7 +1121,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1028,7 +1136,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1040,7 +1151,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1052,7 +1166,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1064,7 +1181,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1076,7 +1196,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1088,7 +1211,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1100,7 +1226,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1112,7 +1241,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1124,7 +1256,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1136,7 +1271,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1148,7 +1286,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1160,7 +1301,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1172,7 +1316,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1184,7 +1331,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1196,7 +1346,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1208,7 +1361,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1220,7 +1376,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1232,7 +1391,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1244,7 +1406,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1256,7 +1421,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1268,7 +1436,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1280,7 +1451,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1292,7 +1466,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1303,7 +1480,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1315,7 +1495,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1327,7 +1510,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1339,7 +1525,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1351,7 +1540,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1363,7 +1555,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1375,7 +1570,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1387,7 +1585,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1403,7 +1604,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1418,7 +1622,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1432,7 +1639,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1447,7 +1657,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1462,7 +1675,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1477,7 +1693,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1492,7 +1711,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1507,7 +1729,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1522,7 +1747,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1537,7 +1765,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1552,7 +1783,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1567,7 +1801,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1582,7 +1819,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1597,7 +1837,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1612,7 +1855,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1627,7 +1873,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1639,7 +1888,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1651,7 +1903,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1663,7 +1918,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1675,7 +1933,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1687,7 +1948,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1699,7 +1963,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1711,7 +1978,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1723,7 +1993,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1735,7 +2008,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1747,7 +2023,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1759,7 +2038,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1771,7 +2053,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1783,7 +2068,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1795,7 +2083,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1807,7 +2098,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1819,7 +2113,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1830,7 +2127,10 @@ "matches": [ "last snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1842,7 +2142,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1854,7 +2157,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } }, { "snapshot": { @@ -1866,7 +2172,10 @@ "last snapshot", "hourly snapshot" ], - "counters": {"Last": -1, "Hourly": -1} + "counters": { + "last": -1, + "hourly": -1 + } } ] } \ No newline at end of file diff --git a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_38 b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_38 index 6bfdd57f1..f5d7136d4 100644 --- a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_38 +++ b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_38 @@ -507,7 +507,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -518,7 +520,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -529,7 +533,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -540,7 +546,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -551,7 +559,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -562,7 +572,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -573,7 +585,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -584,7 +598,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -595,7 +611,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -606,7 +624,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -617,7 +637,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -628,7 +650,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -639,7 +663,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -650,7 +676,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -661,7 +689,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -672,7 +702,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -683,7 +715,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -694,7 +728,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -705,7 +741,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -716,7 +754,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -727,7 +767,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -738,7 +780,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -749,7 +793,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -760,7 +806,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -771,7 +819,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -782,7 +832,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -793,7 +845,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -804,7 +858,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -815,7 +871,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -826,7 +884,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -837,7 +897,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -848,7 +910,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -859,7 +923,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -870,7 +936,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -881,7 +949,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -892,7 +962,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -903,7 +975,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -914,7 +988,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -925,7 +1001,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -936,7 +1014,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -947,7 +1027,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -958,7 +1040,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -969,7 +1053,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -980,7 +1066,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -991,7 +1079,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1002,7 +1092,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1013,7 +1105,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1024,7 +1118,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1035,7 +1131,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1046,7 +1144,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1057,7 +1157,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1068,7 +1170,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1079,7 +1183,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1090,7 +1196,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1101,7 +1209,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1112,7 +1222,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1127,7 +1239,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1141,7 +1255,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1155,7 +1271,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1169,7 +1287,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1183,7 +1303,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1197,7 +1319,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1211,7 +1335,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1225,7 +1351,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1239,7 +1367,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1253,7 +1383,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1267,7 +1399,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1281,7 +1415,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1295,7 +1431,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1309,7 +1447,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1323,7 +1463,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1334,7 +1476,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1345,7 +1489,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1356,7 +1502,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1367,7 +1515,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1378,7 +1528,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1389,7 +1541,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1400,7 +1554,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1411,7 +1567,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1422,7 +1580,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1433,7 +1593,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1444,7 +1606,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1455,7 +1619,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1466,7 +1632,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1477,7 +1645,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1488,7 +1658,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1499,7 +1671,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1510,7 +1684,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1521,7 +1697,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } }, { "snapshot": { @@ -1532,7 +1710,9 @@ "matches": [ "hourly snapshot" ], - "counters": {"Hourly": -1} + "counters": { + "hourly": -1 + } } ] -} +} \ No newline at end of file diff --git a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_39 b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_39 index 4b111503b..f5fb4b1bf 100644 --- a/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_39 +++ b/mover-restic/restic/internal/restic/testdata/policy_keep_snapshots_39 @@ -74,10 +74,15 @@ "matches": [ "daily snapshot", "weekly snapshot", - "monthly snapshot", - "yearly snapshot" + "monthly snapshot", + "yearly snapshot" ], - "counters": {"Daily": 2, "Weekly": 1, "Monthly": -1, "Yearly": -1} + "counters": { + "daily": 2, + "weekly": 1, + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -89,7 +94,11 @@ "daily snapshot", "weekly snapshot" ], - "counters": {"Daily": 1, "Monthly": -1, "Yearly": -1} + "counters": { + "daily": 1, + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -100,7 +109,10 @@ "matches": [ "daily snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -112,7 +124,10 @@ "monthly snapshot", "yearly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -123,7 +138,10 @@ "matches": [ "monthly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -134,7 +152,10 @@ "matches": [ "monthly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -145,7 +166,10 @@ "matches": [ "monthly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -157,7 +181,10 @@ "monthly snapshot", "yearly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -171,7 +198,10 @@ "matches": [ "monthly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -182,7 +212,10 @@ "matches": [ "monthly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -193,7 +226,10 @@ "matches": [ "monthly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } }, { "snapshot": { @@ -205,7 +241,10 @@ "monthly snapshot", "yearly snapshot" ], - "counters": {"Monthly": -1, "Yearly": -1} + "counters": { + "monthly": -1, + "yearly": -1 + } } ] } \ No newline at end of file diff --git a/mover-restic/restic/internal/restic/testing.go b/mover-restic/restic/internal/restic/testing.go index 004df627c..8f86a7b2c 100644 --- a/mover-restic/restic/internal/restic/testing.go +++ b/mover-restic/restic/internal/restic/testing.go @@ -187,3 +187,22 @@ func ParseDurationOrPanic(s string) Duration { return d } + +// TestLoadAllSnapshots returns a list of all snapshots in the repo. +// If a snapshot ID is in excludeIDs, it will not be included in the result. +func TestLoadAllSnapshots(ctx context.Context, repo ListerLoaderUnpacked, excludeIDs IDSet) (snapshots Snapshots, err error) { + err = ForAllSnapshots(ctx, repo, repo, excludeIDs, func(id ID, sn *Snapshot, err error) error { + if err != nil { + return err + } + + snapshots = append(snapshots, sn) + return nil + }) + + if err != nil { + return nil, err + } + + return snapshots, nil +} diff --git a/mover-restic/restic/internal/restic/testing_test.go b/mover-restic/restic/internal/restic/testing_test.go index 760a53a52..0a0c43892 100644 --- a/mover-restic/restic/internal/restic/testing_test.go +++ b/mover-restic/restic/internal/restic/testing_test.go @@ -17,32 +17,13 @@ const ( testDepth = 2 ) -// LoadAllSnapshots returns a list of all snapshots in the repo. -// If a snapshot ID is in excludeIDs, it will not be included in the result. -func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs restic.IDSet) (snapshots restic.Snapshots, err error) { - err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, excludeIDs, func(id restic.ID, sn *restic.Snapshot, err error) error { - if err != nil { - return err - } - - snapshots = append(snapshots, sn) - return nil - }) - - if err != nil { - return nil, err - } - - return snapshots, nil -} - func TestCreateSnapshot(t *testing.T) { repo := repository.TestRepository(t) for i := 0; i < testCreateSnapshots; i++ { restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } - snapshots, err := loadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) + snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, restic.NewIDSet()) if err != nil { t.Fatal(err) } @@ -64,7 +45,7 @@ func TestCreateSnapshot(t *testing.T) { t.Fatalf("snapshot has zero tree ID") } - checker.TestCheckRepo(t, repo) + checker.TestCheckRepo(t, repo, false) } func BenchmarkTestCreateSnapshot(t *testing.B) { diff --git a/mover-restic/restic/internal/restic/tree_stream.go b/mover-restic/restic/internal/restic/tree_stream.go index 4110a5e8d..123295533 100644 --- a/mover-restic/restic/internal/restic/tree_stream.go +++ b/mover-restic/restic/internal/restic/tree_stream.go @@ -77,7 +77,7 @@ func filterTrees(ctx context.Context, repo Loader, trees IDs, loaderChan chan<- continue } - treeSize, found := repo.LookupBlobSize(nextTreeID.ID, TreeBlob) + treeSize, found := repo.LookupBlobSize(TreeBlob, nextTreeID.ID) if found && treeSize > 50*1024*1024 { loadCh = hugeTreeLoaderChan } else { diff --git a/mover-restic/restic/internal/restic/tree_test.go b/mover-restic/restic/internal/restic/tree_test.go index da674eb1c..8e0b3587a 100644 --- a/mover-restic/restic/internal/restic/tree_test.go +++ b/mover-restic/restic/internal/restic/tree_test.go @@ -86,7 +86,7 @@ func TestNodeComparison(t *testing.T) { fi, err := os.Lstat("tree_test.go") rtest.OK(t, err) - node, err := restic.NodeFromFileInfo("tree_test.go", fi) + node, err := restic.NodeFromFileInfo("tree_test.go", fi, false) rtest.OK(t, err) n2 := *node @@ -127,7 +127,7 @@ func TestTreeEqualSerialization(t *testing.T) { for _, fn := range files[:i] { fi, err := os.Lstat(fn) rtest.OK(t, err) - node, err := restic.NodeFromFileInfo(fn, fi) + node, err := restic.NodeFromFileInfo(fn, fi, false) rtest.OK(t, err) rtest.OK(t, tree.Insert(node)) @@ -181,7 +181,7 @@ func testLoadTree(t *testing.T, version uint) { } // archive a few files - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) rtest.OK(t, repo.Flush(context.Background())) @@ -199,7 +199,7 @@ func benchmarkLoadTree(t *testing.B, version uint) { } // archive a few files - repo := repository.TestRepositoryWithVersion(t, version) + repo, _ := repository.TestRepositoryWithVersion(t, version) sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil) rtest.OK(t, repo.Flush(context.Background())) diff --git a/mover-restic/restic/internal/restorer/filerestorer.go b/mover-restic/restic/internal/restorer/filerestorer.go index 99a460321..e517e6284 100644 --- a/mover-restic/restic/internal/restorer/filerestorer.go +++ b/mover-restic/restic/internal/restorer/filerestorer.go @@ -7,7 +7,6 @@ import ( "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -15,11 +14,6 @@ import ( "github.com/restic/restic/internal/ui/restore" ) -// TODO if a blob is corrupt, there may be good blob copies in other packs -// TODO evaluate if it makes sense to split download and processing workers -// pro: can (slowly) read network and decrypt/write files concurrently -// con: each worker needs to keep one pack in memory - const ( largeFileBlobCount = 25 ) @@ -32,6 +26,7 @@ type fileInfo struct { size int64 location string // file on local filesystem relative to restorer basedir blobs interface{} // blobs of the file + state *fileState } type fileBlobInfo struct { @@ -45,11 +40,12 @@ type packInfo struct { files map[*fileInfo]struct{} // set of files that use blobs from this pack } +type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error + // fileRestorer restores set of files type fileRestorer struct { - key *crypto.Key - idx func(restic.BlobHandle) []restic.PackedBlob - packLoader repository.BackendLoadFn + idx func(restic.BlobType, restic.ID) []restic.PackedBlob + blobsLoader blobsLoaderFn workerCount int filesWriter *filesWriter @@ -57,55 +53,60 @@ type fileRestorer struct { sparse bool progress *restore.Progress + allowRecursiveDelete bool + dst string files []*fileInfo Error func(string, error) error } func newFileRestorer(dst string, - packLoader repository.BackendLoadFn, - key *crypto.Key, - idx func(restic.BlobHandle) []restic.PackedBlob, + blobsLoader blobsLoaderFn, + idx func(restic.BlobType, restic.ID) []restic.PackedBlob, connections uint, sparse bool, + allowRecursiveDelete bool, progress *restore.Progress) *fileRestorer { // as packs are streamed the concurrency is limited by IO workerCount := int(connections) return &fileRestorer{ - key: key, - idx: idx, - packLoader: packLoader, - filesWriter: newFilesWriter(workerCount), - zeroChunk: repository.ZeroChunk(), - sparse: sparse, - progress: progress, - workerCount: workerCount, - dst: dst, - Error: restorerAbortOnAllErrors, + idx: idx, + blobsLoader: blobsLoader, + filesWriter: newFilesWriter(workerCount, allowRecursiveDelete), + zeroChunk: repository.ZeroChunk(), + sparse: sparse, + progress: progress, + allowRecursiveDelete: allowRecursiveDelete, + workerCount: workerCount, + dst: dst, + Error: restorerAbortOnAllErrors, } } -func (r *fileRestorer) addFile(location string, content restic.IDs, size int64) { - r.files = append(r.files, &fileInfo{location: location, blobs: content, size: size}) +func (r *fileRestorer) addFile(location string, content restic.IDs, size int64, state *fileState) { + r.files = append(r.files, &fileInfo{location: location, blobs: content, size: size, state: state}) } func (r *fileRestorer) targetPath(location string) string { return filepath.Join(r.dst, location) } -func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob)) error { +func (r *fileRestorer) forEachBlob(blobIDs []restic.ID, fn func(packID restic.ID, packBlob restic.Blob, idx int, fileOffset int64)) error { if len(blobIDs) == 0 { return nil } - for _, blobID := range blobIDs { - packs := r.idx(restic.BlobHandle{ID: blobID, Type: restic.DataBlob}) + fileOffset := int64(0) + for i, blobID := range blobIDs { + packs := r.idx(restic.DataBlob, blobID) if len(packs) == 0 { return errors.Errorf("Unknown blob %s", blobID.String()) } - fn(packs[0].PackID, packs[0].Blob) + pb := packs[0] + fn(pb.PackID, pb.Blob, i, fileOffset) + fileOffset += int64(pb.DataLength()) } return nil @@ -126,12 +127,19 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { var packsMap map[restic.ID][]fileBlobInfo if largeFile { packsMap = make(map[restic.ID][]fileBlobInfo) + file.blobs = packsMap } - fileOffset := int64(0) - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob) { - if largeFile { - packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset}) - fileOffset += int64(blob.DataLength()) + restoredBlobs := false + err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { + if !file.state.HasMatchingBlob(idx) { + if largeFile { + packsMap[packID] = append(packsMap[packID], fileBlobInfo{id: blob.ID, offset: fileOffset}) + } + restoredBlobs = true + } else { + r.reportBlobProgress(file, uint64(blob.DataLength())) + // completely ignore blob + return } pack, ok := packs[packID] if !ok { @@ -147,20 +155,38 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { file.sparse = r.sparse } }) + if err != nil { + // repository index is messed up, can't do anything + return err + } + if len(fileBlobs) == 1 { // no need to preallocate files with a single block, thus we can always consider them to be sparse // in addition, a short chunk will never match r.zeroChunk which would prevent sparseness for short files file.sparse = r.sparse } - - if err != nil { - // repository index is messed up, can't do anything - return err + if file.state != nil { + // The restorer currently cannot punch new holes into an existing files. + // Thus sections that contained data but should be sparse after restoring + // the snapshot would still contain the old data resulting in a corrupt restore. + file.sparse = false } - if largeFile { - file.blobs = packsMap + + // empty file or one with already uptodate content. Make sure that the file size is correct + if !restoredBlobs { + err := r.truncateFileToSize(file.location, file.size) + if errFile := r.sanitizeError(file, err); errFile != nil { + return errFile + } + + // the progress events were already sent for non-zero size files + if file.size == 0 { + r.reportBlobProgress(file, 0) + } } } + // drop no longer necessary file list + r.files = nil wg, ctx := errgroup.WithContext(ctx) downloadCh := make(chan *packInfo) @@ -179,6 +205,7 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { // the main restore loop wg.Go(func() error { + defer close(downloadCh) for _, id := range packOrder { pack := packs[id] // allow garbage collection of packInfo @@ -190,13 +217,20 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { debug.Log("Scheduled download pack %s", pack.id.Str()) } } - close(downloadCh) return nil }) return wg.Wait() } +func (r *fileRestorer) truncateFileToSize(location string, size int64) error { + f, err := createFile(r.targetPath(location), size, false, r.allowRecursiveDelete) + if err != nil { + return err + } + return f.Close() +} + type blobToFileOffsetsMapping map[restic.ID]struct { files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file blob restic.Blob @@ -216,12 +250,10 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { blobInfo.files[file] = append(blobInfo.files[file], fileOffset) } if fileBlobs, ok := file.blobs.(restic.IDs); ok { - fileOffset := int64(0) - err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob) { - if packID.Equal(pack.id) { + err := r.forEachBlob(fileBlobs, func(packID restic.ID, blob restic.Blob, idx int, fileOffset int64) { + if packID.Equal(pack.id) && !file.state.HasMatchingBlob(idx) { addBlob(blob, fileOffset) } - fileOffset += int64(blob.DataLength()) }) if err != nil { // restoreFiles should have caught this error before @@ -229,7 +261,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { } } else if packsMap, ok := file.blobs.(map[restic.ID][]fileBlobInfo); ok { for _, blob := range packsMap[pack.id] { - idxPacks := r.idx(restic.BlobHandle{ID: blob.id, Type: restic.DataBlob}) + idxPacks := r.idx(restic.DataBlob, blob.id) for _, idxPack := range idxPacks { if idxPack.PackID.Equal(pack.id) { addBlob(idxPack.Blob, blob.offset) @@ -242,41 +274,18 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - for _, entry := range blobs { - occurrences := 0 - for _, offsets := range entry.files { - occurrences += len(offsets) - } - // With a maximum blob size of 8MB, the normal blob streaming has to write - // at most 800MB for a single blob. This should be short enough to avoid - // network connection timeouts. Based on a quick test, a limit of 100 only - // selects a very small number of blobs (the number of references per blob - // - aka. `count` - seem to follow a expontential distribution) - if occurrences > 100 { - // process frequently referenced blobs first as these can take a long time to write - // which can cause backend connections to time out - delete(blobs, entry.blob.ID) - partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry} - err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs) - if err := r.reportError(blobs, processedBlobs, err); err != nil { - return err - } - } - } - - if len(blobs) == 0 { - return nil - } - err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) return r.reportError(blobs, processedBlobs, err) } func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { - if err != nil { - err = r.Error(file.location, err) + switch err { + case nil, context.Canceled, context.DeadlineExceeded: + // Context errors are permanent. + return err + default: + return r.Error(file.location, err) } - return err } func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet, err error) error { @@ -310,7 +319,7 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, for _, entry := range blobs { blobList = append(blobList, entry.blob) } - return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, + return r.blobsLoader(ctx, packID, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { processedBlobs.Insert(h) blob := blobs[h.ID] @@ -324,6 +333,11 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, } for file, offsets := range blob.files { for _, offset := range offsets { + // avoid long cancelation delays for frequently used blobs + if ctx.Err() != nil { + return ctx.Err() + } + writeToFile := func() error { // this looks overly complicated and needs explanation // two competing requirements: @@ -341,11 +355,7 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, createSize = file.size } writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) - - if r.progress != nil { - r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) - } - + r.reportBlobProgress(file, uint64(len(blobData))) return writeErr } err := r.sanitizeError(file, writeToFile()) @@ -357,3 +367,11 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, return nil }) } + +func (r *fileRestorer) reportBlobProgress(file *fileInfo, blobSize uint64) { + action := restore.ActionFileUpdated + if file.state == nil { + action = restore.ActionFileRestored + } + r.progress.AddProgress(file.location, action, uint64(blobSize), uint64(file.size)) +} diff --git a/mover-restic/restic/internal/restorer/filerestorer_test.go b/mover-restic/restic/internal/restorer/filerestorer_test.go index 8d4e2d4d2..f594760e4 100644 --- a/mover-restic/restic/internal/restorer/filerestorer_test.go +++ b/mover-restic/restic/internal/restorer/filerestorer_test.go @@ -4,13 +4,11 @@ import ( "bytes" "context" "fmt" - "io" "os" + "sort" "testing" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -26,11 +24,6 @@ type TestFile struct { } type TestRepo struct { - key *crypto.Key - - // pack names and ids - packsNameToID map[string]restic.ID - packsIDToName map[restic.ID]string packsIDToData map[restic.ID][]byte // blobs and files @@ -39,11 +32,11 @@ type TestRepo struct { filesPathToContent map[string]string // - loader repository.BackendLoadFn + loader blobsLoaderFn } -func (i *TestRepo) Lookup(bh restic.BlobHandle) []restic.PackedBlob { - packs := i.blobs[bh.ID] +func (i *TestRepo) Lookup(tpe restic.BlobType, id restic.ID) []restic.PackedBlob { + packs := i.blobs[id] return packs } @@ -58,16 +51,6 @@ func newTestRepo(content []TestFile) *TestRepo { blobs map[restic.ID]restic.Blob } packs := make(map[string]Pack) - - key := crypto.NewRandomKey() - seal := func(data []byte) []byte { - ciphertext := crypto.NewBlobBuffer(len(data)) - ciphertext = ciphertext[:0] // truncate the slice - nonce := crypto.NewRandomNonce() - ciphertext = append(ciphertext, nonce...) - return key.Seal(ciphertext, nonce, data, nil) - } - filesPathToContent := make(map[string]string) for _, file := range content { @@ -85,14 +68,15 @@ func newTestRepo(content []TestFile) *TestRepo { // calculate blob id and add to the pack as necessary blobID := restic.Hash([]byte(blob.data)) if _, found := pack.blobs[blobID]; !found { - blobData := seal([]byte(blob.data)) + blobData := []byte(blob.data) pack.blobs[blobID] = restic.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: blobID, }, - Length: uint(len(blobData)), - Offset: uint(len(pack.data)), + Length: uint(len(blobData)), + UncompressedLength: uint(len(blobData)), + Offset: uint(len(pack.data)), } pack.data = append(pack.data, blobData...) } @@ -103,15 +87,11 @@ func newTestRepo(content []TestFile) *TestRepo { } blobs := make(map[restic.ID][]restic.PackedBlob) - packsIDToName := make(map[restic.ID]string) packsIDToData := make(map[restic.ID][]byte) - packsNameToID := make(map[string]restic.ID) for _, pack := range packs { packID := restic.Hash(pack.data) - packsIDToName[packID] = pack.name packsIDToData[packID] = pack.data - packsNameToID[pack.name] = packID for blobID, blob := range pack.blobs { blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID}) } @@ -127,30 +107,44 @@ func newTestRepo(content []TestFile) *TestRepo { } repo := &TestRepo{ - key: key, - packsIDToName: packsIDToName, packsIDToData: packsIDToData, - packsNameToID: packsNameToID, blobs: blobs, files: files, filesPathToContent: filesPathToContent, } - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - packID, err := restic.ParseID(h.Name) - if err != nil { - return err + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + blobs = append([]restic.Blob{}, blobs...) + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + + for _, blob := range blobs { + found := false + for _, e := range repo.blobs[blob.ID] { + if packID == e.PackID { + found = true + buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] + err := handleBlobFn(e.BlobHandle, buf, nil) + if err != nil { + return err + } + } + } + if !found { + return fmt.Errorf("missing blob: %v", blob) + } } - rd := bytes.NewReader(repo.packsIDToData[packID][int(offset) : int(offset)+length]) - return fn(rd) + return nil } return repo } func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) { + t.Helper() repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, false, nil) if files == nil { r.files = repo.files @@ -169,6 +163,7 @@ func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files ma } func verifyRestore(t *testing.T, r *fileRestorer, repo *TestRepo) { + t.Helper() for _, file := range r.files { target := r.targetPath(file.location) data, err := os.ReadFile(target) @@ -211,6 +206,10 @@ func TestFileRestorerBasic(t *testing.T) { {"data3-1", "pack3-1"}, }, }, + { + name: "empty", + blobs: []TestBlob{}, + }, }, nil, sparse) } } @@ -282,62 +281,17 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, nil) r.files = repo.files err := r.restoreFiles(context.TODO()) rtest.Assert(t, errors.Is(err, loadError), "got %v, expected contained error %v", err, loadError) } -func TestDownloadError(t *testing.T) { - for i := 0; i < 100; i += 10 { - testPartialDownloadError(t, i) - } -} - -func testPartialDownloadError(t *testing.T, part int) { - tempdir := rtest.TempDir(t) - content := []TestFile{ - { - name: "file1", - blobs: []TestBlob{ - {"data1-1", "pack1"}, - {"data1-2", "pack1"}, - {"data1-3", "pack1"}, - }, - }} - - repo := newTestRepo(content) - - // loader always returns an error - loader := repo.loader - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only load partial data to execise fault handling in different places - err := loader(ctx, h, length*part/100, offset, fn) - if err == nil { - return nil - } - fmt.Println("Retry after error", err) - return loader(ctx, h, length, offset, fn) - } - - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) - r.files = repo.files - r.Error = func(s string, e error) error { - // ignore errors as in the `restore` command - fmt.Println("error during restore", s, e) - return nil - } - - err := r.restoreFiles(context.TODO()) - rtest.OK(t, err) - verifyRestore(t, r, repo) -} - func TestFatalDownloadError(t *testing.T) { tempdir := rtest.TempDir(t) content := []TestFile{ @@ -360,12 +314,19 @@ func TestFatalDownloadError(t *testing.T) { repo := newTestRepo(content) loader := repo.loader - repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only return half the data to break file2 - return loader(ctx, h, length/2, offset, fn) + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + ctr := 0 + return loader(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if ctr < 2 { + ctr++ + return handleBlobFn(blob, buf, err) + } + // break file2 + return errors.New("failed to load blob") + }) } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, false, nil) r.files = repo.files var errors []string diff --git a/mover-restic/restic/internal/restorer/fileswriter.go b/mover-restic/restic/internal/restorer/fileswriter.go index 589aa502a..962f66619 100644 --- a/mover-restic/restic/internal/restorer/fileswriter.go +++ b/mover-restic/restic/internal/restorer/fileswriter.go @@ -1,11 +1,15 @@ package restorer import ( + "fmt" + stdfs "io/fs" "os" "sync" + "syscall" "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" ) @@ -15,7 +19,8 @@ import ( // TODO I am not 100% convinced this is necessary, i.e. it may be okay // to use multiple os.File to write to the same target file type filesWriter struct { - buckets []filesWriterBucket + buckets []filesWriterBucket + allowRecursiveDelete bool } type filesWriterBucket struct { @@ -29,16 +34,135 @@ type partialFile struct { sparse bool } -func newFilesWriter(count int) *filesWriter { +func newFilesWriter(count int, allowRecursiveDelete bool) *filesWriter { buckets := make([]filesWriterBucket, count) for b := 0; b < count; b++ { buckets[b].files = make(map[string]*partialFile) } return &filesWriter{ - buckets: buckets, + buckets: buckets, + allowRecursiveDelete: allowRecursiveDelete, } } +func openFile(path string) (*os.File, error) { + f, err := fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + if !fi.Mode().IsRegular() { + _ = f.Close() + return nil, fmt.Errorf("unexpected file type %v at %q", fi.Mode().Type(), path) + } + return f, nil +} + +func createFile(path string, createSize int64, sparse bool, allowRecursiveDelete bool) (*os.File, error) { + f, err := fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_NOFOLLOW, 0600) + if err != nil && fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(path); err != nil { + return nil, err + } + if f, err = fs.OpenFile(path, fs.O_WRONLY|fs.O_NOFOLLOW, 0600); err != nil { + return nil, err + } + } else if err != nil && (errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.EISDIR)) { + // symlink or directory, try to remove it later on + f = nil + } else if err != nil { + return nil, err + } + + var fi stdfs.FileInfo + if f != nil { + // stat to check that we've opened a regular file + fi, err = f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + } + + mustReplace := f == nil || !fi.Mode().IsRegular() + if !mustReplace { + ex := fs.ExtendedStat(fi) + if ex.Links > 1 { + // there is no efficient way to find out which other files might be linked to this file + // thus nuke the existing file and start with a fresh one + mustReplace = true + } + } + + if mustReplace { + // close handle if we still have it + if f != nil { + if err := f.Close(); err != nil { + return nil, err + } + } + + // not what we expected, try to get rid of it + if allowRecursiveDelete { + if err := fs.RemoveAll(path); err != nil { + return nil, err + } + } else { + if err := fs.Remove(path); err != nil { + return nil, err + } + } + // create a new file, pass O_EXCL to make sure there are no surprises + f, err = fs.OpenFile(path, fs.O_CREATE|fs.O_WRONLY|fs.O_EXCL|fs.O_NOFOLLOW, 0600) + if err != nil { + return nil, err + } + fi, err = f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + } + + return ensureSize(f, fi, createSize, sparse) +} + +func ensureSize(f *os.File, fi stdfs.FileInfo, createSize int64, sparse bool) (*os.File, error) { + if sparse { + err := truncateSparse(f, createSize) + if err != nil { + _ = f.Close() + return nil, err + } + } else if fi.Size() > createSize { + // file is too long must shorten it + err := f.Truncate(createSize) + if err != nil { + _ = f.Close() + return nil, err + } + } else if createSize > 0 { + err := fs.PreallocateFile(f, createSize) + if err != nil { + // Just log the preallocate error but don't let it cause the restore process to fail. + // Preallocate might return an error if the filesystem (implementation) does not + // support preallocation or our parameters combination to the preallocate call + // This should yield a syscall.ENOTSUP error, but some other errors might also + // show up. + debug.Log("Failed to preallocate %v with size %v: %v", f.Name(), createSize, err) + } + } + return f, nil +} + func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error { bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))] @@ -50,41 +174,20 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.files[path].users++ return wr, nil } - - var flags int + var f *os.File + var err error if createSize >= 0 { - flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY - } else { - flags = os.O_WRONLY - } - - f, err := os.OpenFile(path, flags, 0600) - if err != nil { + f, err = createFile(path, createSize, sparse, w.allowRecursiveDelete) + if err != nil { + return nil, err + } + } else if f, err = openFile(path); err != nil { return nil, err } wr := &partialFile{File: f, users: 1, sparse: sparse} bucket.files[path] = wr - if createSize >= 0 { - if sparse { - err = truncateSparse(f, createSize) - if err != nil { - return nil, err - } - } else { - err := fs.PreallocateFile(wr.File, createSize) - if err != nil { - // Just log the preallocate error but don't let it cause the restore process to fail. - // Preallocate might return an error if the filesystem (implementation) does not - // support preallocation or our parameters combination to the preallocate call - // This should yield a syscall.ENOTSUP error, but some other errors might also - // show up. - debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err) - } - } - } - return wr, nil } diff --git a/mover-restic/restic/internal/restorer/fileswriter_other_test.go b/mover-restic/restic/internal/restorer/fileswriter_other_test.go new file mode 100644 index 000000000..530a190e5 --- /dev/null +++ b/mover-restic/restic/internal/restorer/fileswriter_other_test.go @@ -0,0 +1,10 @@ +//go:build !windows +// +build !windows + +package restorer + +import "syscall" + +func notEmptyDirError() error { + return syscall.ENOTEMPTY +} diff --git a/mover-restic/restic/internal/restorer/fileswriter_test.go b/mover-restic/restic/internal/restorer/fileswriter_test.go index 7beb9a2dc..c69847927 100644 --- a/mover-restic/restic/internal/restorer/fileswriter_test.go +++ b/mover-restic/restic/internal/restorer/fileswriter_test.go @@ -1,15 +1,19 @@ package restorer import ( + "fmt" "os" + "path/filepath" + "runtime" "testing" + "github.com/restic/restic/internal/errors" rtest "github.com/restic/restic/internal/test" ) func TestFilesWriterBasic(t *testing.T) { dir := rtest.TempDir(t) - w := newFilesWriter(1) + w := newFilesWriter(1, false) f1 := dir + "/f1" f2 := dir + "/f2" @@ -34,3 +38,133 @@ func TestFilesWriterBasic(t *testing.T) { rtest.OK(t, err) rtest.Equals(t, []byte{2, 2}, buf) } + +func TestFilesWriterRecursiveOverwrite(t *testing.T) { + path := filepath.Join(t.TempDir(), "test") + + // create filled directory + rtest.OK(t, os.Mkdir(path, 0o700)) + rtest.OK(t, os.WriteFile(filepath.Join(path, "file"), []byte("data"), 0o400)) + + // must error if recursive delete is not allowed + w := newFilesWriter(1, false) + err := w.writeToFile(path, []byte{1}, 0, 2, false) + rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexepected error got %v", err) + rtest.Equals(t, 0, len(w.buckets[0].files)) + + // must replace directory + w = newFilesWriter(1, true) + rtest.OK(t, w.writeToFile(path, []byte{1, 1}, 0, 2, false)) + rtest.Equals(t, 0, len(w.buckets[0].files)) + + buf, err := os.ReadFile(path) + rtest.OK(t, err) + rtest.Equals(t, []byte{1, 1}, buf) +} + +func TestCreateFile(t *testing.T) { + basepath := filepath.Join(t.TempDir(), "test") + + scenarios := []struct { + name string + create func(t testing.TB, path string) + check func(t testing.TB, path string) + err error + }{ + { + name: "file", + create: func(t testing.TB, path string) { + rtest.OK(t, os.WriteFile(path, []byte("test-test-test-data"), 0o400)) + }, + }, + { + name: "empty dir", + create: func(t testing.TB, path string) { + rtest.OK(t, os.Mkdir(path, 0o400)) + }, + }, + { + name: "symlink", + create: func(t testing.TB, path string) { + rtest.OK(t, os.Symlink("./something", path)) + }, + }, + { + name: "filled dir", + create: func(t testing.TB, path string) { + rtest.OK(t, os.Mkdir(path, 0o700)) + rtest.OK(t, os.WriteFile(filepath.Join(path, "file"), []byte("data"), 0o400)) + }, + err: notEmptyDirError(), + }, + { + name: "hardlinks", + create: func(t testing.TB, path string) { + rtest.OK(t, os.WriteFile(path, []byte("test-test-test-data"), 0o400)) + rtest.OK(t, os.Link(path, path+"h")) + }, + check: func(t testing.TB, path string) { + if runtime.GOOS == "windows" { + // hardlinks are not supported on windows + return + } + + data, err := os.ReadFile(path + "h") + rtest.OK(t, err) + rtest.Equals(t, "test-test-test-data", string(data), "unexpected content change") + }, + }, + } + + tests := []struct { + size int64 + isSparse bool + }{ + {5, false}, + {21, false}, + {100, false}, + {5, true}, + {21, true}, + {100, true}, + } + + for i, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + for j, test := range tests { + path := basepath + fmt.Sprintf("%v%v", i, j) + sc.create(t, path) + f, err := createFile(path, test.size, test.isSparse, false) + if sc.err == nil { + rtest.OK(t, err) + fi, err := f.Stat() + rtest.OK(t, err) + rtest.Assert(t, fi.Mode().IsRegular(), "wrong filetype %v", fi.Mode()) + rtest.Assert(t, fi.Size() <= test.size, "unexpected file size expected %v, got %v", test.size, fi.Size()) + rtest.OK(t, f.Close()) + if sc.check != nil { + sc.check(t, path) + } + } else { + rtest.Assert(t, errors.Is(err, sc.err), "unexpected error got %v expected %v", err, sc.err) + } + rtest.OK(t, os.RemoveAll(path)) + } + }) + } +} + +func TestCreateFileRecursiveDelete(t *testing.T) { + path := filepath.Join(t.TempDir(), "test") + + // create filled directory + rtest.OK(t, os.Mkdir(path, 0o700)) + rtest.OK(t, os.WriteFile(filepath.Join(path, "file"), []byte("data"), 0o400)) + + // replace it + f, err := createFile(path, 42, false, true) + rtest.OK(t, err) + fi, err := f.Stat() + rtest.OK(t, err) + rtest.Assert(t, fi.Mode().IsRegular(), "wrong filetype %v", fi.Mode()) + rtest.OK(t, f.Close()) +} diff --git a/mover-restic/restic/internal/restorer/fileswriter_windows_test.go b/mover-restic/restic/internal/restorer/fileswriter_windows_test.go new file mode 100644 index 000000000..ec2b062f0 --- /dev/null +++ b/mover-restic/restic/internal/restorer/fileswriter_windows_test.go @@ -0,0 +1,7 @@ +package restorer + +import "syscall" + +func notEmptyDirError() error { + return syscall.ERROR_DIR_NOT_EMPTY +} diff --git a/mover-restic/restic/internal/restorer/hardlinks_index.go b/mover-restic/restic/internal/restorer/hardlinks_index.go index 9cf45975a..d069fb4cb 100644 --- a/mover-restic/restic/internal/restorer/hardlinks_index.go +++ b/mover-restic/restic/internal/restorer/hardlinks_index.go @@ -10,20 +10,20 @@ type HardlinkKey struct { } // HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names. -type HardlinkIndex struct { +type HardlinkIndex[T any] struct { m sync.Mutex - Index map[HardlinkKey]string + Index map[HardlinkKey]T } // NewHardlinkIndex create a new index for hard links -func NewHardlinkIndex() *HardlinkIndex { - return &HardlinkIndex{ - Index: make(map[HardlinkKey]string), +func NewHardlinkIndex[T any]() *HardlinkIndex[T] { + return &HardlinkIndex[T]{ + Index: make(map[HardlinkKey]T), } } -// Has checks wether the link already exist in the index. -func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool { +// Has checks whether the link already exist in the index. +func (idx *HardlinkIndex[T]) Has(inode uint64, device uint64) bool { idx.m.Lock() defer idx.m.Unlock() _, ok := idx.Index[HardlinkKey{inode, device}] @@ -32,25 +32,25 @@ func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool { } // Add adds a link to the index. -func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) { +func (idx *HardlinkIndex[T]) Add(inode uint64, device uint64, value T) { idx.m.Lock() defer idx.m.Unlock() _, ok := idx.Index[HardlinkKey{inode, device}] if !ok { - idx.Index[HardlinkKey{inode, device}] = name + idx.Index[HardlinkKey{inode, device}] = value } } -// GetFilename obtains the filename from the index. -func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string { +// Value obtains the filename from the index. +func (idx *HardlinkIndex[T]) Value(inode uint64, device uint64) T { idx.m.Lock() defer idx.m.Unlock() return idx.Index[HardlinkKey{inode, device}] } // Remove removes a link from the index. -func (idx *HardlinkIndex) Remove(inode uint64, device uint64) { +func (idx *HardlinkIndex[T]) Remove(inode uint64, device uint64) { idx.m.Lock() defer idx.m.Unlock() delete(idx.Index, HardlinkKey{inode, device}) diff --git a/mover-restic/restic/internal/restorer/hardlinks_index_test.go b/mover-restic/restic/internal/restorer/hardlinks_index_test.go index 75a2b83ee..31ce938f9 100644 --- a/mover-restic/restic/internal/restorer/hardlinks_index_test.go +++ b/mover-restic/restic/internal/restorer/hardlinks_index_test.go @@ -10,15 +10,15 @@ import ( // TestHardLinks contains various tests for HardlinkIndex. func TestHardLinks(t *testing.T) { - idx := restorer.NewHardlinkIndex() + idx := restorer.NewHardlinkIndex[string]() idx.Add(1, 2, "inode1-file1-on-device2") idx.Add(2, 3, "inode2-file2-on-device3") - sresult := idx.GetFilename(1, 2) + sresult := idx.Value(1, 2) rtest.Equals(t, sresult, "inode1-file1-on-device2") - sresult = idx.GetFilename(2, 3) + sresult = idx.Value(2, 3) rtest.Equals(t, sresult, "inode2-file2-on-device3") bresult := idx.Has(1, 2) diff --git a/mover-restic/restic/internal/restorer/restorer.go b/mover-restic/restic/internal/restorer/restorer.go index 3c60aca1b..cd3fd076d 100644 --- a/mover-restic/restic/internal/restorer/restorer.go +++ b/mover-restic/restic/internal/restorer/restorer.go @@ -2,6 +2,8 @@ package restorer import ( "context" + "fmt" + "io" "os" "path/filepath" "sync/atomic" @@ -17,27 +19,88 @@ import ( // Restorer is used to restore a snapshot to a directory. type Restorer struct { - repo restic.Repository - sn *restic.Snapshot - sparse bool + repo restic.Repository + sn *restic.Snapshot + opts Options - progress *restoreui.Progress + fileList map[string]bool - Error func(location string, err error) error - SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) + Error func(location string, err error) error + Warn func(message string) + // SelectFilter determines whether the item is selectedForRestore or whether a childMayBeSelected. + // selectedForRestore must not depend on isDir as `removeUnexpectedFiles` always passes false to isDir. + SelectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) } -var restorerAbortOnAllErrors = func(location string, err error) error { return err } +var restorerAbortOnAllErrors = func(_ string, err error) error { return err } + +type Options struct { + DryRun bool + Sparse bool + Progress *restoreui.Progress + Overwrite OverwriteBehavior + Delete bool +} + +type OverwriteBehavior int + +// Constants for different overwrite behavior +const ( + OverwriteAlways OverwriteBehavior = iota + // OverwriteIfChanged is like OverwriteAlways except that it skips restoring the content + // of files with matching size&mtime. Metadata is always restored. + OverwriteIfChanged + OverwriteIfNewer + OverwriteNever + OverwriteInvalid +) + +// Set implements the method needed for pflag command flag parsing. +func (c *OverwriteBehavior) Set(s string) error { + switch s { + case "always": + *c = OverwriteAlways + case "if-changed": + *c = OverwriteIfChanged + case "if-newer": + *c = OverwriteIfNewer + case "never": + *c = OverwriteNever + default: + *c = OverwriteInvalid + return fmt.Errorf("invalid overwrite behavior %q, must be one of (always|if-newer|never)", s) + } + + return nil +} + +func (c *OverwriteBehavior) String() string { + switch *c { + case OverwriteAlways: + return "always" + case OverwriteIfChanged: + return "if-changed" + case OverwriteIfNewer: + return "if-newer" + case OverwriteNever: + return "never" + default: + return "invalid" + } + +} +func (c *OverwriteBehavior) Type() string { + return "behavior" +} // NewRestorer creates a restorer preloaded with the content from the snapshot id. -func NewRestorer(repo restic.Repository, sn *restic.Snapshot, sparse bool, - progress *restoreui.Progress) *Restorer { +func NewRestorer(repo restic.Repository, sn *restic.Snapshot, opts Options) *Restorer { r := &Restorer{ repo: repo, - sparse: sparse, + opts: opts, + fileList: make(map[string]bool), Error: restorerAbortOnAllErrors, - SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true }, - progress: progress, + SelectFilter: func(string, bool) (bool, bool) { return true, true }, sn: sn, } @@ -47,30 +110,78 @@ func NewRestorer(repo restic.Repository, sn *restic.Snapshot, sparse bool, type treeVisitor struct { enterDir func(node *restic.Node, target, location string) error visitNode func(node *restic.Node, target, location string) error - leaveDir func(node *restic.Node, target, location string) error + // 'entries' contains all files the snapshot contains for this node. This also includes files + // ignored by the SelectFilter. + leaveDir func(node *restic.Node, target, location string, entries []string) error +} + +func (res *Restorer) sanitizeError(location string, err error) error { + switch err { + case nil, context.Canceled, context.DeadlineExceeded: + // Context errors are permanent. + return err + default: + return res.Error(location, err) + } } // traverseTree traverses a tree from the repo and calls treeVisitor. // target is the path in the file system, location within the snapshot. -func (res *Restorer) traverseTree(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) (hasRestored bool, err error) { +func (res *Restorer) traverseTree(ctx context.Context, target string, treeID restic.ID, visitor treeVisitor) error { + location := string(filepath.Separator) + + if visitor.enterDir != nil { + err := res.sanitizeError(location, visitor.enterDir(nil, target, location)) + if err != nil { + return err + } + } + childFilenames, hasRestored, err := res.traverseTreeInner(ctx, target, location, treeID, visitor) + if err != nil { + return err + } + if hasRestored && visitor.leaveDir != nil { + err = res.sanitizeError(location, visitor.leaveDir(nil, target, location, childFilenames)) + } + + return err +} + +func (res *Restorer) traverseTreeInner(ctx context.Context, target, location string, treeID restic.ID, visitor treeVisitor) (filenames []string, hasRestored bool, err error) { debug.Log("%v %v %v", target, location, treeID) tree, err := restic.LoadTree(ctx, res.repo, treeID) if err != nil { debug.Log("error loading tree %v: %v", treeID, err) - return hasRestored, res.Error(location, err) + return nil, hasRestored, res.sanitizeError(location, err) } - for _, node := range tree.Nodes { + if res.opts.Delete { + filenames = make([]string, 0, len(tree.Nodes)) + } + for i, node := range tree.Nodes { + if ctx.Err() != nil { + return nil, hasRestored, ctx.Err() + } + + // allow GC of tree node + tree.Nodes[i] = nil + if res.opts.Delete { + // just track all files included in the tree node to simplify the control flow. + // tracking too many files does not matter except for a slightly elevated memory usage + filenames = append(filenames, node.Name) + } // ensure that the node name does not contain anything that refers to a // top-level directory. nodeName := filepath.Base(filepath.Join(string(filepath.Separator), node.Name)) if nodeName != node.Name { debug.Log("node %q has invalid name %q", node.Name, nodeName) - err := res.Error(location, errors.Errorf("invalid child node name %s", node.Name)) + err := res.sanitizeError(location, errors.Errorf("invalid child node name %s", node.Name)) if err != nil { - return hasRestored, err + return nil, hasRestored, err } + // force disable deletion to prevent unexpected behavior + res.opts.Delete = false continue } @@ -80,10 +191,12 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string, if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) { debug.Log("target: %v %v", target, nodeTarget) debug.Log("node %q has invalid target path %q", node.Name, nodeTarget) - err := res.Error(nodeLocation, errors.New("node has invalid path")) + err := res.sanitizeError(nodeLocation, errors.New("node has invalid path")) if err != nil { - return hasRestored, err + return nil, hasRestored, err } + // force disable deletion to prevent unexpected behavior + res.opts.Delete = false continue } @@ -92,44 +205,35 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string, continue } - selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, nodeTarget, node) + selectedForRestore, childMayBeSelected := res.SelectFilter(nodeLocation, node.Type == "dir") debug.Log("SelectFilter returned %v %v for %q", selectedForRestore, childMayBeSelected, nodeLocation) if selectedForRestore { hasRestored = true } - sanitizeError := func(err error) error { - switch err { - case nil, context.Canceled, context.DeadlineExceeded: - // Context errors are permanent. - return err - default: - return res.Error(nodeLocation, err) - } - } - if node.Type == "dir" { if node.Subtree == nil { - return hasRestored, errors.Errorf("Dir without subtree in tree %v", treeID.Str()) + return nil, hasRestored, errors.Errorf("Dir without subtree in tree %v", treeID.Str()) } if selectedForRestore && visitor.enterDir != nil { - err = sanitizeError(visitor.enterDir(node, nodeTarget, nodeLocation)) + err = res.sanitizeError(nodeLocation, visitor.enterDir(node, nodeTarget, nodeLocation)) if err != nil { - return hasRestored, err + return nil, hasRestored, err } } // keep track of restored child status // so metadata of the current directory are restored on leaveDir childHasRestored := false + var childFilenames []string if childMayBeSelected { - childHasRestored, err = res.traverseTree(ctx, nodeTarget, nodeLocation, *node.Subtree, visitor) - err = sanitizeError(err) + childFilenames, childHasRestored, err = res.traverseTreeInner(ctx, nodeTarget, nodeLocation, *node.Subtree, visitor) + err = res.sanitizeError(nodeLocation, err) if err != nil { - return hasRestored, err + return nil, hasRestored, err } // inform the parent directory to restore parent metadata on leaveDir if needed if childHasRestored { @@ -140,9 +244,9 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string, // metadata need to be restore when leaving the directory in both cases // selected for restore or any child of any subtree have been restored if (selectedForRestore || childHasRestored) && visitor.leaveDir != nil { - err = sanitizeError(visitor.leaveDir(node, nodeTarget, nodeLocation)) + err = res.sanitizeError(nodeLocation, visitor.leaveDir(node, nodeTarget, nodeLocation, childFilenames)) if err != nil { - return hasRestored, err + return nil, hasRestored, err } } @@ -150,35 +254,40 @@ func (res *Restorer) traverseTree(ctx context.Context, target, location string, } if selectedForRestore { - err = sanitizeError(visitor.visitNode(node, nodeTarget, nodeLocation)) + err = res.sanitizeError(nodeLocation, visitor.visitNode(node, nodeTarget, nodeLocation)) if err != nil { - return hasRestored, err + return nil, hasRestored, err } } } - return hasRestored, nil + return filenames, hasRestored, nil } func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, target, location string) error { - debug.Log("restoreNode %v %v %v", node.Name, target, location) - - err := node.CreateAt(ctx, target, res.repo) - if err != nil { - debug.Log("node.CreateAt(%s) error %v", target, err) - return err - } + if !res.opts.DryRun { + debug.Log("restoreNode %v %v %v", node.Name, target, location) + if err := fs.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.Wrap(err, "RemoveNode") + } - if res.progress != nil { - res.progress.AddProgress(location, 0, 0) + err := node.CreateAt(ctx, target, res.repo) + if err != nil { + debug.Log("node.CreateAt(%s) error %v", target, err) + return err + } } + res.opts.Progress.AddProgress(location, restoreui.ActionOtherRestored, 0, 0) return res.restoreNodeMetadataTo(node, target, location) } func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { + if res.opts.DryRun { + return nil + } debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := node.RestoreMetadata(target) + err := node.RestoreMetadata(target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } @@ -186,37 +295,40 @@ func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location s } func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location string) error { - if err := fs.Remove(path); !os.IsNotExist(err) { - return errors.Wrap(err, "RemoveCreateHardlink") - } - err := fs.Link(target, path) - if err != nil { - return errors.WithStack(err) - } - - if res.progress != nil { - res.progress.AddProgress(location, 0, 0) + if !res.opts.DryRun { + if err := fs.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.Wrap(err, "RemoveCreateHardlink") + } + err := fs.Link(target, path) + if err != nil { + return errors.WithStack(err) + } } + res.opts.Progress.AddProgress(location, restoreui.ActionOtherRestored, 0, 0) // TODO investigate if hardlinks have separate metadata on any supported system return res.restoreNodeMetadataTo(node, path, location) } -func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error { - wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return err - } - err = wr.Close() - if err != nil { - return err +func (res *Restorer) ensureDir(target string) error { + if res.opts.DryRun { + return nil } - if res.progress != nil { - res.progress.AddProgress(location, 0, 0) + fi, err := fs.Lstat(target) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("failed to check for directory: %w", err) + } + if err == nil && !fi.IsDir() { + // try to cleanup unexpected file + if err := fs.Remove(target); err != nil { + return fmt.Errorf("failed to remove stale item: %w", err) + } } - return res.restoreNodeMetadataTo(node, target, location) + // create parent dir with default permissions + // second pass #leaveDir restores dir metadata after visiting/restoring all children + return fs.MkdirAll(target, 0700) } // RestoreTo creates the directories and files in the snapshot below dst. @@ -230,105 +342,126 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } } - idx := NewHardlinkIndex() - filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, - res.repo.Connections(), res.sparse, res.progress) + if !res.opts.DryRun { + // ensure that the target directory exists and is actually a directory + // Using ensureDir is too aggressive here as it also removes unexpected files + if err := fs.MkdirAll(dst, 0700); err != nil { + return fmt.Errorf("cannot create target directory: %w", err) + } + } + + idx := NewHardlinkIndex[string]() + filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.LookupBlob, + res.repo.Connections(), res.opts.Sparse, res.opts.Delete, res.opts.Progress) filerestorer.Error = res.Error debug.Log("first pass for %q", dst) + var buf []byte + // first tree pass: create directories and collect all files to restore - _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ - enterDir: func(node *restic.Node, target, location string) error { + err = res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{ + enterDir: func(_ *restic.Node, target, location string) error { debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) - if res.progress != nil { - res.progress.AddFile(0) + if location != string(filepath.Separator) { + res.opts.Progress.AddFile(0) } - // create dir with default permissions - // #leaveDir restores dir metadata after visiting all children - return fs.MkdirAll(target, 0700) + return res.ensureDir(target) }, visitNode: func(node *restic.Node, target, location string) error { debug.Log("first pass, visitNode: mkdir %q, leaveDir on second pass should restore metadata", location) - // create parent dir with default permissions - // second pass #leaveDir restores dir metadata after visiting/restoring all children - err := fs.MkdirAll(filepath.Dir(target), 0700) - if err != nil { + if err := res.ensureDir(filepath.Dir(target)); err != nil { return err } if node.Type != "file" { - if res.progress != nil { - res.progress.AddFile(0) - } + res.opts.Progress.AddFile(0) return nil } - if node.Size == 0 { - if res.progress != nil { - res.progress.AddFile(node.Size) - } - return nil // deal with empty files later - } - if node.Links > 1 { if idx.Has(node.Inode, node.DeviceID) { - if res.progress != nil { - // a hardlinked file does not increase the restore size - res.progress.AddFile(0) - } + // a hardlinked file does not increase the restore size + res.opts.Progress.AddFile(0) return nil } idx.Add(node.Inode, node.DeviceID, location) } - if res.progress != nil { - res.progress.AddFile(node.Size) - } - - filerestorer.addFile(location, node.Content, int64(node.Size)) - - return nil + buf, err = res.withOverwriteCheck(ctx, node, target, location, false, buf, func(updateMetadataOnly bool, matches *fileState) error { + if updateMetadataOnly { + res.opts.Progress.AddSkippedFile(location, node.Size) + } else { + res.opts.Progress.AddFile(node.Size) + if !res.opts.DryRun { + filerestorer.addFile(location, node.Content, int64(node.Size), matches) + } else { + action := restoreui.ActionFileUpdated + if matches == nil { + action = restoreui.ActionFileRestored + } + // immediately mark as completed + res.opts.Progress.AddProgress(location, action, node.Size, node.Size) + } + } + res.trackFile(location, updateMetadataOnly) + return nil + }) + return err }, }) if err != nil { return err } - err = filerestorer.restoreFiles(ctx) - if err != nil { - return err + if !res.opts.DryRun { + err = filerestorer.restoreFiles(ctx) + if err != nil { + return err + } } debug.Log("second pass for %q", dst) // second tree pass: restore special files and filesystem metadata - _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ + err = res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{ visitNode: func(node *restic.Node, target, location string) error { debug.Log("second pass, visitNode: restore node %q", location) if node.Type != "file" { - return res.restoreNodeTo(ctx, node, target, location) + _, err := res.withOverwriteCheck(ctx, node, target, location, false, nil, func(_ bool, _ *fileState) error { + return res.restoreNodeTo(ctx, node, target, location) + }) + return err } - // create empty files, but not hardlinks to empty files - if node.Size == 0 && (node.Links < 2 || !idx.Has(node.Inode, node.DeviceID)) { - if node.Links > 1 { - idx.Add(node.Inode, node.DeviceID, location) + if idx.Has(node.Inode, node.DeviceID) && idx.Value(node.Inode, node.DeviceID) != location { + _, err := res.withOverwriteCheck(ctx, node, target, location, true, nil, func(_ bool, _ *fileState) error { + return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.Value(node.Inode, node.DeviceID)), target, location) + }) + return err + } + + if _, ok := res.hasRestoredFile(location); ok { + return res.restoreNodeMetadataTo(node, target, location) + } + // don't touch skipped files + return nil + }, + leaveDir: func(node *restic.Node, target, location string, expectedFilenames []string) error { + if res.opts.Delete { + if err := res.removeUnexpectedFiles(target, location, expectedFilenames); err != nil { + return err } - return res.restoreEmptyFileAt(node, target, location) } - if idx.Has(node.Inode, node.DeviceID) && idx.GetFilename(node.Inode, node.DeviceID) != location { - return res.restoreHardlinkAt(node, filerestorer.targetPath(idx.GetFilename(node.Inode, node.DeviceID)), target, location) + if node == nil { + return nil } - return res.restoreNodeMetadataTo(node, target, location) - }, - leaveDir: func(node *restic.Node, target, location string) error { err := res.restoreNodeMetadataTo(node, target, location) - if err == nil && res.progress != nil { - res.progress.AddProgress(location, 0, 0) + if err == nil { + res.opts.Progress.AddProgress(location, restoreui.ActionDirRestored, 0, 0) } return err }, @@ -336,6 +469,108 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return err } +func (res *Restorer) removeUnexpectedFiles(target, location string, expectedFilenames []string) error { + if !res.opts.Delete { + panic("internal error") + } + + entries, err := fs.Readdirnames(fs.Local{}, target, fs.O_NOFOLLOW) + if errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + keep := map[string]struct{}{} + for _, name := range expectedFilenames { + keep[toComparableFilename(name)] = struct{}{} + } + + for _, entry := range entries { + if _, ok := keep[toComparableFilename(entry)]; ok { + continue + } + + nodeTarget := filepath.Join(target, entry) + nodeLocation := filepath.Join(location, entry) + + if target == nodeTarget || !fs.HasPathPrefix(target, nodeTarget) { + return fmt.Errorf("skipping deletion due to invalid filename: %v", entry) + } + + // TODO pass a proper value to the isDir parameter once this becomes relevant for the filters + selectedForRestore, _ := res.SelectFilter(nodeLocation, false) + // only delete files that were selected for restore + if selectedForRestore { + res.opts.Progress.ReportDeletedFile(nodeLocation) + if !res.opts.DryRun { + if err := fs.RemoveAll(nodeTarget); err != nil { + return err + } + } + } + } + + return nil +} + +func (res *Restorer) trackFile(location string, metadataOnly bool) { + res.fileList[location] = metadataOnly +} + +func (res *Restorer) hasRestoredFile(location string) (metadataOnly bool, ok bool) { + metadataOnly, ok = res.fileList[location] + return metadataOnly, ok +} + +func (res *Restorer) withOverwriteCheck(ctx context.Context, node *restic.Node, target, location string, isHardlink bool, buf []byte, cb func(updateMetadataOnly bool, matches *fileState) error) ([]byte, error) { + overwrite, err := shouldOverwrite(res.opts.Overwrite, node, target) + if err != nil { + return buf, err + } else if !overwrite { + size := node.Size + if isHardlink { + size = 0 + } + res.opts.Progress.AddSkippedFile(location, size) + return buf, nil + } + + var matches *fileState + updateMetadataOnly := false + if node.Type == "file" && !isHardlink { + // if a file fails to verify, then matches is nil which results in restoring from scratch + matches, buf, _ = res.verifyFile(ctx, target, node, false, res.opts.Overwrite == OverwriteIfChanged, buf) + // skip files that are already correct completely + updateMetadataOnly = !matches.NeedsRestore() + } + + return buf, cb(updateMetadataOnly, matches) +} + +func shouldOverwrite(overwrite OverwriteBehavior, node *restic.Node, destination string) (bool, error) { + if overwrite == OverwriteAlways || overwrite == OverwriteIfChanged { + return true, nil + } + + fi, err := fs.Lstat(destination) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + return false, err + } + + if overwrite == OverwriteIfNewer { + // return if node is newer + return node.ModTime.After(fi.ModTime()), nil + } else if overwrite == OverwriteNever { + // file exists + return false, nil + } + panic("unknown overwrite behavior") +} + // Snapshot returns the snapshot this restorer is configured to use. func (res *Restorer) Snapshot() *restic.Snapshot { return res.sn @@ -365,11 +600,14 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { g.Go(func() error { defer close(work) - _, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ + err := res.traverseTree(ctx, dst, *res.sn.Tree, treeVisitor{ visitNode: func(node *restic.Node, target, location string) error { if node.Type != "file" { return nil } + if metadataOnly, ok := res.hasRestoredFile(location); !ok || metadataOnly { + return nil + } select { case <-ctx.Done(): return ctx.Err() @@ -385,10 +623,8 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { g.Go(func() (err error) { var buf []byte for job := range work { - buf, err = res.verifyFile(job.path, job.node, buf) - if err != nil { - err = res.Error(job.path, err) - } + _, buf, err = res.verifyFile(ctx, job.path, job.node, true, false, buf) + err = res.sanitizeError(job.path, err) if err != nil || ctx.Err() != nil { break } @@ -401,34 +637,75 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { return int(nchecked), g.Wait() } +type fileState struct { + blobMatches []bool + sizeMatches bool +} + +func (s *fileState) NeedsRestore() bool { + if s == nil { + return true + } + if !s.sizeMatches { + return true + } + for _, match := range s.blobMatches { + if !match { + return true + } + } + return false +} + +func (s *fileState) HasMatchingBlob(i int) bool { + if s == nil || s.blobMatches == nil { + return false + } + return i < len(s.blobMatches) && s.blobMatches[i] +} + // Verify that the file target has the contents of node. // // buf and the first return value are scratch space, passed around for reuse. // Reusing buffers prevents the verifier goroutines allocating all of RAM and // flushing the filesystem cache (at least on Linux). -func (res *Restorer) verifyFile(target string, node *restic.Node, buf []byte) ([]byte, error) { - f, err := os.Open(target) +func (res *Restorer) verifyFile(ctx context.Context, target string, node *restic.Node, failFast bool, trustMtime bool, buf []byte) (*fileState, []byte, error) { + f, err := fs.OpenFile(target, fs.O_RDONLY|fs.O_NOFOLLOW, 0) if err != nil { - return buf, err + return nil, buf, err } defer func() { _ = f.Close() }() fi, err := f.Stat() + sizeMatches := true switch { case err != nil: - return buf, err + return nil, buf, err + case !fi.Mode().IsRegular(): + return nil, buf, errors.Errorf("Expected %s to be a regular file", target) case int64(node.Size) != fi.Size(): - return buf, errors.Errorf("Invalid file size for %s: expected %d, got %d", - target, node.Size, fi.Size()) + if failFast { + return nil, buf, errors.Errorf("Invalid file size for %s: expected %d, got %d", + target, node.Size, fi.Size()) + } + sizeMatches = false + } + + if trustMtime && fi.ModTime().Equal(node.ModTime) && sizeMatches { + return &fileState{nil, sizeMatches}, buf, nil } + matches := make([]bool, len(node.Content)) var offset int64 - for _, blobID := range node.Content { - length, found := res.repo.LookupBlobSize(blobID, restic.DataBlob) + for i, blobID := range node.Content { + if ctx.Err() != nil { + return nil, buf, ctx.Err() + } + length, found := res.repo.LookupBlobSize(restic.DataBlob, blobID) if !found { - return buf, errors.Errorf("Unable to fetch blob %s", blobID) + return nil, buf, errors.Errorf("Unable to fetch blob %s", blobID) } if length > uint(cap(buf)) { @@ -437,16 +714,21 @@ func (res *Restorer) verifyFile(target string, node *restic.Node, buf []byte) ([ buf = buf[:length] _, err = f.ReadAt(buf, offset) + if err == io.EOF && !failFast { + sizeMatches = false + break + } if err != nil { - return buf, err + return nil, buf, err } - if !blobID.Equal(restic.Hash(buf)) { - return buf, errors.Errorf( + matches[i] = blobID.Equal(restic.Hash(buf)) + if failFast && !matches[i] { + return nil, buf, errors.Errorf( "Unexpected content in %s, starting at offset %d", target, offset) } offset += int64(length) } - return buf, nil + return &fileState{matches, sizeMatches}, buf, nil } diff --git a/mover-restic/restic/internal/restorer/restorer_test.go b/mover-restic/restic/internal/restorer/restorer_test.go index 6c45d5556..9c02afe68 100644 --- a/mover-restic/restic/internal/restorer/restorer_test.go +++ b/mover-restic/restic/internal/restorer/restorer_test.go @@ -3,20 +3,26 @@ package restorer import ( "bytes" "context" + "encoding/json" + "fmt" "io" "math" "os" "path/filepath" + "reflect" "runtime" "strings" + "syscall" "testing" "time" "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" ) @@ -27,24 +33,40 @@ type Snapshot struct { } type File struct { - Data string - Links uint64 - Inode uint64 - Mode os.FileMode + Data string + DataParts []string + Links uint64 + Inode uint64 + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes +} + +type Symlink struct { + Target string ModTime time.Time } type Dir struct { - Nodes map[string]Node - Mode os.FileMode - ModTime time.Time + Nodes map[string]Node + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes +} + +type FileAttributes struct { + ReadOnly bool + Hidden bool + System bool + Archive bool + Encrypted bool } -func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { +func saveFile(t testing.TB, repo restic.BlobSaver, data string) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - id, _, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(node.Data), restic.ID{}, false) + id, _, _, err := repo.SaveBlob(ctx, restic.DataBlob, []byte(data), restic.ID{}, false) if err != nil { t.Fatal(err) } @@ -52,7 +74,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { return id } -func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode uint64) restic.ID { +func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -61,37 +83,58 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode inode++ switch node := n.(type) { case File: - fi := n.(File).Inode + fi := node.Inode if fi == 0 { fi = inode } - lc := n.(File).Links + lc := node.Links if lc == 0 { lc = 1 } fc := []restic.ID{} - if len(n.(File).Data) > 0 { - fc = append(fc, saveFile(t, repo, node)) + size := 0 + if len(node.Data) > 0 { + size = len(node.Data) + fc = append(fc, saveFile(t, repo, node.Data)) + } else if len(node.DataParts) > 0 { + for _, part := range node.DataParts { + fc = append(fc, saveFile(t, repo, part)) + size += len(part) + } } mode := node.Mode if mode == 0 { mode = 0644 } err := tree.Insert(&restic.Node{ - Type: "file", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Content: fc, - Size: uint64(len(n.(File).Data)), - Inode: fi, - Links: lc, + Type: "file", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Content: fc, + Size: uint64(size), + Inode: fi, + Links: lc, + GenericAttributes: getGenericAttributes(node.attributes, false), + }) + rtest.OK(t, err) + case Symlink: + err := tree.Insert(&restic.Node{ + Type: "symlink", + Mode: os.ModeSymlink | 0o777, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + LinkTarget: node.Target, + Inode: inode, + Links: 1, }) rtest.OK(t, err) case Dir: - id := saveDir(t, repo, node.Nodes, inode) + id := saveDir(t, repo, node.Nodes, inode, getGenericAttributes) mode := node.Mode if mode == 0 { @@ -99,13 +142,14 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode } err := tree.Insert(&restic.Node{ - Type: "dir", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Subtree: &id, + Type: "dir", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Subtree: &id, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) default: @@ -121,13 +165,13 @@ func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode return id } -func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*restic.Snapshot, restic.ID) { +func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) (*restic.Snapshot, restic.ID) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - treeID := saveDir(t, repo, snapshot.Nodes, 1000) + treeID := saveDir(t, repo, snapshot.Nodes, 1000, getGenericAttributes) err := repo.Flush(ctx) if err != nil { t.Fatal(err) @@ -147,13 +191,18 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res return sn, id } +var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + // No-op + return nil +} + func TestRestorer(t *testing.T) { var tests = []struct { Snapshot Files map[string]string ErrorsMust map[string]map[string]struct{} ErrorsMay map[string]map[string]struct{} - Select func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) + Select func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) }{ // valid test cases { @@ -245,7 +294,7 @@ func TestRestorer(t *testing.T) { Files: map[string]string{ "dir/file": "content: file\n", }, - Select: func(item, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + Select: func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { switch item { case filepath.FromSlash("/dir"): childMayBeSelected = true @@ -322,25 +371,19 @@ func TestRestorer(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) - res := NewRestorer(repo, sn, false, nil) + res := NewRestorer(repo, sn, Options{}) tempdir := rtest.TempDir(t) // make sure we're creating a new subdir of the tempdir tempdir = filepath.Join(tempdir, "target") - res.SelectFilter = func(item, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - t.Logf("restore %v to %v", item, dstpath) - if !fs.HasPathPrefix(tempdir, dstpath) { - t.Errorf("would restore %v to %v, which is not within the target dir %v", - item, dstpath, tempdir) - return false, false - } - + res.SelectFilter = func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { + t.Logf("restore %v", item) if test.Select != nil { - return test.Select(item, dstpath, node) + return test.Select(item, isDir) } return true, true @@ -439,10 +482,10 @@ func TestRestorerRelative(t *testing.T) { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) - res := NewRestorer(repo, sn, false, nil) + res := NewRestorer(repo, sn, Options{}) tempdir := rtest.TempDir(t) cleanup := rtest.Chdir(t, tempdir) @@ -488,16 +531,17 @@ func TestRestorerRelative(t *testing.T) { type TraverseTreeCheck func(testing.TB) treeVisitor type TreeVisit struct { - funcName string // name of the function - location string // location passed to the function + funcName string // name of the function + location string // location passed to the function + files []string // file list passed to the function } func checkVisitOrder(list []TreeVisit) TraverseTreeCheck { var pos int return func(t testing.TB) treeVisitor { - check := func(funcName string) func(*restic.Node, string, string) error { - return func(node *restic.Node, target, location string) error { + check := func(funcName string) func(*restic.Node, string, string, []string) error { + return func(node *restic.Node, target, location string, expectedFilenames []string) error { if pos >= len(list) { t.Errorf("step %v, %v(%v): expected no more than %d function calls", pos, funcName, location, len(list)) pos++ @@ -515,14 +559,24 @@ func checkVisitOrder(list []TreeVisit) TraverseTreeCheck { t.Errorf("step %v: want location %v, got %v", pos, list[pos].location, location) } + if !reflect.DeepEqual(expectedFilenames, v.files) { + t.Errorf("step %v: want files %v, got %v", pos, list[pos].files, expectedFilenames) + } + pos++ return nil } } + checkNoFilename := func(funcName string) func(*restic.Node, string, string) error { + f := check(funcName) + return func(node *restic.Node, target, location string) error { + return f(node, target, location, nil) + } + } return treeVisitor{ - enterDir: check("enterDir"), - visitNode: check("visitNode"), + enterDir: checkNoFilename("enterDir"), + visitNode: checkNoFilename("visitNode"), leaveDir: check("leaveDir"), } } @@ -531,7 +585,7 @@ func checkVisitOrder(list []TreeVisit) TraverseTreeCheck { func TestRestorerTraverseTree(t *testing.T) { var tests = []struct { Snapshot - Select func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) + Select func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) Visitor TraverseTreeCheck }{ { @@ -547,17 +601,19 @@ func TestRestorerTraverseTree(t *testing.T) { "foo": File{Data: "content: foo\n"}, }, }, - Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { + Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) { return true, true }, Visitor: checkVisitOrder([]TreeVisit{ - {"enterDir", "/dir"}, - {"visitNode", "/dir/otherfile"}, - {"enterDir", "/dir/subdir"}, - {"visitNode", "/dir/subdir/file"}, - {"leaveDir", "/dir/subdir"}, - {"leaveDir", "/dir"}, - {"visitNode", "/foo"}, + {"enterDir", "/", nil}, + {"enterDir", "/dir", nil}, + {"visitNode", "/dir/otherfile", nil}, + {"enterDir", "/dir/subdir", nil}, + {"visitNode", "/dir/subdir/file", nil}, + {"leaveDir", "/dir/subdir", []string{"file"}}, + {"leaveDir", "/dir", []string{"otherfile", "subdir"}}, + {"visitNode", "/foo", nil}, + {"leaveDir", "/", []string{"dir", "foo"}}, }), }, @@ -574,14 +630,16 @@ func TestRestorerTraverseTree(t *testing.T) { "foo": File{Data: "content: foo\n"}, }, }, - Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { + Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) { if item == "/foo" { return true, false } return false, false }, Visitor: checkVisitOrder([]TreeVisit{ - {"visitNode", "/foo"}, + {"enterDir", "/", nil}, + {"visitNode", "/foo", nil}, + {"leaveDir", "/", []string{"dir", "foo"}}, }), }, { @@ -596,14 +654,16 @@ func TestRestorerTraverseTree(t *testing.T) { }}, }, }, - Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { + Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) { if item == "/aaa" { return true, false } return false, false }, Visitor: checkVisitOrder([]TreeVisit{ - {"visitNode", "/aaa"}, + {"enterDir", "/", nil}, + {"visitNode", "/aaa", nil}, + {"leaveDir", "/", []string{"aaa", "dir"}}, }), }, @@ -620,19 +680,21 @@ func TestRestorerTraverseTree(t *testing.T) { "foo": File{Data: "content: foo\n"}, }, }, - Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { + Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) { if strings.HasPrefix(item, "/dir") { return true, true } return false, false }, Visitor: checkVisitOrder([]TreeVisit{ - {"enterDir", "/dir"}, - {"visitNode", "/dir/otherfile"}, - {"enterDir", "/dir/subdir"}, - {"visitNode", "/dir/subdir/file"}, - {"leaveDir", "/dir/subdir"}, - {"leaveDir", "/dir"}, + {"enterDir", "/", nil}, + {"enterDir", "/dir", nil}, + {"visitNode", "/dir/otherfile", nil}, + {"enterDir", "/dir/subdir", nil}, + {"visitNode", "/dir/subdir/file", nil}, + {"leaveDir", "/dir/subdir", []string{"file"}}, + {"leaveDir", "/dir", []string{"otherfile", "subdir"}}, + {"leaveDir", "/", []string{"dir", "foo"}}, }), }, @@ -649,7 +711,7 @@ func TestRestorerTraverseTree(t *testing.T) { "foo": File{Data: "content: foo\n"}, }, }, - Select: func(item string, dstpath string, node *restic.Node) (selectForRestore bool, childMayBeSelected bool) { + Select: func(item string, isDir bool) (selectForRestore bool, childMayBeSelected bool) { switch item { case "/dir": return false, true @@ -660,8 +722,10 @@ func TestRestorerTraverseTree(t *testing.T) { } }, Visitor: checkVisitOrder([]TreeVisit{ - {"visitNode", "/dir/otherfile"}, - {"leaveDir", "/dir"}, + {"enterDir", "/", nil}, + {"visitNode", "/dir/otherfile", nil}, + {"leaveDir", "/dir", []string{"otherfile", "subdir"}}, + {"leaveDir", "/", []string{"dir", "foo"}}, }), }, } @@ -669,9 +733,10 @@ func TestRestorerTraverseTree(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, test.Snapshot) + sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) - res := NewRestorer(repo, sn, false, nil) + // set Delete option to enable tracking filenames in a directory + res := NewRestorer(repo, sn, Options{Delete: true}) res.SelectFilter = test.Select @@ -682,7 +747,7 @@ func TestRestorerTraverseTree(t *testing.T) { // make sure we're creating a new subdir of the tempdir target := filepath.Join(tempdir, "target") - _, err := res.traverseTree(ctx, target, string(filepath.Separator), *sn.Tree, test.Visitor(t)) + err := res.traverseTree(ctx, target, *sn.Tree, test.Visitor(t)) if err != nil { t.Fatal(err) } @@ -745,11 +810,11 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) - res := NewRestorer(repo, sn, false, nil) + res := NewRestorer(repo, sn, Options{}) - res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + res.SelectFilter = func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { switch filepath.ToSlash(item) { case "/dir": childMayBeSelected = true @@ -791,7 +856,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { } } -// VerifyFiles must not report cancelation of its context through res.Error. +// VerifyFiles must not report cancellation of its context through res.Error. func TestVerifyCancel(t *testing.T) { snapshot := Snapshot{ Nodes: map[string]Node{ @@ -800,9 +865,9 @@ func TestVerifyCancel(t *testing.T) { } repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, snapshot) + sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) - res := NewRestorer(repo, sn, false, nil) + res := NewRestorer(repo, sn, Options{}) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) @@ -840,11 +905,11 @@ func TestRestorerSparseFiles(t *testing.T) { rtest.OK(t, err) arch := archiver.New(repo, target, archiver.Options{}) - sn, _, err := arch.Snapshot(context.Background(), []string{"/zeros"}, + sn, _, _, err := arch.Snapshot(context.Background(), []string{"/zeros"}, archiver.SnapshotOptions{}) rtest.OK(t, err) - res := NewRestorer(repo, sn, true, nil) + res := NewRestorer(repo, sn, Options{Sparse: true}) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) @@ -875,3 +940,569 @@ func TestRestorerSparseFiles(t *testing.T) { t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse", len(zeros), blocks, 100*sparsity) } + +func saveSnapshotsAndOverwrite(t *testing.T, baseSnapshot Snapshot, overwriteSnapshot Snapshot, baseOptions, overwriteOptions Options) string { + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // base snapshot + sn, id := saveSnapshot(t, repo, baseSnapshot, noopGetGenericAttributes) + t.Logf("base snapshot saved as %v", id.Str()) + + res := NewRestorer(repo, sn, baseOptions) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + + // overwrite snapshot + sn, id = saveSnapshot(t, repo, overwriteSnapshot, noopGetGenericAttributes) + t.Logf("overwrite snapshot saved as %v", id.Str()) + res = NewRestorer(repo, sn, overwriteOptions) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + + _, err := res.VerifyFiles(ctx, tempdir) + rtest.OK(t, err) + + return tempdir +} + +func TestRestorerSparseOverwrite(t *testing.T) { + baseSnapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: new\n"}, + }, + } + var zero [14]byte + sparseSnapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: string(zero[:])}, + }, + } + + opts := Options{Sparse: true, Overwrite: OverwriteAlways} + saveSnapshotsAndOverwrite(t, baseSnapshot, sparseSnapshot, opts, opts) +} + +type printerMock struct { + s restoreui.State +} + +func (p *printerMock) Update(_ restoreui.State, _ time.Duration) { +} +func (p *printerMock) CompleteItem(action restoreui.ItemAction, item string, size uint64) { +} +func (p *printerMock) Finish(s restoreui.State, _ time.Duration) { + p.s = s +} + +func TestRestorerOverwriteBehavior(t *testing.T) { + baseTime := time.Now() + baseSnapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: foo\n", ModTime: baseTime}, + "dirtest": Dir{ + Nodes: map[string]Node{ + "file": File{Data: "content: file\n", ModTime: baseTime}, + "foo": File{Data: "content: foobar", ModTime: baseTime}, + }, + ModTime: baseTime, + }, + }, + } + overwriteSnapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: new\n", ModTime: baseTime.Add(time.Second)}, + "dirtest": Dir{ + Nodes: map[string]Node{ + "file": File{Data: "content: file2\n", ModTime: baseTime.Add(-time.Second)}, + "foo": File{Data: "content: foo", ModTime: baseTime}, + }, + }, + }, + } + + var tests = []struct { + Overwrite OverwriteBehavior + Files map[string]string + Progress restoreui.State + }{ + { + Overwrite: OverwriteAlways, + Files: map[string]string{ + "foo": "content: new\n", + "dirtest/file": "content: file2\n", + "dirtest/foo": "content: foo", + }, + Progress: restoreui.State{ + FilesFinished: 4, + FilesTotal: 4, + FilesSkipped: 0, + AllBytesWritten: 40, + AllBytesTotal: 40, + AllBytesSkipped: 0, + }, + }, + { + Overwrite: OverwriteIfChanged, + Files: map[string]string{ + "foo": "content: new\n", + "dirtest/file": "content: file2\n", + "dirtest/foo": "content: foo", + }, + Progress: restoreui.State{ + FilesFinished: 4, + FilesTotal: 4, + FilesSkipped: 0, + AllBytesWritten: 40, + AllBytesTotal: 40, + AllBytesSkipped: 0, + }, + }, + { + Overwrite: OverwriteIfNewer, + Files: map[string]string{ + "foo": "content: new\n", + "dirtest/file": "content: file\n", + "dirtest/foo": "content: foobar", + }, + Progress: restoreui.State{ + FilesFinished: 2, + FilesTotal: 2, + FilesSkipped: 2, + AllBytesWritten: 13, + AllBytesTotal: 13, + AllBytesSkipped: 27, + }, + }, + { + Overwrite: OverwriteNever, + Files: map[string]string{ + "foo": "content: foo\n", + "dirtest/file": "content: file\n", + "dirtest/foo": "content: foobar", + }, + Progress: restoreui.State{ + FilesFinished: 1, + FilesTotal: 1, + FilesSkipped: 3, + AllBytesWritten: 0, + AllBytesTotal: 0, + AllBytesSkipped: 40, + }, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + mock := &printerMock{} + progress := restoreui.NewProgress(mock, 0) + tempdir := saveSnapshotsAndOverwrite(t, baseSnapshot, overwriteSnapshot, Options{}, Options{Overwrite: test.Overwrite, Progress: progress}) + + for filename, content := range test.Files { + data, err := os.ReadFile(filepath.Join(tempdir, filepath.FromSlash(filename))) + if err != nil { + t.Errorf("unable to read file %v: %v", filename, err) + continue + } + + if !bytes.Equal(data, []byte(content)) { + t.Errorf("file %v has wrong content: want %q, got %q", filename, content, data) + } + } + + progress.Finish() + rtest.Equals(t, test.Progress, mock.s) + }) + } +} + +func TestRestorerOverwritePartial(t *testing.T) { + parts := make([]string, 100) + size := 0 + for i := 0; i < len(parts); i++ { + parts[i] = fmt.Sprint(i) + size += len(parts[i]) + if i < 8 { + // small file + size += len(parts[i]) + } + } + + // the data of both snapshots is stored in different pack files + // thus both small an foo in the overwriteSnapshot contain blobs from + // two different pack files. This tests basic handling of blobs from + // different pack files. + baseTime := time.Now() + baseSnapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{DataParts: parts[0:5], ModTime: baseTime}, + "small": File{DataParts: parts[0:5], ModTime: baseTime}, + }, + } + overwriteSnapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{DataParts: parts, ModTime: baseTime}, + "small": File{DataParts: parts[0:8], ModTime: baseTime}, + }, + } + + mock := &printerMock{} + progress := restoreui.NewProgress(mock, 0) + saveSnapshotsAndOverwrite(t, baseSnapshot, overwriteSnapshot, Options{}, Options{Overwrite: OverwriteAlways, Progress: progress}) + progress.Finish() + rtest.Equals(t, restoreui.State{ + FilesFinished: 2, + FilesTotal: 2, + FilesSkipped: 0, + AllBytesWritten: uint64(size), + AllBytesTotal: uint64(size), + AllBytesSkipped: 0, + }, mock.s) +} + +func TestRestorerOverwriteSpecial(t *testing.T) { + baseTime := time.Now() + baseSnapshot := Snapshot{ + Nodes: map[string]Node{ + "dirtest": Dir{ModTime: baseTime}, + "link": Symlink{Target: "foo", ModTime: baseTime}, + "file": File{Data: "content: file\n", Inode: 42, Links: 2, ModTime: baseTime}, + "hardlink": File{Data: "content: file\n", Inode: 42, Links: 2, ModTime: baseTime}, + "newdir": File{Data: "content: dir\n", ModTime: baseTime}, + }, + } + overwriteSnapshot := Snapshot{ + Nodes: map[string]Node{ + "dirtest": Symlink{Target: "foo", ModTime: baseTime}, + "link": File{Data: "content: link\n", Inode: 42, Links: 2, ModTime: baseTime.Add(time.Second)}, + "file": Symlink{Target: "foo2", ModTime: baseTime}, + "hardlink": File{Data: "content: link\n", Inode: 42, Links: 2, ModTime: baseTime.Add(time.Second)}, + "newdir": Dir{ModTime: baseTime}, + }, + } + + files := map[string]string{ + "link": "content: link\n", + "hardlink": "content: link\n", + } + links := map[string]string{ + "dirtest": "foo", + "file": "foo2", + } + + opts := Options{Overwrite: OverwriteAlways} + tempdir := saveSnapshotsAndOverwrite(t, baseSnapshot, overwriteSnapshot, opts, opts) + + for filename, content := range files { + data, err := os.ReadFile(filepath.Join(tempdir, filepath.FromSlash(filename))) + if err != nil { + t.Errorf("unable to read file %v: %v", filename, err) + continue + } + + if !bytes.Equal(data, []byte(content)) { + t.Errorf("file %v has wrong content: want %q, got %q", filename, content, data) + } + } + for filename, target := range links { + link, err := fs.Readlink(filepath.Join(tempdir, filepath.FromSlash(filename))) + rtest.OK(t, err) + rtest.Equals(t, link, target, "wrong symlink target") + } +} + +func TestRestoreModified(t *testing.T) { + // overwrite files between snapshots and also change their filesize + snapshots := []Snapshot{ + { + Nodes: map[string]Node{ + "foo": File{Data: "content: foo\n", ModTime: time.Now()}, + "bar": File{Data: "content: a\n", ModTime: time.Now()}, + }, + }, + { + Nodes: map[string]Node{ + "foo": File{Data: "content: a\n", ModTime: time.Now()}, + "bar": File{Data: "content: bar\n", ModTime: time.Now()}, + }, + }, + } + + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + for _, snapshot := range snapshots { + sn, id := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) + t.Logf("snapshot saved as %v", id.Str()) + + res := NewRestorer(repo, sn, Options{Overwrite: OverwriteIfChanged}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + n, err := res.VerifyFiles(ctx, tempdir) + rtest.OK(t, err) + rtest.Equals(t, 2, n, "unexpected number of verified files") + } +} + +func TestRestoreIfChanged(t *testing.T) { + origData := "content: foo\n" + modData := "content: bar\n" + rtest.Equals(t, len(modData), len(origData), "broken testcase") + snapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: origData, ModTime: time.Now()}, + }, + } + + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sn, id := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) + t.Logf("snapshot saved as %v", id.Str()) + + res := NewRestorer(repo, sn, Options{}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + + // modify file but maintain size and timestamp + path := filepath.Join(tempdir, "foo") + f, err := os.OpenFile(path, os.O_RDWR, 0) + rtest.OK(t, err) + fi, err := f.Stat() + rtest.OK(t, err) + _, err = f.Write([]byte(modData)) + rtest.OK(t, err) + rtest.OK(t, f.Close()) + var utimes = [...]syscall.Timespec{ + syscall.NsecToTimespec(fi.ModTime().UnixNano()), + syscall.NsecToTimespec(fi.ModTime().UnixNano()), + } + rtest.OK(t, syscall.UtimesNano(path, utimes[:])) + + for _, overwrite := range []OverwriteBehavior{OverwriteIfChanged, OverwriteAlways} { + res = NewRestorer(repo, sn, Options{Overwrite: overwrite}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + data, err := os.ReadFile(path) + rtest.OK(t, err) + if overwrite == OverwriteAlways { + // restore should notice the changed file content + rtest.Equals(t, origData, string(data), "expected original file content") + } else { + // restore should not have noticed the changed file content + rtest.Equals(t, modData, string(data), "expected modified file content") + } + } +} + +func TestRestoreDryRun(t *testing.T) { + snapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: foo\n", Links: 2, Inode: 42}, + "foo2": File{Data: "content: foo\n", Links: 2, Inode: 42}, + "dirtest": Dir{ + Nodes: map[string]Node{ + "file": File{Data: "content: file\n"}, + }, + }, + "link": Symlink{Target: "foo"}, + }, + } + + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sn, id := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) + t.Logf("snapshot saved as %v", id.Str()) + + res := NewRestorer(repo, sn, Options{DryRun: true}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + + _, err := os.Stat(tempdir) + rtest.Assert(t, errors.Is(err, os.ErrNotExist), "expected no file to be created, got %v", err) +} + +func TestRestoreDryRunDelete(t *testing.T) { + snapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: foo\n"}, + }, + } + + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + tempfile := filepath.Join(tempdir, "existing") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rtest.OK(t, os.Mkdir(tempdir, 0o755)) + f, err := os.Create(tempfile) + rtest.OK(t, err) + rtest.OK(t, f.Close()) + + sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) + res := NewRestorer(repo, sn, Options{DryRun: true, Delete: true}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + + _, err = os.Stat(tempfile) + rtest.Assert(t, err == nil, "expected file to still exist, got error %v", err) +} + +func TestRestoreOverwriteDirectory(t *testing.T) { + saveSnapshotsAndOverwrite(t, + Snapshot{ + Nodes: map[string]Node{ + "dir": Dir{ + Mode: normalizeFileMode(0755 | os.ModeDir), + Nodes: map[string]Node{ + "anotherfile": File{Data: "content: file\n"}, + }, + }, + }, + }, + Snapshot{ + Nodes: map[string]Node{ + "dir": File{Data: "content: file\n"}, + }, + }, + Options{}, + Options{Delete: true}, + ) +} + +func TestRestoreDelete(t *testing.T) { + repo := repository.TestRepository(t) + tempdir := rtest.TempDir(t) + + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: map[string]Node{ + "dir": Dir{ + Mode: normalizeFileMode(0755 | os.ModeDir), + Nodes: map[string]Node{ + "file1": File{Data: "content: file\n"}, + "anotherfile": File{Data: "content: file\n"}, + }, + }, + "dir2": Dir{ + Mode: normalizeFileMode(0755 | os.ModeDir), + Nodes: map[string]Node{ + "anotherfile": File{Data: "content: file\n"}, + }, + }, + "anotherfile": File{Data: "content: file\n"}, + }, + }, noopGetGenericAttributes) + + // should delete files that no longer exist in the snapshot + deleteSn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: map[string]Node{ + "dir": Dir{ + Mode: normalizeFileMode(0755 | os.ModeDir), + Nodes: map[string]Node{ + "file1": File{Data: "content: file\n"}, + }, + }, + }, + }, noopGetGenericAttributes) + + tests := []struct { + selectFilter func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) + fileState map[string]bool + }{ + { + selectFilter: nil, + fileState: map[string]bool{ + "dir": true, + filepath.Join("dir", "anotherfile"): false, + filepath.Join("dir", "file1"): true, + "dir2": false, + filepath.Join("dir2", "anotherfile"): false, + "anotherfile": false, + }, + }, + { + selectFilter: func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { + return false, false + }, + fileState: map[string]bool{ + "dir": true, + filepath.Join("dir", "anotherfile"): true, + filepath.Join("dir", "file1"): true, + "dir2": true, + filepath.Join("dir2", "anotherfile"): true, + "anotherfile": true, + }, + }, + { + selectFilter: func(item string, isDir bool) (selectedForRestore bool, childMayBeSelected bool) { + switch item { + case filepath.FromSlash("/dir"): + selectedForRestore = true + case filepath.FromSlash("/dir2"): + selectedForRestore = true + } + return + }, + fileState: map[string]bool{ + "dir": true, + filepath.Join("dir", "anotherfile"): true, + filepath.Join("dir", "file1"): true, + "dir2": false, + filepath.Join("dir2", "anotherfile"): false, + "anotherfile": true, + }, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + res := NewRestorer(repo, sn, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + + res = NewRestorer(repo, deleteSn, Options{Delete: true}) + if test.selectFilter != nil { + res.SelectFilter = test.selectFilter + } + err = res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + + for fn, shouldExist := range test.fileState { + _, err := os.Stat(filepath.Join(tempdir, fn)) + if shouldExist { + rtest.OK(t, err) + } else { + rtest.Assert(t, errors.Is(err, os.ErrNotExist), "file %v: unexpected error got %v, expected ErrNotExist", fn, err) + } + } + }) + } +} + +func TestRestoreToFile(t *testing.T) { + snapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: foo\n"}, + }, + } + + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + + // create a file in the place of the target directory + rtest.OK(t, os.WriteFile(tempdir, []byte{}, 0o700)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) + res := NewRestorer(repo, sn, Options{}) + err := res.RestoreTo(ctx, tempdir) + rtest.Assert(t, strings.Contains(err.Error(), "cannot create target directory"), "unexpected error %v", err) +} diff --git a/mover-restic/restic/internal/restorer/restorer_unix.go b/mover-restic/restic/internal/restorer/restorer_unix.go new file mode 100644 index 000000000..7316f7b5d --- /dev/null +++ b/mover-restic/restic/internal/restorer/restorer_unix.go @@ -0,0 +1,10 @@ +//go:build !windows +// +build !windows + +package restorer + +// toComparableFilename returns a filename suitable for equality checks. On Windows, it returns the +// uppercase version of the string. On all other systems, it returns the unmodified filename. +func toComparableFilename(path string) string { + return path +} diff --git a/mover-restic/restic/internal/restorer/restorer_unix_test.go b/mover-restic/restic/internal/restorer/restorer_unix_test.go index 2c30a6b64..27d990af4 100644 --- a/mover-restic/restic/internal/restorer/restorer_unix_test.go +++ b/mover-restic/restic/internal/restorer/restorer_unix_test.go @@ -5,6 +5,7 @@ package restorer import ( "context" + "io/fs" "os" "path/filepath" "syscall" @@ -12,12 +13,11 @@ import ( "time" "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" restoreui "github.com/restic/restic/internal/ui/restore" ) -func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { +func TestRestorerRestoreEmptyHardlinkedFields(t *testing.T) { repo := repository.TestRepository(t) sn, _ := saveSnapshot(t, repo, Snapshot{ @@ -29,13 +29,9 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) - res := NewRestorer(repo, sn, false, nil) - - res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - return true, true - } + res := NewRestorer(repo, sn, Options{}) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) @@ -69,20 +65,15 @@ func getBlockCount(t *testing.T, filename string) int64 { return st.Blocks } -type printerMock struct { - filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 +func TestRestorerProgressBar(t *testing.T) { + testRestorerProgressBar(t, false) } -func (p *printerMock) Update(_, _, _, _ uint64, _ time.Duration) { -} -func (p *printerMock) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) { - p.filesFinished = filesFinished - p.filesTotal = filesTotal - p.allBytesWritten = allBytesWritten - p.allBytesTotal = allBytesTotal +func TestRestorerProgressBarDryRun(t *testing.T) { + testRestorerProgressBar(t, true) } -func TestRestorerProgressBar(t *testing.T) { +func testRestorerProgressBar(t *testing.T, dryRun bool) { repo := repository.TestRepository(t) sn, _ := saveSnapshot(t, repo, Snapshot{ @@ -95,14 +86,11 @@ func TestRestorerProgressBar(t *testing.T) { }, "file2": File{Links: 1, Inode: 2, Data: "example"}, }, - }) + }, noopGetGenericAttributes) mock := &printerMock{} progress := restoreui.NewProgress(mock, 0) - res := NewRestorer(repo, sn, false, progress) - res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { - return true, true - } + res := NewRestorer(repo, sn, Options{Progress: progress, DryRun: dryRun}) tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) @@ -112,12 +100,43 @@ func TestRestorerProgressBar(t *testing.T) { rtest.OK(t, err) progress.Finish() - const filesFinished = 4 - const filesTotal = filesFinished - const allBytesWritten = 10 - const allBytesTotal = allBytesWritten - rtest.Assert(t, mock.filesFinished == filesFinished, "filesFinished: expected %v, got %v", filesFinished, mock.filesFinished) - rtest.Assert(t, mock.filesTotal == filesTotal, "filesTotal: expected %v, got %v", filesTotal, mock.filesTotal) - rtest.Assert(t, mock.allBytesWritten == allBytesWritten, "allBytesWritten: expected %v, got %v", allBytesWritten, mock.allBytesWritten) - rtest.Assert(t, mock.allBytesTotal == allBytesTotal, "allBytesTotal: expected %v, got %v", allBytesTotal, mock.allBytesTotal) + rtest.Equals(t, restoreui.State{ + FilesFinished: 4, + FilesTotal: 4, + FilesSkipped: 0, + AllBytesWritten: 10, + AllBytesTotal: 10, + AllBytesSkipped: 0, + }, mock.s) +} + +func TestRestorePermissions(t *testing.T) { + snapshot := Snapshot{ + Nodes: map[string]Node{ + "foo": File{Data: "content: foo\n", Mode: 0o600, ModTime: time.Now()}, + }, + } + + repo := repository.TestRepository(t) + tempdir := filepath.Join(rtest.TempDir(t), "target") + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sn, id := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) + t.Logf("snapshot saved as %v", id.Str()) + + res := NewRestorer(repo, sn, Options{}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + + for _, overwrite := range []OverwriteBehavior{OverwriteIfChanged, OverwriteAlways} { + // tamper with permissions + path := filepath.Join(tempdir, "foo") + rtest.OK(t, os.Chmod(path, 0o700)) + + res = NewRestorer(repo, sn, Options{Overwrite: overwrite}) + rtest.OK(t, res.RestoreTo(ctx, tempdir)) + fi, err := os.Stat(path) + rtest.OK(t, err) + rtest.Equals(t, fs.FileMode(0o600), fi.Mode().Perm(), "unexpected permissions") + } } diff --git a/mover-restic/restic/internal/restorer/restorer_windows.go b/mover-restic/restic/internal/restorer/restorer_windows.go new file mode 100644 index 000000000..72337d8ae --- /dev/null +++ b/mover-restic/restic/internal/restorer/restorer_windows.go @@ -0,0 +1,13 @@ +//go:build windows +// +build windows + +package restorer + +import "strings" + +// toComparableFilename returns a filename suitable for equality checks. On Windows, it returns the +// uppercase version of the string. On all other systems, it returns the unmodified filename. +func toComparableFilename(path string) string { + // apparently NTFS internally uppercases filenames for comparision + return strings.ToUpper(path) +} diff --git a/mover-restic/restic/internal/restorer/restorer_windows_test.go b/mover-restic/restic/internal/restorer/restorer_windows_test.go index 3ec4b1f11..3f6c8472b 100644 --- a/mover-restic/restic/internal/restorer/restorer_windows_test.go +++ b/mover-restic/restic/internal/restorer/restorer_windows_test.go @@ -4,11 +4,21 @@ package restorer import ( + "context" + "encoding/json" "math" + "os" + "path" + "path/filepath" "syscall" "testing" + "time" "unsafe" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -33,3 +43,533 @@ func getBlockCount(t *testing.T, filename string) int64 { return int64(math.Ceil(float64(result) / 512)) } + +type DataStreamInfo struct { + name string + data string +} + +type NodeInfo struct { + DataStreamInfo + parentDir string + attributes FileAttributes + Exists bool + IsDirectory bool +} + +func TestFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, false) +} + +func TestEmptyFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, true) +} + +func testFileAttributeCombination(t *testing.T, isEmpty bool) { + t.Parallel() + //Generate combination of 5 attributes. + attributeCombinations := generateCombinations(5, []bool{}) + + fileName := "TestFile.txt" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required file information + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: false, + } + + //Get the current test name + testName := getCombinationTestName(fileInfo, fileName, fileInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, fileInfo.attributes) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } +} + +func generateCombinations(n int, prefix []bool) [][]bool { + if n == 0 { + // Return a slice containing the current permutation + return [][]bool{append([]bool{}, prefix...)} + } + + // Generate combinations with True + prefixTrue := append(prefix, true) + permsTrue := generateCombinations(n-1, prefixTrue) + + // Generate combinations with False + prefixFalse := append(prefix, false) + permsFalse := generateCombinations(n-1, prefixFalse) + + // Combine combinations with True and False + return append(permsTrue, permsFalse...) +} + +func getDataStreamInfo(isEmpty bool, fileName string) DataStreamInfo { + var dataStreamInfo DataStreamInfo + if isEmpty { + dataStreamInfo = DataStreamInfo{ + name: fileName, + } + } else { + dataStreamInfo = DataStreamInfo{ + name: fileName, + data: "Main file data stream.", + } + } + return dataStreamInfo +} + +func getFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + ReadOnly: values[0], + Hidden: values[1], + System: values[2], + Archive: values[3], + Encrypted: values[4], + } +} + +func getCombinationTestName(fi NodeInfo, fileName string, overwriteAttr FileAttributes) string { + if fi.attributes.ReadOnly { + fileName += "-ReadOnly" + } + if fi.attributes.Hidden { + fileName += "-Hidden" + } + if fi.attributes.System { + fileName += "-System" + } + if fi.attributes.Archive { + fileName += "-Archive" + } + if fi.attributes.Encrypted { + fileName += "-Encrypted" + } + if fi.Exists { + fileName += "-Overwrite" + if overwriteAttr.ReadOnly { + fileName += "-R" + } + if overwriteAttr.Hidden { + fileName += "-H" + } + if overwriteAttr.System { + fileName += "-S" + } + if overwriteAttr.Archive { + fileName += "-A" + } + if overwriteAttr.Encrypted { + fileName += "-E" + } + } + return fileName +} + +func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAttributes) string { + testDir := t.TempDir() + res, _ := setupWithFileAttributes(t, fileInfo, testDir, existingFileAttr) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, testDir) + rtest.OK(t, err) + + mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name) + //Verify restore + verifyFileAttributes(t, mainFilePath, fileInfo.attributes) + return mainFilePath +} + +func setupWithFileAttributes(t *testing.T, nodeInfo NodeInfo, testDir string, existingFileAttr FileAttributes) (*Restorer, []int) { + t.Helper() + if nodeInfo.Exists { + if !nodeInfo.IsDirectory { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir), os.ModeDir) + rtest.OK(t, err) + filepath := path.Join(testDir, nodeInfo.parentDir, nodeInfo.name) + if existingFileAttr.Encrypted { + err := createEncryptedFileWriteData(filepath, nodeInfo) + rtest.OK(t, err) + } else { + // Write the data to the file + file, err := os.OpenFile(path.Clean(filepath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + rtest.OK(t, err) + _, err = file.Write([]byte(nodeInfo.data)) + rtest.OK(t, err) + + err = file.Close() + rtest.OK(t, err) + } + } else { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name), os.ModeDir) + rtest.OK(t, err) + } + + pathPointer, err := syscall.UTF16PtrFromString(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name)) + rtest.OK(t, err) + syscall.SetFileAttributes(pathPointer, getAttributeValue(&existingFileAttr)) + } + + index := 0 + + order := []int{} + streams := []DataStreamInfo{} + if !nodeInfo.IsDirectory { + order = append(order, index) + index++ + streams = append(streams, nodeInfo.DataStreamInfo) + } + return setup(t, getNodes(nodeInfo.parentDir, nodeInfo.name, order, streams, nodeInfo.IsDirectory, &nodeInfo.attributes)), order +} + +func createEncryptedFileWriteData(filepath string, fileInfo NodeInfo) (err error) { + var ptr *uint16 + if ptr, err = windows.UTF16PtrFromString(filepath); err != nil { + return err + } + var handle windows.Handle + //Create the file with encrypted flag + if handle, err = windows.CreateFile(ptr, uint32(windows.GENERIC_READ|windows.GENERIC_WRITE), uint32(windows.FILE_SHARE_READ), nil, uint32(windows.CREATE_ALWAYS), windows.FILE_ATTRIBUTE_ENCRYPTED, 0); err != nil { + return err + } + //Write data to file + if _, err = windows.Write(handle, []byte(fileInfo.data)); err != nil { + return err + } + //Close handle + return windows.CloseHandle(handle) +} + +func setup(t *testing.T, nodesMap map[string]Node) *Restorer { + repo := repository.TestRepository(t) + getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + if attr == nil { + return + } + + fileattr := getAttributeValue(attr) + + if isDir { + //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes + fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY + } + attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + test.OK(t, err) + return attrs + } + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: nodesMap, + }, getFileAttributes) + res := NewRestorer(repo, sn, Options{}) + return res +} + +func getAttributeValue(attr *FileAttributes) uint32 { + var fileattr uint32 + if attr.ReadOnly { + fileattr |= windows.FILE_ATTRIBUTE_READONLY + } + if attr.Hidden { + fileattr |= windows.FILE_ATTRIBUTE_HIDDEN + } + if attr.Encrypted { + fileattr |= windows.FILE_ATTRIBUTE_ENCRYPTED + } + if attr.Archive { + fileattr |= windows.FILE_ATTRIBUTE_ARCHIVE + } + if attr.System { + fileattr |= windows.FILE_ATTRIBUTE_SYSTEM + } + return fileattr +} + +func getNodes(dir string, mainNodeName string, order []int, streams []DataStreamInfo, isDirectory bool, attributes *FileAttributes) map[string]Node { + var mode os.FileMode + if isDirectory { + mode = os.FileMode(2147484159) + } else { + if attributes != nil && attributes.ReadOnly { + mode = os.FileMode(0o444) + } else { + mode = os.FileMode(0o666) + } + } + + getFileNodes := func() map[string]Node { + nodes := map[string]Node{} + if isDirectory { + //Add a directory node at the same level as the other streams + nodes[mainNodeName] = Dir{ + ModTime: time.Now(), + attributes: attributes, + Mode: mode, + } + } + + if len(streams) > 0 { + for _, index := range order { + stream := streams[index] + + var attr *FileAttributes = nil + if mainNodeName == stream.name { + attr = attributes + } else if attributes != nil && attributes.Encrypted { + //Set encrypted attribute + attr = &FileAttributes{Encrypted: true} + } + + nodes[stream.name] = File{ + ModTime: time.Now(), + Data: stream.data, + Mode: mode, + attributes: attr, + } + } + } + return nodes + } + + return map[string]Node{ + dir: Dir{ + Mode: normalizeFileMode(0750 | mode), + ModTime: time.Now(), + Nodes: getFileNodes(), + }, + } +} + +func verifyFileAttributes(t *testing.T, mainFilePath string, attr FileAttributes) { + ptr, err := windows.UTF16PtrFromString(mainFilePath) + rtest.OK(t, err) + //Get file attributes using syscall + fileAttributes, err := syscall.GetFileAttributes(ptr) + rtest.OK(t, err) + //Test positive and negative scenarios + if attr.ReadOnly { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY != 0, "Expected read only attribute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY == 0, "Unexpected read only attribute.") + } + if attr.Hidden { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN != 0, "Expected hidden attribute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN == 0, "Unexpected hidden attribute.") + } + if attr.System { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM != 0, "Expected system attribute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM == 0, "Unexpected system attribute.") + } + if attr.Archive { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE != 0, "Expected archive attribute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE == 0, "Unexpected archive attribute.") + } + if attr.Encrypted { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED != 0, "Expected encrypted attribute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED == 0, "Unexpected encrypted attribute.") + } +} + +func verifyFileRestores(isEmpty bool, mainFilePath string, t *testing.T, fileInfo NodeInfo) { + if isEmpty { + _, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + } else { + + verifyMainFileRestore(t, mainFilePath, fileInfo) + } +} + +func verifyMainFileRestore(t *testing.T, mainFilePath string, fileInfo NodeInfo) { + fi, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + + size := fi.Size() + rtest.Assert(t, size > 0, "The file "+fileInfo.name+" exists but is empty") + + content, err := os.ReadFile(mainFilePath) + rtest.OK(t, err) + rtest.Assert(t, string(content) == fileInfo.data, "The file "+fileInfo.name+" exists but the content is not overwritten") +} + +func TestDirAttributeCombination(t *testing.T) { + t.Parallel() + attributeCombinations := generateCombinations(4, []bool{}) + + dirName := "TestDir" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required directory information + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: false, + IsDirectory: true, + } + + //Get the current test name + testName := getCombinationTestName(dirInfo, dirName, dirInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } +} + +func getDirFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + // readonly not valid for directories + Hidden: values[0], + System: values[1], + Archive: values[2], + Encrypted: values[3], + } +} + +func TestFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, false) +} + +func TestEmptyFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, true) +} + +func testFileAttributeCombinationsOverwrite(t *testing.T, isEmpty bool) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(5, []bool{}) + //Get overwrite file attribute combinations + overwriteCombinations := generateCombinations(5, []bool{}) + + fileName := "TestOverwriteFile" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: true, + } + + overwriteFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteFileAttributes = append(overwriteFileAttributes, getFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combination + for _, overwriteFileAttr := range overwriteFileAttributes { + //Get the test name + testName := getCombinationTestName(fileInfo, fileName, overwriteFileAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, overwriteFileAttr) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } + } +} + +func TestDirAttributeCombinationsOverwrite(t *testing.T) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(4, []bool{}) + //Get overwrite dir attribute combinations + overwriteCombinations := generateCombinations(4, []bool{}) + + dirName := "TestOverwriteDir" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: true, + IsDirectory: true, + } + + overwriteDirFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteDirFileAttributes = append(overwriteDirFileAttributes, getDirFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combinations + for _, overwriteDirAttr := range overwriteDirFileAttributes { + //Get the test name + testName := getCombinationTestName(dirInfo, dirName, overwriteDirAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } + } +} + +func TestRestoreDeleteCaseInsensitive(t *testing.T) { + repo := repository.TestRepository(t) + tempdir := rtest.TempDir(t) + + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: map[string]Node{ + "anotherfile": File{Data: "content: file\n"}, + }, + }, noopGetGenericAttributes) + + // should delete files that no longer exist in the snapshot + deleteSn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: map[string]Node{ + "AnotherfilE": File{Data: "content: file\n"}, + }, + }, noopGetGenericAttributes) + + res := NewRestorer(repo, sn, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + + res = NewRestorer(repo, deleteSn, Options{Delete: true}) + err = res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + + // anotherfile must still exist + _, err = os.Stat(filepath.Join(tempdir, "anotherfile")) + rtest.OK(t, err) +} diff --git a/mover-restic/restic/internal/restorer/sparsewrite.go b/mover-restic/restic/internal/restorer/sparsewrite.go index 2c1f234de..ae354f64f 100644 --- a/mover-restic/restic/internal/restorer/sparsewrite.go +++ b/mover-restic/restic/internal/restorer/sparsewrite.go @@ -1,6 +1,3 @@ -//go:build !windows -// +build !windows - package restorer import ( diff --git a/mover-restic/restic/internal/test/helpers.go b/mover-restic/restic/internal/test/helpers.go index 65e3e36ec..3387d36df 100644 --- a/mover-restic/restic/internal/test/helpers.go +++ b/mover-restic/restic/internal/test/helpers.go @@ -3,7 +3,9 @@ package test import ( "compress/bzip2" "compress/gzip" + "fmt" "io" + "math/rand" "os" "os/exec" "path/filepath" @@ -11,8 +13,6 @@ import ( "testing" "github.com/restic/restic/internal/errors" - - mrand "math/rand" ) // Assert fails the test if the condition is false. @@ -47,10 +47,22 @@ func OKs(tb testing.TB, errs []error) { } // Equals fails the test if exp is not equal to act. -func Equals(tb testing.TB, exp, act interface{}) { +// msg is optional message to be printed, first param being format string and rest being arguments. +func Equals(tb testing.TB, exp, act interface{}, msgs ...string) { tb.Helper() if !reflect.DeepEqual(exp, act) { - tb.Fatalf("\033[31m\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) + var msgString string + length := len(msgs) + if length == 1 { + msgString = msgs[0] + } else if length > 1 { + args := make([]interface{}, length-1) + for i, msg := range msgs[1:] { + args[i] = msg + } + msgString = fmt.Sprintf(msgs[0], args...) + } + tb.Fatalf("\033[31m\n\n\t"+msgString+"\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) } } @@ -58,7 +70,7 @@ func Equals(tb testing.TB, exp, act interface{}) { func Random(seed, count int) []byte { p := make([]byte, count) - rnd := mrand.New(mrand.NewSource(int64(seed))) + rnd := rand.New(rand.NewSource(int64(seed))) for i := 0; i < len(p); i += 8 { val := rnd.Int63() diff --git a/mover-restic/restic/internal/ui/backup/json.go b/mover-restic/restic/internal/ui/backup/json.go index 10f0e91fa..64b5de13b 100644 --- a/mover-restic/restic/internal/ui/backup/json.go +++ b/mover-restic/restic/internal/ui/backup/json.go @@ -163,7 +163,12 @@ func (b *JSONProgress) ReportTotal(start time.Time, s archiver.ScanStats) { } // Finish prints the finishing messages. -func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) { +func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) { + id := "" + // empty if snapshot creation was skipped + if !snapshotID.IsNull() { + id = snapshotID.String() + } b.print(summaryOutput{ MessageType: "summary", FilesNew: summary.Files.New, @@ -175,10 +180,11 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su DataBlobs: summary.ItemStats.DataBlobs, TreeBlobs: summary.ItemStats.TreeBlobs, DataAdded: summary.ItemStats.DataSize + summary.ItemStats.TreeSize, + DataAddedPacked: summary.ItemStats.DataSizeInRepo + summary.ItemStats.TreeSizeInRepo, TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged, TotalBytesProcessed: summary.ProcessedBytes, TotalDuration: time.Since(start).Seconds(), - SnapshotID: snapshotID.String(), + SnapshotID: id, DryRun: dryRun, }) } @@ -230,9 +236,10 @@ type summaryOutput struct { DataBlobs int `json:"data_blobs"` TreeBlobs int `json:"tree_blobs"` DataAdded uint64 `json:"data_added"` + DataAddedPacked uint64 `json:"data_added_packed"` TotalFilesProcessed uint `json:"total_files_processed"` TotalBytesProcessed uint64 `json:"total_bytes_processed"` TotalDuration float64 `json:"total_duration"` // in seconds - SnapshotID string `json:"snapshot_id"` + SnapshotID string `json:"snapshot_id,omitempty"` DryRun bool `json:"dry_run,omitempty"` } diff --git a/mover-restic/restic/internal/ui/backup/progress.go b/mover-restic/restic/internal/ui/backup/progress.go index 4362a8c83..1d494bf14 100644 --- a/mover-restic/restic/internal/ui/backup/progress.go +++ b/mover-restic/restic/internal/ui/backup/progress.go @@ -17,7 +17,7 @@ type ProgressPrinter interface { ScannerError(item string, err error) error CompleteItem(messageType string, item string, s archiver.ItemStats, d time.Duration) ReportTotal(start time.Time, s archiver.ScanStats) - Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) + Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) Reset() P(msg string, args ...interface{}) @@ -28,16 +28,6 @@ type Counter struct { Files, Dirs, Bytes uint64 } -type Summary struct { - Files, Dirs struct { - New uint - Changed uint - Unchanged uint - } - ProcessedBytes uint64 - archiver.ItemStats -} - // Progress reports progress for the `backup` command. type Progress struct { progress.Updater @@ -52,7 +42,6 @@ type Progress struct { processed, total Counter errors uint - summary Summary printer ProgressPrinter } @@ -63,7 +52,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { printer: printer, estimator: *newRateEstimator(time.Now()), } - p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) { + p.Updater = *progress.NewUpdater(interval, func(_ time.Duration, final bool) { if final { p.printer.Reset() } else { @@ -126,16 +115,6 @@ func (p *Progress) CompleteBlob(bytes uint64) { // CompleteItem is the status callback function for the archiver when a // file/dir has been saved successfully. func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) { - p.mu.Lock() - p.summary.ItemStats.Add(s) - - // for the last item "/", current is nil - if current != nil { - p.summary.ProcessedBytes += current.Size - } - - p.mu.Unlock() - if current == nil { // error occurred, tell the status display to remove the line p.mu.Lock() @@ -153,21 +132,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a switch { case previous == nil: p.printer.CompleteItem("dir new", item, s, d) - p.mu.Lock() - p.summary.Dirs.New++ - p.mu.Unlock() - case previous.Equals(*current): p.printer.CompleteItem("dir unchanged", item, s, d) - p.mu.Lock() - p.summary.Dirs.Unchanged++ - p.mu.Unlock() - default: p.printer.CompleteItem("dir modified", item, s, d) - p.mu.Lock() - p.summary.Dirs.Changed++ - p.mu.Unlock() } case "file": @@ -179,21 +147,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a switch { case previous == nil: p.printer.CompleteItem("file new", item, s, d) - p.mu.Lock() - p.summary.Files.New++ - p.mu.Unlock() - case previous.Equals(*current): p.printer.CompleteItem("file unchanged", item, s, d) - p.mu.Lock() - p.summary.Files.Unchanged++ - p.mu.Unlock() - default: p.printer.CompleteItem("file modified", item, s, d) - p.mu.Lock() - p.summary.Files.Changed++ - p.mu.Unlock() } } } @@ -213,8 +170,8 @@ func (p *Progress) ReportTotal(item string, s archiver.ScanStats) { } // Finish prints the finishing messages. -func (p *Progress) Finish(snapshotID restic.ID, dryrun bool) { +func (p *Progress) Finish(snapshotID restic.ID, summary *archiver.Summary, dryrun bool) { // wait for the status update goroutine to shut down p.Updater.Done() - p.printer.Finish(snapshotID, p.start, &p.summary, dryrun) + p.printer.Finish(snapshotID, p.start, summary, dryrun) } diff --git a/mover-restic/restic/internal/ui/backup/progress_test.go b/mover-restic/restic/internal/ui/backup/progress_test.go index 79a56c91e..6b242a0f3 100644 --- a/mover-restic/restic/internal/ui/backup/progress_test.go +++ b/mover-restic/restic/internal/ui/backup/progress_test.go @@ -33,11 +33,10 @@ func (p *mockPrinter) CompleteItem(messageType string, _ string, _ archiver.Item } func (p *mockPrinter) ReportTotal(_ time.Time, _ archiver.ScanStats) {} -func (p *mockPrinter) Finish(id restic.ID, _ time.Time, summary *Summary, _ bool) { +func (p *mockPrinter) Finish(id restic.ID, _ time.Time, _ *archiver.Summary, _ bool) { p.Lock() defer p.Unlock() - _ = *summary // Should not be nil. p.id = id } @@ -64,7 +63,7 @@ func TestProgress(t *testing.T) { time.Sleep(10 * time.Millisecond) id := restic.NewRandomID() - prog.Finish(id, false) + prog.Finish(id, nil, false) if !prnt.dirUnchanged { t.Error(`"dir unchanged" event not seen`) diff --git a/mover-restic/restic/internal/ui/backup/text.go b/mover-restic/restic/internal/ui/backup/text.go index 215982cd4..f96746739 100644 --- a/mover-restic/restic/internal/ui/backup/text.go +++ b/mover-restic/restic/internal/ui/backup/text.go @@ -121,12 +121,12 @@ func (b *TextProgress) ReportTotal(start time.Time, s archiver.ScanStats) { // Reset status func (b *TextProgress) Reset() { if b.term.CanUpdateStatus() { - b.term.SetStatus([]string{""}) + b.term.SetStatus(nil) } } // Finish prints the finishing messages. -func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *Summary, dryRun bool) { +func (b *TextProgress) Finish(id restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) { b.P("\n") b.P("Files: %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged) b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged) @@ -145,4 +145,12 @@ func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *Summary, dr ui.FormatBytes(summary.ProcessedBytes), ui.FormatDuration(time.Since(start)), ) + + if !dryRun { + if id.IsNull() { + b.P("skipped creating snapshot\n") + } else { + b.P("snapshot %s saved\n", id.Str()) + } + } } diff --git a/mover-restic/restic/internal/ui/format.go b/mover-restic/restic/internal/ui/format.go index d2e0a4d2b..de650607d 100644 --- a/mover-restic/restic/internal/ui/format.go +++ b/mover-restic/restic/internal/ui/format.go @@ -8,6 +8,8 @@ import ( "math/bits" "strconv" "time" + + "golang.org/x/text/width" ) func FormatBytes(c uint64) string { @@ -105,3 +107,24 @@ func ToJSONString(status interface{}) string { } return buf.String() } + +// TerminalDisplayWidth returns the number of terminal cells needed to display s +func TerminalDisplayWidth(s string) int { + width := 0 + for _, r := range s { + width += terminalDisplayRuneWidth(r) + } + + return width +} + +func terminalDisplayRuneWidth(r rune) int { + switch width.LookupRune(r).Kind() { + case width.EastAsianWide, width.EastAsianFullwidth: + return 2 + case width.EastAsianNarrow, width.EastAsianHalfwidth, width.EastAsianAmbiguous, width.Neutral: + return 1 + default: + return 0 + } +} diff --git a/mover-restic/restic/internal/ui/format_test.go b/mover-restic/restic/internal/ui/format_test.go index 4223d4e20..d595026c4 100644 --- a/mover-restic/restic/internal/ui/format_test.go +++ b/mover-restic/restic/internal/ui/format_test.go @@ -84,3 +84,21 @@ func TestParseBytesInvalid(t *testing.T) { test.Equals(t, int64(0), v) } } + +func TestTerminalDisplayWidth(t *testing.T) { + for _, c := range []struct { + input string + want int + }{ + {"foo", 3}, + {"aéb", 3}, + {"ab", 3}, + {"a’b", 3}, + {"aあb", 4}, + } { + if got := TerminalDisplayWidth(c.input); got != c.want { + t.Errorf("wrong display width for '%s', want %d, got %d", c.input, c.want, got) + } + } + +} diff --git a/mover-restic/restic/internal/ui/message.go b/mover-restic/restic/internal/ui/message.go index 75e54b019..38cdaf301 100644 --- a/mover-restic/restic/internal/ui/message.go +++ b/mover-restic/restic/internal/ui/message.go @@ -1,6 +1,10 @@ package ui -import "github.com/restic/restic/internal/ui/termstatus" +import ( + "fmt" + + "github.com/restic/restic/internal/ui/termstatus" +) // Message reports progress with messages of different verbosity. type Message struct { @@ -19,27 +23,27 @@ func NewMessage(term *termstatus.Terminal, verbosity uint) *Message { // E reports an error func (m *Message) E(msg string, args ...interface{}) { - m.term.Errorf(msg, args...) + m.term.Error(fmt.Sprintf(msg, args...)) } // P prints a message if verbosity >= 1, this is used for normal messages which // are not errors. func (m *Message) P(msg string, args ...interface{}) { if m.v >= 1 { - m.term.Printf(msg, args...) + m.term.Print(fmt.Sprintf(msg, args...)) } } // V prints a message if verbosity >= 2, this is used for verbose messages. func (m *Message) V(msg string, args ...interface{}) { if m.v >= 2 { - m.term.Printf(msg, args...) + m.term.Print(fmt.Sprintf(msg, args...)) } } // VV prints a message if verbosity >= 3, this is used for debug messages. func (m *Message) VV(msg string, args ...interface{}) { if m.v >= 3 { - m.term.Printf(msg, args...) + m.term.Print(fmt.Sprintf(msg, args...)) } } diff --git a/mover-restic/restic/internal/ui/progress/printer.go b/mover-restic/restic/internal/ui/progress/printer.go new file mode 100644 index 000000000..a2bc4c4b5 --- /dev/null +++ b/mover-restic/restic/internal/ui/progress/printer.go @@ -0,0 +1,65 @@ +package progress + +import "testing" + +// A Printer can can return a new counter or print messages +// at different log levels. +// It must be safe to call its methods from concurrent goroutines. +type Printer interface { + NewCounter(description string) *Counter + + E(msg string, args ...interface{}) + P(msg string, args ...interface{}) + V(msg string, args ...interface{}) + VV(msg string, args ...interface{}) +} + +// NoopPrinter discards all messages +type NoopPrinter struct{} + +var _ Printer = (*NoopPrinter)(nil) + +func (*NoopPrinter) NewCounter(_ string) *Counter { + return nil +} + +func (*NoopPrinter) E(_ string, _ ...interface{}) {} + +func (*NoopPrinter) P(_ string, _ ...interface{}) {} + +func (*NoopPrinter) V(_ string, _ ...interface{}) {} + +func (*NoopPrinter) VV(_ string, _ ...interface{}) {} + +// TestPrinter prints messages during testing +type TestPrinter struct { + t testing.TB +} + +func NewTestPrinter(t testing.TB) *TestPrinter { + return &TestPrinter{ + t: t, + } +} + +var _ Printer = (*TestPrinter)(nil) + +func (p *TestPrinter) NewCounter(_ string) *Counter { + return nil +} + +func (p *TestPrinter) E(msg string, args ...interface{}) { + p.t.Logf("error: "+msg, args...) +} + +func (p *TestPrinter) P(msg string, args ...interface{}) { + p.t.Logf("print: "+msg, args...) +} + +func (p *TestPrinter) V(msg string, args ...interface{}) { + p.t.Logf("verbose: "+msg, args...) +} + +func (p *TestPrinter) VV(msg string, args ...interface{}) { + p.t.Logf("verbose2: "+msg, args...) +} diff --git a/mover-restic/restic/internal/ui/restore/json.go b/mover-restic/restic/internal/ui/restore/json.go index c1b95b00b..c248a7951 100644 --- a/mover-restic/restic/internal/ui/restore/json.go +++ b/mover-restic/restic/internal/ui/restore/json.go @@ -7,12 +7,14 @@ import ( ) type jsonPrinter struct { - terminal term + terminal term + verbosity uint } -func NewJSONProgress(terminal term) ProgressPrinter { +func NewJSONProgress(terminal term, verbosity uint) ProgressPrinter { return &jsonPrinter{ - terminal: terminal, + terminal: terminal, + verbosity: verbosity, } } @@ -20,31 +22,67 @@ func (t *jsonPrinter) print(status interface{}) { t.terminal.Print(ui.ToJSONString(status)) } -func (t *jsonPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { +func (t *jsonPrinter) Update(p State, duration time.Duration) { status := statusUpdate{ MessageType: "status", SecondsElapsed: uint64(duration / time.Second), - TotalFiles: filesTotal, - FilesRestored: filesFinished, - TotalBytes: allBytesTotal, - BytesRestored: allBytesWritten, + TotalFiles: p.FilesTotal, + FilesRestored: p.FilesFinished, + FilesSkipped: p.FilesSkipped, + TotalBytes: p.AllBytesTotal, + BytesRestored: p.AllBytesWritten, + BytesSkipped: p.AllBytesSkipped, } - if allBytesTotal > 0 { - status.PercentDone = float64(allBytesWritten) / float64(allBytesTotal) + if p.AllBytesTotal > 0 { + status.PercentDone = float64(p.AllBytesWritten) / float64(p.AllBytesTotal) } t.print(status) } -func (t *jsonPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { +func (t *jsonPrinter) CompleteItem(messageType ItemAction, item string, size uint64) { + if t.verbosity < 3 { + return + } + + var action string + switch messageType { + case ActionDirRestored: + action = "restored" + case ActionFileRestored: + action = "restored" + case ActionOtherRestored: + action = "restored" + case ActionFileUpdated: + action = "updated" + case ActionFileUnchanged: + action = "unchanged" + case ActionDeleted: + action = "deleted" + default: + panic("unknown message type") + } + + status := verboseUpdate{ + MessageType: "verbose_status", + Action: action, + Item: item, + Size: size, + } + t.print(status) +} + +func (t *jsonPrinter) Finish(p State, duration time.Duration) { status := summaryOutput{ MessageType: "summary", SecondsElapsed: uint64(duration / time.Second), - TotalFiles: filesTotal, - FilesRestored: filesFinished, - TotalBytes: allBytesTotal, - BytesRestored: allBytesWritten, + TotalFiles: p.FilesTotal, + FilesRestored: p.FilesFinished, + FilesSkipped: p.FilesSkipped, + TotalBytes: p.AllBytesTotal, + BytesRestored: p.AllBytesWritten, + BytesSkipped: p.AllBytesSkipped, } t.print(status) } @@ -55,8 +93,17 @@ type statusUpdate struct { PercentDone float64 `json:"percent_done"` TotalFiles uint64 `json:"total_files,omitempty"` FilesRestored uint64 `json:"files_restored,omitempty"` + FilesSkipped uint64 `json:"files_skipped,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"` BytesRestored uint64 `json:"bytes_restored,omitempty"` + BytesSkipped uint64 `json:"bytes_skipped,omitempty"` +} + +type verboseUpdate struct { + MessageType string `json:"message_type"` // "verbose_status" + Action string `json:"action"` + Item string `json:"item"` + Size uint64 `json:"size"` } type summaryOutput struct { @@ -64,6 +111,8 @@ type summaryOutput struct { SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` TotalFiles uint64 `json:"total_files,omitempty"` FilesRestored uint64 `json:"files_restored,omitempty"` + FilesSkipped uint64 `json:"files_skipped,omitempty"` TotalBytes uint64 `json:"total_bytes,omitempty"` BytesRestored uint64 `json:"bytes_restored,omitempty"` + BytesSkipped uint64 `json:"bytes_skipped,omitempty"` } diff --git a/mover-restic/restic/internal/ui/restore/json_test.go b/mover-restic/restic/internal/ui/restore/json_test.go index 7bcabb4d7..06a70d5dc 100644 --- a/mover-restic/restic/internal/ui/restore/json_test.go +++ b/mover-restic/restic/internal/ui/restore/json_test.go @@ -7,23 +7,56 @@ import ( "github.com/restic/restic/internal/test" ) -func TestJSONPrintUpdate(t *testing.T) { +func createJSONProgress() (*mockTerm, ProgressPrinter) { term := &mockTerm{} - printer := NewJSONProgress(term) - printer.Update(3, 11, 29, 47, 5*time.Second) + printer := NewJSONProgress(term, 3) + return term, printer +} + +func TestJSONPrintUpdate(t *testing.T) { + term, printer := createJSONProgress() + printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) } +func TestJSONPrintUpdateWithSkipped(t *testing.T) { + term, printer := createJSONProgress() + printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.output) +} + func TestJSONPrintSummaryOnSuccess(t *testing.T) { - term := &mockTerm{} - printer := NewJSONProgress(term) - printer.Finish(11, 11, 47, 47, 5*time.Second) + term, printer := createJSONProgress() + printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second) test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output) } func TestJSONPrintSummaryOnErrors(t *testing.T) { - term := &mockTerm{} - printer := NewJSONProgress(term) - printer.Finish(3, 11, 29, 47, 5*time.Second) + term, printer := createJSONProgress() + printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) } + +func TestJSONPrintSummaryOnSuccessWithSkipped(t *testing.T) { + term, printer := createJSONProgress() + printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.output) +} + +func TestJSONPrintCompleteItem(t *testing.T) { + for _, data := range []struct { + action ItemAction + size uint64 + expected string + }{ + {ActionDirRestored, 0, "{\"message_type\":\"verbose_status\",\"action\":\"restored\",\"item\":\"test\",\"size\":0}\n"}, + {ActionFileRestored, 123, "{\"message_type\":\"verbose_status\",\"action\":\"restored\",\"item\":\"test\",\"size\":123}\n"}, + {ActionFileUpdated, 123, "{\"message_type\":\"verbose_status\",\"action\":\"updated\",\"item\":\"test\",\"size\":123}\n"}, + {ActionFileUnchanged, 123, "{\"message_type\":\"verbose_status\",\"action\":\"unchanged\",\"item\":\"test\",\"size\":123}\n"}, + {ActionDeleted, 0, "{\"message_type\":\"verbose_status\",\"action\":\"deleted\",\"item\":\"test\",\"size\":0}\n"}, + } { + term, printer := createJSONProgress() + printer.CompleteItem(data.action, "test", data.size) + test.Equals(t, []string{data.expected}, term.output) + } +} diff --git a/mover-restic/restic/internal/ui/restore/progress.go b/mover-restic/restic/internal/ui/restore/progress.go index f2bd5d38b..67b15f07e 100644 --- a/mover-restic/restic/internal/ui/restore/progress.go +++ b/mover-restic/restic/internal/ui/restore/progress.go @@ -7,15 +7,21 @@ import ( "github.com/restic/restic/internal/ui/progress" ) +type State struct { + FilesFinished uint64 + FilesTotal uint64 + FilesSkipped uint64 + AllBytesWritten uint64 + AllBytesTotal uint64 + AllBytesSkipped uint64 +} + type Progress struct { updater progress.Updater m sync.Mutex progressInfoMap map[string]progressInfoEntry - filesFinished uint64 - filesTotal uint64 - allBytesWritten uint64 - allBytesTotal uint64 + s State started time.Time printer ProgressPrinter @@ -32,10 +38,23 @@ type term interface { } type ProgressPrinter interface { - Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) - Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) + Update(progress State, duration time.Duration) + CompleteItem(action ItemAction, item string, size uint64) + Finish(progress State, duration time.Duration) } +type ItemAction string + +// Constants for the different CompleteItem actions. +const ( + ActionDirRestored ItemAction = "dir restored" + ActionFileRestored ItemAction = "file restored" + ActionFileUpdated ItemAction = "file updated" + ActionFileUnchanged ItemAction = "file unchanged" + ActionOtherRestored ItemAction = "other restored" + ActionDeleted ItemAction = "deleted" +) + func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { p := &Progress{ progressInfoMap: make(map[string]progressInfoEntry), @@ -51,23 +70,31 @@ func (p *Progress) update(runtime time.Duration, final bool) { defer p.m.Unlock() if !final { - p.printer.Update(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) + p.printer.Update(p.s, runtime) } else { - p.printer.Finish(p.filesFinished, p.filesTotal, p.allBytesWritten, p.allBytesTotal, runtime) + p.printer.Finish(p.s, runtime) } } // AddFile starts tracking a new file with the given size func (p *Progress) AddFile(size uint64) { + if p == nil { + return + } + p.m.Lock() defer p.m.Unlock() - p.filesTotal++ - p.allBytesTotal += size + p.s.FilesTotal++ + p.s.AllBytesTotal += size } // AddProgress accumulates the number of bytes written for a file -func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTotal uint64) { +func (p *Progress) AddProgress(name string, action ItemAction, bytesWrittenPortion uint64, bytesTotal uint64) { + if p == nil { + return + } + p.m.Lock() defer p.m.Unlock() @@ -78,11 +105,38 @@ func (p *Progress) AddProgress(name string, bytesWrittenPortion uint64, bytesTot entry.bytesWritten += bytesWrittenPortion p.progressInfoMap[name] = entry - p.allBytesWritten += bytesWrittenPortion + p.s.AllBytesWritten += bytesWrittenPortion if entry.bytesWritten == entry.bytesTotal { delete(p.progressInfoMap, name) - p.filesFinished++ + p.s.FilesFinished++ + + p.printer.CompleteItem(action, name, bytesTotal) + } +} + +func (p *Progress) AddSkippedFile(name string, size uint64) { + if p == nil { + return } + + p.m.Lock() + defer p.m.Unlock() + + p.s.FilesSkipped++ + p.s.AllBytesSkipped += size + + p.printer.CompleteItem(ActionFileUnchanged, name, size) +} + +func (p *Progress) ReportDeletedFile(name string) { + if p == nil { + return + } + + p.m.Lock() + defer p.m.Unlock() + + p.printer.CompleteItem(ActionDeleted, name, 0) } func (p *Progress) Finish() { diff --git a/mover-restic/restic/internal/ui/restore/progress_test.go b/mover-restic/restic/internal/ui/restore/progress_test.go index 9e625aa20..4a6304741 100644 --- a/mover-restic/restic/internal/ui/restore/progress_test.go +++ b/mover-restic/restic/internal/ui/restore/progress_test.go @@ -8,7 +8,7 @@ import ( ) type printerTraceEntry struct { - filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64 + progress State duration time.Duration isFinished bool @@ -16,122 +16,177 @@ type printerTraceEntry struct { type printerTrace []printerTraceEntry +type itemTraceEntry struct { + action ItemAction + item string + size uint64 +} + +type itemTrace []itemTraceEntry type mockPrinter struct { trace printerTrace + items itemTrace } const mockFinishDuration = 42 * time.Second -func (p *mockPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { - p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, duration, false}) +func (p *mockPrinter) Update(progress State, duration time.Duration) { + p.trace = append(p.trace, printerTraceEntry{progress, duration, false}) +} +func (p *mockPrinter) CompleteItem(action ItemAction, item string, size uint64) { + p.items = append(p.items, itemTraceEntry{action, item, size}) } -func (p *mockPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, _ time.Duration) { - p.trace = append(p.trace, printerTraceEntry{filesFinished, filesTotal, allBytesWritten, allBytesTotal, mockFinishDuration, true}) +func (p *mockPrinter) Finish(progress State, _ time.Duration) { + p.trace = append(p.trace, printerTraceEntry{progress, mockFinishDuration, true}) } -func testProgress(fn func(progress *Progress) bool) printerTrace { +func testProgress(fn func(progress *Progress) bool) (printerTrace, itemTrace) { printer := &mockPrinter{} progress := NewProgress(printer, 0) final := fn(progress) progress.update(0, final) trace := append(printerTrace{}, printer.trace...) + items := append(itemTrace{}, printer.items...) // cleanup to avoid goroutine leak, but copy trace first progress.Finish() - return trace + return trace, items } func TestNew(t *testing.T) { - result := testProgress(func(progress *Progress) bool { + result, items := testProgress(func(progress *Progress) bool { return false }) test.Equals(t, printerTrace{ - printerTraceEntry{0, 0, 0, 0, 0, false}, + printerTraceEntry{State{0, 0, 0, 0, 0, 0}, 0, false}, }, result) + test.Equals(t, itemTrace{}, items) } func TestAddFile(t *testing.T) { fileSize := uint64(100) - result := testProgress(func(progress *Progress) bool { + result, items := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) return false }) test.Equals(t, printerTrace{ - printerTraceEntry{0, 1, 0, fileSize, 0, false}, + printerTraceEntry{State{0, 1, 0, 0, fileSize, 0}, 0, false}, }, result) + test.Equals(t, itemTrace{}, items) } func TestFirstProgressOnAFile(t *testing.T) { expectedBytesWritten := uint64(5) expectedBytesTotal := uint64(100) - result := testProgress(func(progress *Progress) bool { + result, items := testProgress(func(progress *Progress) bool { progress.AddFile(expectedBytesTotal) - progress.AddProgress("test", expectedBytesWritten, expectedBytesTotal) + progress.AddProgress("test", ActionFileUpdated, expectedBytesWritten, expectedBytesTotal) return false }) test.Equals(t, printerTrace{ - printerTraceEntry{0, 1, expectedBytesWritten, expectedBytesTotal, 0, false}, + printerTraceEntry{State{0, 1, 0, expectedBytesWritten, expectedBytesTotal, 0}, 0, false}, }, result) + test.Equals(t, itemTrace{}, items) } func TestLastProgressOnAFile(t *testing.T) { fileSize := uint64(100) - result := testProgress(func(progress *Progress) bool { + result, items := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) - progress.AddProgress("test", 30, fileSize) - progress.AddProgress("test", 35, fileSize) - progress.AddProgress("test", 35, fileSize) + progress.AddProgress("test", ActionFileUpdated, 30, fileSize) + progress.AddProgress("test", ActionFileUpdated, 35, fileSize) + progress.AddProgress("test", ActionFileUpdated, 35, fileSize) return false }) test.Equals(t, printerTrace{ - printerTraceEntry{1, 1, fileSize, fileSize, 0, false}, + printerTraceEntry{State{1, 1, 0, fileSize, fileSize, 0}, 0, false}, }, result) + test.Equals(t, itemTrace{ + itemTraceEntry{action: ActionFileUpdated, item: "test", size: fileSize}, + }, items) } func TestLastProgressOnLastFile(t *testing.T) { fileSize := uint64(100) - result := testProgress(func(progress *Progress) bool { + result, items := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(50) - progress.AddProgress("test1", 50, 50) - progress.AddProgress("test2", 50, fileSize) - progress.AddProgress("test2", 50, fileSize) + progress.AddProgress("test1", ActionFileUpdated, 50, 50) + progress.AddProgress("test2", ActionFileUpdated, 50, fileSize) + progress.AddProgress("test2", ActionFileUpdated, 50, fileSize) return false }) test.Equals(t, printerTrace{ - printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, 0, false}, + printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, 0, false}, }, result) + test.Equals(t, itemTrace{ + itemTraceEntry{action: ActionFileUpdated, item: "test1", size: 50}, + itemTraceEntry{action: ActionFileUpdated, item: "test2", size: fileSize}, + }, items) } func TestSummaryOnSuccess(t *testing.T) { fileSize := uint64(100) - result := testProgress(func(progress *Progress) bool { + result, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(50) - progress.AddProgress("test1", 50, 50) - progress.AddProgress("test2", fileSize, fileSize) + progress.AddProgress("test1", ActionFileUpdated, 50, 50) + progress.AddProgress("test2", ActionFileUpdated, fileSize, fileSize) return true }) test.Equals(t, printerTrace{ - printerTraceEntry{2, 2, 50 + fileSize, 50 + fileSize, mockFinishDuration, true}, + printerTraceEntry{State{2, 2, 0, 50 + fileSize, 50 + fileSize, 0}, mockFinishDuration, true}, }, result) } func TestSummaryOnErrors(t *testing.T) { fileSize := uint64(100) - result := testProgress(func(progress *Progress) bool { + result, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(50) - progress.AddProgress("test1", 50, 50) - progress.AddProgress("test2", fileSize/2, fileSize) + progress.AddProgress("test1", ActionFileUpdated, 50, 50) + progress.AddProgress("test2", ActionFileUpdated, fileSize/2, fileSize) + return true + }) + test.Equals(t, printerTrace{ + printerTraceEntry{State{1, 2, 0, 50 + fileSize/2, 50 + fileSize, 0}, mockFinishDuration, true}, + }, result) +} + +func TestSkipFile(t *testing.T) { + fileSize := uint64(100) + + result, items := testProgress(func(progress *Progress) bool { + progress.AddSkippedFile("test", fileSize) return true }) test.Equals(t, printerTrace{ - printerTraceEntry{1, 2, 50 + fileSize/2, 50 + fileSize, mockFinishDuration, true}, + printerTraceEntry{State{0, 0, 1, 0, 0, fileSize}, mockFinishDuration, true}, }, result) + test.Equals(t, itemTrace{ + itemTraceEntry{ActionFileUnchanged, "test", fileSize}, + }, items) +} + +func TestProgressTypes(t *testing.T) { + fileSize := uint64(100) + + _, items := testProgress(func(progress *Progress) bool { + progress.AddFile(fileSize) + progress.AddFile(0) + progress.AddProgress("dir", ActionDirRestored, fileSize, fileSize) + progress.AddProgress("new", ActionFileRestored, 0, 0) + progress.ReportDeletedFile("del") + return true + }) + test.Equals(t, itemTrace{ + itemTraceEntry{ActionDirRestored, "dir", fileSize}, + itemTraceEntry{ActionFileRestored, "new", 0}, + itemTraceEntry{ActionDeleted, "del", 0}, + }, items) } diff --git a/mover-restic/restic/internal/ui/restore/text.go b/mover-restic/restic/internal/ui/restore/text.go index 2647bb28b..ec512f369 100644 --- a/mover-restic/restic/internal/ui/restore/text.go +++ b/mover-restic/restic/internal/ui/restore/text.go @@ -8,39 +8,77 @@ import ( ) type textPrinter struct { - terminal term + terminal term + verbosity uint } -func NewTextProgress(terminal term) ProgressPrinter { +func NewTextProgress(terminal term, verbosity uint) ProgressPrinter { return &textPrinter{ - terminal: terminal, + terminal: terminal, + verbosity: verbosity, } } -func (t *textPrinter) Update(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { +func (t *textPrinter) Update(p State, duration time.Duration) { timeLeft := ui.FormatDuration(duration) - formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) - formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) - allPercent := ui.FormatPercent(allBytesWritten, allBytesTotal) + formattedAllBytesWritten := ui.FormatBytes(p.AllBytesWritten) + formattedAllBytesTotal := ui.FormatBytes(p.AllBytesTotal) + allPercent := ui.FormatPercent(p.AllBytesWritten, p.AllBytesTotal) progress := fmt.Sprintf("[%s] %s %v files/dirs %s, total %v files/dirs %v", - timeLeft, allPercent, filesFinished, formattedAllBytesWritten, filesTotal, formattedAllBytesTotal) + timeLeft, allPercent, p.FilesFinished, formattedAllBytesWritten, p.FilesTotal, formattedAllBytesTotal) + if p.FilesSkipped > 0 { + progress += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped)) + } t.terminal.SetStatus([]string{progress}) } -func (t *textPrinter) Finish(filesFinished, filesTotal, allBytesWritten, allBytesTotal uint64, duration time.Duration) { - t.terminal.SetStatus([]string{}) +func (t *textPrinter) CompleteItem(messageType ItemAction, item string, size uint64) { + if t.verbosity < 3 { + return + } + + var action string + switch messageType { + case ActionDirRestored: + action = "restored" + case ActionFileRestored: + action = "restored" + case ActionOtherRestored: + action = "restored" + case ActionFileUpdated: + action = "updated" + case ActionFileUnchanged: + action = "unchanged" + case ActionDeleted: + action = "deleted" + default: + panic("unknown message type") + } + + if messageType == ActionDirRestored || messageType == ActionOtherRestored || messageType == ActionDeleted { + t.terminal.Print(fmt.Sprintf("%-9v %v", action, item)) + } else { + t.terminal.Print(fmt.Sprintf("%-9v %v with size %v", action, item, ui.FormatBytes(size))) + } +} + +func (t *textPrinter) Finish(p State, duration time.Duration) { + t.terminal.SetStatus(nil) timeLeft := ui.FormatDuration(duration) - formattedAllBytesTotal := ui.FormatBytes(allBytesTotal) + formattedAllBytesTotal := ui.FormatBytes(p.AllBytesTotal) var summary string - if filesFinished == filesTotal && allBytesWritten == allBytesTotal { - summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", filesTotal, formattedAllBytesTotal, timeLeft) + if p.FilesFinished == p.FilesTotal && p.AllBytesWritten == p.AllBytesTotal { + summary = fmt.Sprintf("Summary: Restored %d files/dirs (%s) in %s", p.FilesTotal, formattedAllBytesTotal, timeLeft) } else { - formattedAllBytesWritten := ui.FormatBytes(allBytesWritten) + formattedAllBytesWritten := ui.FormatBytes(p.AllBytesWritten) summary = fmt.Sprintf("Summary: Restored %d / %d files/dirs (%s / %s) in %s", - filesFinished, filesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) + p.FilesFinished, p.FilesTotal, formattedAllBytesWritten, formattedAllBytesTotal, timeLeft) + } + if p.FilesSkipped > 0 { + summary += fmt.Sprintf(", skipped %v files/dirs %v", p.FilesSkipped, ui.FormatBytes(p.AllBytesSkipped)) } t.terminal.Print(summary) diff --git a/mover-restic/restic/internal/ui/restore/text_test.go b/mover-restic/restic/internal/ui/restore/text_test.go index fc03904ff..b198a27df 100644 --- a/mover-restic/restic/internal/ui/restore/text_test.go +++ b/mover-restic/restic/internal/ui/restore/text_test.go @@ -19,23 +19,57 @@ func (m *mockTerm) SetStatus(lines []string) { m.output = append([]string{}, lines...) } -func TestPrintUpdate(t *testing.T) { +func createTextProgress() (*mockTerm, ProgressPrinter) { term := &mockTerm{} - printer := NewTextProgress(term) - printer.Update(3, 11, 29, 47, 5*time.Second) + printer := NewTextProgress(term, 3) + return term, printer +} + +func TestPrintUpdate(t *testing.T) { + term, printer := createTextProgress() + printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output) } +func TestPrintUpdateWithSkipped(t *testing.T) { + term, printer := createTextProgress() + printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second) + test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.output) +} + func TestPrintSummaryOnSuccess(t *testing.T) { - term := &mockTerm{} - printer := NewTextProgress(term) - printer.Finish(11, 11, 47, 47, 5*time.Second) + term, printer := createTextProgress() + printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second) test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output) } func TestPrintSummaryOnErrors(t *testing.T) { - term := &mockTerm{} - printer := NewTextProgress(term) - printer.Finish(3, 11, 29, 47, 5*time.Second) + term, printer := createTextProgress() + printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second) test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output) } + +func TestPrintSummaryOnSuccessWithSkipped(t *testing.T) { + term, printer := createTextProgress() + printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second) + test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs 59 B"}, term.output) +} + +func TestPrintCompleteItem(t *testing.T) { + for _, data := range []struct { + action ItemAction + size uint64 + expected string + }{ + {ActionDirRestored, 0, "restored test"}, + {ActionFileRestored, 123, "restored test with size 123 B"}, + {ActionOtherRestored, 0, "restored test"}, + {ActionFileUpdated, 123, "updated test with size 123 B"}, + {ActionFileUnchanged, 123, "unchanged test with size 123 B"}, + {ActionDeleted, 0, "deleted test"}, + } { + term, printer := createTextProgress() + printer.CompleteItem(data.action, "test", data.size) + test.Equals(t, []string{data.expected}, term.output) + } +} diff --git a/mover-restic/restic/internal/ui/stdio_wrapper.go b/mover-restic/restic/internal/ui/stdio_wrapper.go deleted file mode 100644 index 42f4cc545..000000000 --- a/mover-restic/restic/internal/ui/stdio_wrapper.go +++ /dev/null @@ -1,72 +0,0 @@ -package ui - -import ( - "bytes" - "io" - - "github.com/restic/restic/internal/ui/termstatus" -) - -// StdioWrapper provides stdout and stderr integration with termstatus. -type StdioWrapper struct { - stdout *lineWriter - stderr *lineWriter -} - -// NewStdioWrapper initializes a new stdio wrapper that can be used in place of -// os.Stdout or os.Stderr. -func NewStdioWrapper(term *termstatus.Terminal) *StdioWrapper { - return &StdioWrapper{ - stdout: newLineWriter(term.Print), - stderr: newLineWriter(term.Error), - } -} - -// Stdout returns a writer that is line buffered and can be used in place of -// os.Stdout. On Close(), the remaining bytes are written, followed by a line -// break. -func (w *StdioWrapper) Stdout() io.WriteCloser { - return w.stdout -} - -// Stderr returns a writer that is line buffered and can be used in place of -// os.Stderr. On Close(), the remaining bytes are written, followed by a line -// break. -func (w *StdioWrapper) Stderr() io.WriteCloser { - return w.stderr -} - -type lineWriter struct { - buf *bytes.Buffer - print func(string) -} - -var _ io.WriteCloser = &lineWriter{} - -func newLineWriter(print func(string)) *lineWriter { - return &lineWriter{buf: bytes.NewBuffer(nil), print: print} -} - -func (w *lineWriter) Write(data []byte) (n int, err error) { - n, err = w.buf.Write(data) - if err != nil { - return n, err - } - - // look for line breaks - buf := w.buf.Bytes() - i := bytes.LastIndexByte(buf, '\n') - if i != -1 { - w.print(string(buf[:i+1])) - w.buf.Next(i + 1) - } - - return n, err -} - -func (w *lineWriter) Close() error { - if w.buf.Len() > 0 { - w.print(string(append(w.buf.Bytes(), '\n'))) - } - return nil -} diff --git a/mover-restic/restic/internal/ui/table/table.go b/mover-restic/restic/internal/ui/table/table.go index c3ae47f54..1c535cadb 100644 --- a/mover-restic/restic/internal/ui/table/table.go +++ b/mover-restic/restic/internal/ui/table/table.go @@ -6,6 +6,8 @@ import ( "strings" "text/template" + + "github.com/restic/restic/internal/ui" ) // Table contains data for a table to be printed. @@ -89,7 +91,7 @@ func printLine(w io.Writer, print func(io.Writer, string) error, sep string, dat } // apply padding - pad := widths[fieldNum] - len(v) + pad := widths[fieldNum] - ui.TerminalDisplayWidth(v) if pad > 0 { v += strings.Repeat(" ", pad) } @@ -139,16 +141,18 @@ func (t *Table) Write(w io.Writer) error { columnWidths := make([]int, columns) for i, desc := range t.columns { for _, line := range strings.Split(desc, "\n") { - if columnWidths[i] < len(line) { - columnWidths[i] = len(desc) + width := ui.TerminalDisplayWidth(line) + if columnWidths[i] < width { + columnWidths[i] = width } } } for _, line := range lines { for i, content := range line { for _, l := range strings.Split(content, "\n") { - if columnWidths[i] < len(l) { - columnWidths[i] = len(l) + width := ui.TerminalDisplayWidth(l) + if columnWidths[i] < width { + columnWidths[i] = width } } } @@ -159,7 +163,7 @@ func (t *Table) Write(w io.Writer) error { for _, width := range columnWidths { totalWidth += width } - totalWidth += (columns - 1) * len(t.CellSeparator) + totalWidth += (columns - 1) * ui.TerminalDisplayWidth(t.CellSeparator) // write header if len(t.columns) > 0 { diff --git a/mover-restic/restic/internal/ui/table/table_test.go b/mover-restic/restic/internal/ui/table/table_test.go index db116bbc5..2902860b9 100644 --- a/mover-restic/restic/internal/ui/table/table_test.go +++ b/mover-restic/restic/internal/ui/table/table_test.go @@ -29,6 +29,21 @@ first column ---------------------- data: first data field ---------------------- +`, + }, + { + func(t testing.TB) *Table { + table := New() + table.AddColumn("first\ncolumn", "{{.First}}") + table.AddRow(struct{ First string }{"data"}) + return table + }, + ` +first +column +------ +data +------ `, }, { @@ -126,7 +141,7 @@ foo 2018-08-19 22:22:22 xxx other /home/user/other Time string Tags, Dirs []string } - table.AddRow(data{"foo", "2018-08-19 22:22:22", []string{"work", "go"}, []string{"/home/user/work", "/home/user/go"}}) + table.AddRow(data{"foo", "2018-08-19 22:22:22", []string{"work", "go’s"}, []string{"/home/user/work", "/home/user/go"}}) table.AddRow(data{"foo", "2018-08-19 22:22:22", []string{"other"}, []string{"/home/user/other"}}) table.AddRow(data{"foo", "2018-08-19 22:22:22", []string{"other", "bar"}, []string{"/home/user/other"}}) return table @@ -135,7 +150,7 @@ foo 2018-08-19 22:22:22 xxx other /home/user/other host name time zz tags dirs ------------------------------------------------------------ foo 2018-08-19 22:22:22 xxx work /home/user/work - go /home/user/go + go’s /home/user/go foo 2018-08-19 22:22:22 xxx other /home/user/other foo 2018-08-19 22:22:22 xxx other /home/user/other bar diff --git a/mover-restic/restic/internal/ui/termstatus/status.go b/mover-restic/restic/internal/ui/termstatus/status.go index 6e8ddfe7c..39654cc8c 100644 --- a/mover-restic/restic/internal/ui/termstatus/status.go +++ b/mover-restic/restic/internal/ui/termstatus/status.go @@ -105,7 +105,7 @@ func (t *Terminal) run(ctx context.Context) { select { case <-ctx.Done(): if !IsProcessBackground(t.fd) { - t.undoStatus(len(status)) + t.writeStatus([]string{}) } return @@ -235,30 +235,6 @@ func (t *Terminal) runWithoutStatus(ctx context.Context) { } } -func (t *Terminal) undoStatus(lines int) { - for i := 0; i < lines; i++ { - t.clearCurrentLine(t.wr, t.fd) - - _, err := t.wr.WriteRune('\n') - if err != nil { - fmt.Fprintf(os.Stderr, "write failed: %v\n", err) - } - - // flush is needed so that the current line is updated - err = t.wr.Flush() - if err != nil { - fmt.Fprintf(os.Stderr, "flush failed: %v\n", err) - } - } - - t.moveCursorUp(t.wr, t.fd, lines) - - err := t.wr.Flush() - if err != nil { - fmt.Fprintf(os.Stderr, "flush failed: %v\n", err) - } -} - func (t *Terminal) print(line string, isErr bool) { // make sure the line ends with a line break if line[len(line)-1] != '\n' { @@ -276,23 +252,11 @@ func (t *Terminal) Print(line string) { t.print(line, false) } -// Printf uses fmt.Sprintf to write a line to the terminal. -func (t *Terminal) Printf(msg string, args ...interface{}) { - s := fmt.Sprintf(msg, args...) - t.Print(s) -} - // Error writes an error to the terminal. func (t *Terminal) Error(line string) { t.print(line, true) } -// Errorf uses fmt.Sprintf to write an error line to the terminal. -func (t *Terminal) Errorf(msg string, args ...interface{}) { - s := fmt.Sprintf(msg, args...) - t.Error(s) -} - // Truncate s to fit in width (number of terminal cells) w. // If w is negative, returns the empty string. func Truncate(s string, w int) string { @@ -325,7 +289,7 @@ func Truncate(s string, w int) string { // Guess whether the first rune in s would occupy two terminal cells // instead of one. This cannot be determined exactly without knowing -// the terminal font, so we treat all ambigous runes as full-width, +// the terminal font, so we treat all ambiguous runes as full-width, // i.e., two cells. func wideRune(s string) (wide bool, utfsize uint) { prop, size := width.LookupString(s) @@ -351,11 +315,8 @@ func sanitizeLines(lines []string, width int) []string { // SetStatus updates the status lines. // The lines should not contain newlines; this method adds them. +// Pass nil or an empty array to remove the status lines. func (t *Terminal) SetStatus(lines []string) { - if len(lines) == 0 { - return - } - // only truncate interactive status output var width int if t.canUpdateStatus { diff --git a/mover-restic/restic/internal/ui/termstatus/status_test.go b/mover-restic/restic/internal/ui/termstatus/status_test.go index b59063076..2a17a905a 100644 --- a/mover-restic/restic/internal/ui/termstatus/status_test.go +++ b/mover-restic/restic/internal/ui/termstatus/status_test.go @@ -32,6 +32,15 @@ func TestSetStatus(t *testing.T) { term.SetStatus([]string{"first"}) exp := home + clear + "first" + home + term.SetStatus([]string{""}) + exp += home + clear + "" + home + + term.SetStatus([]string{}) + exp += home + clear + "" + home + + // already empty status + term.SetStatus([]string{}) + term.SetStatus([]string{"foo", "bar", "baz"}) exp += home + clear + "foo\n" + home + clear + "bar\n" + home + clear + "baz" + home + up + up @@ -39,11 +48,10 @@ func TestSetStatus(t *testing.T) { term.SetStatus([]string{"quux", "needs\nquote"}) exp += home + clear + "quux\n" + home + clear + "\"needs\\nquote\"\n" + - home + clear + home + up + up // Third line implicit. + home + clear + home + up + up // Clear third line cancel() - exp += home + clear + "\n" + home + clear + "\n" + - home + up + up // Status cleared. + exp += home + clear + "\n" + home + clear + home + up // Status cleared <-term.closed rtest.Equals(t, exp, buf.String()) diff --git a/mover-restic/restic/internal/ui/termstatus/stdio_wrapper.go b/mover-restic/restic/internal/ui/termstatus/stdio_wrapper.go new file mode 100644 index 000000000..233610ba3 --- /dev/null +++ b/mover-restic/restic/internal/ui/termstatus/stdio_wrapper.go @@ -0,0 +1,47 @@ +package termstatus + +import ( + "bytes" + "io" +) + +// WrapStdio returns line-buffering replacements for os.Stdout and os.Stderr. +// On Close, the remaining bytes are written, followed by a line break. +func WrapStdio(term *Terminal) (stdout, stderr io.WriteCloser) { + return newLineWriter(term.Print), newLineWriter(term.Error) +} + +type lineWriter struct { + buf bytes.Buffer + print func(string) +} + +var _ io.WriteCloser = &lineWriter{} + +func newLineWriter(print func(string)) *lineWriter { + return &lineWriter{print: print} +} + +func (w *lineWriter) Write(data []byte) (n int, err error) { + n, err = w.buf.Write(data) + if err != nil { + return n, err + } + + // look for line breaks + buf := w.buf.Bytes() + i := bytes.LastIndexByte(buf, '\n') + if i != -1 { + w.print(string(buf[:i+1])) + w.buf.Next(i + 1) + } + + return n, err +} + +func (w *lineWriter) Close() error { + if w.buf.Len() > 0 { + w.print(string(append(w.buf.Bytes(), '\n'))) + } + return nil +} diff --git a/mover-restic/restic/internal/ui/stdio_wrapper_test.go b/mover-restic/restic/internal/ui/termstatus/stdio_wrapper_test.go similarity index 98% rename from mover-restic/restic/internal/ui/stdio_wrapper_test.go rename to mover-restic/restic/internal/ui/termstatus/stdio_wrapper_test.go index b95d9180d..1e214f1f4 100644 --- a/mover-restic/restic/internal/ui/stdio_wrapper_test.go +++ b/mover-restic/restic/internal/ui/termstatus/stdio_wrapper_test.go @@ -1,4 +1,4 @@ -package ui +package termstatus import ( "strings" diff --git a/mover-restic/restic/internal/ui/termstatus/terminal_windows.go b/mover-restic/restic/internal/ui/termstatus/terminal_windows.go index 7bf5b0a37..3603f16a3 100644 --- a/mover-restic/restic/internal/ui/termstatus/terminal_windows.go +++ b/mover-restic/restic/internal/ui/termstatus/terminal_windows.go @@ -9,8 +9,8 @@ import ( "syscall" "unsafe" - "golang.org/x/crypto/ssh/terminal" "golang.org/x/sys/windows" + "golang.org/x/term" ) // clearCurrentLine removes all characters from the current line and resets the @@ -74,7 +74,7 @@ func windowsMoveCursorUp(_ io.Writer, fd uintptr, n int) { // isWindowsTerminal return true if the file descriptor is a windows terminal (cmd, psh). func isWindowsTerminal(fd uintptr) bool { - return terminal.IsTerminal(int(fd)) + return term.IsTerminal(int(fd)) } func isPipe(fd uintptr) bool { diff --git a/mover-restic/restic/internal/walker/rewriter.go b/mover-restic/restic/internal/walker/rewriter.go index 649857032..6c27b26ac 100644 --- a/mover-restic/restic/internal/walker/rewriter.go +++ b/mover-restic/restic/internal/walker/rewriter.go @@ -11,6 +11,12 @@ import ( type NodeRewriteFunc func(node *restic.Node, path string) *restic.Node type FailedTreeRewriteFunc func(nodeID restic.ID, path string, err error) (restic.ID, error) +type QueryRewrittenSizeFunc func() SnapshotSize + +type SnapshotSize struct { + FileCount uint + FileSize uint64 +} type RewriteOpts struct { // return nil to remove the node @@ -39,19 +45,42 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { } // setup default implementations if rw.opts.RewriteNode == nil { - rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { + rw.opts.RewriteNode = func(node *restic.Node, _ string) *restic.Node { return node } } if rw.opts.RewriteFailedTree == nil { // fail with error by default - rw.opts.RewriteFailedTree = func(nodeID restic.ID, path string, err error) (restic.ID, error) { + rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (restic.ID, error) { return restic.ID{}, err } } return rw } +func NewSnapshotSizeRewriter(rewriteNode NodeRewriteFunc) (*TreeRewriter, QueryRewrittenSizeFunc) { + var count uint + var size uint64 + + t := NewTreeRewriter(RewriteOpts{ + RewriteNode: func(node *restic.Node, path string) *restic.Node { + node = rewriteNode(node, path) + if node != nil && node.Type == "file" { + count++ + size += node.Size + } + return node + }, + DisableNodeCache: true, + }) + + ss := func() SnapshotSize { + return SnapshotSize{count, size} + } + + return t, ss +} + type BlobLoadSaver interface { restic.BlobSaver restic.BlobLoader diff --git a/mover-restic/restic/internal/walker/rewriter_test.go b/mover-restic/restic/internal/walker/rewriter_test.go index 716217ac6..f05e50f9b 100644 --- a/mover-restic/restic/internal/walker/rewriter_test.go +++ b/mover-restic/restic/internal/walker/rewriter_test.go @@ -69,7 +69,7 @@ func checkRewriteItemOrder(want []string) checkRewriteFunc { } } -// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceedes in the correct order. +// checkRewriteSkips excludes nodes if path is in skipFor, it checks that rewriting proceeds in the correct order. func checkRewriteSkips(skipFor map[string]struct{}, want []string, disableCache bool) checkRewriteFunc { var pos int @@ -303,6 +303,60 @@ func TestRewriter(t *testing.T) { } } +func TestSnapshotSizeQuery(t *testing.T) { + tree := TestTree{ + "foo": TestFile{Size: 21}, + "bar": TestFile{Size: 21}, + "subdir": TestTree{ + "subfile": TestFile{Size: 21}, + }, + } + newTree := TestTree{ + "foo": TestFile{Size: 42}, + "subdir": TestTree{ + "subfile": TestFile{Size: 42}, + }, + } + t.Run("", func(t *testing.T) { + repo, root := BuildTreeMap(tree) + expRepo, expRoot := BuildTreeMap(newTree) + modrepo := WritableTreeMap{repo} + + ctx, cancel := context.WithCancel(context.TODO()) + defer cancel() + + rewriteNode := func(node *restic.Node, path string) *restic.Node { + if path == "/bar" { + return nil + } + if node.Type == "file" { + node.Size += 21 + } + return node + } + rewriter, querySize := NewSnapshotSizeRewriter(rewriteNode) + newRoot, err := rewriter.RewriteTree(ctx, modrepo, "/", root) + if err != nil { + t.Error(err) + } + + ss := querySize() + + test.Equals(t, uint(2), ss.FileCount, "snapshot file count mismatch") + test.Equals(t, uint64(84), ss.FileSize, "snapshot size mismatch") + + // verifying against the expected tree root also implicitly checks the structural integrity + if newRoot != expRoot { + t.Error("hash mismatch") + fmt.Println("Got") + modrepo.Dump() + fmt.Println("Expected") + WritableTreeMap{expRepo}.Dump() + } + }) + +} + func TestRewriterFailOnUnknownFields(t *testing.T) { tm := WritableTreeMap{TreeMap{}} node := []byte(`{"nodes":[{"name":"subfile","type":"file","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","uid":0,"gid":0,"content":null,"unknown_field":42}]}`) diff --git a/mover-restic/restic/internal/walker/walker.go b/mover-restic/restic/internal/walker/walker.go index 4c4e7f5ab..091b05489 100644 --- a/mover-restic/restic/internal/walker/walker.go +++ b/mover-restic/restic/internal/walker/walker.go @@ -21,21 +21,22 @@ var ErrSkipNode = errors.New("skip this node") // When the special value ErrSkipNode is returned and node is a dir node, it is // not walked. When the node is not a dir node, the remaining items in this // tree are skipped. -// -// Setting ignore to true tells Walk that it should not visit the node again. -// For tree nodes, this means that the function is not called for the -// referenced tree. If the node is not a tree, and all nodes in the current -// tree have ignore set to true, the current tree will not be visited again. -// When err is not nil and different from ErrSkipNode, the value returned for -// ignore is ignored. -type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (ignore bool, err error) +type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error) + +type WalkVisitor struct { + // If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode` + // was returned. This function is mandatory + ProcessNode WalkFunc + // Optional callback + LeaveDir func(path string) +} // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. -func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { +func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error { tree, err := restic.LoadTree(ctx, repo, root) - _, err = walkFn(root, "/", nil, err) + err = visitor.ProcessNode(root, "/", nil, err) if err != nil { if err == ErrSkipNode { @@ -44,24 +45,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTre return err } - if ignoreTrees == nil { - ignoreTrees = restic.NewIDSet() - } - - _, err = walk(ctx, repo, "/", root, tree, ignoreTrees, walkFn) - return err + return walk(ctx, repo, "/", root, tree, visitor) } // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { - var allNodesIgnored = true - - if len(tree.Nodes) == 0 { - allNodesIgnored = false - } - +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) { sort.Slice(tree.Nodes, func(i, j int) bool { return tree.Nodes[i].Name < tree.Nodes[j].Name }) @@ -70,68 +60,44 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree p := path.Join(prefix, node.Name) if node.Type == "" { - return false, errors.Errorf("node type is empty for node %q", node.Name) + return errors.Errorf("node type is empty for node %q", node.Name) } if node.Type != "dir" { - ignore, err := walkFn(parentTreeID, p, node, nil) + err := visitor.ProcessNode(parentTreeID, p, node, nil) if err != nil { if err == ErrSkipNode { // skip the remaining entries in this tree - return allNodesIgnored, nil + break } - return false, err - } - - if !ignore { - allNodesIgnored = false + return err } continue } if node.Subtree == nil { - return false, errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) - } - - if ignoreTrees.Has(*node.Subtree) { - continue + return errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - ignore, err := walkFn(parentTreeID, p, node, err) + err = visitor.ProcessNode(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { - if ignore { - ignoreTrees.Insert(*node.Subtree) - } continue } - return false, err - } - - if ignore { - ignoreTrees.Insert(*node.Subtree) - } - - if !ignore { - allNodesIgnored = false } - ignore, err = walk(ctx, repo, p, *node.Subtree, subtree, ignoreTrees, walkFn) + err = walk(ctx, repo, p, *node.Subtree, subtree, visitor) if err != nil { - return false, err - } - - if ignore { - ignoreTrees.Insert(*node.Subtree) + return err } + } - if !ignore { - allNodesIgnored = false - } + if visitor.LeaveDir != nil { + visitor.LeaveDir(prefix) } - return allNodesIgnored, nil + return nil } diff --git a/mover-restic/restic/internal/walker/walker_test.go b/mover-restic/restic/internal/walker/walker_test.go index 54cc69792..75f80e57f 100644 --- a/mover-restic/restic/internal/walker/walker_test.go +++ b/mover-restic/restic/internal/walker/walker_test.go @@ -13,7 +13,7 @@ import ( // TestTree is used to construct a list of trees for testing the walker. type TestTree map[string]interface{} -// TestNode is used to test the walker. +// TestFile is used to test the walker. type TestFile struct { Size uint64 } @@ -93,28 +93,32 @@ func (t TreeMap) Connections() uint { // checkFunc returns a function suitable for walking the tree to check // something, and a function which will check the final result. -type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) +type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) // checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkItemOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false, nil + return nil } if path != want[pos] { t.Errorf("wrong path found, want %q, got %q", want[pos], path) } pos++ - return false, nil + return nil + } + + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) } final = func(t testing.TB) { @@ -123,30 +127,30 @@ func checkItemOrder(want []string) checkFunc { } } - return walker, final + return walker, leaveDir, final } } // checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'. func checkParentTreeOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(want) { t.Errorf("additional unexpected parent tree ID found: %v", treeID) - return false, nil + return nil } if treeID.String() != want[pos] { t.Errorf("wrong parent tree ID found, want %q, got %q", want[pos], treeID.String()) } pos++ - return false, nil + return nil } final = func(t testing.TB) { @@ -155,7 +159,7 @@ func checkParentTreeOrder(want []string) checkFunc { } } - return walker, final + return walker, nil, final } } @@ -164,16 +168,16 @@ func checkParentTreeOrder(want []string) checkFunc { func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(wantPaths) { t.Errorf("additional unexpected path found: %v", path) - return false, nil + return nil } if path != wantPaths[pos] { @@ -182,50 +186,14 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { pos++ if _, ok := skipFor[path]; ok { - return false, ErrSkipNode + return ErrSkipNode } - return false, nil + return nil } - final = func(t testing.TB) { - if pos != len(wantPaths) { - t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) - } - } - - return walker, final - } -} - -// checkIgnore returns ErrSkipNode if path is in skipFor and sets ignore according -// to ignoreFor. It checks that the paths the walk func is called for are exactly -// the ones in wantPaths. -func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPaths []string) checkFunc { - var pos int - - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { - if err != nil { - t.Errorf("error walking %v: %v", path, err) - return false, err - } - - if pos >= len(wantPaths) { - t.Errorf("additional unexpected path found: %v", path) - return ignoreFor[path], nil - } - - if path != wantPaths[pos] { - t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path) - } - pos++ - - if _, ok := skipFor[path]; ok { - return ignoreFor[path], ErrSkipNode - } - - return ignoreFor[path], nil + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) } final = func(t testing.TB) { @@ -234,7 +202,7 @@ func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPat } } - return walker, final + return walker, leaveDir, final } } @@ -256,6 +224,8 @@ func TestWalker(t *testing.T) { "/foo", "/subdir", "/subdir/subfile", + "leave: /subdir", + "leave: /", }), checkParentTreeOrder([]string{ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree / @@ -270,16 +240,14 @@ func TestWalker(t *testing.T) { "/", "/foo", "/subdir", + "leave: /", }, ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir": true, + checkSkipFor( + map[string]struct{}{ + "/": {}, }, []string{ "/", - "/foo", - "/subdir", - "/subdir/subfile", }, ), }, @@ -303,10 +271,14 @@ func TestWalker(t *testing.T) { "/foo", "/subdir1", "/subdir1/subfile1", + "leave: /subdir1", "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }), checkParentTreeOrder([]string{ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree / @@ -329,6 +301,9 @@ func TestWalker(t *testing.T) { "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -342,6 +317,8 @@ func TestWalker(t *testing.T) { "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -350,6 +327,7 @@ func TestWalker(t *testing.T) { }, []string{ "/", "/foo", + "leave: /", }, ), }, @@ -382,15 +360,19 @@ func TestWalker(t *testing.T) { "/subdir1/subfile1", "/subdir1/subfile2", "/subdir1/subfile3", + "leave: /subdir1", "/subdir2", "/subdir2/subfile1", "/subdir2/subfile2", "/subdir2/subfile3", + "leave: /subdir2", "/subdir3", "/subdir3/subfile1", "/subdir3/subfile2", "/subdir3/subfile3", + "leave: /subdir3", "/zzz other", + "leave: /", }), checkParentTreeOrder([]string{ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / @@ -409,81 +391,6 @@ func TestWalker(t *testing.T) { "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir3 "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / }), - checkIgnore( - map[string]struct{}{ - "/subdir1": {}, - }, map[string]bool{ - "/subdir1": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir1": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{ - "/subdir2": {}, - }, map[string]bool{ - "/subdir2": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/subdir2", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir1/subfile1": true, - "/subdir1/subfile2": true, - "/subdir1/subfile3": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir2/subfile1": true, - "/subdir2/subfile2": true, - "/subdir2/subfile3": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/subdir2", - "/subdir2/subfile1", - "/subdir2/subfile2", - "/subdir2/subfile3", - "/zzz other", - }, - ), }, }, { @@ -503,45 +410,23 @@ func TestWalker(t *testing.T) { checkItemOrder([]string{ "/", "/subdir1", + "leave: /subdir1", "/subdir2", + "leave: /subdir2", "/subdir3", "/subdir3/file", + "leave: /subdir3", "/subdir4", "/subdir4/file", + "leave: /subdir4", "/subdir5", + "leave: /subdir5", "/subdir6", + "leave: /subdir6", + "leave: /", }), }, }, - { - tree: TestTree{ - "subdir1": TestTree{}, - "subdir2": TestTree{}, - "subdir3": TestTree{ - "file": TestFile{}, - }, - "subdir4": TestTree{}, - "subdir5": TestTree{ - "file": TestFile{}, - }, - "subdir6": TestTree{}, - }, - checks: []checkFunc{ - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir2": true, - }, []string{ - "/", - "/subdir1", - "/subdir2", - "/subdir3", - "/subdir3/file", - "/subdir5", - "/subdir5/file", - }, - ), - }, - }, } for _, test := range tests { @@ -552,8 +437,11 @@ func TestWalker(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - fn, last := check(t) - err := Walk(ctx, repo, root, restic.NewIDSet(), fn) + fn, leaveDir, last := check(t) + err := Walk(ctx, repo, root, WalkVisitor{ + ProcessNode: fn, + LeaveDir: leaveDir, + }) if err != nil { t.Error(err) } From c3e546894d4170aa1e5fb1d552306915bd047020 Mon Sep 17 00:00:00 2001 From: Tesshu Flower Date: Thu, 5 Sep 2024 23:28:22 -0400 Subject: [PATCH 2/2] Update restic to v0.17.1 Signed-off-by: Tesshu Flower --- mover-restic/SOURCE_VERSIONS | 2 +- .../restic/.github/workflows/docker.yml | 2 +- .../restic/.github/workflows/tests.yml | 18 +- mover-restic/restic/CHANGELOG.md | 225 +++++++++++++++++ mover-restic/restic/VERSION | 2 +- .../changelog/0.17.1_2024-09-05/issue-2004 | 18 ++ .../changelog/0.17.1_2024-09-05/issue-4795 | 8 + .../changelog/0.17.1_2024-09-05/issue-4934 | 11 + .../changelog/0.17.1_2024-09-05/issue-4944 | 9 + .../changelog/0.17.1_2024-09-05/issue-4945 | 10 + .../changelog/0.17.1_2024-09-05/issue-4953 | 7 + .../changelog/0.17.1_2024-09-05/issue-4957 | 8 + .../changelog/0.17.1_2024-09-05/issue-4969 | 7 + .../changelog/0.17.1_2024-09-05/issue-4970 | 15 ++ .../changelog/0.17.1_2024-09-05/issue-4975 | 7 + .../changelog/0.17.1_2024-09-05/issue-5004 | 12 + .../changelog/0.17.1_2024-09-05/issue-5005 | 16 ++ .../changelog/0.17.1_2024-09-05/pull-4958 | 7 + .../changelog/0.17.1_2024-09-05/pull-4959 | 6 + .../changelog/0.17.1_2024-09-05/pull-4977 | 16 ++ .../changelog/0.17.1_2024-09-05/pull-4980 | 12 + .../changelog/0.17.1_2024-09-05/pull-5018 | 13 + mover-restic/restic/cmd/restic/cmd_backup.go | 2 + mover-restic/restic/cmd/restic/cmd_cache.go | 1 + mover-restic/restic/cmd/restic/cmd_cat.go | 11 +- mover-restic/restic/cmd/restic/cmd_check.go | 2 + mover-restic/restic/cmd/restic/cmd_copy.go | 3 + mover-restic/restic/cmd/restic/cmd_debug.go | 7 +- mover-restic/restic/cmd/restic/cmd_diff.go | 30 ++- mover-restic/restic/cmd/restic/cmd_dump.go | 6 + .../restic/cmd/restic/cmd_features.go | 2 +- mover-restic/restic/cmd/restic/cmd_find.go | 6 + mover-restic/restic/cmd/restic/cmd_forget.go | 6 + mover-restic/restic/cmd/restic/cmd_init.go | 1 + .../cmd/restic/cmd_init_integration_test.go | 7 + mover-restic/restic/cmd/restic/cmd_key.go | 2 + mover-restic/restic/cmd/restic/cmd_key_add.go | 1 + .../restic/cmd/restic/cmd_key_list.go | 1 + .../restic/cmd/restic/cmd_key_passwd.go | 1 + .../restic/cmd/restic/cmd_key_remove.go | 1 + mover-restic/restic/cmd/restic/cmd_list.go | 2 + mover-restic/restic/cmd/restic/cmd_ls.go | 2 + mover-restic/restic/cmd/restic/cmd_migrate.go | 7 + mover-restic/restic/cmd/restic/cmd_mount.go | 2 + .../cmd/restic/cmd_mount_integration_test.go | 6 + mover-restic/restic/cmd/restic/cmd_options.go | 2 +- mover-restic/restic/cmd/restic/cmd_prune.go | 2 + .../cmd/restic/cmd_prune_integration_test.go | 5 +- mover-restic/restic/cmd/restic/cmd_recover.go | 6 + mover-restic/restic/cmd/restic/cmd_repair.go | 6 +- .../restic/cmd/restic/cmd_repair_index.go | 1 + .../restic/cmd/restic/cmd_repair_packs.go | 1 + .../restic/cmd/restic/cmd_repair_snapshots.go | 1 + mover-restic/restic/cmd/restic/cmd_restore.go | 10 +- mover-restic/restic/cmd/restic/cmd_rewrite.go | 2 + .../restic/cmd/restic/cmd_self_update.go | 1 + .../restic/cmd/restic/cmd_snapshots.go | 10 + mover-restic/restic/cmd/restic/cmd_stats.go | 12 + mover-restic/restic/cmd/restic/cmd_tag.go | 2 + mover-restic/restic/cmd/restic/cmd_unlock.go | 1 + mover-restic/restic/cmd/restic/global.go | 5 +- .../restic/cmd/restic/integration_test.go | 4 +- mover-restic/restic/cmd/restic/main.go | 21 ++ .../restic/cmd/restic/secondary_repo.go | 2 +- .../restic/doc/030_preparing_a_new_repo.rst | 124 ++++------ mover-restic/restic/doc/040_backup.rst | 56 +++-- .../restic/doc/045_working_with_repos.rst | 7 + mover-restic/restic/doc/075_scripting.rst | 19 +- mover-restic/restic/doc/bash-completion.sh | 229 ++++++++++++++++++ mover-restic/restic/doc/design.rst | 22 +- mover-restic/restic/doc/faq.rst | 16 +- mover-restic/restic/doc/man/restic-backup.1 | 5 + mover-restic/restic/doc/man/restic-cache.1 | 4 + mover-restic/restic/doc/man/restic-cat.1 | 5 + mover-restic/restic/doc/man/restic-check.1 | 5 + mover-restic/restic/doc/man/restic-copy.1 | 7 +- mover-restic/restic/doc/man/restic-diff.1 | 5 + mover-restic/restic/doc/man/restic-dump.1 | 5 + mover-restic/restic/doc/man/restic-features.1 | 146 +++++++++++ mover-restic/restic/doc/man/restic-find.1 | 5 + mover-restic/restic/doc/man/restic-forget.1 | 5 + mover-restic/restic/doc/man/restic-generate.1 | 4 + mover-restic/restic/doc/man/restic-init.1 | 6 +- mover-restic/restic/doc/man/restic-key-add.1 | 5 + mover-restic/restic/doc/man/restic-key-list.1 | 5 + .../restic/doc/man/restic-key-passwd.1 | 5 + .../restic/doc/man/restic-key-remove.1 | 5 + mover-restic/restic/doc/man/restic-key.1 | 4 + mover-restic/restic/doc/man/restic-list.1 | 5 + mover-restic/restic/doc/man/restic-ls.1 | 5 + mover-restic/restic/doc/man/restic-migrate.1 | 5 + mover-restic/restic/doc/man/restic-mount.1 | 5 + mover-restic/restic/doc/man/restic-options.1 | 135 +++++++++++ mover-restic/restic/doc/man/restic-prune.1 | 5 + mover-restic/restic/doc/man/restic-recover.1 | 5 + .../restic/doc/man/restic-repair-index.1 | 5 + .../restic/doc/man/restic-repair-packs.1 | 5 + .../restic/doc/man/restic-repair-snapshots.1 | 5 + mover-restic/restic/doc/man/restic-repair.1 | 4 + mover-restic/restic/doc/man/restic-restore.1 | 5 + mover-restic/restic/doc/man/restic-rewrite.1 | 5 + .../restic/doc/man/restic-self-update.1 | 5 + .../restic/doc/man/restic-snapshots.1 | 5 + mover-restic/restic/doc/man/restic-stats.1 | 5 + mover-restic/restic/doc/man/restic-tag.1 | 5 + mover-restic/restic/doc/man/restic-unlock.1 | 4 + mover-restic/restic/doc/man/restic-version.1 | 4 + mover-restic/restic/doc/man/restic.1 | 6 +- mover-restic/restic/doc/manual_rest.rst | 16 +- .../restic/internal/archiver/archiver.go | 10 +- .../restic/internal/archiver/archiver_test.go | 86 +++++++ .../restic/internal/backend/cache/backend.go | 47 +++- .../internal/backend/cache/backend_test.go | 75 ++++++ .../restic/internal/backend/http_transport.go | 9 +- .../restic/internal/backend/rest/rest.go | 6 + .../internal/backend/retry/backend_retry.go | 5 +- .../backend/retry/backend_retry_test.go | 24 ++ .../restic/internal/backend/sftp/sftp.go | 4 + .../internal/backend/watchdog_roundtriper.go | 3 + .../backend/watchdog_roundtriper_test.go | 6 +- mover-restic/restic/internal/fs/ea_windows.go | 33 +++ .../restic/internal/fs/ea_windows_test.go | 76 ++++++ mover-restic/restic/internal/fs/file.go | 2 +- .../restic/internal/fs/file_windows.go | 2 +- .../restic/internal/fs/fs_reader_command.go | 4 + .../internal/fs/fs_reader_command_test.go | 5 + mover-restic/restic/internal/fs/sd_windows.go | 33 +-- mover-restic/restic/internal/fuse/dir.go | 8 + mover-restic/restic/internal/fuse/file.go | 6 +- .../restic/internal/fuse/snapshots_dir.go | 4 + .../restic/internal/repository/check.go | 4 + .../internal/repository/index/master_index.go | 3 + .../restic/internal/repository/repository.go | 15 +- mover-restic/restic/internal/restic/lock.go | 27 ++- mover-restic/restic/internal/restic/node.go | 36 +-- .../restic/internal/restic/node_test.go | 35 ++- .../restic/internal/restic/node_windows.go | 152 ++++++++++-- .../internal/restic/node_windows_test.go | 196 +++++++++++++++ .../restic/internal/restic/snapshot_find.go | 4 + .../restic/internal/restorer/filerestorer.go | 4 + .../internal/restorer/fileswriter_test.go | 2 +- .../restic/internal/restorer/restorer.go | 33 ++- .../restic/internal/restorer/restorer_test.go | 103 ++++++-- .../internal/restorer/restorer_unix_test.go | 10 +- .../internal/restorer/restorer_windows.go | 2 +- .../restorer/restorer_windows_test.go | 6 +- .../restic/internal/ui/backup/json.go | 21 +- .../restic/internal/ui/backup/json_test.go | 27 +++ .../restic/internal/ui/backup/text.go | 14 +- .../restic/internal/ui/backup/text_test.go | 27 +++ mover-restic/restic/internal/ui/message.go | 6 +- mover-restic/restic/internal/ui/mock.go | 22 ++ .../restic/internal/ui/restore/json.go | 29 ++- .../restic/internal/ui/restore/json_test.go | 24 +- .../restic/internal/ui/restore/progress.go | 17 +- .../internal/ui/restore/progress_test.go | 55 ++++- .../restic/internal/ui/restore/text.go | 24 +- .../restic/internal/ui/restore/text_test.go | 36 ++- mover-restic/restic/internal/ui/terminal.go | 10 + .../restic/internal/walker/rewriter.go | 4 + mover-restic/restic/internal/walker/walker.go | 4 + 161 files changed, 2615 insertions(+), 362 deletions(-) create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-2004 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4795 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4934 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4944 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4945 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4953 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4957 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4969 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4970 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4975 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5004 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5005 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4958 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4959 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4977 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4980 create mode 100644 mover-restic/restic/changelog/0.17.1_2024-09-05/pull-5018 create mode 100644 mover-restic/restic/doc/man/restic-features.1 create mode 100644 mover-restic/restic/doc/man/restic-options.1 create mode 100644 mover-restic/restic/internal/ui/backup/json_test.go create mode 100644 mover-restic/restic/internal/ui/backup/text_test.go create mode 100644 mover-restic/restic/internal/ui/mock.go create mode 100644 mover-restic/restic/internal/ui/terminal.go diff --git a/mover-restic/SOURCE_VERSIONS b/mover-restic/SOURCE_VERSIONS index b86a53d0a..3d3b7dd0a 100644 --- a/mover-restic/SOURCE_VERSIONS +++ b/mover-restic/SOURCE_VERSIONS @@ -1,2 +1,2 @@ -https://github.com/restic/restic.git v0.17.0 277c8f5029a12bd882c2c1d2088f435caec67bb8 +https://github.com/restic/restic.git v0.17.1 975aa41e1e6a1c88deb501451f23cbdbb013f1da https://github.com/minio/minio-go.git v7.0.66 5415e6c72a71610108fe05ee747ac760dd40094f diff --git a/mover-restic/restic/.github/workflows/docker.yml b/mover-restic/restic/.github/workflows/docker.yml index a943d1b15..a24660b45 100644 --- a/mover-restic/restic/.github/workflows/docker.yml +++ b/mover-restic/restic/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/mover-restic/restic/.github/workflows/tests.yml b/mover-restic/restic/.github/workflows/tests.yml index 3ca7a9edb..2ffeb5ff2 100644 --- a/mover-restic/restic/.github/workflows/tests.yml +++ b/mover-restic/restic/.github/workflows/tests.yml @@ -66,6 +66,9 @@ jobs: GOPROXY: https://proxy.golang.org steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ matrix.go }} uses: actions/setup-go@v5 with: @@ -139,9 +142,6 @@ jobs: echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH if: matrix.os == 'windows-latest' - - name: Check out code - uses: actions/checkout@v4 - - name: Build with build.go run: | go run build.go @@ -230,14 +230,14 @@ jobs: name: Cross Compile for subset ${{ matrix.subset }} steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} - - name: Check out code - uses: actions/checkout@v4 - - name: Cross-compile for subset ${{ matrix.subset }} run: | mkdir build-output build-output-debug @@ -252,14 +252,14 @@ jobs: # allow annotating code in the PR checks: write steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Set up Go ${{ env.latest_go }} uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} - - name: Check out code - uses: actions/checkout@v4 - - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: diff --git a/mover-restic/restic/CHANGELOG.md b/mover-restic/restic/CHANGELOG.md index 2a6926755..9a5393915 100644 --- a/mover-restic/restic/CHANGELOG.md +++ b/mover-restic/restic/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.17.1](#changelog-for-restic-0171-2024-09-05) * [Changelog for 0.17.0](#changelog-for-restic-0170-2024-07-26) * [Changelog for 0.16.5](#changelog-for-restic-0165-2024-07-01) * [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) @@ -35,6 +36,230 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.17.1 (2024-09-05) +The following sections list the changes in restic 0.17.1 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #2004: Correctly handle volume names in `backup` command on Windows + * Fix #4945: Include missing backup error text with `--json` + * Fix #4953: Correctly handle long paths on older Windows versions + * Fix #4957: Fix delayed cancellation of certain commands + * Fix #4958: Don't ignore metadata-setting errors during restore + * Fix #4969: Correctly restore timestamp for files with resource forks on macOS + * Fix #4975: Prevent `backup --stdin-from-command` from panicking + * Fix #4980: Skip extended attribute processing on unsupported Windows volumes + * Fix #5004: Fix spurious "A Required Privilege Is Not Held by the Client" error + * Fix #5005: Fix rare failures to retry locking a repository + * Fix #5018: Improve HTTP/2 support for REST backend + * Chg #4953: Also back up files with incomplete metadata + * Enh #4795: Display progress bar for `restore --verify` + * Enh #4934: Automatically clear removed snapshots from cache + * Enh #4944: Print JSON-formatted errors during `restore --json` + * Enh #4959: Return exit code 12 for "bad password" errors + * Enh #4970: Make timeout for stuck requests customizable + +## Details + + * Bugfix #2004: Correctly handle volume names in `backup` command on Windows + + On Windows, when the specified backup target only included the volume name + without a trailing slash, for example, `C:`, then restoring the resulting + snapshot would result in an error. Note that using `C:\` as backup target worked + correctly. + + Specifying volume names is now handled correctly. To restore snapshots created + before this bugfix, use the : syntax. For example, to restore + a snapshot with ID `12345678` that backed up `C:`, use the following command: + + ``` + restic restore 12345678:/C/C:./ --target output/folder + ``` + + https://github.com/restic/restic/issues/2004 + https://github.com/restic/restic/pull/5028 + + * Bugfix #4945: Include missing backup error text with `--json` + + Previously, when running a backup with the `--json` option, restic failed to + include the actual error message in the output, resulting in `"error": {}` being + displayed. + + This has now been fixed, and restic now includes the error text in JSON output. + + https://github.com/restic/restic/issues/4945 + https://github.com/restic/restic/pull/4946 + + * Bugfix #4953: Correctly handle long paths on older Windows versions + + On older Windows versions, like Windows Server 2012, restic 0.17.0 failed to + back up files with long paths. This problem has now been resolved. + + https://github.com/restic/restic/issues/4953 + https://github.com/restic/restic/pull/4954 + + * Bugfix #4957: Fix delayed cancellation of certain commands + + Since restic 0.17.0, some commands did not immediately respond to cancellation + via Ctrl-C (SIGINT) and continued running for a short period. The most affected + commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This is now resolved. + + https://github.com/restic/restic/issues/4957 + https://github.com/restic/restic/pull/4960 + + * Bugfix #4958: Don't ignore metadata-setting errors during restore + + Previously, restic used to ignore errors when setting timestamps, attributes, or + file modes during a restore. It now reports those errors, except for permission + related errors when running without root privileges. + + https://github.com/restic/restic/pull/4958 + + * Bugfix #4969: Correctly restore timestamp for files with resource forks on macOS + + On macOS, timestamps were not restored for files with resource forks. This has + now been fixed. + + https://github.com/restic/restic/issues/4969 + https://github.com/restic/restic/pull/5006 + + * Bugfix #4975: Prevent `backup --stdin-from-command` from panicking + + Restic would previously crash if `--stdin-from-command` was specified without + providing a command. This issue has now been fixed. + + https://github.com/restic/restic/issues/4975 + https://github.com/restic/restic/pull/4976 + + * Bugfix #4980: Skip extended attribute processing on unsupported Windows volumes + + With restic 0.17.0, backups of certain Windows paths, such as network drives, + failed due to errors while fetching extended attributes. + + Restic now skips extended attribute processing for volumes where they are not + supported. + + https://github.com/restic/restic/issues/4955 + https://github.com/restic/restic/issues/4950 + https://github.com/restic/restic/pull/4980 + https://github.com/restic/restic/pull/4998 + + * Bugfix #5004: Fix spurious "A Required Privilege Is Not Held by the Client" error + + On Windows, creating a backup could sometimes trigger the following error: + + ``` + error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client. + ``` + + This has now been fixed. + + https://github.com/restic/restic/issues/5004 + https://github.com/restic/restic/pull/5019 + + * Bugfix #5005: Fix rare failures to retry locking a repository + + Restic 0.17.0 could in rare cases fail to retry locking a repository if one of + the lock files failed to load, resulting in the error: + + ``` + unable to create lock in backend: circuit breaker open for file + ``` + + This issue has now been addressed. The error handling now properly retries the + locking operation. In addition, restic waits a few seconds between locking + retries to increase chances of successful locking. + + https://github.com/restic/restic/issues/5005 + https://github.com/restic/restic/pull/5011 + https://github.com/restic/restic/pull/5012 + + * Bugfix #5018: Improve HTTP/2 support for REST backend + + If `rest-server` tried to gracefully shut down an HTTP/2 connection still in use + by the client, it could result in the following error: + + ``` + http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error + ``` + + This issue has now been resolved. + + https://github.com/restic/restic/pull/5018 + https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367 + + * Change #4953: Also back up files with incomplete metadata + + If restic failed to read extended metadata for a file or folder during a backup, + then the file or folder was not included in the resulting snapshot. Instead, a + warning message was printed along with returning exit code 3 once the backup was + finished. + + Now, restic also includes items for which the extended metadata could not be + read in a snapshot. The warning message has been updated to: + + ``` + incomplete metadata for /path/to/file:
+ ``` + + https://github.com/restic/restic/issues/4953 + https://github.com/restic/restic/pull/4977 + + * Enhancement #4795: Display progress bar for `restore --verify` + + When the `restore` command is run with `--verify`, it now displays a progress + bar while the verification step is running. The progress bar is not shown when + the `--json` flag is specified. + + https://github.com/restic/restic/issues/4795 + https://github.com/restic/restic/pull/4989 + + * Enhancement #4934: Automatically clear removed snapshots from cache + + Previously, restic only removed snapshots from the cache on the host where the + `forget` command was executed. On other hosts that use the same repository, the + old snapshots remained in the cache. + + Restic now automatically clears old snapshots from the local cache of the + current host. + + https://github.com/restic/restic/issues/4934 + https://github.com/restic/restic/pull/4981 + + * Enhancement #4944: Print JSON-formatted errors during `restore --json` + + Restic used to print any `restore` errors directly to the console as freeform + text messages, even when using the `--json` option. + + Now, when `--json` is specified, restic prints them as JSON formatted messages. + + https://github.com/restic/restic/issues/4944 + https://github.com/restic/restic/pull/4946 + + * Enhancement #4959: Return exit code 12 for "bad password" errors + + Restic now returns exit code 12 when it cannot open the repository due to an + incorrect password. + + https://github.com/restic/restic/pull/4959 + + * Enhancement #4970: Make timeout for stuck requests customizable + + Restic monitors connections to the backend to detect stuck requests. If a + request does not return any data within five minutes, restic assumes the request + is stuck and retries it. However, for large repositories this timeout might be + insufficient to collect a list of all files, causing the following error: + + `List(data) returned error, retrying after 1s: [...]: request timeout` + + It is now possible to increase the timeout using the `--stuck-request-timeout` + option. + + https://github.com/restic/restic/issues/4970 + https://github.com/restic/restic/pull/5014 + + # Changelog for restic 0.17.0 (2024-07-26) The following sections list the changes in restic 0.17.0 relevant to restic users. The changes are ordered by importance. diff --git a/mover-restic/restic/VERSION b/mover-restic/restic/VERSION index c5523bd09..7cca7711a 100644 --- a/mover-restic/restic/VERSION +++ b/mover-restic/restic/VERSION @@ -1 +1 @@ -0.17.0 +0.17.1 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-2004 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-2004 new file mode 100644 index 000000000..5372eeb8c --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-2004 @@ -0,0 +1,18 @@ +Bugfix: Correctly handle volume names in `backup` command on Windows + +On Windows, when the specified backup target only included the volume +name without a trailing slash, for example, `C:`, then restoring the +resulting snapshot would result in an error. Note that using `C:\` +as backup target worked correctly. + +Specifying volume names is now handled correctly. To restore snapshots +created before this bugfix, use the : syntax. For +example, to restore a snapshot with ID `12345678` that backed up `C:`, +use the following command: + +``` +restic restore 12345678:/C/C:./ --target output/folder +``` + +https://github.com/restic/restic/issues/2004 +https://github.com/restic/restic/pull/5028 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4795 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4795 new file mode 100644 index 000000000..ff86f0931 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4795 @@ -0,0 +1,8 @@ +Enhancement: Display progress bar for `restore --verify` + +When the `restore` command is run with `--verify`, it now displays a progress +bar while the verification step is running. The progress bar is not shown when +the `--json` flag is specified. + +https://github.com/restic/restic/issues/4795 +https://github.com/restic/restic/pull/4989 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4934 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4934 new file mode 100644 index 000000000..df77109a7 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4934 @@ -0,0 +1,11 @@ +Enhancement: Automatically clear removed snapshots from cache + +Previously, restic only removed snapshots from the cache on the host where the +`forget` command was executed. On other hosts that use the same repository, the +old snapshots remained in the cache. + +Restic now automatically clears old snapshots from the local cache of the +current host. + +https://github.com/restic/restic/issues/4934 +https://github.com/restic/restic/pull/4981 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4944 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4944 new file mode 100644 index 000000000..95ae24c03 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4944 @@ -0,0 +1,9 @@ +Enhancement: Print JSON-formatted errors during `restore --json` + +Restic used to print any `restore` errors directly to the console as freeform +text messages, even when using the `--json` option. + +Now, when `--json` is specified, restic prints them as JSON formatted messages. + +https://github.com/restic/restic/issues/4944 +https://github.com/restic/restic/pull/4946 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4945 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4945 new file mode 100644 index 000000000..a7a483fed --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4945 @@ -0,0 +1,10 @@ +Bugfix: Include missing backup error text with `--json` + +Previously, when running a backup with the `--json` option, restic failed to +include the actual error message in the output, resulting in `"error": {}` +being displayed. + +This has now been fixed, and restic now includes the error text in JSON output. + +https://github.com/restic/restic/issues/4945 +https://github.com/restic/restic/pull/4946 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4953 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4953 new file mode 100644 index 000000000..c542377fc --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4953 @@ -0,0 +1,7 @@ +Bugfix: Correctly handle long paths on older Windows versions + +On older Windows versions, like Windows Server 2012, restic 0.17.0 failed to +back up files with long paths. This problem has now been resolved. + +https://github.com/restic/restic/issues/4953 +https://github.com/restic/restic/pull/4954 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4957 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4957 new file mode 100644 index 000000000..59c73b5c7 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4957 @@ -0,0 +1,8 @@ +Bugfix: Fix delayed cancellation of certain commands + +Since restic 0.17.0, some commands did not immediately respond to cancellation +via Ctrl-C (SIGINT) and continued running for a short period. The most affected +commands were `diff`,`find`, `ls`, `stats` and `rewrite`. This is now resolved. + +https://github.com/restic/restic/issues/4957 +https://github.com/restic/restic/pull/4960 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4969 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4969 new file mode 100644 index 000000000..d92392a20 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4969 @@ -0,0 +1,7 @@ +Bugfix: Correctly restore timestamp for files with resource forks on macOS + +On macOS, timestamps were not restored for files with resource forks. This has +now been fixed. + +https://github.com/restic/restic/issues/4969 +https://github.com/restic/restic/pull/5006 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4970 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4970 new file mode 100644 index 000000000..422ae3c25 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4970 @@ -0,0 +1,15 @@ +Enhancement: Make timeout for stuck requests customizable + +Restic monitors connections to the backend to detect stuck requests. If a +request does not return any data within five minutes, restic assumes the +request is stuck and retries it. However, for large repositories this timeout +might be insufficient to collect a list of all files, causing the following +error: + +`List(data) returned error, retrying after 1s: [...]: request timeout` + +It is now possible to increase the timeout using the `--stuck-request-timeout` +option. + +https://github.com/restic/restic/issues/4970 +https://github.com/restic/restic/pull/5014 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4975 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4975 new file mode 100644 index 000000000..614642c06 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-4975 @@ -0,0 +1,7 @@ +Bugfix: Prevent `backup --stdin-from-command` from panicking + +Restic would previously crash if `--stdin-from-command` was specified without +providing a command. This issue has now been fixed. + +https://github.com/restic/restic/issues/4975 +https://github.com/restic/restic/pull/4976 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5004 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5004 new file mode 100644 index 000000000..72e98a9a4 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5004 @@ -0,0 +1,12 @@ +Bugfix: Fix spurious "A Required Privilege Is Not Held by the Client" error + +On Windows, creating a backup could sometimes trigger the following error: + +``` +error: nodeFromFileInfo [...]: get named security info failed with: a required privilege is not held by the client. +``` + +This has now been fixed. + +https://github.com/restic/restic/issues/5004 +https://github.com/restic/restic/pull/5019 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5005 b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5005 new file mode 100644 index 000000000..16ac83b4a --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/issue-5005 @@ -0,0 +1,16 @@ +Bugfix: Fix rare failures to retry locking a repository + +Restic 0.17.0 could in rare cases fail to retry locking a repository if one of +the lock files failed to load, resulting in the error: + +``` +unable to create lock in backend: circuit breaker open for file +``` + +This issue has now been addressed. The error handling now properly retries the +locking operation. In addition, restic waits a few seconds between locking +retries to increase chances of successful locking. + +https://github.com/restic/restic/issues/5005 +https://github.com/restic/restic/pull/5011 +https://github.com/restic/restic/pull/5012 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4958 b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4958 new file mode 100644 index 000000000..dae9b2c8e --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4958 @@ -0,0 +1,7 @@ +Bugfix: Don't ignore metadata-setting errors during restore + +Previously, restic used to ignore errors when setting timestamps, attributes, +or file modes during a restore. It now reports those errors, except for +permission related errors when running without root privileges. + +https://github.com/restic/restic/pull/4958 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4959 b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4959 new file mode 100644 index 000000000..80b2780b2 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4959 @@ -0,0 +1,6 @@ +Enhancement: Return exit code 12 for "bad password" errors + +Restic now returns exit code 12 when it cannot open the repository due to an +incorrect password. + +https://github.com/restic/restic/pull/4959 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4977 b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4977 new file mode 100644 index 000000000..781576a56 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4977 @@ -0,0 +1,16 @@ +Change: Also back up files with incomplete metadata + +If restic failed to read extended metadata for a file or folder during a +backup, then the file or folder was not included in the resulting snapshot. +Instead, a warning message was printed along with returning exit code 3 once +the backup was finished. + +Now, restic also includes items for which the extended metadata could not be +read in a snapshot. The warning message has been updated to: + +``` +incomplete metadata for /path/to/file:
+``` + +https://github.com/restic/restic/issues/4953 +https://github.com/restic/restic/pull/4977 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4980 b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4980 new file mode 100644 index 000000000..b51ee8d59 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-4980 @@ -0,0 +1,12 @@ +Bugfix: Skip extended attribute processing on unsupported Windows volumes + +With restic 0.17.0, backups of certain Windows paths, such as network drives, +failed due to errors while fetching extended attributes. + +Restic now skips extended attribute processing for volumes where they are not +supported. + +https://github.com/restic/restic/pull/4980 +https://github.com/restic/restic/pull/4998 +https://github.com/restic/restic/issues/4955 +https://github.com/restic/restic/issues/4950 diff --git a/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-5018 b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-5018 new file mode 100644 index 000000000..ca600c3e1 --- /dev/null +++ b/mover-restic/restic/changelog/0.17.1_2024-09-05/pull-5018 @@ -0,0 +1,13 @@ +Bugfix: Improve HTTP/2 support for REST backend + +If `rest-server` tried to gracefully shut down an HTTP/2 connection still in +use by the client, it could result in the following error: + +``` +http2: Transport: cannot retry err [http2: Transport received Server's graceful shutdown GOAWAY] after Request.Body was written; define Request.GetBody to avoid this error +``` + +This issue has now been resolved. + +https://github.com/restic/restic/pull/5018 +https://forum.restic.net/t/receiving-http2-goaway-messages-with-windows-restic-v0-17-0/8367 diff --git a/mover-restic/restic/cmd/restic/cmd_backup.go b/mover-restic/restic/cmd/restic/cmd_backup.go index 9957b5784..562108a33 100644 --- a/mover-restic/restic/cmd/restic/cmd_backup.go +++ b/mover-restic/restic/cmd/restic/cmd_backup.go @@ -43,6 +43,7 @@ Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, PreRun: func(_ *cobra.Command, _ []string) { if backupOptions.Host == "" { @@ -54,6 +55,7 @@ Exit status is 11 if the repository is already locked. backupOptions.Host = hostname } }, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() diff --git a/mover-restic/restic/cmd/restic/cmd_cache.go b/mover-restic/restic/cmd/restic/cmd_cache.go index e71d38365..e54c73451 100644 --- a/mover-restic/restic/cmd/restic/cmd_cache.go +++ b/mover-restic/restic/cmd/restic/cmd_cache.go @@ -28,6 +28,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, args []string) error { return runCache(cacheOptions, globalOptions, args) diff --git a/mover-restic/restic/cmd/restic/cmd_cat.go b/mover-restic/restic/cmd/restic/cmd_cat.go index 693c26790..6160c54df 100644 --- a/mover-restic/restic/cmd/restic/cmd_cat.go +++ b/mover-restic/restic/cmd/restic/cmd_cat.go @@ -12,6 +12,8 @@ import ( "github.com/restic/restic/internal/restic" ) +var catAllowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"} + var cmdCat = &cobra.Command{ Use: "cat [flags] [masterkey|config|pack ID|blob ID|snapshot ID|index ID|key ID|lock ID|tree snapshot:subfolder]", Short: "Print internal objects to stdout", @@ -25,11 +27,14 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runCat(cmd.Context(), globalOptions, args) }, + ValidArgs: catAllowedCmds, } func init() { @@ -37,21 +42,19 @@ func init() { } func validateCatArgs(args []string) error { - var allowedCmds = []string{"config", "index", "snapshot", "key", "masterkey", "lock", "pack", "blob", "tree"} - if len(args) < 1 { return errors.Fatal("type not specified") } validType := false - for _, v := range allowedCmds { + for _, v := range catAllowedCmds { if v == args[0] { validType = true break } } if !validType { - return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(allowedCmds, "|")) + return errors.Fatalf("invalid type %q, must be one of [%s]", args[0], strings.Join(catAllowedCmds, "|")) } if args[0] != "masterkey" && args[0] != "config" && len(args) != 2 { diff --git a/mover-restic/restic/cmd/restic/cmd_check.go b/mover-restic/restic/cmd/restic/cmd_check.go index 9cccc0609..dcf7f27df 100644 --- a/mover-restic/restic/cmd/restic/cmd_check.go +++ b/mover-restic/restic/cmd/restic/cmd_check.go @@ -39,7 +39,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() diff --git a/mover-restic/restic/cmd/restic/cmd_copy.go b/mover-restic/restic/cmd/restic/cmd_copy.go index d7761174a..cd92193ac 100644 --- a/mover-restic/restic/cmd/restic/cmd_copy.go +++ b/mover-restic/restic/cmd/restic/cmd_copy.go @@ -38,7 +38,10 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, + DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runCopy(cmd.Context(), copyOptions, globalOptions, args) }, diff --git a/mover-restic/restic/cmd/restic/cmd_debug.go b/mover-restic/restic/cmd/restic/cmd_debug.go index 74c21df24..b92192492 100644 --- a/mover-restic/restic/cmd/restic/cmd_debug.go +++ b/mover-restic/restic/cmd/restic/cmd_debug.go @@ -29,8 +29,10 @@ import ( ) var cmdDebug = &cobra.Command{ - Use: "debug", - Short: "Debug commands", + Use: "debug", + Short: "Debug commands", + GroupID: cmdGroupDefault, + DisableAutoGenTag: true, } var cmdDebugDump = &cobra.Command{ @@ -47,6 +49,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_diff.go b/mover-restic/restic/cmd/restic/cmd_diff.go index 6488a7c35..594e387e8 100644 --- a/mover-restic/restic/cmd/restic/cmd_diff.go +++ b/mover-restic/restic/cmd/restic/cmd_diff.go @@ -43,7 +43,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runDiff(cmd.Context(), diffOptions, globalOptions, args) @@ -177,6 +179,10 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b } for _, node := range tree.Nodes { + if ctx.Err() != nil { + return ctx.Err() + } + name := path.Join(prefix, node.Name) if node.Type == "dir" { name += "/" @@ -187,13 +193,13 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b if node.Type == "dir" { err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree) - if err != nil { + if err != nil && err != context.Canceled { Warnf("error: %v\n", err) } } } - return nil + return ctx.Err() } func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error { @@ -204,17 +210,21 @@ func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id rest } for _, node := range tree.Nodes { + if ctx.Err() != nil { + return ctx.Err() + } + addBlobs(blobs, node) if node.Type == "dir" { err := c.collectDir(ctx, blobs, *node.Subtree) - if err != nil { + if err != nil && err != context.Canceled { Warnf("error: %v\n", err) } } } - return nil + return ctx.Err() } func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) { @@ -255,6 +265,10 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2) for _, name := range names { + if ctx.Err() != nil { + return ctx.Err() + } + node1, t1 := tree1Nodes[name] node2, t2 := tree2Nodes[name] @@ -304,7 +318,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref } else { err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree) } - if err != nil { + if err != nil && err != context.Canceled { Warnf("error: %v\n", err) } } @@ -318,7 +332,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref if node1.Type == "dir" { err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree) - if err != nil { + if err != nil && err != context.Canceled { Warnf("error: %v\n", err) } } @@ -332,14 +346,14 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref if node2.Type == "dir" { err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree) - if err != nil { + if err != nil && err != context.Canceled { Warnf("error: %v\n", err) } } } } - return nil + return ctx.Err() } func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_dump.go b/mover-restic/restic/cmd/restic/cmd_dump.go index 7e1efa3ae..7d6652e17 100644 --- a/mover-restic/restic/cmd/restic/cmd_dump.go +++ b/mover-restic/restic/cmd/restic/cmd_dump.go @@ -38,7 +38,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runDump(cmd.Context(), dumpOptions, globalOptions, args) @@ -85,6 +87,10 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade item := filepath.Join(prefix, pathComponents[0]) l := len(pathComponents) for _, node := range tree.Nodes { + if ctx.Err() != nil { + return ctx.Err() + } + // If dumping something in the highest level it will just take the // first item it finds and dump that according to the switch case below. if node.Name == pathComponents[0] { diff --git a/mover-restic/restic/cmd/restic/cmd_features.go b/mover-restic/restic/cmd/restic/cmd_features.go index 497013696..a2f04be31 100644 --- a/mover-restic/restic/cmd/restic/cmd_features.go +++ b/mover-restic/restic/cmd/restic/cmd_features.go @@ -31,7 +31,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, - Hidden: true, + GroupID: cmdGroupAdvanced, DisableAutoGenTag: true, RunE: func(_ *cobra.Command, args []string) error { if len(args) != 0 { diff --git a/mover-restic/restic/cmd/restic/cmd_find.go b/mover-restic/restic/cmd/restic/cmd_find.go index 4f9549ca4..cb5c0e5e0 100644 --- a/mover-restic/restic/cmd/restic/cmd_find.go +++ b/mover-restic/restic/cmd/restic/cmd_find.go @@ -37,7 +37,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runFind(cmd.Context(), findOptions, globalOptions, args) @@ -377,6 +379,10 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { if node.Type == "file" && f.blobIDs != nil { for _, id := range node.Content { + if ctx.Err() != nil { + return ctx.Err() + } + idStr := id.String() if _, ok := f.blobIDs[idStr]; !ok { // Look for short ID form diff --git a/mover-restic/restic/cmd/restic/cmd_forget.go b/mover-restic/restic/cmd/restic/cmd_forget.go index 87738b518..58a9d25b7 100644 --- a/mover-restic/restic/cmd/restic/cmd_forget.go +++ b/mover-restic/restic/cmd/restic/cmd_forget.go @@ -39,7 +39,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() @@ -246,6 +248,10 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption printer.P("Applying Policy: %v\n", policy) for k, snapshotGroup := range snapshotGroups { + if ctx.Err() != nil { + return ctx.Err() + } + if gopts.Verbose >= 1 && !gopts.JSON { err = PrintSnapshotGroupHeader(globalOptions.stdout, k) if err != nil { diff --git a/mover-restic/restic/cmd/restic/cmd_init.go b/mover-restic/restic/cmd/restic/cmd_init.go index 3c0319e55..2a2aae1dc 100644 --- a/mover-restic/restic/cmd/restic/cmd_init.go +++ b/mover-restic/restic/cmd/restic/cmd_init.go @@ -26,6 +26,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runInit(cmd.Context(), initOptions, globalOptions, args) diff --git a/mover-restic/restic/cmd/restic/cmd_init_integration_test.go b/mover-restic/restic/cmd/restic/cmd_init_integration_test.go index 9b5eed6e0..4795d5510 100644 --- a/mover-restic/restic/cmd/restic/cmd_init_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_init_integration_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "os" + "path/filepath" "testing" "github.com/restic/restic/internal/repository" @@ -16,6 +18,11 @@ func testRunInit(t testing.TB, opts GlobalOptions) { rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil)) t.Logf("repository initialized at %v", opts.Repo) + + // create temporary junk files to verify that restic does not trip over them + for _, path := range []string{"index", "snapshots", "keys", "locks", filepath.Join("data", "00")} { + rtest.OK(t, os.WriteFile(filepath.Join(opts.Repo, path, "tmp12345"), []byte("junk file"), 0o600)) + } } func TestInitCopyChunkerParams(t *testing.T) { diff --git a/mover-restic/restic/cmd/restic/cmd_key.go b/mover-restic/restic/cmd/restic/cmd_key.go index c687eca53..a94caa0d8 100644 --- a/mover-restic/restic/cmd/restic/cmd_key.go +++ b/mover-restic/restic/cmd/restic/cmd_key.go @@ -11,6 +11,8 @@ var cmdKey = &cobra.Command{ The "key" command allows you to set multiple access keys or passwords per repository. `, + DisableAutoGenTag: true, + GroupID: cmdGroupDefault, } func init() { diff --git a/mover-restic/restic/cmd/restic/cmd_key_add.go b/mover-restic/restic/cmd/restic/cmd_key_add.go index c9f0ef233..2737410a0 100644 --- a/mover-restic/restic/cmd/restic/cmd_key_add.go +++ b/mover-restic/restic/cmd/restic/cmd_key_add.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, } diff --git a/mover-restic/restic/cmd/restic/cmd_key_list.go b/mover-restic/restic/cmd/restic/cmd_key_list.go index ae751a487..1c70cce8a 100644 --- a/mover-restic/restic/cmd/restic/cmd_key_list.go +++ b/mover-restic/restic/cmd/restic/cmd_key_list.go @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_key_passwd.go b/mover-restic/restic/cmd/restic/cmd_key_passwd.go index 723acaaab..9bb141749 100644 --- a/mover-restic/restic/cmd/restic/cmd_key_passwd.go +++ b/mover-restic/restic/cmd/restic/cmd_key_passwd.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, } diff --git a/mover-restic/restic/cmd/restic/cmd_key_remove.go b/mover-restic/restic/cmd/restic/cmd_key_remove.go index c4c24fdb7..3cb2e0bd7 100644 --- a/mover-restic/restic/cmd/restic/cmd_key_remove.go +++ b/mover-restic/restic/cmd/restic/cmd_key_remove.go @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_list.go b/mover-restic/restic/cmd/restic/cmd_list.go index 060bca871..1a4791e31 100644 --- a/mover-restic/restic/cmd/restic/cmd_list.go +++ b/mover-restic/restic/cmd/restic/cmd_list.go @@ -23,8 +23,10 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runList(cmd.Context(), globalOptions, args) }, diff --git a/mover-restic/restic/cmd/restic/cmd_ls.go b/mover-restic/restic/cmd/restic/cmd_ls.go index 76e192b6c..69e278103 100644 --- a/mover-restic/restic/cmd/restic/cmd_ls.go +++ b/mover-restic/restic/cmd/restic/cmd_ls.go @@ -43,8 +43,10 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runLs(cmd.Context(), lsOptions, globalOptions, args) }, diff --git a/mover-restic/restic/cmd/restic/cmd_migrate.go b/mover-restic/restic/cmd/restic/cmd_migrate.go index e89980050..5c3e425ed 100644 --- a/mover-restic/restic/cmd/restic/cmd_migrate.go +++ b/mover-restic/restic/cmd/restic/cmd_migrate.go @@ -26,8 +26,10 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() defer cancel() @@ -74,8 +76,10 @@ func checkMigrations(ctx context.Context, repo restic.Repository, printer progre func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string, term *termstatus.Terminal, printer progress.Printer) error { var firsterr error for _, name := range args { + found := false for _, m := range migrations.All { if m.Name() == name { + found = true ok, reason, err := m.Check(ctx, repo) if err != nil { return err @@ -119,6 +123,9 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio printer.P("migration %v: success\n", m.Name()) } } + if !found { + printer.E("unknown migration %v", name) + } } return firsterr diff --git a/mover-restic/restic/cmd/restic/cmd_mount.go b/mover-restic/restic/cmd/restic/cmd_mount.go index 3e0b159be..2f57a6d1f 100644 --- a/mover-restic/restic/cmd/restic/cmd_mount.go +++ b/mover-restic/restic/cmd/restic/cmd_mount.go @@ -68,8 +68,10 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, + GroupID: cmdGroupDefault, RunE: func(cmd *cobra.Command, args []string) error { return runMount(cmd.Context(), mountOptions, globalOptions, args) }, diff --git a/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go b/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go index d764b4e4f..c5f4d193a 100644 --- a/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_mount_integration_test.go @@ -13,6 +13,7 @@ import ( "time" systemFuse "github.com/anacrolix/fuse" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -205,6 +206,11 @@ func TestMountSameTimestamps(t *testing.T) { t.Skip("Skipping fuse tests") } + debugEnabled := debug.TestLogToStderr(t) + if debugEnabled { + defer debug.TestDisableLog(t) + } + env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.backendTestHook = nil diff --git a/mover-restic/restic/cmd/restic/cmd_options.go b/mover-restic/restic/cmd/restic/cmd_options.go index 4cd574b68..9c07b2626 100644 --- a/mover-restic/restic/cmd/restic/cmd_options.go +++ b/mover-restic/restic/cmd/restic/cmd_options.go @@ -20,7 +20,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, - Hidden: true, + GroupID: cmdGroupAdvanced, DisableAutoGenTag: true, Run: func(_ *cobra.Command, _ []string) { fmt.Printf("All Extended Options:\n") diff --git a/mover-restic/restic/cmd/restic/cmd_prune.go b/mover-restic/restic/cmd/restic/cmd_prune.go index 7e706ccf8..e8473bd6f 100644 --- a/mover-restic/restic/cmd/restic/cmd_prune.go +++ b/mover-restic/restic/cmd/restic/cmd_prune.go @@ -32,7 +32,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { term, cancel := setupTermstatus() diff --git a/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go b/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go index 746eb5cc9..536ec40d8 100644 --- a/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go +++ b/mover-restic/restic/cmd/restic/cmd_prune_integration_test.go @@ -146,10 +146,9 @@ func TestPruneWithDamagedRepository(t *testing.T) { env.gopts.backendTestHook = oldHook }() // prune should fail - rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { + rtest.Equals(t, repository.ErrPacksMissing, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error { return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term) - }) == repository.ErrPacksMissing, - "prune should have reported index not complete error") + }), "prune should have reported index not complete error") } // Test repos for edge cases diff --git a/mover-restic/restic/cmd/restic/cmd_recover.go b/mover-restic/restic/cmd/restic/cmd_recover.go index 5e4744bb6..a6ef59cc2 100644 --- a/mover-restic/restic/cmd/restic/cmd_recover.go +++ b/mover-restic/restic/cmd/restic/cmd_recover.go @@ -26,7 +26,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { return runRecover(cmd.Context(), globalOptions) @@ -118,6 +120,10 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { return nil } + if ctx.Err() != nil { + return ctx.Err() + } + tree := restic.NewTree(len(roots)) for id := range roots { var subtreeID = id diff --git a/mover-restic/restic/cmd/restic/cmd_repair.go b/mover-restic/restic/cmd/restic/cmd_repair.go index aefe02f3c..6a1a1f9dc 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair.go +++ b/mover-restic/restic/cmd/restic/cmd_repair.go @@ -5,8 +5,10 @@ import ( ) var cmdRepair = &cobra.Command{ - Use: "repair", - Short: "Repair the repository", + Use: "repair", + Short: "Repair the repository", + GroupID: cmdGroupDefault, + DisableAutoGenTag: true, } func init() { diff --git a/mover-restic/restic/cmd/restic/cmd_repair_index.go b/mover-restic/restic/cmd/restic/cmd_repair_index.go index e6b6e9fa5..83c1bfa7f 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_index.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_index.go @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_repair_packs.go b/mover-restic/restic/cmd/restic/cmd_repair_packs.go index b0afefb2d..290c3734e 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_packs.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_packs.go @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go b/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go index fc221ebea..385854312 100644 --- a/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go +++ b/mover-restic/restic/cmd/restic/cmd_repair_snapshots.go @@ -41,6 +41,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_restore.go b/mover-restic/restic/cmd/restic/cmd_restore.go index 89942f4cf..c58b0b80d 100644 --- a/mover-restic/restic/cmd/restic/cmd_restore.go +++ b/mover-restic/restic/cmd/restic/cmd_restore.go @@ -36,7 +36,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { term, cancel := setupTermstatus() @@ -164,9 +166,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, totalErrors := 0 res.Error = func(location string, err error) error { - msg.E("ignoring error for %s: %s\n", location, err) totalErrors++ - return nil + return progress.Error(location, err) } res.Warn = func(message string) { msg.E("Warning: %s\n", message) @@ -221,7 +222,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, msg.P("restoring %s to %s\n", res.Snapshot(), opts.Target) } - err = res.RestoreTo(ctx, opts.Target) + countRestoredFiles, err := res.RestoreTo(ctx, opts.Target) if err != nil { return err } @@ -238,7 +239,8 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, } var count int t0 := time.Now() - count, err = res.VerifyFiles(ctx, opts.Target) + bar := newTerminalProgressMax(!gopts.Quiet && !gopts.JSON && stdoutIsTerminal(), 0, "files verified", term) + count, err = res.VerifyFiles(ctx, opts.Target, countRestoredFiles, bar) if err != nil { return err } diff --git a/mover-restic/restic/cmd/restic/cmd_rewrite.go b/mover-restic/restic/cmd/restic/cmd_rewrite.go index 463720ee1..7788016b7 100644 --- a/mover-restic/restic/cmd/restic/cmd_rewrite.go +++ b/mover-restic/restic/cmd/restic/cmd_rewrite.go @@ -42,7 +42,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runRewrite(cmd.Context(), rewriteOptions, globalOptions, args) diff --git a/mover-restic/restic/cmd/restic/cmd_self_update.go b/mover-restic/restic/cmd/restic/cmd_self_update.go index 0fce41241..09c86bf2c 100644 --- a/mover-restic/restic/cmd/restic/cmd_self_update.go +++ b/mover-restic/restic/cmd/restic/cmd_self_update.go @@ -28,6 +28,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/mover-restic/restic/cmd/restic/cmd_snapshots.go b/mover-restic/restic/cmd/restic/cmd_snapshots.go index 9112e1b95..42677918f 100644 --- a/mover-restic/restic/cmd/restic/cmd_snapshots.go +++ b/mover-restic/restic/cmd/restic/cmd_snapshots.go @@ -27,7 +27,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args) @@ -81,6 +83,10 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } for k, list := range snapshotGroups { + if ctx.Err() != nil { + return ctx.Err() + } + if opts.Last { // This branch should be removed in the same time // that --last. @@ -101,6 +107,10 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions } for k, list := range snapshotGroups { + if ctx.Err() != nil { + return ctx.Err() + } + if grouped { err := PrintSnapshotGroupHeader(globalOptions.stdout, k) if err != nil { diff --git a/mover-restic/restic/cmd/restic/cmd_stats.go b/mover-restic/restic/cmd/restic/cmd_stats.go index ab333e6ef..56cf213d1 100644 --- a/mover-restic/restic/cmd/restic/cmd_stats.go +++ b/mover-restic/restic/cmd/restic/cmd_stats.go @@ -53,7 +53,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runStats(cmd.Context(), statsOptions, globalOptions, args) @@ -70,10 +72,20 @@ type StatsOptions struct { var statsOptions StatsOptions +func must(err error) { + if err != nil { + panic(fmt.Sprintf("error during setup: %v", err)) + } +} + func init() { cmdRoot.AddCommand(cmdStats) f := cmdStats.Flags() f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data") + must(cmdStats.RegisterFlagCompletionFunc("mode", func(_ *cobra.Command, _ []string, _ string) ([]string, cobra.ShellCompDirective) { + return []string{countModeRestoreSize, countModeUniqueFilesByContents, countModeBlobsPerFile, countModeRawData}, cobra.ShellCompDirectiveDefault + })) + initMultiSnapshotFilter(f, &statsOptions.SnapshotFilter, true) } diff --git a/mover-restic/restic/cmd/restic/cmd_tag.go b/mover-restic/restic/cmd/restic/cmd_tag.go index ea73955f0..c7bf725e9 100644 --- a/mover-restic/restic/cmd/restic/cmd_tag.go +++ b/mover-restic/restic/cmd/restic/cmd_tag.go @@ -29,7 +29,9 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { return runTag(cmd.Context(), tagOptions, globalOptions, args) diff --git a/mover-restic/restic/cmd/restic/cmd_unlock.go b/mover-restic/restic/cmd/restic/cmd_unlock.go index 96eef7e02..d87cde065 100644 --- a/mover-restic/restic/cmd/restic/cmd_unlock.go +++ b/mover-restic/restic/cmd/restic/cmd_unlock.go @@ -19,6 +19,7 @@ EXIT STATUS Exit status is 0 if the command was successful. Exit status is 1 if there was any error. `, + GroupID: cmdGroupDefault, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { return runUnlock(cmd.Context(), unlockOptions, globalOptions) diff --git a/mover-restic/restic/cmd/restic/global.go b/mover-restic/restic/cmd/restic/global.go index 080863da7..9df009d8c 100644 --- a/mover-restic/restic/cmd/restic/global.go +++ b/mover-restic/restic/cmd/restic/global.go @@ -47,7 +47,7 @@ import ( // to a missing backend storage location or config file var ErrNoRepository = errors.New("repository does not exist") -var version = "0.17.0" +var version = "0.17.1" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" @@ -140,6 +140,7 @@ func init() { f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)") f.StringVar(&globalOptions.HTTPUserAgent, "http-user-agent", "", "set a http user agent for outgoing http requests") + f.DurationVar(&globalOptions.StuckRequestTimeout, "stuck-request-timeout", 5*time.Minute, "`duration` after which to retry stuck requests") // Use our "generate" command instead of the cobra provided "completion" command cmdRoot.CompletionOptions.DisableDefaultCmd = true @@ -493,7 +494,7 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } } if err != nil { - if errors.IsFatal(err) { + if errors.IsFatal(err) || errors.Is(err, repository.ErrNoKeyFound) { return nil, err } return nil, errors.Fatalf("%s", err) diff --git a/mover-restic/restic/cmd/restic/integration_test.go b/mover-restic/restic/cmd/restic/integration_test.go index 4cecec6bc..df95031dc 100644 --- a/mover-restic/restic/cmd/restic/integration_test.go +++ b/mover-restic/restic/cmd/restic/integration_test.go @@ -80,7 +80,7 @@ func TestListOnce(t *testing.T) { defer cleanup() env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { - return newListOnceBackend(r), nil + return newOrderedListOnceBackend(r), nil } pruneOpts := PruneOptions{MaxUnused: "0"} checkOpts := CheckOptions{ReadData: true, CheckUnused: true} @@ -148,7 +148,7 @@ func TestFindListOnce(t *testing.T) { defer cleanup() env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { - return newListOnceBackend(r), nil + return newOrderedListOnceBackend(r), nil } testSetupBackupData(t, env) diff --git a/mover-restic/restic/cmd/restic/main.go b/mover-restic/restic/cmd/restic/main.go index 5818221a5..26e45bb38 100644 --- a/mover-restic/restic/cmd/restic/main.go +++ b/mover-restic/restic/cmd/restic/main.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/options" + "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" ) @@ -82,6 +83,22 @@ The full documentation can be found at https://restic.readthedocs.io/ . }, } +var cmdGroupDefault = "default" +var cmdGroupAdvanced = "advanced" + +func init() { + cmdRoot.AddGroup( + &cobra.Group{ + ID: cmdGroupDefault, + Title: "Available Commands:", + }, + &cobra.Group{ + ID: cmdGroupAdvanced, + Title: "Advanced Options:", + }, + ) +} + // Distinguish commands that need the password from those that work without, // so we don't run $RESTIC_PASSWORD_COMMAND for no reason (it might prompt the // user for authentication). @@ -138,6 +155,8 @@ func main() { fmt.Fprintf(os.Stderr, "Warning: %v\n", err) case errors.IsFatal(err): fmt.Fprintf(os.Stderr, "%v\n", err) + case errors.Is(err, repository.ErrNoKeyFound): + fmt.Fprintf(os.Stderr, "Fatal: %v\n", err) case err != nil: fmt.Fprintf(os.Stderr, "%+v\n", err) @@ -160,6 +179,8 @@ func main() { exitCode = 10 case restic.IsAlreadyLocked(err): exitCode = 11 + case errors.Is(err, repository.ErrNoKeyFound): + exitCode = 12 case errors.Is(err, context.Canceled): exitCode = 130 default: diff --git a/mover-restic/restic/cmd/restic/secondary_repo.go b/mover-restic/restic/cmd/restic/secondary_repo.go index 9a3eb5fe2..44621afa1 100644 --- a/mover-restic/restic/cmd/restic/secondary_repo.go +++ b/mover-restic/restic/cmd/restic/secondary_repo.go @@ -50,7 +50,7 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)") f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)") f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)") - f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository, must be passed to every restic command (insecure)") + f.BoolVar(&opts.InsecureNoPassword, "from-insecure-no-password", false, "use an empty password for the source repository (insecure)") opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY") opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE") diff --git a/mover-restic/restic/doc/030_preparing_a_new_repo.rst b/mover-restic/restic/doc/030_preparing_a_new_repo.rst index 87975f9fa..fd5b31127 100644 --- a/mover-restic/restic/doc/030_preparing_a_new_repo.rst +++ b/mover-restic/restic/doc/030_preparing_a_new_repo.rst @@ -249,28 +249,22 @@ while creating the bucket. $ export AWS_ACCESS_KEY_ID= $ export AWS_SECRET_ACCESS_KEY= +When using temporary credentials make sure to include the session token via +the environment variable ``AWS_SESSION_TOKEN``. + You can then easily initialize a repository that uses your Amazon S3 as -a backend. If the bucket does not exist it will be created in the -default location: +a backend. Make sure to use the endpoint for the correct region. The example +uses ``us-east-1``. If the bucket does not exist it will be created in that region: .. code-block:: console - $ restic -r s3:s3.amazonaws.com/bucket_name init + $ restic -r s3:s3.us-east-1.amazonaws.com/bucket_name init enter password for new repository: enter password again: - created restic repository eefee03bbd at s3:s3.amazonaws.com/bucket_name + created restic repository eefee03bbd at s3:s3.us-east-1.amazonaws.com/bucket_name Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. -If needed, you can manually specify the region to use by either setting the -environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option -parameter like ``-o s3.region="us-east-1"``. If the region is not specified, -the default region is used. Afterwards, the S3 server (at least for AWS, -``s3.amazonaws.com``) will redirect restic to the correct endpoint. - -When using temporary credentials make sure to include the session token via -then environment variable ``AWS_SESSION_TOKEN``. - Until version 0.8.0, restic used a default prefix of ``restic``, so the files in the bucket were placed in a directory named ``restic``. If you want to access a repository created with an older version of restic, specify the path @@ -278,25 +272,14 @@ after the bucket name like this: .. code-block:: console - $ restic -r s3:s3.amazonaws.com/bucket_name/restic [...] + $ restic -r s3:s3.us-east-1.amazonaws.com/bucket_name/restic [...] -For an S3-compatible server that is not Amazon (like Minio, see below), -or is only available via HTTP, you can specify the URL to the server -like this: ``s3:http://server:port/bucket_name``. - .. note:: restic expects `path-style URLs `__ - like for example ``s3.us-west-2.amazonaws.com/bucket_name``. + like for example ``s3.us-west-2.amazonaws.com/bucket_name`` for Amazon S3. Virtual-hosted–style URLs like ``bucket_name.s3.us-west-2.amazonaws.com``, where the bucket name is part of the hostname are not supported. These must be converted to path-style URLs instead, for example ``s3.us-west-2.amazonaws.com/bucket_name``. - -.. note:: Certain S3-compatible servers do not properly implement the - ``ListObjectsV2`` API, most notably Ceph versions before v14.2.5. On these - backends, as a temporary workaround, you can provide the - ``-o s3.list-objects-v1=true`` option to use the older - ``ListObjects`` API instead. This option may be removed in future - versions of restic. - + See below for configuration options for S3-compatible storage from other providers. Minio Server ************ @@ -321,81 +304,66 @@ this command. .. code-block:: console - $ ./restic -r s3:http://localhost:9000/restic init + $ restic -r s3:http://localhost:9000/restic init enter password for new repository: enter password again: - created restic repository 6ad29560f5 at s3:http://localhost:9000/restic1 + created restic repository 6ad29560f5 at s3:http://localhost:9000/restic Please note that knowledge of your password is required to access the repository. Losing your password means that your data is irrecoverably lost. -Wasabi -************ +S3-compatible Storage +********************* -`Wasabi `__ is a low cost Amazon S3 conformant object storage provider. -Due to its S3 conformance, Wasabi can be used as a storage provider for a restic repository. +For an S3-compatible server that is not Amazon, you can specify the URL to the server +like this: ``s3:https://server:port/bucket_name``. -- Create a Wasabi bucket using the `Wasabi Console `__. -- Determine the correct Wasabi service URL for your bucket `here `__. +If needed, you can manually specify the region to use by either setting the +environment variable ``AWS_DEFAULT_REGION`` or calling restic with an option +parameter like ``-o s3.region="us-east-1"``. If the region is not specified, +the default region ``us-east-1`` is used. -You must first setup the following environment variables with the -credentials of your Wasabi account. +To select between path-style and virtual-hosted access, the extended option +``-o s3.bucket-lookup=auto`` can be used. It supports the following values: -.. code-block:: console +- ``auto``: Default behavior. Uses ``dns`` for Amazon and Google endpoints. Uses + ``path`` for all other endpoints +- ``dns``: Use virtual-hosted-style bucket access +- ``path``: Use path-style bucket access - $ export AWS_ACCESS_KEY_ID= - $ export AWS_SECRET_ACCESS_KEY= +Certain S3-compatible servers do not properly implement the ``ListObjectsV2`` API, +most notably Ceph versions before v14.2.5. On these backends, as a temporary +workaround, you can provide the ``-o s3.list-objects-v1=true`` option to use the +older ``ListObjects`` API instead. This option may be removed in future versions +of restic. -Now you can easily initialize restic to use Wasabi as a backend with -this command. +Wasabi +****** + +S3 storage from `Wasabi `__ can be used as follows. + +- Determine the correct Wasabi service URL for your bucket `here `__. +- Set environment variables with the necessary account credentials .. code-block:: console - $ ./restic -r s3:https:/// init - enter password for new repository: - enter password again: - created restic repository xxxxxxxxxx at s3:https:/// - Please note that knowledge of your password is required to access - the repository. Losing your password means that your data is irrecoverably lost. + $ export AWS_ACCESS_KEY_ID= + $ export AWS_SECRET_ACCESS_KEY= + $ restic -r s3:https:/// init Alibaba Cloud (Aliyun) Object Storage System (OSS) ************************************************** -`Alibaba OSS `__ is an -encrypted, secure, cost-effective, and easy-to-use object storage -service that enables you to store, back up, and archive large amounts -of data in the cloud. - -Alibaba OSS is S3 compatible so it can be used as a storage provider -for a restic repository with a couple of extra parameters. +S3 storage from `Alibaba OSS `__ can be used as follows. -- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` -- You'll need the region name too - this will be something like ``oss-eu-west-1`` - -You must first setup the following environment variables with the -credentials of your Alibaba OSS account. +- Determine the correct `Alibaba OSS region endpoint `__ - this will be something like ``oss-eu-west-1.aliyuncs.com`` +- You will need the region name too - this will be something like ``oss-eu-west-1`` +- Set environment variables with the necessary account credentials .. code-block:: console $ export AWS_ACCESS_KEY_ID= $ export AWS_SECRET_ACCESS_KEY= - -Now you can easily initialize restic to use Alibaba OSS as a backend with -this command. - -.. code-block:: console - - $ ./restic -o s3.bucket-lookup=dns -o s3.region= -r s3:https:/// init - enter password for new backend: - enter password again: - created restic backend xxxxxxxxxx at s3:https:/// - Please note that knowledge of your password is required to access - the repository. Losing your password means that your data is irrecoverably lost. - -For example with an actual endpoint: - -.. code-block:: console - - $ restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init + $ restic -o s3.bucket-lookup=dns -o s3.region= -r s3:https:/// init OpenStack Swift *************** diff --git a/mover-restic/restic/doc/040_backup.rst b/mover-restic/restic/doc/040_backup.rst index 81d99e071..696b235cc 100644 --- a/mover-restic/restic/doc/040_backup.rst +++ b/mover-restic/restic/doc/040_backup.rst @@ -584,11 +584,13 @@ Reading data from a command Sometimes, it can be useful to directly save the output of a program, for example, ``mysqldump`` so that the SQL can later be restored. Restic supports this mode of operation; just supply the option ``--stdin-from-command`` when using the -``backup`` action, and write the command in place of the files/directories: +``backup`` action, and write the command in place of the files/directories. To prevent +restic from interpreting the arguments for the command, make sure to add ``--`` before +the command starts: .. code-block:: console - $ restic -r /srv/restic-repo backup --stdin-from-command mysqldump [...] + $ restic -r /srv/restic-repo backup --stdin-from-command -- mysqldump --host example mydb [...] This command creates a new snapshot based on the standard output of ``mysqldump``. By default, the command's standard output is saved in a file named ``stdin``. @@ -596,7 +598,7 @@ A different name can be specified with ``--stdin-filename``: .. code-block:: console - $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command mysqldump [...] + $ restic -r /srv/restic-repo backup --stdin-filename production.sql --stdin-from-command -- mysqldump --host example mydb [...] Restic uses the command exit code to determine whether the command succeeded. A non-zero exit code from the command causes restic to cancel the backup. This causes @@ -684,6 +686,30 @@ created as it would only be written at the very (successful) end of the backup operation. Previous snapshots will still be there and will still work. +Exit status codes +***************** + +Restic returns an exit status code after the backup command is run: + +* 0 when the backup was successful (snapshot with all source files created) +* 1 when there was a fatal error (no snapshot created) +* 3 when some source files could not be read (incomplete snapshot with remaining files created) +* further exit codes are documented in :ref:`exit-codes`. + +Fatal errors occur for example when restic is unable to write to the backup destination, when +there are network connectivity issues preventing successful communication, or when an invalid +password or command line argument is provided. When restic returns this exit status code, one +should not expect a snapshot to have been created. + +Source file read errors occur when restic fails to read one or more files or directories that +it was asked to back up, e.g. due to permission problems. Restic displays the number of source +file read errors that occurred while running the backup. If there are errors of this type, +restic will still try to complete the backup run with all the other files, and create a +snapshot that then contains all but the unreadable files. + +For use of these exit status codes in scripts and other automation tools, see :ref:`exit-codes`. +To manually inspect the exit code in e.g. Linux, run ``echo $?``. + Environment Variables ********************* @@ -702,6 +728,7 @@ environment variables. The following lists these environment variables: RESTIC_TLS_CLIENT_CERT Location of TLS client certificate and private key (replaces --tls-client-cert) RESTIC_CACHE_DIR Location of the cache directory RESTIC_COMPRESSION Compression mode (only available for repository format version 2) + RESTIC_HOST Only consider snapshots for this host / Set the hostname for the snapshot manually (replaces --host) RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated RESTIC_PACK_SIZE Target size for pack files RESTIC_READ_CONCURRENCY Concurrency for file reads @@ -771,26 +798,3 @@ See :ref:`caching` for the rules concerning cache locations when The external programs that restic may execute include ``rclone`` (for rclone backends) and ``ssh`` (for the SFTP backend). These may respond to further environment variables and configuration files; see their respective manuals. - -Exit status codes -***************** - -Restic returns one of the following exit status codes after the backup command is run: - -* 0 when the backup was successful (snapshot with all source files created) -* 1 when there was a fatal error (no snapshot created) -* 3 when some source files could not be read (incomplete snapshot with remaining files created) - -Fatal errors occur for example when restic is unable to write to the backup destination, when -there are network connectivity issues preventing successful communication, or when an invalid -password or command line argument is provided. When restic returns this exit status code, one -should not expect a snapshot to have been created. - -Source file read errors occur when restic fails to read one or more files or directories that -it was asked to back up, e.g. due to permission problems. Restic displays the number of source -file read errors that occurred while running the backup. If there are errors of this type, -restic will still try to complete the backup run with all the other files, and create a -snapshot that then contains all but the unreadable files. - -One can use these exit status codes in scripts and other automation tools, to make them aware of -the outcome of the backup run. To manually inspect the exit code in e.g. Linux, run ``echo $?``. diff --git a/mover-restic/restic/doc/045_working_with_repos.rst b/mover-restic/restic/doc/045_working_with_repos.rst index 8dba8439f..f31e75c84 100644 --- a/mover-restic/restic/doc/045_working_with_repos.rst +++ b/mover-restic/restic/doc/045_working_with_repos.rst @@ -305,6 +305,13 @@ In order to preview the changes which ``rewrite`` would make, you can use the modifying the repository. Instead restic will only print the actions it would perform. +.. note:: The ``rewrite`` command verifies that it does not modify snapshots in + unexpected ways and fails with an ``cannot encode tree at "[...]" without loosing information`` + error otherwise. This can occur when rewriting a snapshot created by a newer + version of restic or some third-party implementation. + + To convert a snapshot into the format expected by the ``rewrite`` command + use ``restic repair snapshots ``. Modifying metadata of snapshots =============================== diff --git a/mover-restic/restic/doc/075_scripting.rst b/mover-restic/restic/doc/075_scripting.rst index 87ae4fcf4..9fa0da6d0 100644 --- a/mover-restic/restic/doc/075_scripting.rst +++ b/mover-restic/restic/doc/075_scripting.rst @@ -39,6 +39,8 @@ Note that restic will also return exit code ``1`` if a different error is encoun If there are no errors, restic will return a zero exit code and print the repository metadata. +.. _exit-codes: + Exit codes ********** @@ -63,6 +65,8 @@ a more specific description. +-----+----------------------------------------------------+ | 11 | Failed to lock repository (since restic 0.17.0) | +-----+----------------------------------------------------+ +| 12 | Wrong password (since restic 0.17.1) | ++-----+----------------------------------------------------+ | 130 | Restic was interrupted using SIGINT or SIGSTOP | +-----+----------------------------------------------------+ @@ -139,7 +143,7 @@ Error +----------------------+-------------------------------------------+ | ``message_type`` | Always "error" | +----------------------+-------------------------------------------+ -| ``error`` | Error message | +| ``error.message`` | Error message | +----------------------+-------------------------------------------+ | ``during`` | What restic was trying to do | +----------------------+-------------------------------------------+ @@ -539,6 +543,19 @@ Status |``bytes_skipped`` | Total size of skipped files | +----------------------+------------------------------------------------------------+ +Error +^^^^^ + ++----------------------+-------------------------------------------+ +| ``message_type`` | Always "error" | ++----------------------+-------------------------------------------+ +| ``error.message`` | Error message | ++----------------------+-------------------------------------------+ +| ``during`` | Always "restore" | ++----------------------+-------------------------------------------+ +| ``item`` | Usually, the path of the problematic file | ++----------------------+-------------------------------------------+ + Verbose Status ^^^^^^^^^^^^^^ diff --git a/mover-restic/restic/doc/bash-completion.sh b/mover-restic/restic/doc/bash-completion.sh index 9d64871ca..0517fdf7c 100644 --- a/mover-restic/restic/doc/bash-completion.sh +++ b/mover-restic/restic/doc/bash-completion.sh @@ -516,6 +516,8 @@ _restic_backup() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -592,6 +594,8 @@ _restic_cache() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -660,6 +664,8 @@ _restic_cat() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -667,6 +673,15 @@ _restic_cat() must_have_one_flag=() must_have_one_noun=() + must_have_one_noun+=("blob") + must_have_one_noun+=("config") + must_have_one_noun+=("index") + must_have_one_noun+=("key") + must_have_one_noun+=("lock") + must_have_one_noun+=("masterkey") + must_have_one_noun+=("pack") + must_have_one_noun+=("snapshot") + must_have_one_noun+=("tree") noun_aliases=() } @@ -736,6 +751,8 @@ _restic_check() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -840,6 +857,8 @@ _restic_copy() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -910,6 +929,8 @@ _restic_diff() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1004,6 +1025,78 @@ _restic_dump() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_features() +{ + last_command="restic_features" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1122,6 +1215,8 @@ _restic_find() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1298,6 +1393,8 @@ _restic_forget() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1386,6 +1483,8 @@ _restic_generate() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1450,6 +1549,8 @@ _restic_help() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1547,6 +1648,8 @@ _restic_init() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1629,6 +1732,8 @@ _restic_key_add() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1693,6 +1798,8 @@ _restic_key_help() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1762,6 +1869,8 @@ _restic_key_list() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1844,6 +1953,8 @@ _restic_key_passwd() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1912,6 +2023,8 @@ _restic_key_remove() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -1985,6 +2098,8 @@ _restic_key() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2053,6 +2168,8 @@ _restic_list() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2145,6 +2262,8 @@ _restic_ls() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2217,6 +2336,8 @@ _restic_migrate() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2313,6 +2434,78 @@ _restic_mount() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") + flags+=("--tls-client-cert=") + two_word_flags+=("--tls-client-cert") + flags+=("--verbose") + flags+=("-v") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + +_restic_options() +{ + last_command="restic_options" + + command_aliases=() + + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--help") + flags+=("-h") + local_nonpersistent_flags+=("--help") + local_nonpersistent_flags+=("-h") + flags+=("--cacert=") + two_word_flags+=("--cacert") + flags+=("--cache-dir=") + two_word_flags+=("--cache-dir") + flags+=("--cleanup-cache") + flags+=("--compression=") + two_word_flags+=("--compression") + flags+=("--http-user-agent=") + two_word_flags+=("--http-user-agent") + flags+=("--insecure-no-password") + flags+=("--insecure-tls") + flags+=("--json") + flags+=("--key-hint=") + two_word_flags+=("--key-hint") + flags+=("--limit-download=") + two_word_flags+=("--limit-download") + flags+=("--limit-upload=") + two_word_flags+=("--limit-upload") + flags+=("--no-cache") + flags+=("--no-extra-verify") + flags+=("--no-lock") + flags+=("--option=") + two_word_flags+=("--option") + two_word_flags+=("-o") + flags+=("--pack-size=") + two_word_flags+=("--pack-size") + flags+=("--password-command=") + two_word_flags+=("--password-command") + flags+=("--password-file=") + two_word_flags+=("--password-file") + two_word_flags+=("-p") + flags+=("--quiet") + flags+=("-q") + flags+=("--repo=") + two_word_flags+=("--repo") + two_word_flags+=("-r") + flags+=("--repository-file=") + two_word_flags+=("--repository-file") + flags+=("--retry-lock=") + two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2403,6 +2596,8 @@ _restic_prune() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2471,6 +2666,8 @@ _restic_recover() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2535,6 +2732,8 @@ _restic_repair_help() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2606,6 +2805,8 @@ _restic_repair_index() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2674,6 +2875,8 @@ _restic_repair_packs() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2762,6 +2965,8 @@ _restic_repair_snapshots() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2834,6 +3039,8 @@ _restic_repair() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -2970,6 +3177,8 @@ _restic_restore() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3084,6 +3293,8 @@ _restic_rewrite() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3156,6 +3367,8 @@ _restic_self-update() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3252,6 +3465,8 @@ _restic_snapshots() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3288,6 +3503,8 @@ _restic_stats() local_nonpersistent_flags+=("-H") flags+=("--mode=") two_word_flags+=("--mode") + flags_with_completion+=("--mode") + flags_completion+=("__restic_handle_go_custom_completion") local_nonpersistent_flags+=("--mode") local_nonpersistent_flags+=("--mode=") flags+=("--path=") @@ -3338,6 +3555,8 @@ _restic_stats() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3432,6 +3651,8 @@ _restic_tag() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3502,6 +3723,8 @@ _restic_unlock() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3570,6 +3793,8 @@ _restic_version() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") @@ -3594,6 +3819,7 @@ _restic_root_command() commands+=("copy") commands+=("diff") commands+=("dump") + commands+=("features") commands+=("find") commands+=("forget") commands+=("generate") @@ -3604,6 +3830,7 @@ _restic_root_command() commands+=("ls") commands+=("migrate") commands+=("mount") + commands+=("options") commands+=("prune") commands+=("recover") commands+=("repair") @@ -3666,6 +3893,8 @@ _restic_root_command() two_word_flags+=("--repository-file") flags+=("--retry-lock=") two_word_flags+=("--retry-lock") + flags+=("--stuck-request-timeout=") + two_word_flags+=("--stuck-request-timeout") flags+=("--tls-client-cert=") two_word_flags+=("--tls-client-cert") flags+=("--verbose") diff --git a/mover-restic/restic/doc/design.rst b/mover-restic/restic/doc/design.rst index 7fb8b71b2..c974e997a 100644 --- a/mover-restic/restic/doc/design.rst +++ b/mover-restic/restic/doc/design.rst @@ -126,8 +126,8 @@ the option ``-o local.layout=default``, valid values are ``default`` and ``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the s3 backend ``s3.layout``. -S3 Legacy Layout ----------------- +S3 Legacy Layout (deprecated) +----------------------------- Unfortunately during development the Amazon S3 backend uses slightly different paths (directory names use singular instead of plural for ``key``, @@ -152,8 +152,7 @@ the ``data`` directory. The S3 Legacy repository layout looks like this: /snapshot └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec -The S3 backend understands and accepts both forms, new backends are -always created with the default layout for compatibility reasons. +Restic 0.17 is the last version that supports the legacy layout. Pack Format =========== @@ -234,7 +233,9 @@ Individual files for the index, locks or snapshots are encrypted and authenticated like Data and Tree Blobs, so the outer structure is ``IV || Ciphertext || MAC`` again. In repository format version 1 the plaintext always consists of a JSON document which must either be an -object or an array. +object or an array. The JSON encoder must deterministically encode the +document and should match the behavior of the Go standard library implementation +in ``encoding/json``. Repository format version 2 adds support for compression. The plaintext now starts with a header to indicate the encoding version to distinguish @@ -473,6 +474,10 @@ A snapshot references a tree by the SHA-256 hash of the JSON string representation of its contents. Trees and data are saved in pack files in a subdirectory of the directory ``data``. +The JSON encoder must deterministically encode the document and should +match the behavior of the Go standard library implementation in ``encoding/json``. +This ensures that trees can be properly deduplicated. + The command ``restic cat blob`` can be used to inspect the tree referenced above (piping the output of the command to ``jq .`` so that the JSON is indented): @@ -507,12 +512,11 @@ this metadata is generated: - The name is quoted using `strconv.Quote `__ before being saved. This handles non-unicode names, but also changes the representation of names containing ``"`` or ``\``. - - The filemode saved is the mode defined by `fs.FileMode `__ masked by ``os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky`` - -When the entry references a directory, the field ``subtree`` contains the plain text -ID of another tree object. +- When the entry references a directory, the field ``subtree`` contains the plain text + ID of another tree object. +- Check the implementation for a full struct definition. When the command ``restic cat blob`` is used, the plaintext ID is needed to print a tree. The tree referenced above can be dumped as follows: diff --git a/mover-restic/restic/doc/faq.rst b/mover-restic/restic/doc/faq.rst index 19879d817..74dd77d71 100644 --- a/mover-restic/restic/doc/faq.rst +++ b/mover-restic/restic/doc/faq.rst @@ -90,7 +90,7 @@ The error here is that the tilde ``~`` in ``"~/documents"`` didn't get expanded /home/john/documents $ echo "~/documents" - ~/document + ~/documents $ echo "$HOME/documents" /home/john/documents @@ -228,3 +228,17 @@ Restic backup command fails to find a valid file in Windows If the name of a file in Windows contains an invalid character, Restic will not be able to read the file. To solve this issue, consider renaming the particular file. + +What can I do in case of "request timeout" errors? +-------------------------------------------------- + +Restic monitors connections to the backend to detect stuck requests. If a request +does not return any data within five minutes, restic assumes the request is stuck and +retries it. However, for large repositories it sometimes takes longer than that to +collect a list of all files, causing the following error: + +:: + + List(data) returned error, retrying after 1s: [...]: request timeout + +In this case you can increase the timeout using the ``--stuck-request-timeout`` option. diff --git a/mover-restic/restic/doc/man/restic-backup.1 b/mover-restic/restic/doc/man/restic-backup.1 index cda4aadff..a84b955ba 100644 --- a/mover-restic/restic/doc/man/restic-backup.1 +++ b/mover-restic/restic/doc/man/restic-backup.1 @@ -24,6 +24,7 @@ Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -229,6 +230,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-cache.1 b/mover-restic/restic/doc/man/restic-cache.1 index f868b8a6b..fb23fe8a9 100644 --- a/mover-restic/restic/doc/man/restic-cache.1 +++ b/mover-restic/restic/doc/man/restic-cache.1 @@ -129,6 +129,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-cat.1 b/mover-restic/restic/doc/man/restic-cat.1 index 2298c58cf..cab1b85a5 100644 --- a/mover-restic/restic/doc/man/restic-cat.1 +++ b/mover-restic/restic/doc/man/restic-cat.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -119,6 +120,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-check.1 b/mover-restic/restic/doc/man/restic-check.1 index c0d1b07a8..60d17a313 100644 --- a/mover-restic/restic/doc/man/restic-check.1 +++ b/mover-restic/restic/doc/man/restic-check.1 @@ -27,6 +27,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -136,6 +137,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-copy.1 b/mover-restic/restic/doc/man/restic-copy.1 index 63b67e5e7..96c394139 100644 --- a/mover-restic/restic/doc/man/restic-copy.1 +++ b/mover-restic/restic/doc/man/restic-copy.1 @@ -36,12 +36,13 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS .PP \fB--from-insecure-no-password\fP[=false] - use an empty password for the source repository, must be passed to every restic command (insecure) + use an empty password for the source repository (insecure) .PP \fB--from-key-hint\fP="" @@ -169,6 +170,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-diff.1 b/mover-restic/restic/doc/man/restic-diff.1 index f4ffa2737..f4c8a1d14 100644 --- a/mover-restic/restic/doc/man/restic-diff.1 +++ b/mover-restic/restic/doc/man/restic-diff.1 @@ -49,6 +49,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -150,6 +151,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-dump.1 b/mover-restic/restic/doc/man/restic-dump.1 index 00cb3c8b6..657570f6d 100644 --- a/mover-restic/restic/doc/man/restic-dump.1 +++ b/mover-restic/restic/doc/man/restic-dump.1 @@ -34,6 +34,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -151,6 +152,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-features.1 b/mover-restic/restic/doc/man/restic-features.1 new file mode 100644 index 000000000..b288f655a --- /dev/null +++ b/mover-restic/restic/doc/man/restic-features.1 @@ -0,0 +1,146 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-features - Print list of feature flags + + +.SH SYNOPSIS +.PP +\fBrestic features [flags]\fP + + +.SH DESCRIPTION +.PP +The "features" command prints a list of supported feature flags. + +.PP +To pass feature flags to restic, set the RESTIC_FEATURES environment variable +to "featureA=true,featureB=false". Specifying an unknown feature flag is an error. + +.PP +A feature can either be in alpha, beta, stable or deprecated state. +An \fIalpha\fP feature is disabled by default and may change in arbitrary ways between restic versions or be removed. +A \fIbeta\fP feature is enabled by default, but still can change in minor ways or be removed. +A \fIstable\fP feature is always enabled and cannot be disabled. The flag will be removed in a future restic version. +A \fIdeprecated\fP feature is always disabled and cannot be enabled. The flag will be removed in a future restic version. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for features + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic(1)\fP diff --git a/mover-restic/restic/doc/man/restic-find.1 b/mover-restic/restic/doc/man/restic-find.1 index 2d81decd3..e8d974527 100644 --- a/mover-restic/restic/doc/man/restic-find.1 +++ b/mover-restic/restic/doc/man/restic-find.1 @@ -165,6 +165,10 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) @@ -190,6 +194,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .EE diff --git a/mover-restic/restic/doc/man/restic-forget.1 b/mover-restic/restic/doc/man/restic-forget.1 index 55705288f..058dbee25 100644 --- a/mover-restic/restic/doc/man/restic-forget.1 +++ b/mover-restic/restic/doc/man/restic-forget.1 @@ -36,6 +36,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -237,6 +238,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-generate.1 b/mover-restic/restic/doc/man/restic-generate.1 index f2db39bac..f17a6fcd0 100644 --- a/mover-restic/restic/doc/man/restic-generate.1 +++ b/mover-restic/restic/doc/man/restic-generate.1 @@ -138,6 +138,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-init.1 b/mover-restic/restic/doc/man/restic-init.1 index de439add5..50fa00b71 100644 --- a/mover-restic/restic/doc/man/restic-init.1 +++ b/mover-restic/restic/doc/man/restic-init.1 @@ -29,7 +29,7 @@ Exit status is 1 if there was any error. .PP \fB--from-insecure-no-password\fP[=false] - use an empty password for the source repository, must be passed to every restic command (insecure) + use an empty password for the source repository (insecure) .PP \fB--from-key-hint\fP="" @@ -149,6 +149,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-key-add.1 b/mover-restic/restic/doc/man/restic-key-add.1 index 6a24e1e67..ff33408b4 100644 --- a/mover-restic/restic/doc/man/restic-key-add.1 +++ b/mover-restic/restic/doc/man/restic-key-add.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -135,6 +136,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-key-list.1 b/mover-restic/restic/doc/man/restic-key-list.1 index a00b116b9..7deb05793 100644 --- a/mover-restic/restic/doc/man/restic-key-list.1 +++ b/mover-restic/restic/doc/man/restic-key-list.1 @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -121,6 +122,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-key-passwd.1 b/mover-restic/restic/doc/man/restic-key-passwd.1 index 42315d72a..68e81edd9 100644 --- a/mover-restic/restic/doc/man/restic-key-passwd.1 +++ b/mover-restic/restic/doc/man/restic-key-passwd.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -136,6 +137,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-key-remove.1 b/mover-restic/restic/doc/man/restic-key-remove.1 index 6ee826059..ff1a0ceb9 100644 --- a/mover-restic/restic/doc/man/restic-key-remove.1 +++ b/mover-restic/restic/doc/man/restic-key-remove.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -120,6 +121,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-key.1 b/mover-restic/restic/doc/man/restic-key.1 index 43da808cc..4fd1f6caf 100644 --- a/mover-restic/restic/doc/man/restic-key.1 +++ b/mover-restic/restic/doc/man/restic-key.1 @@ -112,6 +112,10 @@ per repository. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-list.1 b/mover-restic/restic/doc/man/restic-list.1 index f8a1db005..29945e859 100644 --- a/mover-restic/restic/doc/man/restic-list.1 +++ b/mover-restic/restic/doc/man/restic-list.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -119,6 +120,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-ls.1 b/mover-restic/restic/doc/man/restic-ls.1 index 6cc662583..b990d2ec8 100644 --- a/mover-restic/restic/doc/man/restic-ls.1 +++ b/mover-restic/restic/doc/man/restic-ls.1 @@ -37,6 +37,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -162,6 +163,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-migrate.1 b/mover-restic/restic/doc/man/restic-migrate.1 index 2272294bf..c0fa2dbc1 100644 --- a/mover-restic/restic/doc/man/restic-migrate.1 +++ b/mover-restic/restic/doc/man/restic-migrate.1 @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -125,6 +126,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-mount.1 b/mover-restic/restic/doc/man/restic-mount.1 index a256d2a5f..5ec59391d 100644 --- a/mover-restic/restic/doc/man/restic-mount.1 +++ b/mover-restic/restic/doc/man/restic-mount.1 @@ -64,6 +64,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -193,6 +194,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-options.1 b/mover-restic/restic/doc/man/restic-options.1 new file mode 100644 index 000000000..8ea8bea63 --- /dev/null +++ b/mover-restic/restic/doc/man/restic-options.1 @@ -0,0 +1,135 @@ +.nh +.TH "restic backup" "1" "Jan 2017" "generated by \fBrestic generate\fR" "" + +.SH NAME +.PP +restic-options - Print list of extended options + + +.SH SYNOPSIS +.PP +\fBrestic options [flags]\fP + + +.SH DESCRIPTION +.PP +The "options" command prints a list of extended options. + + +.SH EXIT STATUS +.PP +Exit status is 0 if the command was successful. +Exit status is 1 if there was any error. + + +.SH OPTIONS +.PP +\fB-h\fP, \fB--help\fP[=false] + help for options + + +.SH OPTIONS INHERITED FROM PARENT COMMANDS +.PP +\fB--cacert\fP=[] + \fBfile\fR to load root certificates from (default: use system certificates or $RESTIC_CACERT) + +.PP +\fB--cache-dir\fP="" + set the cache \fBdirectory\fR\&. (default: use system default cache directory) + +.PP +\fB--cleanup-cache\fP[=false] + auto remove old cache directories + +.PP +\fB--compression\fP=auto + compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION) + +.PP +\fB--http-user-agent\fP="" + set a http user agent for outgoing http requests + +.PP +\fB--insecure-no-password\fP[=false] + use an empty password for the repository, must be passed to every restic command (insecure) + +.PP +\fB--insecure-tls\fP[=false] + skip TLS certificate verification when connecting to the repository (insecure) + +.PP +\fB--json\fP[=false] + set output mode to JSON for commands that support it + +.PP +\fB--key-hint\fP="" + \fBkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT) + +.PP +\fB--limit-download\fP=0 + limits downloads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--limit-upload\fP=0 + limits uploads to a maximum \fBrate\fR in KiB/s. (default: unlimited) + +.PP +\fB--no-cache\fP[=false] + do not use a local cache + +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + +.PP +\fB--no-lock\fP[=false] + do not lock the repository, this allows some operations on read-only repositories + +.PP +\fB-o\fP, \fB--option\fP=[] + set extended option (\fBkey=value\fR, can be specified multiple times) + +.PP +\fB--pack-size\fP=0 + set target pack \fBsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE) + +.PP +\fB--password-command\fP="" + shell \fBcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND) + +.PP +\fB-p\fP, \fB--password-file\fP="" + \fBfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE) + +.PP +\fB-q\fP, \fB--quiet\fP[=false] + do not output comprehensive progress report + +.PP +\fB-r\fP, \fB--repo\fP="" + \fBrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY) + +.PP +\fB--repository-file\fP="" + \fBfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE) + +.PP +\fB--retry-lock\fP=0s + retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) + +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + +.PP +\fB--tls-client-cert\fP="" + path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) + +.PP +\fB-v\fP, \fB--verbose\fP[=0] + be verbose (specify multiple times or a level using --verbose=n``, max level/times is 2) + + +.SH SEE ALSO +.PP +\fBrestic(1)\fP diff --git a/mover-restic/restic/doc/man/restic-prune.1 b/mover-restic/restic/doc/man/restic-prune.1 index 7e16748ab..1ee262b61 100644 --- a/mover-restic/restic/doc/man/restic-prune.1 +++ b/mover-restic/restic/doc/man/restic-prune.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -148,6 +149,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-recover.1 b/mover-restic/restic/doc/man/restic-recover.1 index 0529360ae..382a91ceb 100644 --- a/mover-restic/restic/doc/man/restic-recover.1 +++ b/mover-restic/restic/doc/man/restic-recover.1 @@ -24,6 +24,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -121,6 +122,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-repair-index.1 b/mover-restic/restic/doc/man/restic-repair-index.1 index 60327a916..341f90d59 100644 --- a/mover-restic/restic/doc/man/restic-repair-index.1 +++ b/mover-restic/restic/doc/man/restic-repair-index.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -124,6 +125,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-repair-packs.1 b/mover-restic/restic/doc/man/restic-repair-packs.1 index 01a2f6540..d0091725b 100644 --- a/mover-restic/restic/doc/man/restic-repair-packs.1 +++ b/mover-restic/restic/doc/man/restic-repair-packs.1 @@ -23,6 +23,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -120,6 +121,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-repair-snapshots.1 b/mover-restic/restic/doc/man/restic-repair-snapshots.1 index c4439f131..d9e12ddf1 100644 --- a/mover-restic/restic/doc/man/restic-repair-snapshots.1 +++ b/mover-restic/restic/doc/man/restic-repair-snapshots.1 @@ -41,6 +41,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -158,6 +159,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-repair.1 b/mover-restic/restic/doc/man/restic-repair.1 index 7fa313aab..b06562486 100644 --- a/mover-restic/restic/doc/man/restic-repair.1 +++ b/mover-restic/restic/doc/man/restic-repair.1 @@ -111,6 +111,10 @@ Repair the repository \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-restore.1 b/mover-restic/restic/doc/man/restic-restore.1 index 876b18bf8..e9ef4ef94 100644 --- a/mover-restic/restic/doc/man/restic-restore.1 +++ b/mover-restic/restic/doc/man/restic-restore.1 @@ -31,6 +31,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -196,6 +197,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-rewrite.1 b/mover-restic/restic/doc/man/restic-rewrite.1 index d3dd92436..c0d4a7e1a 100644 --- a/mover-restic/restic/doc/man/restic-rewrite.1 +++ b/mover-restic/restic/doc/man/restic-rewrite.1 @@ -39,6 +39,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -180,6 +181,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-self-update.1 b/mover-restic/restic/doc/man/restic-self-update.1 index e6dd4faf2..d475f13cb 100644 --- a/mover-restic/restic/doc/man/restic-self-update.1 +++ b/mover-restic/restic/doc/man/restic-self-update.1 @@ -25,6 +25,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -126,6 +127,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-snapshots.1 b/mover-restic/restic/doc/man/restic-snapshots.1 index 25d5274e3..f59240b44 100644 --- a/mover-restic/restic/doc/man/restic-snapshots.1 +++ b/mover-restic/restic/doc/man/restic-snapshots.1 @@ -22,6 +22,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -143,6 +144,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-stats.1 b/mover-restic/restic/doc/man/restic-stats.1 index fe4074ca5..1e6e79dac 100644 --- a/mover-restic/restic/doc/man/restic-stats.1 +++ b/mover-restic/restic/doc/man/restic-stats.1 @@ -52,6 +52,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -165,6 +166,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-tag.1 b/mover-restic/restic/doc/man/restic-tag.1 index 7ab1911e5..89c677867 100644 --- a/mover-restic/restic/doc/man/restic-tag.1 +++ b/mover-restic/restic/doc/man/restic-tag.1 @@ -29,6 +29,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was any error. Exit status is 10 if the repository does not exist. Exit status is 11 if the repository is already locked. +Exit status is 12 if the password is incorrect. .SH OPTIONS @@ -150,6 +151,10 @@ Exit status is 11 if the repository is already locked. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-unlock.1 b/mover-restic/restic/doc/man/restic-unlock.1 index a24a4f815..74679ef91 100644 --- a/mover-restic/restic/doc/man/restic-unlock.1 +++ b/mover-restic/restic/doc/man/restic-unlock.1 @@ -121,6 +121,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic-version.1 b/mover-restic/restic/doc/man/restic-version.1 index e9df439ed..8d5fe6c65 100644 --- a/mover-restic/restic/doc/man/restic-version.1 +++ b/mover-restic/restic/doc/man/restic-version.1 @@ -118,6 +118,10 @@ Exit status is 1 if there was any error. \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) diff --git a/mover-restic/restic/doc/man/restic.1 b/mover-restic/restic/doc/man/restic.1 index ee423c6ad..bd8009aac 100644 --- a/mover-restic/restic/doc/man/restic.1 +++ b/mover-restic/restic/doc/man/restic.1 @@ -113,6 +113,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB--retry-lock\fP=0s retry to lock the repository if it is already locked, takes a value like 5m or 2h (default: no retries) +.PP +\fB--stuck-request-timeout\fP=5m0s + \fBduration\fR after which to retry stuck requests + .PP \fB--tls-client-cert\fP="" path to a \fBfile\fR containing PEM encoded TLS client certificate and private key (default: $RESTIC_TLS_CLIENT_CERT) @@ -124,4 +128,4 @@ The full documentation can be found at https://restic.readthedocs.io/ . .SH SEE ALSO .PP -\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-repair(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP +\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-features(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-options(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-repair(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP diff --git a/mover-restic/restic/doc/manual_rest.rst b/mover-restic/restic/doc/manual_rest.rst index a7a0f96e0..d1e5817f3 100644 --- a/mover-restic/restic/doc/manual_rest.rst +++ b/mover-restic/restic/doc/manual_rest.rst @@ -8,7 +8,7 @@ Usage help is available: .. code-block:: console - $ ./restic --help + $ restic --help restic is a backup program which allows saving multiple revisions of files and directories in an encrypted repository stored on different backends. @@ -28,8 +28,6 @@ Usage help is available: dump Print a backed-up file to stdout find Find a file, a directory or restic IDs forget Remove snapshots from the repository - generate Generate manual pages and auto-completion files (bash, fish, zsh, powershell) - help Help about any command init Initialize a new repository key Manage keys (passwords) list List objects in the repository @@ -41,11 +39,19 @@ Usage help is available: repair Repair the repository restore Extract the data from a snapshot rewrite Rewrite snapshots to exclude unwanted files - self-update Update the restic binary snapshots List all snapshots stats Scan the repository and show basic statistics tag Modify tags on snapshots unlock Remove locks other processes created + + Advanced Options: + features Print list of feature flags + options Print list of extended options + + Additional Commands: + generate Generate manual pages and auto-completion files (bash, fish, zsh, powershell) + help Help about any command + self-update Update the restic binary version Print version information Flags: @@ -85,7 +91,7 @@ command: .. code-block:: console - $ ./restic backup --help + $ restic backup --help The "backup" command creates a new snapshot and saves the files and directories given as the arguments. diff --git a/mover-restic/restic/internal/archiver/archiver.go b/mover-restic/restic/internal/archiver/archiver.go index d9f089e81..e7c346d3a 100644 --- a/mover-restic/restic/internal/archiver/archiver.go +++ b/mover-restic/restic/internal/archiver/archiver.go @@ -263,7 +263,8 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, // overwrite name to match that within the snapshot node.Name = path.Base(snPath) if err != nil { - return node, fmt.Errorf("nodeFromFileInfo %v: %w", filename, err) + err = fmt.Errorf("incomplete metadata for %v: %w", filename, err) + return node, arch.error(filename, err) } return node, err } @@ -714,7 +715,12 @@ func resolveRelativeTargets(filesys fs.FS, targets []string) ([]string, error) { debug.Log("targets before resolving: %v", targets) result := make([]string, 0, len(targets)) for _, target := range targets { - target = filesys.Clean(target) + if target != "" && filesys.VolumeName(target) == target { + // special case to allow users to also specify a volume name "C:" instead of a path "C:\" + target = target + filesys.Separator() + } else { + target = filesys.Clean(target) + } pc, _ := pathComponents(filesys, target, false) if len(pc) > 0 { result = append(result, target) diff --git a/mover-restic/restic/internal/archiver/archiver_test.go b/mover-restic/restic/internal/archiver/archiver_test.go index f38d5b0de..c54f9ea33 100644 --- a/mover-restic/restic/internal/archiver/archiver_test.go +++ b/mover-restic/restic/internal/archiver/archiver_test.go @@ -3,6 +3,7 @@ package archiver import ( "bytes" "context" + "fmt" "io" "os" "path/filepath" @@ -1447,6 +1448,66 @@ func TestArchiverSnapshot(t *testing.T) { } } +func TestResolveRelativeTargetsSpecial(t *testing.T) { + var tests = []struct { + name string + targets []string + expected []string + win bool + }{ + { + name: "basic relative path", + targets: []string{filepath.FromSlash("some/path")}, + expected: []string{filepath.FromSlash("some/path")}, + }, + { + name: "partial relative path", + targets: []string{filepath.FromSlash("../some/path")}, + expected: []string{filepath.FromSlash("../some/path")}, + }, + { + name: "basic absolute path", + targets: []string{filepath.FromSlash("/some/path")}, + expected: []string{filepath.FromSlash("/some/path")}, + }, + { + name: "volume name", + targets: []string{"C:"}, + expected: []string{"C:\\"}, + win: true, + }, + { + name: "volume root path", + targets: []string{"C:\\"}, + expected: []string{"C:\\"}, + win: true, + }, + { + name: "UNC path", + targets: []string{"\\\\server\\volume"}, + expected: []string{"\\\\server\\volume\\"}, + win: true, + }, + { + name: "UNC path with trailing slash", + targets: []string{"\\\\server\\volume\\"}, + expected: []string{"\\\\server\\volume\\"}, + win: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if test.win && runtime.GOOS != "windows" { + t.Skip("skip test on unix") + } + + targets, err := resolveRelativeTargets(&fs.Local{}, test.targets) + rtest.OK(t, err) + rtest.Equals(t, test.expected, targets) + }) + } +} + func TestArchiverSnapshotSelect(t *testing.T) { var tests = []struct { name string @@ -2338,3 +2399,28 @@ func TestRacyFileSwap(t *testing.T) { t.Errorf("Save() excluded the node, that's unexpected") } } + +func TestMetadataBackupErrorFiltering(t *testing.T) { + tempdir := t.TempDir() + repo := repository.TestRepository(t) + + filename := filepath.Join(tempdir, "file") + rtest.OK(t, os.WriteFile(filename, []byte("example"), 0o600)) + fi, err := os.Stat(filename) + rtest.OK(t, err) + + arch := New(repo, fs.Local{}, Options{}) + + var filteredErr error + replacementErr := fmt.Errorf("replacement") + arch.Error = func(item string, err error) error { + filteredErr = err + return replacementErr + } + + // check that errors from reading extended metadata are properly filtered + node, err := arch.nodeFromFileInfo("file", filename+"invalid", fi, false) + rtest.Assert(t, node != nil, "node is missing") + rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err) + rtest.Assert(t, filteredErr != nil, "missing inner error") +} diff --git a/mover-restic/restic/internal/backend/cache/backend.go b/mover-restic/restic/internal/backend/cache/backend.go index 63bb6f85f..3754266ba 100644 --- a/mover-restic/restic/internal/backend/cache/backend.go +++ b/mover-restic/restic/internal/backend/cache/backend.go @@ -2,11 +2,14 @@ package cache import ( "context" + "fmt" "io" + "os" "sync" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/restic" ) // Backend wraps a restic.Backend and adds a cache. @@ -162,7 +165,9 @@ func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset // try loading from cache without checking that the handle is actually cached inCache, err := b.loadFromCache(h, length, offset, consumer) if inCache { - debug.Log("error loading %v from cache: %v", h, err) + if err != nil { + debug.Log("error loading %v from cache: %v", h, err) + } // the caller must explicitly use cache.Forget() to remove the cache entry return err } @@ -213,3 +218,43 @@ func (b *Backend) IsNotExist(err error) bool { func (b *Backend) Unwrap() backend.Backend { return b.Backend } + +func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(f backend.FileInfo) error) error { + if !b.Cache.canBeCached(t) { + return b.Backend.List(ctx, t, fn) + } + + // will contain the IDs of the files that are in the repository + ids := restic.NewIDSet() + + // wrap the original function to also add the file to the ids set + wrapFn := func(f backend.FileInfo) error { + id, err := restic.ParseID(f.Name) + if err != nil { + // ignore files with invalid name + return nil + } + + ids.Insert(id) + + // execute the original function + return fn(f) + } + + err := b.Backend.List(ctx, t, wrapFn) + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + + // clear the cache for files that are not in the repo anymore, ignore errors + err = b.Cache.Clear(t, ids) + if err != nil { + fmt.Fprintf(os.Stderr, "error clearing %s files in cache: %v\n", t.String(), err) + } + + return nil +} diff --git a/mover-restic/restic/internal/backend/cache/backend_test.go b/mover-restic/restic/internal/backend/cache/backend_test.go index 7addc275d..7f83e40cb 100644 --- a/mover-restic/restic/internal/backend/cache/backend_test.go +++ b/mover-restic/restic/internal/backend/cache/backend_test.go @@ -57,6 +57,13 @@ func randomData(n int) (backend.Handle, []byte) { return h, data } +func list(t testing.TB, be backend.Backend, fn func(backend.FileInfo) error) { + err := be.List(context.TODO(), backend.IndexFile, fn) + if err != nil { + t.Fatal(err) + } +} + func TestBackend(t *testing.T) { be := mem.New() c := TestNewCache(t) @@ -238,3 +245,71 @@ func TestErrorBackend(t *testing.T) { wg.Wait() } + +func TestAutomaticCacheClear(t *testing.T) { + be := mem.New() + c := TestNewCache(t) + wbe := c.Wrap(be) + + // add two handles h1 and h2 + h1, data := randomData(2000) + // save h1 directly to the backend + save(t, be, h1, data) + if c.Has(h1) { + t.Errorf("cache has file1 too early") + } + + h2, data2 := randomData(3000) + + // save h2 directly to the backend + save(t, be, h2, data2) + if c.Has(h2) { + t.Errorf("cache has file2 too early") + } + + loadAndCompare(t, wbe, h1, data) + if !c.Has(h1) { + t.Errorf("cache doesn't have file1 after load") + } + + loadAndCompare(t, wbe, h2, data2) + if !c.Has(h2) { + t.Errorf("cache doesn't have file2 after load") + } + + // remove h1 directly from the backend + remove(t, be, h1) + if !c.Has(h1) { + t.Errorf("file1 not in cache any more, should be removed from cache only after list") + } + + // list all files in the backend + list(t, wbe, func(_ backend.FileInfo) error { return nil }) + + // h1 should be removed from the cache + if c.Has(h1) { + t.Errorf("cache has file1 after remove") + } + + // h2 should still be in the cache + if !c.Has(h2) { + t.Errorf("cache doesn't have file2 after list") + } +} + +func TestAutomaticCacheClearInvalidFilename(t *testing.T) { + be := mem.New() + c := TestNewCache(t) + + data := test.Random(rand.Int(), 42) + h := backend.Handle{ + Type: backend.IndexFile, + Name: "tmp12345", + } + save(t, be, h, data) + + wbe := c.Wrap(be) + + // list all files in the backend + list(t, wbe, func(_ backend.FileInfo) error { return nil }) +} diff --git a/mover-restic/restic/internal/backend/http_transport.go b/mover-restic/restic/internal/backend/http_transport.go index 5162d3571..5a3856e41 100644 --- a/mover-restic/restic/internal/backend/http_transport.go +++ b/mover-restic/restic/internal/backend/http_transport.go @@ -31,6 +31,9 @@ type TransportOptions struct { // Specify Custom User-Agent for the http Client HTTPUserAgent string + + // Timeout after which to retry stuck requests + StuckRequestTimeout time.Duration } // readPEMCertKey reads a file and returns the PEM encoded certificate and key @@ -143,7 +146,11 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) { } if feature.Flag.Enabled(feature.BackendErrorRedesign) { - rt = newWatchdogRoundtripper(rt, 5*time.Minute, 128*1024) + if opts.StuckRequestTimeout == 0 { + opts.StuckRequestTimeout = 5 * time.Minute + } + + rt = newWatchdogRoundtripper(rt, opts.StuckRequestTimeout, 128*1024) } // wrap in the debug round tripper (if active) diff --git a/mover-restic/restic/internal/backend/rest/rest.go b/mover-restic/restic/internal/backend/rest/rest.go index 1af88ec3f..d0a08175b 100644 --- a/mover-restic/restic/internal/backend/rest/rest.go +++ b/mover-restic/restic/internal/backend/rest/rest.go @@ -143,6 +143,12 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR if err != nil { return errors.WithStack(err) } + req.GetBody = func() (io.ReadCloser, error) { + if err := rd.Rewind(); err != nil { + return nil, err + } + return io.NopCloser(rd), nil + } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Accept", ContentTypeV2) diff --git a/mover-restic/restic/internal/backend/retry/backend_retry.go b/mover-restic/restic/internal/backend/retry/backend_retry.go index 8d0f42bfd..92c285c4b 100644 --- a/mover-restic/restic/internal/backend/retry/backend_retry.go +++ b/mover-restic/restic/internal/backend/retry/backend_retry.go @@ -209,9 +209,10 @@ func (be *Backend) Load(ctx context.Context, h backend.Handle, length int, offse return be.Backend.Load(ctx, h, length, offset, consumer) }) - if feature.Flag.Enabled(feature.BackendErrorRedesign) && err != nil && !be.IsPermanentError(err) { + if feature.Flag.Enabled(feature.BackendErrorRedesign) && err != nil && ctx.Err() == nil && !be.IsPermanentError(err) { // We've exhausted the retries, the file is likely inaccessible. By excluding permanent - // errors, not found or truncated files are not recorded. + // errors, not found or truncated files are not recorded. Also ignore errors if the context + // was canceled. be.failedLoads.LoadOrStore(key, time.Now()) } diff --git a/mover-restic/restic/internal/backend/retry/backend_retry_test.go b/mover-restic/restic/internal/backend/retry/backend_retry_test.go index fd76200d4..ffb8ae186 100644 --- a/mover-restic/restic/internal/backend/retry/backend_retry_test.go +++ b/mover-restic/restic/internal/backend/retry/backend_retry_test.go @@ -357,6 +357,30 @@ func TestBackendLoadCircuitBreaker(t *testing.T) { test.Equals(t, notFound, err, "expected circuit breaker to reset, got %v") } +func TestBackendLoadCircuitBreakerCancel(t *testing.T) { + cctx, cancel := context.WithCancel(context.Background()) + be := mock.NewBackend() + be.OpenReaderFn = func(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) { + cancel() + return nil, errors.New("something") + } + nilRd := func(rd io.Reader) (err error) { + return nil + } + + TestFastRetries(t) + retryBackend := New(be, 2, nil, nil) + // canceling the context should not trip the circuit breaker + err := retryBackend.Load(cctx, backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Equals(t, context.Canceled, err, "unexpected error") + + // reset context and check that the cirucit breaker does not return an error + cctx, cancel = context.WithCancel(context.Background()) + defer cancel() + err = retryBackend.Load(cctx, backend.Handle{Name: "other"}, 0, 0, nilRd) + test.Equals(t, context.Canceled, err, "unexpected error") +} + func TestBackendStatNotExists(t *testing.T) { // stat should not retry if the error matches IsNotExist notFound := errors.New("not found") diff --git a/mover-restic/restic/internal/backend/sftp/sftp.go b/mover-restic/restic/internal/backend/sftp/sftp.go index 70fc30a62..efbd0c8d5 100644 --- a/mover-restic/restic/internal/backend/sftp/sftp.go +++ b/mover-restic/restic/internal/backend/sftp/sftp.go @@ -578,6 +578,10 @@ func (r *SFTP) deleteRecursive(ctx context.Context, name string) error { } for _, fi := range entries { + if ctx.Err() != nil { + return ctx.Err() + } + itemName := r.Join(name, fi.Name()) if fi.IsDir() { err := r.deleteRecursive(ctx, itemName) diff --git a/mover-restic/restic/internal/backend/watchdog_roundtriper.go b/mover-restic/restic/internal/backend/watchdog_roundtriper.go index e3e10d7fe..dc270b974 100644 --- a/mover-restic/restic/internal/backend/watchdog_roundtriper.go +++ b/mover-restic/restic/internal/backend/watchdog_roundtriper.go @@ -65,6 +65,9 @@ func (w *watchdogRoundtripper) RoundTrip(req *http.Request) (*http.Response, err resp, err := w.rt.RoundTrip(req) if err != nil { + if isTimeout(err) { + err = errRequestTimeout + } return nil, err } diff --git a/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go b/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go index bc43447e1..f7f90259c 100644 --- a/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go +++ b/mover-restic/restic/internal/backend/watchdog_roundtriper_test.go @@ -135,7 +135,7 @@ func TestUploadTimeout(t *testing.T) { rtest.OK(t, err) resp, err := rt.RoundTrip(req) - rtest.Equals(t, context.Canceled, err) + rtest.Equals(t, errRequestTimeout, err) // make linter happy if resp != nil { rtest.OK(t, resp.Body.Close()) @@ -162,7 +162,7 @@ func TestProcessingTimeout(t *testing.T) { rtest.OK(t, err) resp, err := rt.RoundTrip(req) - rtest.Equals(t, context.Canceled, err) + rtest.Equals(t, errRequestTimeout, err) // make linter happy if resp != nil { rtest.OK(t, resp.Body.Close()) @@ -190,7 +190,7 @@ func TestDownloadTimeout(t *testing.T) { })) defer srv.Close() - rt := newWatchdogRoundtripper(http.DefaultTransport, 10*time.Millisecond, 1024) + rt := newWatchdogRoundtripper(http.DefaultTransport, 25*time.Millisecond, 1024) req, err := http.NewRequestWithContext(context.TODO(), "GET", srv.URL, io.NopCloser(bytes.NewReader(msg))) rtest.OK(t, err) diff --git a/mover-restic/restic/internal/fs/ea_windows.go b/mover-restic/restic/internal/fs/ea_windows.go index 08466c33f..bf7b02fd4 100644 --- a/mover-restic/restic/internal/fs/ea_windows.go +++ b/mover-restic/restic/internal/fs/ea_windows.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "errors" "fmt" + "strings" "syscall" "unsafe" @@ -283,3 +284,35 @@ func setFileEA(handle windows.Handle, iosb *ioStatusBlock, buf *uint8, bufLen ui status = ntStatus(r0) return } + +// PathSupportsExtendedAttributes returns true if the path supports extended attributes. +func PathSupportsExtendedAttributes(path string) (supported bool, err error) { + var fileSystemFlags uint32 + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return false, err + } + err = windows.GetVolumeInformation(utf16Path, nil, 0, nil, nil, &fileSystemFlags, nil, 0) + if err != nil { + return false, err + } + supported = (fileSystemFlags & windows.FILE_SUPPORTS_EXTENDED_ATTRIBUTES) != 0 + return supported, nil +} + +// GetVolumePathName returns the volume path name for the given path. +func GetVolumePathName(path string) (volumeName string, err error) { + utf16Path, err := windows.UTF16PtrFromString(path) + if err != nil { + return "", err + } + // Get the volume path (e.g., "D:") + var volumePath [windows.MAX_PATH + 1]uint16 + err = windows.GetVolumePathName(utf16Path, &volumePath[0], windows.MAX_PATH+1) + if err != nil { + return "", err + } + // Trim any trailing backslashes + volumeName = strings.TrimRight(windows.UTF16ToString(volumePath[:]), "\\") + return volumeName, nil +} diff --git a/mover-restic/restic/internal/fs/ea_windows_test.go b/mover-restic/restic/internal/fs/ea_windows_test.go index b249f43c4..74afd7aa5 100644 --- a/mover-restic/restic/internal/fs/ea_windows_test.go +++ b/mover-restic/restic/internal/fs/ea_windows_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "reflect" + "strings" "syscall" "testing" "unsafe" @@ -245,3 +246,78 @@ func testSetGetEA(t *testing.T, path string, handle windows.Handle, testEAs []Ex t.Fatalf("EAs read from path %s don't match", path) } } + +func TestPathSupportsExtendedAttributes(t *testing.T) { + testCases := []struct { + name string + path string + expected bool + }{ + { + name: "System drive", + path: os.Getenv("SystemDrive") + `\`, + expected: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + supported, err := PathSupportsExtendedAttributes(tc.path) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if supported != tc.expected { + t.Errorf("Expected %v, got %v for path %s", tc.expected, supported, tc.path) + } + }) + } + + // Test with an invalid path + _, err := PathSupportsExtendedAttributes("Z:\\NonExistentPath-UAS664da5s4dyu56das45f5as") + if err == nil { + t.Error("Expected an error for non-existent path, but got nil") + } +} + +func TestGetVolumePathName(t *testing.T) { + tempDirVolume := filepath.VolumeName(os.TempDir()) + testCases := []struct { + name string + path string + expectedPrefix string + }{ + { + name: "Root directory", + path: os.Getenv("SystemDrive") + `\`, + expectedPrefix: os.Getenv("SystemDrive"), + }, + { + name: "Nested directory", + path: os.Getenv("SystemDrive") + `\Windows\System32`, + expectedPrefix: os.Getenv("SystemDrive"), + }, + { + name: "Temp directory", + path: os.TempDir() + `\`, + expectedPrefix: tempDirVolume, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + volumeName, err := GetVolumePathName(tc.path) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if !strings.HasPrefix(volumeName, tc.expectedPrefix) { + t.Errorf("Expected volume name to start with %s, but got %s", tc.expectedPrefix, volumeName) + } + }) + } + + // Test with an invalid path + _, err := GetVolumePathName("Z:\\NonExistentPath") + if err == nil { + t.Error("Expected an error for non-existent path, but got nil") + } +} diff --git a/mover-restic/restic/internal/fs/file.go b/mover-restic/restic/internal/fs/file.go index 929195f1c..85b202dc8 100644 --- a/mover-restic/restic/internal/fs/file.go +++ b/mover-restic/restic/internal/fs/file.go @@ -134,7 +134,7 @@ func IsAccessDenied(err error) bool { // ResetPermissions resets the permissions of the file at the specified path func ResetPermissions(path string) error { // Set the default file permissions - if err := os.Chmod(path, 0600); err != nil { + if err := os.Chmod(fixpath(path), 0600); err != nil { return err } return nil diff --git a/mover-restic/restic/internal/fs/file_windows.go b/mover-restic/restic/internal/fs/file_windows.go index b05068c42..50c7e9938 100644 --- a/mover-restic/restic/internal/fs/file_windows.go +++ b/mover-restic/restic/internal/fs/file_windows.go @@ -85,7 +85,7 @@ func ClearSystem(path string) error { // ClearAttribute removes the specified attribute from the file. func ClearAttribute(path string, attribute uint32) error { - ptr, err := windows.UTF16PtrFromString(path) + ptr, err := windows.UTF16PtrFromString(fixpath(path)) if err != nil { return err } diff --git a/mover-restic/restic/internal/fs/fs_reader_command.go b/mover-restic/restic/internal/fs/fs_reader_command.go index 3830e5811..6d061f641 100644 --- a/mover-restic/restic/internal/fs/fs_reader_command.go +++ b/mover-restic/restic/internal/fs/fs_reader_command.go @@ -29,6 +29,10 @@ type CommandReader struct { } func NewCommandReader(ctx context.Context, args []string, logOutput io.Writer) (*CommandReader, error) { + if len(args) == 0 { + return nil, fmt.Errorf("no command was specified as argument") + } + // Prepare command and stdout command := exec.CommandContext(ctx, args[0], args[1:]...) stdout, err := command.StdoutPipe() diff --git a/mover-restic/restic/internal/fs/fs_reader_command_test.go b/mover-restic/restic/internal/fs/fs_reader_command_test.go index a9028544c..8f0d17b1e 100644 --- a/mover-restic/restic/internal/fs/fs_reader_command_test.go +++ b/mover-restic/restic/internal/fs/fs_reader_command_test.go @@ -34,6 +34,11 @@ func TestCommandReaderInvalid(t *testing.T) { test.Assert(t, err != nil, "missing error") } +func TestCommandReaderEmptyArgs(t *testing.T) { + _, err := fs.NewCommandReader(context.TODO(), []string{}, io.Discard) + test.Assert(t, err != nil, "missing error") +} + func TestCommandReaderOutput(t *testing.T) { reader, err := fs.NewCommandReader(context.TODO(), []string{"echo", "hello world"}, io.Discard) test.OK(t, err) diff --git a/mover-restic/restic/internal/fs/sd_windows.go b/mover-restic/restic/internal/fs/sd_windows.go index 5d98b4ef4..0004f1809 100644 --- a/mover-restic/restic/internal/fs/sd_windows.go +++ b/mover-restic/restic/internal/fs/sd_windows.go @@ -11,6 +11,7 @@ import ( "unsafe" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" "golang.org/x/sys/windows" ) @@ -47,19 +48,20 @@ func GetSecurityDescriptor(filePath string) (securityDescriptor *[]byte, err err var sd *windows.SECURITY_DESCRIPTOR - if lowerPrivileges.Load() { + // store original value to avoid unrelated changes in the error check + useLowerPrivileges := lowerPrivileges.Load() + if useLowerPrivileges { sd, err = getNamedSecurityInfoLow(filePath) } else { sd, err = getNamedSecurityInfoHigh(filePath) } if err != nil { - if !lowerPrivileges.Load() && isHandlePrivilegeNotHeldError(err) { + if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) { // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. lowerPrivileges.Store(true) - sd, err = getNamedSecurityInfoLow(filePath) - if err != nil { - return nil, fmt.Errorf("get low-level named security info failed with: %w", err) - } + return GetSecurityDescriptor(filePath) + } else if errors.Is(err, windows.ERROR_NOT_SUPPORTED) { + return nil, nil } else { return nil, fmt.Errorf("get named security info failed with: %w", err) } @@ -106,20 +108,19 @@ func SetSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { sacl = nil } - if lowerPrivileges.Load() { + // store original value to avoid unrelated changes in the error check + useLowerPrivileges := lowerPrivileges.Load() + if useLowerPrivileges { err = setNamedSecurityInfoLow(filePath, dacl) } else { err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl) } if err != nil { - if !lowerPrivileges.Load() && isHandlePrivilegeNotHeldError(err) { + if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) { // If ERROR_PRIVILEGE_NOT_HELD is encountered, fallback to backups/restores using lower non-admin privileges. lowerPrivileges.Store(true) - err = setNamedSecurityInfoLow(filePath, dacl) - if err != nil { - return fmt.Errorf("set low-level named security info failed with: %w", err) - } + return SetSecurityDescriptor(filePath, securityDescriptor) } else { return fmt.Errorf("set named security info failed with: %w", err) } @@ -129,22 +130,22 @@ func SetSecurityDescriptor(filePath string, securityDescriptor *[]byte) error { // getNamedSecurityInfoHigh gets the higher level SecurityDescriptor which requires admin permissions. func getNamedSecurityInfoHigh(filePath string) (*windows.SECURITY_DESCRIPTOR, error) { - return windows.GetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, highSecurityFlags) + return windows.GetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highSecurityFlags) } // getNamedSecurityInfoLow gets the lower level SecurityDescriptor which requires no admin permissions. func getNamedSecurityInfoLow(filePath string) (*windows.SECURITY_DESCRIPTOR, error) { - return windows.GetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, lowBackupSecurityFlags) + return windows.GetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, lowBackupSecurityFlags) } // setNamedSecurityInfoHigh sets the higher level SecurityDescriptor which requires admin permissions. func setNamedSecurityInfoHigh(filePath string, owner *windows.SID, group *windows.SID, dacl *windows.ACL, sacl *windows.ACL) error { - return windows.SetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, highSecurityFlags, owner, group, dacl, sacl) + return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, highSecurityFlags, owner, group, dacl, sacl) } // setNamedSecurityInfoLow sets the lower level SecurityDescriptor which requires no admin permissions. func setNamedSecurityInfoLow(filePath string, dacl *windows.ACL) error { - return windows.SetNamedSecurityInfo(filePath, windows.SE_FILE_OBJECT, lowRestoreSecurityFlags, nil, nil, dacl, nil) + return windows.SetNamedSecurityInfo(fixpath(filePath), windows.SE_FILE_OBJECT, lowRestoreSecurityFlags, nil, nil, dacl, nil) } // enableBackupPrivilege enables privilege for backing up security descriptors diff --git a/mover-restic/restic/internal/fuse/dir.go b/mover-restic/restic/internal/fuse/dir.go index 763a9640c..fd030295b 100644 --- a/mover-restic/restic/internal/fuse/dir.go +++ b/mover-restic/restic/internal/fuse/dir.go @@ -107,6 +107,10 @@ func (d *dir) open(ctx context.Context) error { } items := make(map[string]*restic.Node) for _, n := range tree.Nodes { + if ctx.Err() != nil { + return ctx.Err() + } + nodes, err := replaceSpecialNodes(ctx, d.root.repo, n) if err != nil { debug.Log(" replaceSpecialNodes(%v) failed: %v", n, err) @@ -171,6 +175,10 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { }) for _, node := range d.items { + if ctx.Err() != nil { + return nil, ctx.Err() + } + name := cleanupNodeName(node.Name) var typ fuse.DirentType switch node.Type { diff --git a/mover-restic/restic/internal/fuse/file.go b/mover-restic/restic/internal/fuse/file.go index e2e0cf9a0..494fca283 100644 --- a/mover-restic/restic/internal/fuse/file.go +++ b/mover-restic/restic/internal/fuse/file.go @@ -66,12 +66,16 @@ func (f *file) Attr(_ context.Context, a *fuse.Attr) error { } -func (f *file) Open(_ context.Context, _ *fuse.OpenRequest, _ *fuse.OpenResponse) (fs.Handle, error) { +func (f *file) Open(ctx context.Context, _ *fuse.OpenRequest, _ *fuse.OpenResponse) (fs.Handle, error) { debug.Log("open file %v with %d blobs", f.node.Name, len(f.node.Content)) var bytes uint64 cumsize := make([]uint64, 1+len(f.node.Content)) for i, id := range f.node.Content { + if ctx.Err() != nil { + return nil, ctx.Err() + } + size, found := f.root.repo.LookupBlobSize(restic.DataBlob, id) if !found { return nil, errors.Errorf("id %v not found in repository", id) diff --git a/mover-restic/restic/internal/fuse/snapshots_dir.go b/mover-restic/restic/internal/fuse/snapshots_dir.go index 7369ea17a..4cae7106c 100644 --- a/mover-restic/restic/internal/fuse/snapshots_dir.go +++ b/mover-restic/restic/internal/fuse/snapshots_dir.go @@ -78,6 +78,10 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { } for name, entry := range meta.names { + if ctx.Err() != nil { + return nil, ctx.Err() + } + d := fuse.Dirent{ Inode: inodeFromName(d.inode, name), Name: name, diff --git a/mover-restic/restic/internal/repository/check.go b/mover-restic/restic/internal/repository/check.go index 9f571cb4f..0a48f2b9d 100644 --- a/mover-restic/restic/internal/repository/check.go +++ b/mover-restic/restic/internal/repository/check.go @@ -95,6 +95,10 @@ func checkPackInner(ctx context.Context, r *Repository, id restic.ID, blobs []re it := newPackBlobIterator(id, newBufReader(bufRd), 0, blobs, r.Key(), dec) for { + if ctx.Err() != nil { + return ctx.Err() + } + val, err := it.Next() if err == errPackEOF { break diff --git a/mover-restic/restic/internal/repository/index/master_index.go b/mover-restic/restic/internal/repository/index/master_index.go index f8e776b23..9b5c4f9f8 100644 --- a/mover-restic/restic/internal/repository/index/master_index.go +++ b/mover-restic/restic/internal/repository/index/master_index.go @@ -456,6 +456,9 @@ func (mi *MasterIndex) Rewrite(ctx context.Context, repo restic.Unpacked, exclud worker := func() error { for idx := range saveCh { idx.Finalize() + if len(idx.packs) == 0 { + continue + } if _, err := idx.SaveIndex(wgCtx, repo); err != nil { return err } diff --git a/mover-restic/restic/internal/repository/repository.go b/mover-restic/restic/internal/repository/repository.go index 838858c38..d408e3105 100644 --- a/mover-restic/restic/internal/repository/repository.go +++ b/mover-restic/restic/internal/repository/repository.go @@ -706,19 +706,10 @@ func (r *Repository) prepareCache() error { return nil } - indexIDs := r.idx.IDs() - debug.Log("prepare cache with %d index files", len(indexIDs)) - - // clear old index files - err := r.Cache.Clear(restic.IndexFile, indexIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "error clearing index files in cache: %v\n", err) - } - packs := r.idx.Packs(restic.NewIDSet()) // clear old packs - err = r.Cache.Clear(restic.PackFile, packs) + err := r.Cache.Clear(restic.PackFile, packs) if err != nil { fmt.Fprintf(os.Stderr, "error clearing pack files in cache: %v\n", err) } @@ -1000,6 +991,10 @@ func streamPackPart(ctx context.Context, beLoad backendLoadFn, loadBlobFn loadBl it := newPackBlobIterator(packID, newByteReader(data), dataStart, blobs, key, dec) for { + if ctx.Err() != nil { + return ctx.Err() + } + val, err := it.Next() if err == errPackEOF { break diff --git a/mover-restic/restic/internal/restic/lock.go b/mover-restic/restic/internal/restic/lock.go index 49c7cedf2..969d0593d 100644 --- a/mover-restic/restic/internal/restic/lock.go +++ b/mover-restic/restic/internal/restic/lock.go @@ -103,10 +103,14 @@ func NewExclusiveLock(ctx context.Context, repo Unpacked) (*Lock, error) { var waitBeforeLockCheck = 200 * time.Millisecond +// delay increases by factor 2 on each retry +var initialWaitBetweenLockRetries = 5 * time.Second + // TestSetLockTimeout can be used to reduce the lock wait timeout for tests. func TestSetLockTimeout(t testing.TB, d time.Duration) { t.Logf("setting lock timeout to %v", d) waitBeforeLockCheck = d + initialWaitBetweenLockRetries = d } func newLock(ctx context.Context, repo Unpacked, excl bool) (*Lock, error) { @@ -170,8 +174,17 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { if l.lockID != nil { checkedIDs.Insert(*l.lockID) } + delay := initialWaitBetweenLockRetries // retry locking a few times - for i := 0; i < 3; i++ { + for i := 0; i < 4; i++ { + if i != 0 { + // sleep between retries to give backend some time to settle + if err := cancelableDelay(ctx, delay); err != nil { + return err + } + delay *= 2 + } + // Store updates in new IDSet to prevent data races var m sync.Mutex newCheckedIDs := NewIDSet(checkedIDs.List()...) @@ -213,6 +226,18 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { return err } +func cancelableDelay(ctx context.Context, delay time.Duration) error { + // delay next try a bit + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + return nil +} + // createLock acquires the lock by creating a file in the repository. func (l *Lock) createLock(ctx context.Context) (ID, error) { id, err := SaveJSONUnpacked(ctx, l.repo, LockFile, l) diff --git a/mover-restic/restic/internal/restic/node.go b/mover-restic/restic/internal/restic/node.go index 51c6071b7..6afdff64a 100644 --- a/mover-restic/restic/internal/restic/node.go +++ b/mover-restic/restic/internal/restic/node.go @@ -229,6 +229,13 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) er func (node Node) RestoreMetadata(path string, warn func(msg string)) error { err := node.restoreMetadata(path, warn) if err != nil { + // It is common to have permission errors for folders like /home + // unless you're running as root, so ignore those. + if os.Geteuid() > 0 && errors.Is(err, os.ErrPermission) { + debug.Log("not running as root, ignoring permission error for %v: %v", + path, err) + return nil + } debug.Log("restoreMetadata(%s) error %v", path, err) } @@ -239,33 +246,26 @@ func (node Node) restoreMetadata(path string, warn func(msg string)) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { - // Like "cp -a" and "rsync -a" do, we only report lchown permission errors - // if we run as root. - if os.Geteuid() > 0 && os.IsPermission(err) { - debug.Log("not running as root, ignoring lchown permission error for %v: %v", - path, err) - } else { - firsterr = errors.WithStack(err) - } - } - - if err := node.RestoreTimestamps(path); err != nil { - debug.Log("error restoring timestamps for dir %v: %v", path, err) - if firsterr != nil { - firsterr = err - } + firsterr = errors.WithStack(err) } if err := node.restoreExtendedAttributes(path); err != nil { debug.Log("error restoring extended attributes for %v: %v", path, err) - if firsterr != nil { + if firsterr == nil { firsterr = err } } if err := node.restoreGenericAttributes(path, warn); err != nil { debug.Log("error restoring generic attributes for %v: %v", path, err) - if firsterr != nil { + if firsterr == nil { + firsterr = err + } + } + + if err := node.RestoreTimestamps(path); err != nil { + debug.Log("error restoring timestamps for %v: %v", path, err) + if firsterr == nil { firsterr = err } } @@ -275,7 +275,7 @@ func (node Node) restoreMetadata(path string, warn func(msg string)) error { // calls above would fail. if node.Type != "symlink" { if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr != nil { + if firsterr == nil { firsterr = errors.WithStack(err) } } diff --git a/mover-restic/restic/internal/restic/node_test.go b/mover-restic/restic/internal/restic/node_test.go index 6e0f31e21..ab7f66e5b 100644 --- a/mover-restic/restic/internal/restic/node_test.go +++ b/mover-restic/restic/internal/restic/node_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -196,6 +197,20 @@ var nodeTests = []Node{ {"user.foo", []byte("bar")}, }, }, + { + Name: "testXattrFileMacOSResourceFork", + Type: "file", + Content: IDs{}, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Mode: 0604, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + ExtendedAttributes: []ExtendedAttribute{ + {"com.apple.ResourceFork", []byte("bar")}, + }, + }, } func TestNodeRestoreAt(t *testing.T) { @@ -215,6 +230,11 @@ func TestNodeRestoreAt(t *testing.T) { extAttrArr[i].Name = strings.ToUpper(extAttrArr[i].Name) } } + for _, attr := range test.ExtendedAttributes { + if strings.HasPrefix(attr.Name, "com.apple.") && runtime.GOOS != "darwin" { + t.Skipf("attr %v only relevant on macOS", attr.Name) + } + } // tempdir might be backed by a filesystem that does not support // extended attributes @@ -228,10 +248,6 @@ func TestNodeRestoreAt(t *testing.T) { rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) - if test.Type == "dir" { - rtest.OK(t, test.RestoreTimestamps(nodePath)) - } - fi, err := os.Lstat(nodePath) rtest.OK(t, err) @@ -382,3 +398,14 @@ func TestSymlinkSerializationFormat(t *testing.T) { test.Assert(t, n2.LinkTargetRaw == nil, "quoted link target is just a helper field and must be unset after decoding") } } + +func TestNodeRestoreMetadataError(t *testing.T) { + tempdir := t.TempDir() + + node := nodeTests[0] + nodePath := filepath.Join(tempdir, node.Name) + + // This will fail because the target file does not exist + err := node.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) }) + test.Assert(t, errors.Is(err, os.ErrNotExist), "failed for an unexpected reason") +} diff --git a/mover-restic/restic/internal/restic/node_windows.go b/mover-restic/restic/internal/restic/node_windows.go index 2785e0412..bce01ccad 100644 --- a/mover-restic/restic/internal/restic/node_windows.go +++ b/mover-restic/restic/internal/restic/node_windows.go @@ -8,6 +8,7 @@ import ( "reflect" "runtime" "strings" + "sync" "syscall" "unsafe" @@ -32,6 +33,16 @@ var ( modAdvapi32 = syscall.NewLazyDLL("advapi32.dll") procEncryptFile = modAdvapi32.NewProc("EncryptFileW") procDecryptFile = modAdvapi32.NewProc("DecryptFileW") + + // eaSupportedVolumesMap is a map of volumes to boolean values indicating if they support extended attributes. + eaSupportedVolumesMap = sync.Map{} +) + +const ( + extendedPathPrefix = `\\?\` + uncPathPrefix = `\\?\UNC\` + globalRootPrefix = `\\?\GLOBALROOT\` + volumeGUIDPrefix = `\\?\Volume{` ) // mknod is not supported on Windows. @@ -351,32 +362,137 @@ func decryptFile(pathPointer *uint16) error { } // fillGenericAttributes fills in the generic attributes for windows like File Attributes, -// Created time etc. +// Created time and Security Descriptors. +// It also checks if the volume supports extended attributes and stores the result in a map +// so that it does not have to be checked again for subsequent calls for paths in the same volume. func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { if strings.Contains(filepath.Base(path), ":") { - //Do not process for Alternate Data Streams in Windows + // Do not process for Alternate Data Streams in Windows // Also do not allow processing of extended attributes for ADS. return false, nil } - if !strings.HasSuffix(filepath.Clean(path), `\`) { - // Do not process file attributes and created time for windows directories like - // C:, D: - // Filepath.Clean(path) ends with '\' for Windows root drives only. - var sd *[]byte - if node.Type == "file" || node.Type == "dir" { - if sd, err = fs.GetSecurityDescriptor(path); err != nil { - return true, err - } + + if strings.HasSuffix(filepath.Clean(path), `\`) { + // filepath.Clean(path) ends with '\' for Windows root volume paths only + // Do not process file attributes, created time and sd for windows root volume paths + // Security descriptors are not supported for root volume paths. + // Though file attributes and created time are supported for root volume paths, + // we ignore them and we do not want to replace them during every restore. + allowExtended, err = checkAndStoreEASupport(path) + if err != nil { + return false, err + } + return allowExtended, nil + } + + var sd *[]byte + if node.Type == "file" || node.Type == "dir" { + // Check EA support and get security descriptor for file/dir only + allowExtended, err = checkAndStoreEASupport(path) + if err != nil { + return false, err + } + if sd, err = fs.GetSecurityDescriptor(path); err != nil { + return allowExtended, err + } + } + // Add Windows attributes + node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{ + CreationTime: getCreationTime(fi, path), + FileAttributes: &stat.FileAttributes, + SecurityDescriptor: sd, + }) + return allowExtended, err +} + +// checkAndStoreEASupport checks if the volume of the path supports extended attributes and stores the result in a map +// If the result is already in the map, it returns the result from the map. +func checkAndStoreEASupport(path string) (isEASupportedVolume bool, err error) { + var volumeName string + volumeName, err = prepareVolumeName(path) + if err != nil { + return false, err + } + + if volumeName != "" { + // First check if the manually prepared volume name is already in the map + eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeName) + if exists { + // Cache hit, immediately return the cached value + return eaSupportedValue.(bool), nil } + // If not found, check if EA is supported with manually prepared volume name + isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeName + `\`) + // If the prepared volume name is not valid, we will fetch the actual volume name next. + if err != nil && !errors.Is(err, windows.DNS_ERROR_INVALID_NAME) { + debug.Log("Error checking if extended attributes are supported for prepared volume name %s: %v", volumeName, err) + // There can be multiple errors like path does not exist, bad network path, etc. + // We just gracefully disallow extended attributes for cases. + return false, nil + } + } + // If an entry is not found, get the actual volume name using the GetVolumePathName function + volumeNameActual, err := fs.GetVolumePathName(path) + if err != nil { + debug.Log("Error getting actual volume name %s for path %s: %v", volumeName, path, err) + // There can be multiple errors like path does not exist, bad network path, etc. + // We just gracefully disallow extended attributes for cases. + return false, nil + } + if volumeNameActual != volumeName { + // If the actual volume name is different, check cache for the actual volume name + eaSupportedValue, exists := eaSupportedVolumesMap.Load(volumeNameActual) + if exists { + // Cache hit, immediately return the cached value + return eaSupportedValue.(bool), nil + } + // If the actual volume name is different and is not in the map, again check if the new volume supports extended attributes with the actual volume name + isEASupportedVolume, err = fs.PathSupportsExtendedAttributes(volumeNameActual + `\`) + // Debug log for cases where the prepared volume name is not valid + if err != nil { + debug.Log("Error checking if extended attributes are supported for actual volume name %s: %v", volumeNameActual, err) + // There can be multiple errors like path does not exist, bad network path, etc. + // We just gracefully disallow extended attributes for cases. + return false, nil + } else { + debug.Log("Checking extended attributes. Prepared volume name: %s, actual volume name: %s, isEASupportedVolume: %v, err: %v", volumeName, volumeNameActual, isEASupportedVolume, err) + } + } + if volumeNameActual != "" { + eaSupportedVolumesMap.Store(volumeNameActual, isEASupportedVolume) + } + return isEASupportedVolume, err +} - // Add Windows attributes - node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{ - CreationTime: getCreationTime(fi, path), - FileAttributes: &stat.FileAttributes, - SecurityDescriptor: sd, - }) +// prepareVolumeName prepares the volume name for different cases in Windows +func prepareVolumeName(path string) (volumeName string, err error) { + // Check if it's an extended length path + if strings.HasPrefix(path, globalRootPrefix) { + // Extract the VSS snapshot volume name eg. `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopyXX` + if parts := strings.SplitN(path, `\`, 7); len(parts) >= 6 { + volumeName = strings.Join(parts[:6], `\`) + } else { + volumeName = filepath.VolumeName(path) + } + } else { + if !strings.HasPrefix(path, volumeGUIDPrefix) { // Handle volume GUID path + if strings.HasPrefix(path, uncPathPrefix) { + // Convert \\?\UNC\ extended path to standard path to get the volume name correctly + path = `\\` + path[len(uncPathPrefix):] + } else if strings.HasPrefix(path, extendedPathPrefix) { + //Extended length path prefix needs to be trimmed to get the volume name correctly + path = path[len(extendedPathPrefix):] + } else { + // Use the absolute path + path, err = filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("failed to get absolute path: %w", err) + } + } + } + volumeName = filepath.VolumeName(path) } - return true, err + return volumeName, nil } // windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection diff --git a/mover-restic/restic/internal/restic/node_windows_test.go b/mover-restic/restic/internal/restic/node_windows_test.go index 4fd57bbb7..6ba25559b 100644 --- a/mover-restic/restic/internal/restic/node_windows_test.go +++ b/mover-restic/restic/internal/restic/node_windows_test.go @@ -12,6 +12,7 @@ import ( "strings" "syscall" "testing" + "time" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" @@ -329,3 +330,198 @@ func TestRestoreExtendedAttributes(t *testing.T) { } } } + +func TestPrepareVolumeName(t *testing.T) { + currentVolume := filepath.VolumeName(func() string { + // Get the current working directory + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get current working directory: %v", err) + } + return pwd + }()) + // Create a temporary directory for the test + tempDir, err := os.MkdirTemp("", "restic_test_"+time.Now().Format("20060102150405")) + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Create a long file name + longFileName := `\Very\Long\Path\That\Exceeds\260\Characters\` + strings.Repeat(`\VeryLongFolderName`, 20) + `\\LongFile.txt` + longFilePath := filepath.Join(tempDir, longFileName) + + tempDirVolume := filepath.VolumeName(tempDir) + // Create the file + content := []byte("This is a test file with a very long name.") + err = os.MkdirAll(filepath.Dir(longFilePath), 0755) + test.OK(t, err) + if err != nil { + t.Fatalf("Failed to create long folder: %v", err) + } + err = os.WriteFile(longFilePath, content, 0644) + test.OK(t, err) + if err != nil { + t.Fatalf("Failed to create long file: %v", err) + } + osVolumeGUIDPath := getOSVolumeGUIDPath(t) + osVolumeGUIDVolume := filepath.VolumeName(osVolumeGUIDPath) + + testCases := []struct { + name string + path string + expectedVolume string + expectError bool + expectedEASupported bool + isRealPath bool + }{ + { + name: "Network drive path", + path: `Z:\Shared\Documents`, + expectedVolume: `Z:`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Subst drive path", + path: `X:\Virtual\Folder`, + expectedVolume: `X:`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Windows reserved path", + path: `\\.\` + os.Getenv("SystemDrive") + `\System32\drivers\etc\hosts`, + expectedVolume: `\\.\` + os.Getenv("SystemDrive"), + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Long UNC path", + path: `\\?\UNC\LongServerName\VeryLongShareName\DeepPath\File.txt`, + expectedVolume: `\\LongServerName\VeryLongShareName`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Volume GUID path", + path: osVolumeGUIDPath, + expectedVolume: osVolumeGUIDVolume, + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Volume GUID path with subfolder", + path: osVolumeGUIDPath + `\Windows`, + expectedVolume: osVolumeGUIDVolume, + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Standard path", + path: os.Getenv("SystemDrive") + `\Users\`, + expectedVolume: os.Getenv("SystemDrive"), + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "Extended length path", + path: longFilePath, + expectedVolume: tempDirVolume, + expectError: false, + expectedEASupported: true, + isRealPath: true, + }, + { + name: "UNC path", + path: `\\server\share\folder`, + expectedVolume: `\\server\share`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Extended UNC path", + path: `\\?\UNC\server\share\folder`, + expectedVolume: `\\server\share`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Volume Shadow Copy path", + path: `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1\Users\test`, + expectedVolume: `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy1`, + expectError: false, + expectedEASupported: false, + }, + { + name: "Relative path", + path: `folder\subfolder`, + + expectedVolume: currentVolume, // Get current volume + expectError: false, + expectedEASupported: true, + }, + { + name: "Empty path", + path: ``, + expectedVolume: currentVolume, + expectError: false, + expectedEASupported: true, + isRealPath: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + isEASupported, err := checkAndStoreEASupport(tc.path) + test.OK(t, err) + test.Equals(t, tc.expectedEASupported, isEASupported) + + volume, err := prepareVolumeName(tc.path) + + if tc.expectError { + test.Assert(t, err != nil, "Expected an error, but got none") + } else { + test.OK(t, err) + } + test.Equals(t, tc.expectedVolume, volume) + + if tc.isRealPath { + isEASupportedVolume, err := fs.PathSupportsExtendedAttributes(volume + `\`) + // If the prepared volume name is not valid, we will next fetch the actual volume name. + test.OK(t, err) + + test.Equals(t, tc.expectedEASupported, isEASupportedVolume) + + actualVolume, err := fs.GetVolumePathName(tc.path) + test.OK(t, err) + test.Equals(t, tc.expectedVolume, actualVolume) + } + }) + } +} + +func getOSVolumeGUIDPath(t *testing.T) string { + // Get the path of the OS drive (usually C:\) + osDrive := os.Getenv("SystemDrive") + "\\" + + // Convert to a volume GUID path + volumeName, err := windows.UTF16PtrFromString(osDrive) + test.OK(t, err) + if err != nil { + return "" + } + + var volumeGUID [windows.MAX_PATH]uint16 + err = windows.GetVolumeNameForVolumeMountPoint(volumeName, &volumeGUID[0], windows.MAX_PATH) + test.OK(t, err) + if err != nil { + return "" + } + + return windows.UTF16ToString(volumeGUID[:]) +} diff --git a/mover-restic/restic/internal/restic/snapshot_find.go b/mover-restic/restic/internal/restic/snapshot_find.go index 6d1ab9a7a..6eb51b237 100644 --- a/mover-restic/restic/internal/restic/snapshot_find.go +++ b/mover-restic/restic/internal/restic/snapshot_find.go @@ -134,6 +134,10 @@ func (f *SnapshotFilter) FindAll(ctx context.Context, be Lister, loader LoaderUn ids := NewIDSet() // Process all snapshot IDs given as arguments. for _, s := range snapshotIDs { + if ctx.Err() != nil { + return ctx.Err() + } + var sn *Snapshot if s == "latest" { if usedFilter { diff --git a/mover-restic/restic/internal/restorer/filerestorer.go b/mover-restic/restic/internal/restorer/filerestorer.go index e517e6284..31234b960 100644 --- a/mover-restic/restic/internal/restorer/filerestorer.go +++ b/mover-restic/restic/internal/restorer/filerestorer.go @@ -122,6 +122,10 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { // create packInfo from fileInfo for _, file := range r.files { + if ctx.Err() != nil { + return ctx.Err() + } + fileBlobs := file.blobs.(restic.IDs) largeFile := len(fileBlobs) > largeFileBlobCount var packsMap map[restic.ID][]fileBlobInfo diff --git a/mover-restic/restic/internal/restorer/fileswriter_test.go b/mover-restic/restic/internal/restorer/fileswriter_test.go index c69847927..9ea8767b8 100644 --- a/mover-restic/restic/internal/restorer/fileswriter_test.go +++ b/mover-restic/restic/internal/restorer/fileswriter_test.go @@ -49,7 +49,7 @@ func TestFilesWriterRecursiveOverwrite(t *testing.T) { // must error if recursive delete is not allowed w := newFilesWriter(1, false) err := w.writeToFile(path, []byte{1}, 0, 2, false) - rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexepected error got %v", err) + rtest.Assert(t, errors.Is(err, notEmptyDirError()), "unexpected error got %v", err) rtest.Equals(t, 0, len(w.buckets[0].files)) // must replace directory diff --git a/mover-restic/restic/internal/restorer/restorer.go b/mover-restic/restic/internal/restorer/restorer.go index cd3fd076d..0e30b82f8 100644 --- a/mover-restic/restic/internal/restorer/restorer.go +++ b/mover-restic/restic/internal/restorer/restorer.go @@ -12,6 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" @@ -333,12 +334,13 @@ func (res *Restorer) ensureDir(target string) error { // RestoreTo creates the directories and files in the snapshot below dst. // Before an item is created, res.Filter is called. -func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { +func (res *Restorer) RestoreTo(ctx context.Context, dst string) (uint64, error) { + restoredFileCount := uint64(0) var err error if !filepath.IsAbs(dst) { dst, err = filepath.Abs(dst) if err != nil { - return errors.Wrap(err, "Abs") + return restoredFileCount, errors.Wrap(err, "Abs") } } @@ -346,7 +348,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { // ensure that the target directory exists and is actually a directory // Using ensureDir is too aggressive here as it also removes unexpected files if err := fs.MkdirAll(dst, 0700); err != nil { - return fmt.Errorf("cannot create target directory: %w", err) + return restoredFileCount, fmt.Errorf("cannot create target directory: %w", err) } } @@ -406,19 +408,22 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } } res.trackFile(location, updateMetadataOnly) + if !updateMetadataOnly { + restoredFileCount++ + } return nil }) return err }, }) if err != nil { - return err + return 0, err } if !res.opts.DryRun { err = filerestorer.restoreFiles(ctx) if err != nil { - return err + return 0, err } } @@ -450,7 +455,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { }, leaveDir: func(node *restic.Node, target, location string, expectedFilenames []string) error { if res.opts.Delete { - if err := res.removeUnexpectedFiles(target, location, expectedFilenames); err != nil { + if err := res.removeUnexpectedFiles(ctx, target, location, expectedFilenames); err != nil { return err } } @@ -466,10 +471,10 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { return err }, }) - return err + return restoredFileCount, err } -func (res *Restorer) removeUnexpectedFiles(target, location string, expectedFilenames []string) error { +func (res *Restorer) removeUnexpectedFiles(ctx context.Context, target, location string, expectedFilenames []string) error { if !res.opts.Delete { panic("internal error") } @@ -487,6 +492,10 @@ func (res *Restorer) removeUnexpectedFiles(target, location string, expectedFile } for _, entry := range entries { + if ctx.Err() != nil { + return ctx.Err() + } + if _, ok := keep[toComparableFilename(entry)]; ok { continue } @@ -583,7 +592,7 @@ const nVerifyWorkers = 8 // have been successfully written to dst. It stops when it encounters an // error. It returns that error and the number of files it has successfully // verified. -func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { +func (res *Restorer) VerifyFiles(ctx context.Context, dst string, countRestoredFiles uint64, p *progress.Counter) (int, error) { type mustCheck struct { node *restic.Node path string @@ -594,6 +603,11 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { work = make(chan mustCheck, 2*nVerifyWorkers) ) + if p != nil { + p.SetMax(countRestoredFiles) + defer p.Done() + } + g, ctx := errgroup.WithContext(ctx) // Traverse tree and send jobs to work. @@ -628,6 +642,7 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { if err != nil || ctx.Err() != nil { break } + p.Add(1) atomic.AddUint64(&nchecked, 1) } return err diff --git a/mover-restic/restic/internal/restorer/restorer_test.go b/mover-restic/restic/internal/restorer/restorer_test.go index 9c02afe68..7d4895068 100644 --- a/mover-restic/restic/internal/restorer/restorer_test.go +++ b/mover-restic/restic/internal/restorer/restorer_test.go @@ -22,6 +22,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" restoreui "github.com/restic/restic/internal/ui/restore" "golang.org/x/sync/errgroup" ) @@ -403,13 +404,13 @@ func TestRestorer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) if err != nil { t.Fatal(err) } if len(test.ErrorsMust)+len(test.ErrorsMay) == 0 { - _, err = res.VerifyFiles(ctx, tempdir) + _, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.OK(t, err) } @@ -501,13 +502,18 @@ func TestRestorerRelative(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, "restore") + countRestoredFiles, err := res.RestoreTo(ctx, "restore") if err != nil { t.Fatal(err) } - nverified, err := res.VerifyFiles(ctx, "restore") + p := progress.NewCounter(time.Second, countRestoredFiles, func(value uint64, total uint64, runtime time.Duration, final bool) {}) + defer p.Done() + nverified, err := res.VerifyFiles(ctx, "restore", countRestoredFiles, p) rtest.OK(t, err) rtest.Equals(t, len(test.Files), nverified) + counterValue, maxValue := p.Get() + rtest.Equals(t, counterValue, uint64(2)) + rtest.Equals(t, maxValue, uint64(2)) for filename, err := range errors { t.Errorf("unexpected error for %v found: %v", filename, err) @@ -524,6 +530,13 @@ func TestRestorerRelative(t *testing.T) { t.Errorf("file %v has wrong content: want %q, got %q", filename, content, data) } } + + // verify that restoring the same snapshot again results in countRestoredFiles == 0 + countRestoredFiles, err = res.RestoreTo(ctx, "restore") + if err != nil { + t.Fatal(err) + } + rtest.Equals(t, uint64(0), countRestoredFiles) }) } } @@ -835,7 +848,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) var testPatterns = []struct { @@ -872,9 +885,9 @@ func TestVerifyCancel(t *testing.T) { tempdir := rtest.TempDir(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - - rtest.OK(t, res.RestoreTo(ctx, tempdir)) - err := os.WriteFile(filepath.Join(tempdir, "foo"), []byte("bar"), 0644) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + err = os.WriteFile(filepath.Join(tempdir, "foo"), []byte("bar"), 0644) rtest.OK(t, err) var errs []error @@ -883,7 +896,7 @@ func TestVerifyCancel(t *testing.T) { return err } - nverified, err := res.VerifyFiles(ctx, tempdir) + nverified, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.Equals(t, 0, nverified) rtest.Assert(t, err != nil, "nil error from VerifyFiles") rtest.Equals(t, 1, len(errs)) @@ -915,7 +928,7 @@ func TestRestorerSparseFiles(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err = res.RestoreTo(ctx, tempdir) + _, err = res.RestoreTo(ctx, tempdir) rtest.OK(t, err) filename := filepath.Join(tempdir, "zeros") @@ -952,15 +965,17 @@ func saveSnapshotsAndOverwrite(t *testing.T, baseSnapshot Snapshot, overwriteSna t.Logf("base snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, baseOptions) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) // overwrite snapshot sn, id = saveSnapshot(t, repo, overwriteSnapshot, noopGetGenericAttributes) t.Logf("overwrite snapshot saved as %v", id.Str()) res = NewRestorer(repo, sn, overwriteOptions) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) - _, err := res.VerifyFiles(ctx, tempdir) + _, err = res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.OK(t, err) return tempdir @@ -989,6 +1004,9 @@ type printerMock struct { func (p *printerMock) Update(_ restoreui.State, _ time.Duration) { } +func (p *printerMock) Error(item string, err error) error { + return nil +} func (p *printerMock) CompleteItem(action restoreui.ItemAction, item string, size uint64) { } func (p *printerMock) Finish(s restoreui.State, _ time.Duration) { @@ -1238,8 +1256,9 @@ func TestRestoreModified(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{Overwrite: OverwriteIfChanged}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) - n, err := res.VerifyFiles(ctx, tempdir) + countRestoredFiles, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) + n, err := res.VerifyFiles(ctx, tempdir, countRestoredFiles, nil) rtest.OK(t, err) rtest.Equals(t, 2, n, "unexpected number of verified files") } @@ -1264,7 +1283,8 @@ func TestRestoreIfChanged(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) // modify file but maintain size and timestamp path := filepath.Join(tempdir, "foo") @@ -1283,7 +1303,8 @@ func TestRestoreIfChanged(t *testing.T) { for _, overwrite := range []OverwriteBehavior{OverwriteIfChanged, OverwriteAlways} { res = NewRestorer(repo, sn, Options{Overwrite: overwrite}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) data, err := os.ReadFile(path) rtest.OK(t, err) if overwrite == OverwriteAlways { @@ -1319,9 +1340,10 @@ func TestRestoreDryRun(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{DryRun: true}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) - _, err := os.Stat(tempdir) + _, err = os.Stat(tempdir) rtest.Assert(t, errors.Is(err, os.ErrNotExist), "expected no file to be created, got %v", err) } @@ -1345,7 +1367,8 @@ func TestRestoreDryRunDelete(t *testing.T) { sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, Options{DryRun: true, Delete: true}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err = res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) _, err = os.Stat(tempfile) rtest.Assert(t, err == nil, "expected file to still exist, got error %v", err) @@ -1463,14 +1486,14 @@ func TestRestoreDelete(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) res = NewRestorer(repo, deleteSn, Options{Delete: true}) if test.selectFilter != nil { res.SelectFilter = test.selectFilter } - err = res.RestoreTo(ctx, tempdir) + _, err = res.RestoreTo(ctx, tempdir) rtest.OK(t, err) for fn, shouldExist := range test.fileState { @@ -1503,6 +1526,40 @@ func TestRestoreToFile(t *testing.T) { sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, Options{}) - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.Assert(t, strings.Contains(err.Error(), "cannot create target directory"), "unexpected error %v", err) } + +func TestRestorerLongPath(t *testing.T) { + tmp := t.TempDir() + + longPath := tmp + for i := 0; i < 20; i++ { + longPath = filepath.Join(longPath, "aaaaaaaaaaaaaaaaaaaa") + } + + rtest.OK(t, os.MkdirAll(longPath, 0o700)) + f, err := fs.OpenFile(filepath.Join(longPath, "file"), fs.O_CREATE|fs.O_RDWR, 0o600) + rtest.OK(t, err) + _, err = f.WriteString("Hello, World!") + rtest.OK(t, err) + rtest.OK(t, f.Close()) + + repo := repository.TestRepository(t) + + local := &fs.Local{} + sc := archiver.NewScanner(local) + rtest.OK(t, sc.Scan(context.TODO(), []string{tmp})) + arch := archiver.New(repo, local, archiver.Options{}) + sn, _, _, err := arch.Snapshot(context.Background(), []string{tmp}, archiver.SnapshotOptions{}) + rtest.OK(t, err) + + res := NewRestorer(repo, sn, Options{}) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + countRestoredFiles, err := res.RestoreTo(ctx, tmp) + rtest.OK(t, err) + _, err = res.VerifyFiles(ctx, tmp, countRestoredFiles, nil) + rtest.OK(t, err) +} diff --git a/mover-restic/restic/internal/restorer/restorer_unix_test.go b/mover-restic/restic/internal/restorer/restorer_unix_test.go index 27d990af4..c4e8149b2 100644 --- a/mover-restic/restic/internal/restorer/restorer_unix_test.go +++ b/mover-restic/restic/internal/restorer/restorer_unix_test.go @@ -37,7 +37,7 @@ func TestRestorerRestoreEmptyHardlinkedFields(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) f1, err := os.Stat(filepath.Join(tempdir, "dirtest/file1")) @@ -96,7 +96,7 @@ func testRestorerProgressBar(t *testing.T, dryRun bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) progress.Finish() @@ -126,7 +126,8 @@ func TestRestorePermissions(t *testing.T) { t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, Options{}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) for _, overwrite := range []OverwriteBehavior{OverwriteIfChanged, OverwriteAlways} { // tamper with permissions @@ -134,7 +135,8 @@ func TestRestorePermissions(t *testing.T) { rtest.OK(t, os.Chmod(path, 0o700)) res = NewRestorer(repo, sn, Options{Overwrite: overwrite}) - rtest.OK(t, res.RestoreTo(ctx, tempdir)) + _, err := res.RestoreTo(ctx, tempdir) + rtest.OK(t, err) fi, err := os.Stat(path) rtest.OK(t, err) rtest.Equals(t, fs.FileMode(0o600), fi.Mode().Perm(), "unexpected permissions") diff --git a/mover-restic/restic/internal/restorer/restorer_windows.go b/mover-restic/restic/internal/restorer/restorer_windows.go index 72337d8ae..9ddc0a932 100644 --- a/mover-restic/restic/internal/restorer/restorer_windows.go +++ b/mover-restic/restic/internal/restorer/restorer_windows.go @@ -8,6 +8,6 @@ import "strings" // toComparableFilename returns a filename suitable for equality checks. On Windows, it returns the // uppercase version of the string. On all other systems, it returns the unmodified filename. func toComparableFilename(path string) string { - // apparently NTFS internally uppercases filenames for comparision + // apparently NTFS internally uppercases filenames for comparison return strings.ToUpper(path) } diff --git a/mover-restic/restic/internal/restorer/restorer_windows_test.go b/mover-restic/restic/internal/restorer/restorer_windows_test.go index 3f6c8472b..4764bed2d 100644 --- a/mover-restic/restic/internal/restorer/restorer_windows_test.go +++ b/mover-restic/restic/internal/restorer/restorer_windows_test.go @@ -181,7 +181,7 @@ func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAtt ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, testDir) + _, err := res.RestoreTo(ctx, testDir) rtest.OK(t, err) mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name) @@ -562,11 +562,11 @@ func TestRestoreDeleteCaseInsensitive(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - err := res.RestoreTo(ctx, tempdir) + _, err := res.RestoreTo(ctx, tempdir) rtest.OK(t, err) res = NewRestorer(repo, deleteSn, Options{Delete: true}) - err = res.RestoreTo(ctx, tempdir) + _, err = res.RestoreTo(ctx, tempdir) rtest.OK(t, err) // anotherfile must still exist diff --git a/mover-restic/restic/internal/ui/backup/json.go b/mover-restic/restic/internal/ui/backup/json.go index 64b5de13b..f4a76afd7 100644 --- a/mover-restic/restic/internal/ui/backup/json.go +++ b/mover-restic/restic/internal/ui/backup/json.go @@ -7,14 +7,13 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" - "github.com/restic/restic/internal/ui/termstatus" ) // JSONProgress reports progress for the `backup` command in JSON. type JSONProgress struct { *ui.Message - term *termstatus.Terminal + term ui.Terminal v uint } @@ -22,7 +21,7 @@ type JSONProgress struct { var _ ProgressPrinter = &JSONProgress{} // NewJSONProgress returns a new backup progress reporter. -func NewJSONProgress(term *termstatus.Terminal, verbosity uint) *JSONProgress { +func NewJSONProgress(term ui.Terminal, verbosity uint) *JSONProgress { return &JSONProgress{ Message: ui.NewMessage(term, verbosity), term: term, @@ -68,7 +67,7 @@ func (b *JSONProgress) Update(total, processed Counter, errors uint, currentFile func (b *JSONProgress) ScannerError(item string, err error) error { b.error(errorUpdate{ MessageType: "error", - Error: err, + Error: errorObject{err.Error()}, During: "scan", Item: item, }) @@ -79,7 +78,7 @@ func (b *JSONProgress) ScannerError(item string, err error) error { func (b *JSONProgress) Error(item string, err error) error { b.error(errorUpdate{ MessageType: "error", - Error: err, + Error: errorObject{err.Error()}, During: "archival", Item: item, }) @@ -206,11 +205,15 @@ type statusUpdate struct { CurrentFiles []string `json:"current_files,omitempty"` } +type errorObject struct { + Message string `json:"message"` +} + type errorUpdate struct { - MessageType string `json:"message_type"` // "error" - Error error `json:"error"` - During string `json:"during"` - Item string `json:"item"` + MessageType string `json:"message_type"` // "error" + Error errorObject `json:"error"` + During string `json:"during"` + Item string `json:"item"` } type verboseUpdate struct { diff --git a/mover-restic/restic/internal/ui/backup/json_test.go b/mover-restic/restic/internal/ui/backup/json_test.go new file mode 100644 index 000000000..b4872efd5 --- /dev/null +++ b/mover-restic/restic/internal/ui/backup/json_test.go @@ -0,0 +1,27 @@ +package backup + +import ( + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" +) + +func createJSONProgress() (*ui.MockTerminal, ProgressPrinter) { + term := &ui.MockTerminal{} + printer := NewJSONProgress(term, 3) + return term, printer +} + +func TestJSONError(t *testing.T) { + term, printer := createJSONProgress() + test.Equals(t, printer.Error("/path", errors.New("error \"message\"")), nil) + test.Equals(t, []string{"{\"message_type\":\"error\",\"error\":{\"message\":\"error \\\"message\\\"\"},\"during\":\"archival\",\"item\":\"/path\"}\n"}, term.Errors) +} + +func TestJSONScannerError(t *testing.T) { + term, printer := createJSONProgress() + test.Equals(t, printer.ScannerError("/path", errors.New("error \"message\"")), nil) + test.Equals(t, []string{"{\"message_type\":\"error\",\"error\":{\"message\":\"error \\\"message\\\"\"},\"during\":\"scan\",\"item\":\"/path\"}\n"}, term.Errors) +} diff --git a/mover-restic/restic/internal/ui/backup/text.go b/mover-restic/restic/internal/ui/backup/text.go index f96746739..097f0d0d8 100644 --- a/mover-restic/restic/internal/ui/backup/text.go +++ b/mover-restic/restic/internal/ui/backup/text.go @@ -15,17 +15,19 @@ import ( type TextProgress struct { *ui.Message - term *termstatus.Terminal + term ui.Terminal + verbosity uint } // assert that Backup implements the ProgressPrinter interface var _ ProgressPrinter = &TextProgress{} // NewTextProgress returns a new backup progress reporter. -func NewTextProgress(term *termstatus.Terminal, verbosity uint) *TextProgress { +func NewTextProgress(term ui.Terminal, verbosity uint) *TextProgress { return &TextProgress{ - Message: ui.NewMessage(term, verbosity), - term: term, + Message: ui.NewMessage(term, verbosity), + term: term, + verbosity: verbosity, } } @@ -73,7 +75,9 @@ func (b *TextProgress) Update(total, processed Counter, errors uint, currentFile // ScannerError is the error callback function for the scanner, it prints the // error in verbose mode and returns nil. func (b *TextProgress) ScannerError(_ string, err error) error { - b.V("scan: %v\n", err) + if b.verbosity >= 2 { + b.E("scan: %v\n", err) + } return nil } diff --git a/mover-restic/restic/internal/ui/backup/text_test.go b/mover-restic/restic/internal/ui/backup/text_test.go new file mode 100644 index 000000000..39338a50c --- /dev/null +++ b/mover-restic/restic/internal/ui/backup/text_test.go @@ -0,0 +1,27 @@ +package backup + +import ( + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" +) + +func createTextProgress() (*ui.MockTerminal, ProgressPrinter) { + term := &ui.MockTerminal{} + printer := NewTextProgress(term, 3) + return term, printer +} + +func TestError(t *testing.T) { + term, printer := createTextProgress() + test.Equals(t, printer.Error("/path", errors.New("error \"message\"")), nil) + test.Equals(t, []string{"error: error \"message\"\n"}, term.Errors) +} + +func TestScannerError(t *testing.T) { + term, printer := createTextProgress() + test.Equals(t, printer.ScannerError("/path", errors.New("error \"message\"")), nil) + test.Equals(t, []string{"scan: error \"message\"\n"}, term.Errors) +} diff --git a/mover-restic/restic/internal/ui/message.go b/mover-restic/restic/internal/ui/message.go index 38cdaf301..6ad5a439e 100644 --- a/mover-restic/restic/internal/ui/message.go +++ b/mover-restic/restic/internal/ui/message.go @@ -2,19 +2,17 @@ package ui import ( "fmt" - - "github.com/restic/restic/internal/ui/termstatus" ) // Message reports progress with messages of different verbosity. type Message struct { - term *termstatus.Terminal + term Terminal v uint } // NewMessage returns a message progress reporter with underlying terminal // term. -func NewMessage(term *termstatus.Terminal, verbosity uint) *Message { +func NewMessage(term Terminal, verbosity uint) *Message { return &Message{ term: term, v: verbosity, diff --git a/mover-restic/restic/internal/ui/mock.go b/mover-restic/restic/internal/ui/mock.go new file mode 100644 index 000000000..5a4debb02 --- /dev/null +++ b/mover-restic/restic/internal/ui/mock.go @@ -0,0 +1,22 @@ +package ui + +type MockTerminal struct { + Output []string + Errors []string +} + +func (m *MockTerminal) Print(line string) { + m.Output = append(m.Output, line) +} + +func (m *MockTerminal) Error(line string) { + m.Errors = append(m.Errors, line) +} + +func (m *MockTerminal) SetStatus(lines []string) { + m.Output = append([]string{}, lines...) +} + +func (m *MockTerminal) CanUpdateStatus() bool { + return true +} diff --git a/mover-restic/restic/internal/ui/restore/json.go b/mover-restic/restic/internal/ui/restore/json.go index c248a7951..72cc38a6e 100644 --- a/mover-restic/restic/internal/ui/restore/json.go +++ b/mover-restic/restic/internal/ui/restore/json.go @@ -7,11 +7,11 @@ import ( ) type jsonPrinter struct { - terminal term + terminal ui.Terminal verbosity uint } -func NewJSONProgress(terminal term, verbosity uint) ProgressPrinter { +func NewJSONProgress(terminal ui.Terminal, verbosity uint) ProgressPrinter { return &jsonPrinter{ terminal: terminal, verbosity: verbosity, @@ -22,6 +22,10 @@ func (t *jsonPrinter) print(status interface{}) { t.terminal.Print(ui.ToJSONString(status)) } +func (t *jsonPrinter) error(status interface{}) { + t.terminal.Error(ui.ToJSONString(status)) +} + func (t *jsonPrinter) Update(p State, duration time.Duration) { status := statusUpdate{ MessageType: "status", @@ -41,6 +45,16 @@ func (t *jsonPrinter) Update(p State, duration time.Duration) { t.print(status) } +func (t *jsonPrinter) Error(item string, err error) error { + t.error(errorUpdate{ + MessageType: "error", + Error: errorObject{err.Error()}, + During: "restore", + Item: item, + }) + return nil +} + func (t *jsonPrinter) CompleteItem(messageType ItemAction, item string, size uint64) { if t.verbosity < 3 { return @@ -99,6 +113,17 @@ type statusUpdate struct { BytesSkipped uint64 `json:"bytes_skipped,omitempty"` } +type errorObject struct { + Message string `json:"message"` +} + +type errorUpdate struct { + MessageType string `json:"message_type"` // "error" + Error errorObject `json:"error"` + During string `json:"during"` + Item string `json:"item"` +} + type verboseUpdate struct { MessageType string `json:"message_type"` // "verbose_status" Action string `json:"action"` diff --git a/mover-restic/restic/internal/ui/restore/json_test.go b/mover-restic/restic/internal/ui/restore/json_test.go index 06a70d5dc..917a48070 100644 --- a/mover-restic/restic/internal/ui/restore/json_test.go +++ b/mover-restic/restic/internal/ui/restore/json_test.go @@ -4,11 +4,13 @@ import ( "testing" "time" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" ) -func createJSONProgress() (*mockTerm, ProgressPrinter) { - term := &mockTerm{} +func createJSONProgress() (*ui.MockTerminal, ProgressPrinter) { + term := &ui.MockTerminal{} printer := NewJSONProgress(term, 3) return term, printer } @@ -16,31 +18,31 @@ func createJSONProgress() (*mockTerm, ProgressPrinter) { func TestJSONPrintUpdate(t *testing.T) { term, printer := createJSONProgress() printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.Output) } func TestJSONPrintUpdateWithSkipped(t *testing.T) { term, printer := createJSONProgress() printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"status\",\"seconds_elapsed\":5,\"percent_done\":0.6170212765957447,\"total_files\":11,\"files_restored\":3,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":29,\"bytes_skipped\":59}\n"}, term.Output) } func TestJSONPrintSummaryOnSuccess(t *testing.T) { term, printer := createJSONProgress() printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"total_bytes\":47,\"bytes_restored\":47}\n"}, term.Output) } func TestJSONPrintSummaryOnErrors(t *testing.T) { term, printer := createJSONProgress() printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":3,\"total_bytes\":47,\"bytes_restored\":29}\n"}, term.Output) } func TestJSONPrintSummaryOnSuccessWithSkipped(t *testing.T) { term, printer := createJSONProgress() printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second) - test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.output) + test.Equals(t, []string{"{\"message_type\":\"summary\",\"seconds_elapsed\":5,\"total_files\":11,\"files_restored\":11,\"files_skipped\":2,\"total_bytes\":47,\"bytes_restored\":47,\"bytes_skipped\":59}\n"}, term.Output) } func TestJSONPrintCompleteItem(t *testing.T) { @@ -57,6 +59,12 @@ func TestJSONPrintCompleteItem(t *testing.T) { } { term, printer := createJSONProgress() printer.CompleteItem(data.action, "test", data.size) - test.Equals(t, []string{data.expected}, term.output) + test.Equals(t, []string{data.expected}, term.Output) } } + +func TestJSONError(t *testing.T) { + term, printer := createJSONProgress() + test.Equals(t, printer.Error("/path", errors.New("error \"message\"")), nil) + test.Equals(t, []string{"{\"message_type\":\"error\",\"error\":{\"message\":\"error \\\"message\\\"\"},\"during\":\"restore\",\"item\":\"/path\"}\n"}, term.Errors) +} diff --git a/mover-restic/restic/internal/ui/restore/progress.go b/mover-restic/restic/internal/ui/restore/progress.go index 67b15f07e..06f4c86aa 100644 --- a/mover-restic/restic/internal/ui/restore/progress.go +++ b/mover-restic/restic/internal/ui/restore/progress.go @@ -32,13 +32,9 @@ type progressInfoEntry struct { bytesTotal uint64 } -type term interface { - Print(line string) - SetStatus(lines []string) -} - type ProgressPrinter interface { Update(progress State, duration time.Duration) + Error(item string, err error) error CompleteItem(action ItemAction, item string, size uint64) Finish(progress State, duration time.Duration) } @@ -139,6 +135,17 @@ func (p *Progress) ReportDeletedFile(name string) { p.printer.CompleteItem(ActionDeleted, name, 0) } +func (p *Progress) Error(item string, err error) error { + if p == nil { + return nil + } + + p.m.Lock() + defer p.m.Unlock() + + return p.printer.Error(item, err) +} + func (p *Progress) Finish() { p.updater.Done() } diff --git a/mover-restic/restic/internal/ui/restore/progress_test.go b/mover-restic/restic/internal/ui/restore/progress_test.go index 4a6304741..b01440bee 100644 --- a/mover-restic/restic/internal/ui/restore/progress_test.go +++ b/mover-restic/restic/internal/ui/restore/progress_test.go @@ -4,6 +4,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" ) @@ -23,9 +24,18 @@ type itemTraceEntry struct { } type itemTrace []itemTraceEntry + +type errorTraceEntry struct { + item string + err error +} + +type errorTrace []errorTraceEntry + type mockPrinter struct { - trace printerTrace - items itemTrace + trace printerTrace + items itemTrace + errors errorTrace } const mockFinishDuration = 42 * time.Second @@ -33,6 +43,10 @@ const mockFinishDuration = 42 * time.Second func (p *mockPrinter) Update(progress State, duration time.Duration) { p.trace = append(p.trace, printerTraceEntry{progress, duration, false}) } +func (p *mockPrinter) Error(item string, err error) error { + p.errors = append(p.errors, errorTraceEntry{item, err}) + return nil +} func (p *mockPrinter) CompleteItem(action ItemAction, item string, size uint64) { p.items = append(p.items, itemTraceEntry{action, item, size}) } @@ -40,20 +54,21 @@ func (p *mockPrinter) Finish(progress State, _ time.Duration) { p.trace = append(p.trace, printerTraceEntry{progress, mockFinishDuration, true}) } -func testProgress(fn func(progress *Progress) bool) (printerTrace, itemTrace) { +func testProgress(fn func(progress *Progress) bool) (printerTrace, itemTrace, errorTrace) { printer := &mockPrinter{} progress := NewProgress(printer, 0) final := fn(progress) progress.update(0, final) trace := append(printerTrace{}, printer.trace...) items := append(itemTrace{}, printer.items...) + errors := append(errorTrace{}, printer.errors...) // cleanup to avoid goroutine leak, but copy trace first progress.Finish() - return trace, items + return trace, items, errors } func TestNew(t *testing.T) { - result, items := testProgress(func(progress *Progress) bool { + result, items, _ := testProgress(func(progress *Progress) bool { return false }) test.Equals(t, printerTrace{ @@ -65,7 +80,7 @@ func TestNew(t *testing.T) { func TestAddFile(t *testing.T) { fileSize := uint64(100) - result, items := testProgress(func(progress *Progress) bool { + result, items, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) return false }) @@ -79,7 +94,7 @@ func TestFirstProgressOnAFile(t *testing.T) { expectedBytesWritten := uint64(5) expectedBytesTotal := uint64(100) - result, items := testProgress(func(progress *Progress) bool { + result, items, _ := testProgress(func(progress *Progress) bool { progress.AddFile(expectedBytesTotal) progress.AddProgress("test", ActionFileUpdated, expectedBytesWritten, expectedBytesTotal) return false @@ -93,7 +108,7 @@ func TestFirstProgressOnAFile(t *testing.T) { func TestLastProgressOnAFile(t *testing.T) { fileSize := uint64(100) - result, items := testProgress(func(progress *Progress) bool { + result, items, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddProgress("test", ActionFileUpdated, 30, fileSize) progress.AddProgress("test", ActionFileUpdated, 35, fileSize) @@ -111,7 +126,7 @@ func TestLastProgressOnAFile(t *testing.T) { func TestLastProgressOnLastFile(t *testing.T) { fileSize := uint64(100) - result, items := testProgress(func(progress *Progress) bool { + result, items, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(50) progress.AddProgress("test1", ActionFileUpdated, 50, 50) @@ -131,7 +146,7 @@ func TestLastProgressOnLastFile(t *testing.T) { func TestSummaryOnSuccess(t *testing.T) { fileSize := uint64(100) - result, _ := testProgress(func(progress *Progress) bool { + result, _, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(50) progress.AddProgress("test1", ActionFileUpdated, 50, 50) @@ -146,7 +161,7 @@ func TestSummaryOnSuccess(t *testing.T) { func TestSummaryOnErrors(t *testing.T) { fileSize := uint64(100) - result, _ := testProgress(func(progress *Progress) bool { + result, _, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(50) progress.AddProgress("test1", ActionFileUpdated, 50, 50) @@ -161,7 +176,7 @@ func TestSummaryOnErrors(t *testing.T) { func TestSkipFile(t *testing.T) { fileSize := uint64(100) - result, items := testProgress(func(progress *Progress) bool { + result, items, _ := testProgress(func(progress *Progress) bool { progress.AddSkippedFile("test", fileSize) return true }) @@ -176,7 +191,7 @@ func TestSkipFile(t *testing.T) { func TestProgressTypes(t *testing.T) { fileSize := uint64(100) - _, items := testProgress(func(progress *Progress) bool { + _, items, _ := testProgress(func(progress *Progress) bool { progress.AddFile(fileSize) progress.AddFile(0) progress.AddProgress("dir", ActionDirRestored, fileSize, fileSize) @@ -190,3 +205,17 @@ func TestProgressTypes(t *testing.T) { itemTraceEntry{ActionDeleted, "del", 0}, }, items) } + +func TestProgressError(t *testing.T) { + err1 := errors.New("err1") + err2 := errors.New("err2") + _, _, errors := testProgress(func(progress *Progress) bool { + test.Equals(t, progress.Error("first", err1), nil) + test.Equals(t, progress.Error("second", err2), nil) + return true + }) + test.Equals(t, errorTrace{ + errorTraceEntry{"first", err1}, + errorTraceEntry{"second", err2}, + }, errors) +} diff --git a/mover-restic/restic/internal/ui/restore/text.go b/mover-restic/restic/internal/ui/restore/text.go index ec512f369..ba0dcd007 100644 --- a/mover-restic/restic/internal/ui/restore/text.go +++ b/mover-restic/restic/internal/ui/restore/text.go @@ -8,14 +8,15 @@ import ( ) type textPrinter struct { - terminal term - verbosity uint + *ui.Message + + terminal ui.Terminal } -func NewTextProgress(terminal term, verbosity uint) ProgressPrinter { +func NewTextProgress(terminal ui.Terminal, verbosity uint) ProgressPrinter { return &textPrinter{ - terminal: terminal, - verbosity: verbosity, + Message: ui.NewMessage(terminal, verbosity), + terminal: terminal, } } @@ -33,11 +34,12 @@ func (t *textPrinter) Update(p State, duration time.Duration) { t.terminal.SetStatus([]string{progress}) } -func (t *textPrinter) CompleteItem(messageType ItemAction, item string, size uint64) { - if t.verbosity < 3 { - return - } +func (t *textPrinter) Error(item string, err error) error { + t.E("ignoring error for %s: %s\n", item, err) + return nil +} +func (t *textPrinter) CompleteItem(messageType ItemAction, item string, size uint64) { var action string switch messageType { case ActionDirRestored: @@ -57,9 +59,9 @@ func (t *textPrinter) CompleteItem(messageType ItemAction, item string, size uin } if messageType == ActionDirRestored || messageType == ActionOtherRestored || messageType == ActionDeleted { - t.terminal.Print(fmt.Sprintf("%-9v %v", action, item)) + t.VV("%-9v %v", action, item) } else { - t.terminal.Print(fmt.Sprintf("%-9v %v with size %v", action, item, ui.FormatBytes(size))) + t.VV("%-9v %v with size %v", action, item, ui.FormatBytes(size)) } } diff --git a/mover-restic/restic/internal/ui/restore/text_test.go b/mover-restic/restic/internal/ui/restore/text_test.go index b198a27df..4ffb1615d 100644 --- a/mover-restic/restic/internal/ui/restore/text_test.go +++ b/mover-restic/restic/internal/ui/restore/text_test.go @@ -4,23 +4,13 @@ import ( "testing" "time" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui" ) -type mockTerm struct { - output []string -} - -func (m *mockTerm) Print(line string) { - m.output = append(m.output, line) -} - -func (m *mockTerm) SetStatus(lines []string) { - m.output = append([]string{}, lines...) -} - -func createTextProgress() (*mockTerm, ProgressPrinter) { - term := &mockTerm{} +func createTextProgress() (*ui.MockTerminal, ProgressPrinter) { + term := &ui.MockTerminal{} printer := NewTextProgress(term, 3) return term, printer } @@ -28,31 +18,31 @@ func createTextProgress() (*mockTerm, ProgressPrinter) { func TestPrintUpdate(t *testing.T) { term, printer := createTextProgress() printer.Update(State{3, 11, 0, 29, 47, 0}, 5*time.Second) - test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.output) + test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B"}, term.Output) } func TestPrintUpdateWithSkipped(t *testing.T) { term, printer := createTextProgress() printer.Update(State{3, 11, 2, 29, 47, 59}, 5*time.Second) - test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.output) + test.Equals(t, []string{"[0:05] 61.70% 3 files/dirs 29 B, total 11 files/dirs 47 B, skipped 2 files/dirs 59 B"}, term.Output) } func TestPrintSummaryOnSuccess(t *testing.T) { term, printer := createTextProgress() printer.Finish(State{11, 11, 0, 47, 47, 0}, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.output) + test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05"}, term.Output) } func TestPrintSummaryOnErrors(t *testing.T) { term, printer := createTextProgress() printer.Finish(State{3, 11, 0, 29, 47, 0}, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.output) + test.Equals(t, []string{"Summary: Restored 3 / 11 files/dirs (29 B / 47 B) in 0:05"}, term.Output) } func TestPrintSummaryOnSuccessWithSkipped(t *testing.T) { term, printer := createTextProgress() printer.Finish(State{11, 11, 2, 47, 47, 59}, 5*time.Second) - test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs 59 B"}, term.output) + test.Equals(t, []string{"Summary: Restored 11 files/dirs (47 B) in 0:05, skipped 2 files/dirs 59 B"}, term.Output) } func TestPrintCompleteItem(t *testing.T) { @@ -70,6 +60,12 @@ func TestPrintCompleteItem(t *testing.T) { } { term, printer := createTextProgress() printer.CompleteItem(data.action, "test", data.size) - test.Equals(t, []string{data.expected}, term.output) + test.Equals(t, []string{data.expected}, term.Output) } } + +func TestError(t *testing.T) { + term, printer := createTextProgress() + test.Equals(t, printer.Error("/path", errors.New("error \"message\"")), nil) + test.Equals(t, []string{"ignoring error for /path: error \"message\"\n"}, term.Errors) +} diff --git a/mover-restic/restic/internal/ui/terminal.go b/mover-restic/restic/internal/ui/terminal.go new file mode 100644 index 000000000..2d9418a61 --- /dev/null +++ b/mover-restic/restic/internal/ui/terminal.go @@ -0,0 +1,10 @@ +package ui + +// Terminal is used to write messages and display status lines which can be +// updated. See termstatus.Terminal for a concrete implementation. +type Terminal interface { + Print(line string) + Error(line string) + SetStatus(lines []string) + CanUpdateStatus() bool +} diff --git a/mover-restic/restic/internal/walker/rewriter.go b/mover-restic/restic/internal/walker/rewriter.go index 6c27b26ac..7e984ae25 100644 --- a/mover-restic/restic/internal/walker/rewriter.go +++ b/mover-restic/restic/internal/walker/rewriter.go @@ -116,6 +116,10 @@ func (t *TreeRewriter) RewriteTree(ctx context.Context, repo BlobLoadSaver, node tb := restic.NewTreeJSONBuilder() for _, node := range curTree.Nodes { + if ctx.Err() != nil { + return restic.ID{}, ctx.Err() + } + path := path.Join(nodepath, node.Name) node = t.opts.RewriteNode(node, path) if node == nil { diff --git a/mover-restic/restic/internal/walker/walker.go b/mover-restic/restic/internal/walker/walker.go index 091b05489..788ece1cf 100644 --- a/mover-restic/restic/internal/walker/walker.go +++ b/mover-restic/restic/internal/walker/walker.go @@ -57,6 +57,10 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree }) for _, node := range tree.Nodes { + if ctx.Err() != nil { + return ctx.Err() + } + p := path.Join(prefix, node.Name) if node.Type == "" {