diff --git a/cmd/torrent_export.go b/cmd/torrent_export.go index 8bdc5f1..6562e5d 100644 --- a/cmd/torrent_export.go +++ b/cmd/torrent_export.go @@ -8,13 +8,19 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/ludviglundgren/qbittorrent-cli/internal/config" fsutil "github.com/ludviglundgren/qbittorrent-cli/internal/fs" + "github.com/ludviglundgren/qbittorrent-cli/pkg/archive" + qbit "github.com/ludviglundgren/qbittorrent-cli/pkg/qbittorrent" + "github.com/ludviglundgren/qbittorrent-cli/pkg/utils" + "github.com/anacrolix/torrent/metainfo" "github.com/autobrr/go-qbittorrent" "github.com/pkg/errors" "github.com/spf13/cobra" + "github.com/zeebo/bencode" ) func RunTorrentExport() *cobra.Command { @@ -28,6 +34,7 @@ func RunTorrentExport() *cobra.Command { f := export{ dry: false, verbose: false, + archive: false, sourceDir: "", exportDir: "", includeCategory: nil, @@ -42,6 +49,7 @@ func RunTorrentExport() *cobra.Command { command.Flags().BoolVar(&f.dry, "dry-run", false, "dry run") command.Flags().BoolVarP(&f.verbose, "verbose", "v", false, "verbose output") + command.Flags().BoolVarP(&f.archive, "archive", "a", false, "archive export dir to .tar.gz") command.Flags().BoolVar(&skipManifest, "skip-manifest", false, "Do not export all used tags and categories into manifest") command.Flags().StringVar(&f.sourceDir, "source", "", "Dir with torrent and fast-resume files (required)") @@ -56,9 +64,21 @@ func RunTorrentExport() *cobra.Command { command.MarkFlagRequired("source") command.MarkFlagRequired("export-dir") + command.MarkFlagsMutuallyExclusive("include-category", "exclude-category") + command.MarkFlagsMutuallyExclusive("include-tag", "exclude-tag") + command.RunE = func(cmd *cobra.Command, args []string) error { - // get torrents from client by categories - config.InitConfig() + var err error + + f.sourceDir, err = utils.ExpandTilde(f.sourceDir) + if err != nil { + return errors.Wrap(err, "could not read source-dir") + } + + f.exportDir, err = utils.ExpandTilde(f.exportDir) + if err != nil { + return errors.Wrap(err, "could not read export-dir") + } if _, err := os.Stat(f.sourceDir); err != nil { if os.IsNotExist(err) { @@ -68,6 +88,11 @@ func RunTorrentExport() *cobra.Command { return err } + log.Printf("Preparing to export torrents using source-dir: %q export-dir: %q\n", f.sourceDir, f.exportDir) + + // get torrents from client by categories + config.InitConfig() + qbtSettings := qbittorrent.Config{ Host: config.Qbit.Addr, Username: config.Qbit.Login, @@ -81,110 +106,74 @@ func RunTorrentExport() *cobra.Command { ctx := cmd.Context() if err := qb.LoginCtx(ctx); err != nil { - fmt.Fprintf(os.Stderr, "ERROR: connection failed: %v\n", err) - os.Exit(1) + return errors.Wrapf(err, "failed to login") } + var torrents []qbittorrent.Torrent + if len(f.includeCategory) > 0 { for _, category := range f.includeCategory { - torrents, err := qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Category: category}) + torrentsByCategory, err := qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{Category: category}) if err != nil { return errors.Wrapf(err, "could not get torrents for category: %s", category) } - for _, tor := range torrents { - // only grab completed torrents - //if tor.Progress != 1 { - // continue - //} - - if tor.Tags != "" { - tags := strings.Split(tor.Tags, ", ") - - // check tags and exclude categories - if len(f.includeTag) > 0 && !containsTag(f.includeTag, tags) { - continue - } - - if len(f.excludeTag) > 0 && containsTag(f.excludeTag, tags) { - continue - } - - for _, tag := range tags { - _, ok := f.tags[tag] - if !ok { - f.tags[tag] = struct{}{} - } - } - - } - - if tor.Category != "" { - f.category[tor.Category] = qbittorrent.Category{ - Name: tor.Category, - SavePath: "", - } - } - - // append hash to map of hashes to gather - f.hashes[strings.ToLower(tor.Hash)] = tor - } + torrents = append(torrents, torrentsByCategory...) } } else { - torrents, err := qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{}) + torrents, err = qb.GetTorrentsCtx(ctx, qbittorrent.TorrentFilterOptions{}) if err != nil { return errors.Wrap(err, "could not get torrents") } + } - for _, tor := range torrents { - // only grab completed torrents - //if tor.Progress != 1 { - // continue - //} + for _, tor := range torrents { + // only grab completed torrents + //if tor.Progress != 1 { + // continue + //} - if len(f.excludeCategory) > 0 && containsCategory(f.excludeCategory, tor.Category) { - continue - } - - if tor.Tags != "" { - tags := strings.Split(tor.Tags, ", ") + if len(f.excludeCategory) > 0 && containsCategory(f.excludeCategory, tor.Category) { + continue + } - // check tags and exclude categories - if len(f.includeTag) > 0 && !containsTag(f.includeTag, tags) { - continue - } + if tor.Tags != "" { + tags := strings.Split(tor.Tags, ", ") - if len(f.excludeTag) > 0 && containsTag(f.excludeTag, tags) { - continue - } + // check tags and exclude categories + if len(f.includeTag) > 0 && !containsTag(f.includeTag, tags) { + continue + } - for _, tag := range tags { - _, ok := f.tags[tag] - if !ok { - f.tags[tag] = struct{}{} - } - } + if len(f.excludeTag) > 0 && containsTag(f.excludeTag, tags) { + continue } - if tor.Category != "" { - f.category[tor.Category] = qbittorrent.Category{ - Name: tor.Category, - SavePath: "", + for _, tag := range tags { + _, ok := f.tags[tag] + if !ok { + f.tags[tag] = struct{}{} } } + } - // append hash to map of hashes to gather - f.hashes[strings.ToLower(tor.Hash)] = tor + if tor.Category != "" { + f.category[tor.Category] = qbittorrent.Category{ + Name: tor.Category, + SavePath: "", + } } + + // append hash to map of hashes to gather + f.hashes[strings.ToLower(tor.Hash)] = tor } if len(f.hashes) == 0 { - fmt.Printf("Could not find any matching torrents to export from (%s)\n", strings.Join(f.includeCategory, ",")) - os.Exit(1) + return errors.Errorf("Could not find any matching torrents to export from client") } - fmt.Printf("Found '%d' matching torrents\n", len(f.hashes)) + log.Printf("Found (%d) matching torrents\n", len(f.hashes)) if err := processExport(f.sourceDir, f.exportDir, f.hashes, f.dry, f.verbose); err != nil { return errors.Wrapf(err, "could not process torrents") @@ -211,18 +200,36 @@ func RunTorrentExport() *cobra.Command { } if f.dry { - fmt.Println("dry-run: successfully wrote manifest to file") + log.Println("dry-run: Saved export manifest to file") } else { - if err := exportManifest(f.hashes, f.tags, f.category); err != nil { - fmt.Printf("could not export manifest: %q\n", err) - os.Exit(1) + if err := exportManifest(f.hashes, f.tags, f.category, f.exportDir); err != nil { + return errors.Wrapf(err, "could not export manifest") } - - fmt.Println("successfully wrote manifest to file") } } - fmt.Println("Successfully exported torrents!") + log.Println("Successfully exported torrents!") + + if f.archive { + currentWorkingDirectory, err := os.Getwd() + if err != nil { + return errors.Wrap(err, "could not get current working directory") + } + + archiveFilename := filepath.Join(currentWorkingDirectory, "qbittorrent-export.tar.gz") + + if f.dry { + log.Printf("dry-run: Archive exported dir to: %s\n", archiveFilename) + } else { + log.Printf("Archive exported dir to: %s\n", archiveFilename) + + if err := archive.TarGzDirectory(f.exportDir, archiveFilename); err != nil { + return errors.Wrapf(err, "could not archive exported torrents") + } + } + + log.Println("Successfully archived exported torrents!") + } return nil } @@ -230,7 +237,7 @@ func RunTorrentExport() *cobra.Command { return command } -func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struct{}, categories map[string]qbittorrent.Category) error { +func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struct{}, categories map[string]qbittorrent.Category, exportDir string) error { data := Manifest{ Tags: make([]string, 0), Categories: []qbittorrent.Category{}, @@ -255,110 +262,192 @@ func exportManifest(hashes map[string]qbittorrent.Torrent, tags map[string]struc }) } - res, err := json.Marshal(data) - if err != nil { - return errors.Wrap(err, "could not marshal manifest to json") - } + //currentWorkingDirectory, err := os.Getwd() + //if err != nil { + // return errors.Wrap(err, "could not get current working directory") + //} - currentWorkingDirectory, err := os.Getwd() - if err != nil { - return err - } + currentTime := time.Now() + timestamp := currentTime.Format("20060102150405") + + // Create a new manifestFile in the current working directory. + manifestFileName := fmt.Sprintf("export-manifest-%s.json", timestamp) - // Create a new file in the current working directory. - fileName := "export-manifest.json" + manifestFilePath := filepath.Join(exportDir, manifestFileName) - file, err := os.Create(filepath.Join(currentWorkingDirectory, fileName)) + manifestFile, err := os.Create(manifestFilePath) if err != nil { - return err + return errors.Wrapf(err, "could not create manifestFile: %s", manifestFilePath) } - defer file.Close() + defer manifestFile.Close() - // Write the string to the file. - _, err = file.WriteString(string(res)) - if err != nil { - return err + // create new encoder with pretty print + encoder := json.NewEncoder(manifestFile) + encoder.SetIndent("", " ") + + if err := encoder.Encode(&data); err != nil { + return errors.Wrap(err, "could not encode manifest to json") } + log.Printf("Saved export manifest to %s\n", manifestFilePath) + return nil } func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.Torrent, dry, verbose bool) error { - exportCount := 0 exportTorrentCount := 0 exportFastresumeCount := 0 // check if export dir exists, if not then lets create it if err := createDirIfNotExists(exportDir); err != nil { - fmt.Printf("could not check if dir %s exists. err: %q\n", exportDir, err) return errors.Wrapf(err, "could not check if dir exists: %s", exportDir) } - // check BT_backup dir, pick torrent and fastresume files by id - err := filepath.Walk(sourceDir, func(dirPath string, info fs.FileInfo, err error) error { + // qbittorrent from v4.5.x removes the announce-urls from the .torrent file so we need to add that back + needTrackerFix := false + + // keep track of processed fastresume files + processedFastResumeHashes := map[string]bool{} + + // check BT_backup dir, pick torrent and fastresume files by hash + err := filepath.WalkDir(sourceDir, func(dirPath string, d fs.DirEntry, err error) error { if err != nil { return err } - if info.IsDir() { + if d.IsDir() { return nil } - fileName := info.Name() + fileName := d.Name() ext := filepath.Ext(fileName) - if !isValidExt(ext) { + if ext != ".torrent" { return nil } torrentHash := fileNameTrimExt(fileName) // if filename not in hashes return and check next - _, ok := hashes[torrentHash] + torrent, ok := hashes[torrentHash] if !ok { return nil } + if verbose { + log.Printf("Processing: %s\n", fileName) + } + if dry { - if verbose { - fmt.Printf("processing: %s\n", fileName) + exportTorrentCount++ + log.Printf("dry-run: [%d/%d] exported: %s %s\n", exportTorrentCount, len(hashes), torrentHash+".torrent", torrent.Name) + + exportFastresumeCount++ + log.Printf("dry-run: [%d/%d] exported: %s %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume", torrent.Name) + + return nil + } + + outFile := filepath.Join(exportDir, fileName) + + // determine if this should be run on first run and the ones after + if (exportTorrentCount == 0 && !needTrackerFix) || needTrackerFix { + + // open file and check if announce is in there. If it's not, open .fastresume and combine before output + torrentFile, err := os.Open(dirPath) + if err != nil { + return errors.Wrapf(err, "could not open torrent file: %s", dirPath) } + defer torrentFile.Close() - exportCount++ + torrentInfo, err := metainfo.Load(torrentFile) + if err != nil { + return errors.Wrapf(err, "could not open file: %s", dirPath) + } - //fmt.Printf("dry-run: (%d/%d) exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) + if torrentInfo.Announce == "" { + needTrackerFix = true + + sourceFastResumeFilePath := filepath.Join(sourceDir, torrentHash+".fastresume") + fastResumeFile, err := os.Open(sourceFastResumeFilePath) + if err != nil { + return errors.Wrapf(err, "could not open fastresume file: %s", sourceFastResumeFilePath) + } + defer fastResumeFile.Close() + + // open fastresume and get announce then // open fastresume and get announce then + var fastResume qbit.Fastresume + if err := bencode.NewDecoder(fastResumeFile).Decode(&fastResume); err != nil { + return errors.Wrapf(err, "could not open file: %s", sourceFastResumeFilePath) + } + + if len(fastResume.Trackers) == 0 { + return errors.New("no trackers found in fastresume") + } + + torrentInfo.Announce = fastResume.Trackers[0][0] + torrentInfo.AnnounceList = fastResume.Trackers + + if len(torrentInfo.UrlList) == 0 && len(fastResume.UrlList) > 0 { + torrentInfo.UrlList = fastResume.UrlList + } + + // copy .fastresume here already since we already have it open + fastresumeFilePath := filepath.Join(exportDir, torrentHash+".fastresume") + newFastResumeFile, err := os.Create(fastresumeFilePath) + if err != nil { + return errors.Wrapf(err, "could not create new fastresume file: %s", fastresumeFilePath) + } + defer newFastResumeFile.Close() + + if err := bencode.NewEncoder(newFastResumeFile).Encode(&fastResume); err != nil { + return errors.Wrapf(err, "could not encode fastresume to file: %s", fastresumeFilePath) + } + + // make sure the fastresume is only written once + processedFastResumeHashes[torrentHash] = true - switch ext { - case ".torrent": - exportTorrentCount++ - fmt.Printf("dry-run: (%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) - case ".fastresume": exportFastresumeCount++ - fmt.Printf("dry-run: (%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) + log.Printf("[%d/%d] exported: %s %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume", torrent.Name) } - } else { - if verbose { - fmt.Printf("processing: %s\n", fileName) + // write new torrent file to destination path + newTorrentFile, err := os.Create(outFile) + if err != nil { + return errors.Wrapf(err, "could not create new torrent file: %s", outFile) } + defer newTorrentFile.Close() - outFile := filepath.Join(exportDir, fileName) - if err := fsutil.CopyFile(dirPath, outFile); err != nil { - return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) + if err := torrentInfo.Write(newTorrentFile); err != nil { + return errors.Wrapf(err, "could not write torrent info into file %s", outFile) } - exportCount++ + // all good lets return for this file + exportTorrentCount++ + log.Printf("[%d/%d] exported: %s %s\n", exportTorrentCount, len(hashes), fileName, torrent.Name) - switch ext { - case ".torrent": - exportTorrentCount++ - fmt.Printf("(%d/%d) exported: %s\n", exportTorrentCount, len(hashes), fileName) - case ".fastresume": - exportFastresumeCount++ - fmt.Printf("(%d/%d) exported: %s\n", exportFastresumeCount, len(hashes), fileName) + return nil + } + + // only do this if !needTrackerFix + if err := fsutil.CopyFile(dirPath, outFile); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, outFile) + } + + exportTorrentCount++ + log.Printf("[%d/%d] exported: %s %s\n", exportTorrentCount, len(hashes), fileName, torrent.Name) + + // process if fastresume has not already been copied + _, ok = processedFastResumeHashes[torrentHash] + if !ok { + fastResumeFilePath := filepath.Join(exportDir, torrentHash+".fastresume") + + if err := fsutil.CopyFile(dirPath, fastResumeFilePath); err != nil { + return errors.Wrapf(err, "could not copy file: %s to %s", dirPath, fastResumeFilePath) } - //fmt.Printf("(%d/%d) exported: %s '%s'\n", exportCount, len(hashes), torrentHash, fileName) + exportFastresumeCount++ + log.Printf("[%d/%d] exported: %s %s\n", exportFastresumeCount, len(hashes), torrentHash+".fastresume", torrent.Name) } return nil @@ -368,11 +457,7 @@ func processExport(sourceDir, exportDir string, hashes map[string]qbittorrent.To return err } - fmt.Printf(` -found (%d) files in total -exported fastresume: %d -exported torrent %d -`, exportCount, exportFastresumeCount, exportTorrentCount) + log.Printf("Exported (%d) files in total: fastresume (%d) torrents (%d)\n", exportFastresumeCount+exportTorrentCount, exportFastresumeCount, exportTorrentCount) return nil } @@ -414,6 +499,7 @@ func createDirIfNotExists(dir string) error { type export struct { dry bool verbose bool + archive bool sourceDir string exportDir string includeCategory []string diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go new file mode 100644 index 0000000..f4357db --- /dev/null +++ b/pkg/archive/archive.go @@ -0,0 +1,70 @@ +package archive + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" +) + +func TarGzDirectory(source, target string) error { + file, err := os.Create(target) + if err != nil { + return fmt.Errorf("failed to create tar.gz file: %w", err) + } + defer file.Close() + + // Create a new gzip writer + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + + // Create a new tar writer + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Create a tar header from file info + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + + // Preserve the directory structure + header.Name, err = filepath.Rel(filepath.Dir(source), path) + if err != nil { + return err + } + + // Write the header to the tar archive + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + + // If it's a file, write its content to the tar archive + if !info.IsDir() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + if _, err := io.Copy(tarWriter, file); err != nil { + return err + } + } + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to add files to %s.tar.gz archive: %w", target, err) + } + + return nil +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index eb14570..4093a15 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,6 +2,8 @@ package utils import ( "fmt" + "os/user" + "path/filepath" "regexp" "strings" ) @@ -23,3 +25,16 @@ func ValidateHash(hashes []string) error { return nil } + +// ExpandTilde expands the ~ in the file path to the home directory +func ExpandTilde(path string) (string, error) { + if strings.HasPrefix(path, "~") { + usr, err := user.Current() + if err != nil { + return "", err + } + homeDir := usr.HomeDir + return filepath.Join(homeDir, path[1:]), nil + } + return path, nil +}