From e57c374d4a156c810ddb6fa4e232b836403f9a07 Mon Sep 17 00:00:00 2001 From: nielash Date: Mon, 11 Dec 2023 07:38:54 -0500 Subject: [PATCH] Initial commit --- .gitignore | 26 +++ LICENSE | 21 +++ Makefile | 5 + README.md | 54 ++++++ bin/cross-compile.go | 415 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 3 + license.txt | 23 +++ main.go | 120 +++++++++++++ 8 files changed, 667 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 bin/cross-compile.go create mode 100644 go.mod create mode 100644 license.txt create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..cabe30a --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work + +# Other +.DS_Store +rclone-permissions-mapper +build \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c7e7acf --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 nielash + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ec7b287 --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +SHELL = bash +VERSION := 1.0 + +compile_all: + go run bin/cross-compile.go -compile-only $(VERSION) diff --git a/README.md b/README.md new file mode 100644 index 0000000..7b2c1f8 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# Rclone Permissions Mapper + +Tool to convert `uid` and `gid` between mac and linux defaults when syncing files via cloud storage using [rclone](https://github.com/rclone/rclone) [`sync`](https://rclone.org/commands/rclone_sync/). + +## Usage +```bash +rclone sync source:path dest:path --metadata-mapper /path/to/rclone-permissions-mapper +``` + +or, to see input and output: +```bash +rclone sync source:path dest:path --metadata-mapper /path/to/rclone-permissions-mapper -v --dump mapper +``` + +## Background +The default UID of the first regular user on macOS is `501`. On Linux, it is usually `1000`. If you [`rclone sync`](https://rclone.org/commands/rclone_sync/) a file created on one to the other (by way of a cloud storage remote) and use the [`--metadata`](https://rclone.org/docs/#m-metadata) flag (without using `sudo`), by default you will probably get an error like this one: + +```text +ERROR : file.txt: Failed to copy: failed to set metadata: failed to change ownership: chown /testing/file.txt.fekayen6.partial: operation not permitted +``` + +This is because it is trying to `chown 1000:1000 /testing/file.txt` when actually it should be `501:20` (or vice versa.) + +This tool uses rclone's new `--metadata-mapper` feature to automatically detect and correct this during the sync. It does so by simply omitting the `uid` and `gid` (when necessary) in the metadata blob it passes back to rclone, so that the default values are kept. + +## Installation +[Download](https://github.com/nielash/rclone-permissions-mapper/releases) and unzip (or build from source with `go build`), and then move the executable to your `$PATH`: +```bash +sudo rclone moveto /Users/yourusername/Downloads/rclone-permissions-mapper-1.0-osx-arm64/rclone-permissions-mapper /usr/local/bin/rclone-permissions-mapper -v +``` + +Test if it's working: + +```bash +echo '{"Metadata": {"hello": "world"}}' | rclone-permissions-mapper +``` +should output: `{"Metadata":{"hello":"world"}}` + +You can test what it will do by giving it different `uid` and `gid` values: +``` bash +echo '{"Metadata":{"gid":"20","uid":"501"}}' | rclone-permissions-mapper +// on mac: {"Metadata":{"gid":"20","uid":"501"}} +// on linux: {"Metadata":{}} + +echo '{"Metadata":{"gid":"1000","uid":"1000"}}' | rclone-permissions-mapper +// on mac: {"Metadata":{}} +// on linux: {"Metadata":{"gid":"20","uid":"501"}} +``` + +## Resources +* [`--metadata-mapper` docs](https://rclone.org/docs/#metadata-mapper) +* [rclone's handling of `uid` and `gid`](https://github.com/rclone/rclone/blob/c69eb84573c85206ab028eda2987180e049ef2e4/backend/local/metadata.go#L113-L128) +* [Downloads](https://github.com/nielash/rclone-permissions-mapper/releases) +* [Rclone Forum](https://forum.rclone.org/) \ No newline at end of file diff --git a/bin/cross-compile.go b/bin/cross-compile.go new file mode 100644 index 0000000..063c7f7 --- /dev/null +++ b/bin/cross-compile.go @@ -0,0 +1,415 @@ +//go:build ignore +// +build ignore + +// lightly adapted from ncw's excellent original at: https://github.com/rclone/rclone/blob/master/bin/cross-compile.go + +package main + +import ( + "flag" + "fmt" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "sync" + "text/template" + "time" +) + +var ( + // Flags + debug = flag.Bool("d", false, "Print commands instead of running them") + parallel = flag.Int("parallel", runtime.NumCPU(), "Number of commands to run in parallel") + copyAs = flag.String("release", "", "Make copies of the releases with this name") + gitLog = flag.String("git-log", "", "git log to include as well") + include = flag.String("include", "^.*$", "os/arch regexp to include") + exclude = flag.String("exclude", "^$", "os/arch regexp to exclude") + cgo = flag.Bool("cgo", false, "Use cgo for the build") + noClean = flag.Bool("no-clean", false, "Don't clean the build directory before running") + tags = flag.String("tags", "", "Space separated list of build tags") + buildmode = flag.String("buildmode", "", "Passed to go build -buildmode flag") + compileOnly = flag.Bool("compile-only", false, "Just build the binary, not the zip") + extraEnv = flag.String("env", "", "comma separated list of VAR=VALUE env vars to set") + macOSSDK = flag.String("macos-sdk", "", "macOS SDK to use") + macOSArch = flag.String("macos-arch", "", "macOS arch to use") + extraCgoCFlags = flag.String("cgo-cflags", "", "extra CGO_CFLAGS") + extraCgoLdFlags = flag.String("cgo-ldflags", "", "extra CGO_LDFLAGS") +) + +// GOOS/GOARCH pairs we build for +// +// If the GOARCH contains a - it is a synthetic arch with more parameters +var osarches = []string{ + /* "windows/386", + "windows/amd64", + "windows/arm64", */ + "darwin/amd64", + "darwin/arm64", + "linux/386", + "linux/amd64", + "linux/arm", + "linux/arm-v6", + "linux/arm-v7", + "linux/arm64", + "linux/mips", + "linux/mipsle", + "freebsd/386", + "freebsd/amd64", + "freebsd/arm", + "freebsd/arm-v6", + "freebsd/arm-v7", + "netbsd/386", + "netbsd/amd64", + "netbsd/arm", + "netbsd/arm-v6", + "netbsd/arm-v7", + "openbsd/386", + "openbsd/amd64", + "plan9/386", + "plan9/amd64", + "solaris/amd64", + "js/wasm", +} + +// Special environment flags for a given arch +var archFlags = map[string][]string{ + "386": {"GO386=softfloat"}, + "mips": {"GOMIPS=softfloat"}, + "mipsle": {"GOMIPS=softfloat"}, + "arm": {"GOARM=5"}, + "arm-v6": {"GOARM=6"}, + "arm-v7": {"GOARM=7"}, +} + +// Map Go architectures to NFPM architectures +// Any missing are passed straight through +var goarchToNfpm = map[string]string{ + "arm": "arm5", + "arm-v6": "arm6", + "arm-v7": "arm7", +} + +// runEnv - run a shell command with env +func runEnv(args, env []string) error { + if *debug { + args = append([]string{"echo"}, args...) + } + cmd := exec.Command(args[0], args[1:]...) + if env != nil { + cmd.Env = append(os.Environ(), env...) + } + if *debug { + log.Printf("args = %v, env = %v\n", args, cmd.Env) + } + out, err := cmd.CombinedOutput() + if err != nil { + log.Print("----------------------------") + log.Printf("Failed to run %v: %v", args, err) + log.Printf("Command output was:\n%s", out) + log.Print("----------------------------") + } + return err +} + +// run a shell command +func run(args ...string) { + err := runEnv(args, nil) + if err != nil { + log.Fatalf("Exiting after error: %v", err) + } +} + +// chdir or die +func chdir(dir string) { + err := os.Chdir(dir) + if err != nil { + log.Fatalf("Couldn't cd into %q: %v", dir, err) + } +} + +// substitute data from go template file in to file out +func substitute(inFile, outFile string, data interface{}) { + t, err := template.ParseFiles(inFile) + if err != nil { + log.Fatalf("Failed to read template file %q: %v", inFile, err) + } + out, err := os.Create(outFile) + if err != nil { + log.Fatalf("Failed to create output file %q: %v", outFile, err) + } + defer func() { + err := out.Close() + if err != nil { + log.Fatalf("Failed to close output file %q: %v", outFile, err) + } + }() + err = t.Execute(out, data) + if err != nil { + log.Fatalf("Failed to substitute template file %q: %v", inFile, err) + } +} + +// build the zip package return its name +func buildZip(dir string) string { + // Now build the zip + run("cp", "-a", "../MANUAL.txt", filepath.Join(dir, "README.txt")) + run("cp", "-a", "../MANUAL.html", filepath.Join(dir, "README.html")) + run("cp", "-a", "../rclone-permissions-mapper.1", dir) + if *gitLog != "" { + run("cp", "-a", *gitLog, dir) + } + zip := dir + ".zip" + run("zip", "-r9", zip, dir) + return zip +} + +// Build .deb and .rpm packages +// +// It returns a list of artifacts it has made +func buildDebAndRpm(dir, version, goarch string) []string { + // Make internal version number acceptable to .deb and .rpm + pkgVersion := version[1:] + pkgVersion = strings.ReplaceAll(pkgVersion, "β", "-beta") + pkgVersion = strings.ReplaceAll(pkgVersion, "-", ".") + nfpmArch, ok := goarchToNfpm[goarch] + if !ok { + nfpmArch = goarch + } + + // Make nfpm.yaml from the template + substitute("../bin/nfpm.yaml", path.Join(dir, "nfpm.yaml"), map[string]string{ + "Version": pkgVersion, + "Arch": nfpmArch, + }) + + // build them + var artifacts []string + for _, pkg := range []string{".deb", ".rpm"} { + artifact := dir + pkg + run("bash", "-c", "cd "+dir+" && nfpm -f nfpm.yaml pkg -t ../"+artifact) + artifacts = append(artifacts, artifact) + } + + return artifacts +} + +// Trip a version suffix off the arch if present +func stripVersion(goarch string) string { + i := strings.Index(goarch, "-") + if i < 0 { + return goarch + } + return goarch[:i] +} + +// run the command returning trimmed output +func runOut(command ...string) string { + out, err := exec.Command(command[0], command[1:]...).Output() + if err != nil { + log.Fatalf("Failed to run %q: %v", command, err) + } + return strings.TrimSpace(string(out)) +} + +// Generate Windows resource system object file (.syso), which can be picked +// up by the following go build for embedding version information and icon +// resources into the executable. +func generateResourceWindows(version, arch string) func() { + sysoPath := fmt.Sprintf("../resource_windows_%s.syso", arch) // Use explicit destination filename, even though it should be same as default, so that we are sure we have the correct reference to it + if err := os.Remove(sysoPath); !os.IsNotExist(err) { + // Note: This one we choose to treat as fatal, to avoid any risk of picking up an old .syso file without noticing. + log.Fatalf("Failed to remove existing Windows %s resource system object file %s: %v", arch, sysoPath, err) + } + args := []string{"go", "run", "../bin/resource_windows.go", "-arch", arch, "-version", version, "-syso", sysoPath} + if err := runEnv(args, nil); err != nil { + log.Printf("Warning: Couldn't generate Windows %s resource system object file, binaries will not have version information or icon embedded", arch) + return nil + } + if _, err := os.Stat(sysoPath); err != nil { + log.Printf("Warning: Couldn't find generated Windows %s resource system object file, binaries will not have version information or icon embedded", arch) + return nil + } + return func() { + if err := os.Remove(sysoPath); err != nil && !os.IsNotExist(err) { + log.Printf("Warning: Couldn't remove generated Windows %s resource system object file %s: %v. Please remove it manually.", arch, sysoPath, err) + } + } +} + +// build the binary in dir returning success or failure +func compileArch(version, goos, goarch, dir string) bool { + log.Printf("Compiling %s/%s into %s", goos, goarch, dir) + goarchBase := stripVersion(goarch) + output := filepath.Join(dir, "rclone-permissions-mapper") + if goos == "windows" { + output += ".exe" + if cleanupFn := generateResourceWindows(version, goarchBase); cleanupFn != nil { + defer cleanupFn() + } + } + err := os.MkdirAll(dir, 0777) + if err != nil { + log.Fatalf("Failed to mkdir: %v", err) + } + args := []string{ + "go", "build", + "--ldflags", "-s -X github.com/rclone-permissions-mapper/rclone-permissions-mapper/fs.Version=" + version, + "-trimpath", + "-o", output, + "-tags", *tags, + } + if *buildmode != "" { + args = append(args, + "-buildmode", *buildmode, + ) + } + args = append(args, + "..", + ) + env := []string{ + "GOOS=" + goos, + "GOARCH=" + goarchBase, + } + if *extraEnv != "" { + env = append(env, strings.Split(*extraEnv, ",")...) + } + var ( + cgoCFlags []string + cgoLdFlags []string + ) + if *macOSSDK != "" { + flag := "-isysroot " + runOut("xcrun", "--sdk", *macOSSDK, "--show-sdk-path") + cgoCFlags = append(cgoCFlags, flag) + cgoLdFlags = append(cgoLdFlags, flag) + } + if *macOSArch != "" { + flag := "-arch " + *macOSArch + cgoCFlags = append(cgoCFlags, flag) + cgoLdFlags = append(cgoLdFlags, flag) + } + if *extraCgoCFlags != "" { + cgoCFlags = append(cgoCFlags, *extraCgoCFlags) + } + if *extraCgoLdFlags != "" { + cgoLdFlags = append(cgoLdFlags, *extraCgoLdFlags) + } + if len(cgoCFlags) > 0 { + env = append(env, "CGO_CFLAGS="+strings.Join(cgoCFlags, " ")) + } + if len(cgoLdFlags) > 0 { + env = append(env, "CGO_LDFLAGS="+strings.Join(cgoLdFlags, " ")) + } + if !*cgo { + env = append(env, "CGO_ENABLED=0") + } else { + env = append(env, "CGO_ENABLED=1") + } + if flags, ok := archFlags[goarch]; ok { + env = append(env, flags...) + } + err = runEnv(args, env) + if err != nil { + log.Printf("Error compiling %s/%s: %v", goos, goarch, err) + return false + } + if !*compileOnly { + if goos != "js" { + artifacts := []string{buildZip(dir)} + // build a .deb and .rpm if appropriate + if goos == "linux" { + artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...) + } + if *copyAs != "" { + for _, artifact := range artifacts { + run("ln", artifact, strings.Replace(artifact, "-"+version, "-"+*copyAs, 1)) + } + } + } + // tidy up + run("rm", "-rf", dir) + } + log.Printf("Done compiling %s/%s", goos, goarch) + return true +} + +func compile(version string) { + start := time.Now() + wg := new(sync.WaitGroup) + run := make(chan func(), *parallel) + for i := 0; i < *parallel; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for f := range run { + f() + } + }() + } + includeRe, err := regexp.Compile(*include) + if err != nil { + log.Fatalf("Bad -include regexp: %v", err) + } + excludeRe, err := regexp.Compile(*exclude) + if err != nil { + log.Fatalf("Bad -exclude regexp: %v", err) + } + compiled := 0 + var failuresMu sync.Mutex + var failures []string + for _, osarch := range osarches { + if excludeRe.MatchString(osarch) || !includeRe.MatchString(osarch) { + continue + } + parts := strings.Split(osarch, "/") + if len(parts) != 2 { + log.Fatalf("Bad osarch %q", osarch) + } + goos, goarch := parts[0], parts[1] + userGoos := goos + if goos == "darwin" { + userGoos = "osx" + } + dir := filepath.Join("rclone-permissions-mapper-" + version + "-" + userGoos + "-" + goarch) + run <- func() { + if !compileArch(version, goos, goarch, dir) { + failuresMu.Lock() + failures = append(failures, goos+"/"+goarch) + failuresMu.Unlock() + } + } + compiled++ + } + close(run) + wg.Wait() + log.Printf("Compiled %d arches in %v", compiled, time.Since(start)) + if len(failures) > 0 { + sort.Strings(failures) + log.Printf("%d compile failures:\n %s\n", len(failures), strings.Join(failures, "\n ")) + os.Exit(1) + } +} + +func main() { + flag.Parse() + args := flag.Args() + if len(args) != 1 { + log.Fatalf("Syntax: %s ", os.Args[0]) + } + version := args[0] + if !*noClean { + run("rm", "-rf", "build") + run("mkdir", "build") + } + chdir("build") + err := os.WriteFile("version.txt", []byte(fmt.Sprintf("rclone-permissions-mapper %s\n", version)), 0666) + if err != nil { + log.Fatalf("Couldn't write version.txt: %v", err) + } + compile(version) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e9ab381 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/nielash/rclone-permissions-mapper + +go 1.21.5 diff --git a/license.txt b/license.txt new file mode 100644 index 0000000..076b365 --- /dev/null +++ b/license.txt @@ -0,0 +1,23 @@ +Copyright (C) 2023 by nielash https://github.com/nielash + +Includes code from rclone (https://github.com/rclone/rclone) +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/main.go b/main.go new file mode 100644 index 0000000..6e0c2e7 --- /dev/null +++ b/main.go @@ -0,0 +1,120 @@ +// convert uid and gid between mac and linux during rclone sync, using --metadata-mapper +// +// usage: rclone sync source:path dest:path --metadata-mapper /path/to/rclone-permissions-mapper +// or, to see input and output: +// rclone sync source:path dest:path --metadata-mapper /path/to/rclone-permissions-mapper -v --dump mapper +// https://rclone.org/docs/#metadata-mapper +// https://github.com/nielash/rclone-permissions-mapper + +package main + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "strconv" +) + +func main() { + // Read the input + var in map[string]any + err := json.NewDecoder(os.Stdin).Decode(&in) + if err != nil { + log.Fatal(err) + } + + // Check the input + metadata, ok := in["Metadata"] + if !ok { + fmt.Fprintf(os.Stderr, "Metadata key not found\n") + os.Exit(1) + } + + // Map the metadata + metadataOut := map[string]string{} + var out = map[string]any{ + "Metadata": metadataOut, + } + + // debug settings + debugging := false // set true to debug + dir := "/" + debugfile := "" + var df *os.File + if debugging { + dir, err = os.UserHomeDir() + if err == nil { + debugfile = filepath.Join(dir, "rclone-permissions-mapper-debug.txt") + } + df, err = os.Create(debugfile) + defer df.Close() + } + debug := func(format string, a ...any) { + if !debugging { + return + } + fmt.Fprintf(df, format, a...) + } + + // loop through the metadata keys + for k, v := range metadata.(map[string]any) { + switch k { + case "error": + fmt.Fprintf(os.Stderr, "Error: %s\n", v) + os.Exit(1) + case "uid": + debug("uid detected! key: %s, val: %v\n", k, v) + uidstr, ok := v.(string) + osuid := os.Getuid() + debug("osuid: %v\n", osuid) + if ok { + uid, err := strconv.Atoi(uidstr) + if err != nil { + fmt.Fprintf(os.Stderr, "Error converting string to int: %v\n", uidstr) + } + // mac is 501 - 999, linux is 1000+ + if (uid >= 1000 && osuid >= 501 && osuid < 1000) || (osuid >= 1000 && uid >= 501 && uid < 1000) { + // unset it + debug("unsetting. key: %s, val: %v\n", k, v) + continue + } + } + case "gid": + debug("gid detected! key: %s, val: %v\n", k, v) + gidstr, ok := v.(string) + if ok { + gid, err := strconv.Atoi(gidstr) + if err != nil { + fmt.Fprintf(os.Stderr, "Error converting string to int: %v\n", gidstr) + } + // mac is 20, linux is 1000 + debug("osgid: %v\n", os.Getgid()) + if gid != os.Getgid() { + // unset it + debug("unsetting. key: %s, val: %v\n", k, v) + continue + } + } + default: + debug("skipping -- key: %s, val: %v\n", k, v) + } + metadataOut[k] = v.(string) + } + + // Write the output + json.NewEncoder(os.Stdout).Encode(&out) + if err != nil { + log.Fatal(err) + } + + if debugging { + debug("final: \n") + bytes, err := json.MarshalIndent(&out, "", "\t") + debug("%v", string(bytes)) + if err != nil { + debug("json err: %v", err.Error()) + } + } +}