diff --git a/modules/command/shepherd_win.go b/modules/command/shepherd_win.go index 517c16d..0c7440b 100644 --- a/modules/command/shepherd_win.go +++ b/modules/command/shepherd_win.go @@ -8,7 +8,7 @@ func setSysProcAttribute(c *exec.Cmd, detached bool) { // placeholders } -func cleanExit(c *exec.Cmd, detached bool) { +func cleanExit(c *exec.Cmd, _ bool) { if c != nil && c.Process != nil { _ = c.Process.Kill() } diff --git a/modules/env/env_unix.go b/modules/env/env_unix.go index 94ee80a..8852ec4 100644 --- a/modules/env/env_unix.go +++ b/modules/env/env_unix.go @@ -2,6 +2,29 @@ package env +import ( + "os" + "path/filepath" + "strings" +) + func InitializeEnv() error { + pathEnv := os.Getenv("PATH") + pathList := strings.Split(pathEnv, string(os.PathListSeparator)) + pathNewList := make([]string, 0, len(pathList)) + seen := make(map[string]bool) + for _, p := range pathList { + cleanedPath := filepath.Clean(p) + if cleanedPath == "." { + continue + } + u := strings.ToLower(cleanedPath) + if seen[u] { + continue + } + seen[u] = true + pathNewList = append(pathNewList, cleanedPath) + } + os.Setenv("PATH", strings.Join(pathNewList, string(os.PathListSeparator))) return nil } diff --git a/modules/env/env_windows.go b/modules/env/env_windows.go index 89ef1f1..cc8bcc0 100644 --- a/modules/env/env_windows.go +++ b/modules/env/env_windows.go @@ -11,45 +11,47 @@ import ( "golang.org/x/sys/windows/registry" ) -// initializeGW todo +// initializeGW: detect git for windows installation func initializeGW() (string, error) { k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\GitForWindows`, registry.QUERY_VALUE) if err != nil { - if k, err = registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\WOW6432Node\GitForWindows`, registry.QUERY_VALUE); err != nil { - return "", err - } + return "", nil } defer k.Close() installPath, _, err := k.GetStringValue("InstallPath") if err != nil { return "", err } - installPath = filepath.Clean(installPath) - git := filepath.Join(installPath, "cmd\\git.exe") - if _, err := os.Stat(git); err != nil { + gitForWindowsBinDir := filepath.Clean(filepath.Join(installPath, "cmd")) + if _, err := os.Stat(filepath.Join(gitForWindowsBinDir, "git.exe")); err != nil { return "", err } - return filepath.Join(installPath, "cmd"), nil + return gitForWindowsBinDir, nil } -// InitializeEnv todo +// InitializeEnv: initialize path env func InitializeEnv() error { - if _, err := exec.LookPath("git"); err == nil { - return nil - } - gitBinDir, err := initializeGW() - if err != nil { - return err - } pathEnv := os.Getenv("PATH") pathList := strings.Split(pathEnv, string(os.PathListSeparator)) pathNewList := make([]string, 0, len(pathList)+2) - pathNewList = append(pathNewList, filepath.Clean(gitBinDir)) - for _, s := range pathList { - cleanedPath := filepath.Clean(s) + if _, err := exec.LookPath("git"); err != nil { + gitForWindowsBinDir, err := initializeGW() + if err != nil { + return err + } + pathNewList = append(pathNewList, filepath.Clean(gitForWindowsBinDir)) + } + seen := make(map[string]bool) + for _, p := range pathList { + cleanedPath := filepath.Clean(p) if cleanedPath == "." { continue } + u := strings.ToLower(cleanedPath) + if seen[u] { + continue + } + seen[u] = true pathNewList = append(pathNewList, cleanedPath) } os.Setenv("PATH", strings.Join(pathNewList, string(os.PathListSeparator))) diff --git a/modules/env/env_windows_test.go b/modules/env/env_windows_test.go new file mode 100644 index 0000000..80e3320 --- /dev/null +++ b/modules/env/env_windows_test.go @@ -0,0 +1,16 @@ +//go:build windows + +package env + +import ( + "fmt" + "os" + "testing" +) + +func TestInitializeEnv(t *testing.T) { + os.Setenv("PATH", os.Getenv("PATH")+";C:\\Windows") + if err := InitializeEnv(); err != nil { + fmt.Fprintf(os.Stderr, "initialize env error: %v\n", err) + } +} diff --git a/modules/strengthen/fs_unix.go b/modules/strengthen/fs_unix.go index 8ae3eda..0f3db45 100644 --- a/modules/strengthen/fs_unix.go +++ b/modules/strengthen/fs_unix.go @@ -2,12 +2,14 @@ package strengthen -import "os" - -func Rename(oldpath, newpath string) error { +import ( + "os" +) + +func FinalizeObject(oldpath string, newpath string) (err error) { + if err = os.Link(oldpath, newpath); err == nil { + _ = os.Remove(oldpath) + return + } return os.Rename(oldpath, newpath) } - -func Remove(name string) error { - return os.Remove(name) -} diff --git a/modules/strengthen/fs_windows.go b/modules/strengthen/fs_windows.go index 58762ac..7f49dab 100644 --- a/modules/strengthen/fs_windows.go +++ b/modules/strengthen/fs_windows.go @@ -3,7 +3,11 @@ package strengthen import ( + "errors" + "os" + "runtime" "syscall" + "time" "unsafe" "golang.org/x/sys/windows" @@ -43,8 +47,15 @@ type FILE_RENAME_INFO struct { FileName [1]uint16 } -// Rename: posix rename semantics -func Rename(oldpath, newpath string) error { +var ( + errUnsupported = map[error]bool{ + windows.ERROR_INVALID_PARAMETER: true, + windows.ERROR_INVALID_FUNCTION: true, + windows.ERROR_NOT_SUPPORTED: true, + } +) + +func posixSemanticsRename(oldpath, newpath string) error { oldPathUTF16, err := windows.UTF16PtrFromString(oldpath) if err != nil { return err @@ -74,28 +85,37 @@ func Rename(oldpath, newpath string) error { return windows.SetFileInformationByHandle(fd, windows.FileRenameInfoEx, &buffer[0], uint32(bufferSize)) } -func removeHiddenAttr(fd windows.Handle) error { +// rename: posix rename semantics +func rename(oldpath, newpath string) error { + err := posixSemanticsRename(oldpath, newpath) + if errUnsupported[err] { + return os.Rename(oldpath, newpath) + } + return err +} + +func removeHideAttrbutes(fd windows.Handle) error { var du FILE_BASIC_INFO if err := windows.GetFileInformationByHandleEx(fd, windows.FileBasicInfo, (*byte)(unsafe.Pointer(&du)), uint32(unsafe.Sizeof(du))); err != nil { return err } du.FileAttributes &^= (windows.FILE_ATTRIBUTE_HIDDEN | windows.FILE_ATTRIBUTE_READONLY) - return windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&du)), uint32(unsafe.Sizeof(&du))) + return windows.SetFileInformationByHandle(fd, windows.FileBasicInfo, (*byte)(unsafe.Pointer(&du)), uint32(unsafe.Sizeof(du))) } func posixSemanticsRemove(fd windows.Handle) error { infoEx := FILE_DISPOSITION_INFO_EX{ - Flags: windows.FILE_DISPOSITION_POSIX_SEMANTICS, + Flags: windows.FILE_DISPOSITION_DELETE | windows.FILE_DISPOSITION_POSIX_SEMANTICS, } var err error - if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(&infoEx))); err == nil { + if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(infoEx))); err == nil { return nil } if err == windows.ERROR_ACCESS_DENIED { - if err := removeHiddenAttr(fd); err != nil { + if err := removeHideAttrbutes(fd); err != nil { return err } - if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(&infoEx))); err == nil { + if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(infoEx))); err == nil { return nil } } @@ -105,19 +125,19 @@ func posixSemanticsRemove(fd windows.Handle) error { info := FILE_DISPOSITION_INFO{ Flags: 0x13, // DELETE } - if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(&info))); err == nil { + if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err == nil { return nil } if err != windows.ERROR_ACCESS_DENIED { return err } - if err := removeHiddenAttr(fd); err != nil { + if err := removeHideAttrbutes(fd); err != nil { return err } - return windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(&info))) + return windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))) } -func Remove(name string) error { +func remove(name string) error { nameUTF16, err := windows.UTF16PtrFromString(name) if err != nil { return err @@ -135,3 +155,90 @@ func Remove(name string) error { defer windows.CloseHandle(fd) return posixSemanticsRemove(fd) } + +var ( + delay = []time.Duration{0, 1, 10, 20, 40} + isWindows = func() bool { + return runtime.GOOS == "windows" + }() +) + +const ( + ERROR_ACCESS_DENIED syscall.Errno = 5 + ERROR_SHARING_VIOLATION syscall.Errno = 32 + ERROR_LOCK_VIOLATION syscall.Errno = 33 +) + +func isRetryErr(err error) bool { + if !isWindows { + return false + } + if os.IsPermission(err) { + return true + } + var errno syscall.Errno + if errors.As(err, &errno) { + switch errno { + case ERROR_ACCESS_DENIED, + ERROR_SHARING_VIOLATION, + ERROR_LOCK_VIOLATION: + return true + } + } + return false +} + +func windowsLink(oldpath, newpath string) (err error) { + for i := 0; i < 2; i++ { + if err = os.Link(oldpath, newpath); err == nil { + _ = os.Remove(oldpath) + return nil + } + if !errors.Is(err, windows.ERROR_ALREADY_EXISTS) { + break + } + if removeErr := os.Remove(newpath); removeErr != nil { + break + } + } + return err +} + +func FinalizeObject(oldpath string, newpath string) (err error) { + if err = windowsLink(oldpath, newpath); err == nil { + return err + } + // no retry rename + if err = rename(oldpath, newpath); err == nil { + return + } + // on Windows and + if !isRetryErr(err) { + return + } + for tries := 0; tries < len(delay); tries++ { + /* + * We assume that some other process had the source or + * destination file open at the wrong moment and retry. + * In order to give the other process a higher chance to + * complete its operation, we give up our time slice now. + * If we have to retry again, we do sleep a bit. + */ + time.Sleep(delay[tries] * time.Millisecond) + _ = os.Chmod(newpath, 0644) // & ~FILE_ATTRIBUTE_READONLY + // retry run + if err = rename(oldpath, newpath); err == nil { + return + } + // Only windows retry + if !isRetryErr(err) { + return + } + } + // FIXME: Windows platform security software can cause some bizarre phenomena, such as star points. + if os.IsPermission(err) { + _, err = os.Stat(newpath) + return + } + return +} diff --git a/modules/strengthen/statfs_test.go b/modules/strengthen/statfs_test.go index 1a860d2..d48f4d3 100644 --- a/modules/strengthen/statfs_test.go +++ b/modules/strengthen/statfs_test.go @@ -8,7 +8,22 @@ import ( func TestGetDiskFreeSpaceEx(t *testing.T) { gb := float64(1024 * 1024 * 1024) - ds, err := GetDiskFreeSpaceEx("/") + cwd, err := os.Getwd() + if err != nil { + return + } + ds, err := GetDiskFreeSpaceEx(cwd) + if err != nil { + fmt.Fprintf(os.Stderr, "usage: %v\n", err) + return + } + fmt.Fprintf(os.Stderr, "disk space total: %0.2f GB. used: %0.2f GB. available: %0.2f GB FS: %s\n", + float64(ds.Total)/gb, float64(ds.Used)/gb, float64(ds.Avail)/gb, ds.FS) +} + +func TestGetDiskFreeSpaceExTemp(t *testing.T) { + gb := float64(1024 * 1024 * 1024) + ds, err := GetDiskFreeSpaceEx(os.TempDir()) if err != nil { fmt.Fprintf(os.Stderr, "usage: %v\n", err) return diff --git a/modules/zeta/backend/file_storer.go b/modules/zeta/backend/file_storer.go index 9833222..e9e025c 100644 --- a/modules/zeta/backend/file_storer.go +++ b/modules/zeta/backend/file_storer.go @@ -7,20 +7,17 @@ import ( "bytes" "context" "encoding/binary" - "errors" "fmt" "io" "io/fs" "os" "path/filepath" - "runtime" "strings" - "syscall" - "time" "github.com/antgroup/hugescm/modules/mime" "github.com/antgroup/hugescm/modules/plumbing" "github.com/antgroup/hugescm/modules/streamio" + "github.com/antgroup/hugescm/modules/strengthen" "github.com/antgroup/hugescm/modules/zeta/backend/storage" "github.com/antgroup/hugescm/modules/zeta/object" ) @@ -228,80 +225,8 @@ func mkdir(paths ...string) error { return nil } -var ( - delay = []time.Duration{0, 1, 10, 20, 40} - isWindows = func() bool { - return runtime.GOOS == "windows" - }() -) - -const ( - ERROR_ACCESS_DENIED syscall.Errno = 5 - ERROR_SHARING_VIOLATION syscall.Errno = 32 - ERROR_LOCK_VIOLATION syscall.Errno = 33 -) - -func isRetryErr(err error) bool { - if !isWindows { - return false - } - if os.IsPermission(err) { - return true - } - var errno syscall.Errno - if errors.As(err, &errno) { - switch errno { - case ERROR_ACCESS_DENIED, - ERROR_SHARING_VIOLATION, - ERROR_LOCK_VIOLATION: - return true - } - } - return false -} - -func finalizeObject0(oldpath string, newpath string) (err error) { - if err = os.Link(oldpath, newpath); err == nil { - _ = os.Remove(oldpath) - return - } - // no retry rename - if err = os.Rename(oldpath, newpath); err == nil { - return - } - // on Windows and - if !isRetryErr(err) { - return - } - for tries := 0; tries < len(delay); tries++ { - /* - * We assume that some other process had the source or - * destination file open at the wrong moment and retry. - * In order to give the other process a higher chance to - * complete its operation, we give up our time slice now. - * If we have to retry again, we do sleep a bit. - */ - time.Sleep(delay[tries] * time.Millisecond) - _ = os.Chmod(newpath, 0644) // & ~FILE_ATTRIBUTE_READONLY - // retry run - if err = os.Rename(oldpath, newpath); err == nil { - return - } - // Only windows retry - if !isRetryErr(err) { - return - } - } - // FIXME: Windows platform security software can cause some bizarre phenomena, such as star points. - if os.IsPermission(err) && isWindows { - _, err = os.Stat(newpath) - return - } - return -} - func finalizeObject(oldpath string, newpath string) (err error) { - if err = finalizeObject0(oldpath, newpath); err == nil { + if err = strengthen.FinalizeObject(oldpath, newpath); err == nil { _ = os.Chmod(newpath, 0444) } return diff --git a/utils/rename/rename_test.go b/utils/rename/rename_test.go new file mode 100644 index 0000000..56a4034 --- /dev/null +++ b/utils/rename/rename_test.go @@ -0,0 +1,100 @@ +//go:build windows + +package rename + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/windows" +) + +func TestPosixRename(t *testing.T) { + a := filepath.Join(os.TempDir(), "ed34a14b-0b09-4078-ac36-71745e4c4084.tmp") + b := filepath.Join(os.TempDir(), "b.txt") + fmt.Fprintf(os.Stderr, "rename %s to %s\n", a, b) + if err := PosixRename(a, b); err != nil { + fmt.Fprintf(os.Stderr, "rename %s to %s error: %v\n", a, b, err) + return + } + fmt.Fprintf(os.Stderr, "rename success\n") +} + +func TestRename(t *testing.T) { + a := filepath.Join(os.TempDir(), "a.txt") + b := filepath.Join(os.TempDir(), "b.txt") + fmt.Fprintf(os.Stderr, "rename %s to %s\n", a, b) + if err := os.Rename(a, b); err != nil { + fmt.Fprintf(os.Stderr, "rename %s to %s error: %v\n", a, b, err) + return + } + fmt.Fprintf(os.Stderr, "rename success\n") +} + +func TestPosixRemove(t *testing.T) { + a := filepath.Join(os.TempDir(), "a.txt") + fmt.Fprintf(os.Stderr, "remove %s\n", a) + if err := Remove(a); err != nil { + fmt.Fprintf(os.Stderr, "remove %s error: %v\n", a, err) + return + } + fmt.Fprintf(os.Stderr, "remove success\n") +} + +func TestRemove(t *testing.T) { + a := filepath.Join(os.TempDir(), "a.txt") + fmt.Fprintf(os.Stderr, "remove %s\n", a) + if err := os.Remove(a); err != nil { + fmt.Fprintf(os.Stderr, "remove %s error: %v\n", a, err) + return + } + fmt.Fprintf(os.Stderr, "remove success\n") +} + +func TestLink(t *testing.T) { + a := filepath.Join(os.TempDir(), "a.txt") + os.Link(a, filepath.Join(os.TempDir(), "cc/b.txt")) + fmt.Fprintf(os.Stderr, "remove %s\n", a) + if err := os.Remove(a); err != nil { + fmt.Fprintf(os.Stderr, "remove %s error: %v\n", a, err) + return + } + fmt.Fprintf(os.Stderr, "remove success\n") +} + +func windowsLink(oldpath, newpath string) (err error) { + for i := 0; i < 2; i++ { + if err = os.Link(oldpath, newpath); err == nil { + _ = os.Remove(oldpath) + return nil + } + if !errors.Is(err, windows.ERROR_ALREADY_EXISTS) { + break + } + if err = os.Remove(newpath); err != nil { + break + } + } + return err +} + +func TestReFsLink(t *testing.T) { + cwd, err := os.Getwd() + if err != nil { + return + } + a := filepath.Join(cwd, "a.txt") + _ = os.WriteFile(a, []byte("hello world\n"), 0644) + if err := windowsLink(a, filepath.Join(cwd, "b.txt")); err != nil { + fmt.Fprintf(os.Stderr, "Link %s error: %v\n", a, err) + } + // fmt.Fprintf(os.Stderr, "remove %s\n", a) + // if err := os.Remove(a); err != nil { + // fmt.Fprintf(os.Stderr, "remove %s error: %v\n", a, err) + // return + // } + // fmt.Fprintf(os.Stderr, "remove success\n") +} diff --git a/utils/rename/rename_windows.go b/utils/rename/rename_windows.go index f44bcc0..0dcf1da 100644 --- a/utils/rename/rename_windows.go +++ b/utils/rename/rename_windows.go @@ -85,32 +85,36 @@ func removeHideAttrbutes(fd windows.Handle) error { return err } du.FileAttributes &^= (windows.FILE_ATTRIBUTE_HIDDEN | windows.FILE_ATTRIBUTE_READONLY) - return windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&du)), uint32(unsafe.Sizeof(&du))) + return windows.SetFileInformationByHandle(fd, windows.FileBasicInfo, (*byte)(unsafe.Pointer(&du)), uint32(unsafe.Sizeof(du))) } func removeInternal(fd windows.Handle) error { infoEx := FILE_DISPOSITION_INFO_EX{ - Flags: windows.FILE_DISPOSITION_POSIX_SEMANTICS, + Flags: windows.FILE_DISPOSITION_DELETE | windows.FILE_DISPOSITION_POSIX_SEMANTICS, } var err error - if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(&infoEx))); err == nil { + if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(infoEx))); err == nil { return nil } if err == windows.ERROR_ACCESS_DENIED { if err := removeHideAttrbutes(fd); err != nil { return err } - if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(&infoEx))); err == nil { + if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfoEx, (*byte)(unsafe.Pointer(&infoEx)), uint32(unsafe.Sizeof(infoEx))); err == nil { return nil } } - if err != windows.ERROR_INVALID_PARAMETER && err != windows.ERROR_INVALID_FUNCTION && err != windows.ERROR_NOT_SUBSTED { + switch { + case err == windows.ERROR_INVALID_PARAMETER: + case err == windows.ERROR_INVALID_FUNCTION: + case err == windows.ERROR_NOT_SUPPORTED: + default: return err } info := FILE_DISPOSITION_INFO{ Flags: 0x13, // DELETE } - if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(&info))); err == nil { + if err = windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))); err == nil { return nil } if err != windows.ERROR_ACCESS_DENIED { @@ -119,7 +123,7 @@ func removeInternal(fd windows.Handle) error { if err := removeHideAttrbutes(fd); err != nil { return err } - return windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(&info))) + return windows.SetFileInformationByHandle(fd, windows.FileDispositionInfo, (*byte)(unsafe.Pointer(&info)), uint32(unsafe.Sizeof(info))) } func Remove(name string) error {