Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

testonly: faster rmr unlink implement #5621

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
+82 −59
Draft

testonly: faster rmr unlink implement #5621

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a85b867
faster rmr unlink implement
anysql Feb 6, 2025
bffb17f
pass parent attr from emptyDir
anysql Feb 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions pkg/meta/base.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ type engine interface {
doLookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr) syscall.Errno
doMknod(ctx Context, parent Ino, name string, _type uint8, mode, cumask uint16, path string, inode *Ino, attr *Attr) syscall.Errno
doLink(ctx Context, inode, parent Ino, name string, attr *Attr) syscall.Errno
doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno
doUnlink(ctx Context, parent Ino, name string, attr *Attr, entry *Entry, pattr *Attr, skipCheckTrash ...bool) syscall.Errno
doRmdir(ctx Context, parent Ino, name string, inode *Ino, skipCheckTrash ...bool) syscall.Errno
doReadlink(ctx Context, inode Ino, noatime bool) (int64, []byte, error)
doReaddir(ctx Context, inode Ino, plus uint8, entries *[]*Entry, limit int) syscall.Errno
Expand Down Expand Up @@ -1344,7 +1344,7 @@ func (m *baseMeta) ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno
return 0
}

func (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno {
func (m *baseMeta) Unlink2(ctx Context, parent Ino, name string, entry *Entry, pattr *Attr, skipCheckTrash ...bool) syscall.Errno {
if parent == RootInode && name == TrashName || isTrash(parent) && ctx.Uid() != 0 {
return syscall.EPERM
}
Expand All @@ -1355,8 +1355,8 @@ func (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCheckTrash .
defer m.timeit("Unlink", time.Now())
parent = m.checkRoot(parent)
var attr Attr
err := m.en.doUnlink(ctx, parent, name, &attr, skipCheckTrash...)
if err == 0 {
err := m.en.doUnlink(ctx, parent, name, &attr, entry, pattr, skipCheckTrash...)
if entry == nil && err == 0 {
var diffLength uint64
if attr.Typ == TypeFile {
diffLength = attr.Length
Expand All @@ -1367,6 +1367,10 @@ func (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCheckTrash .
return err
}

func (m *baseMeta) Unlink(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno {
return m.Unlink2(ctx, parent, name, nil, nil, skipCheckTrash...)
}

func (m *baseMeta) Rmdir(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno {
if name == "." {
return syscall.EINVAL
Expand Down
22 changes: 12 additions & 10 deletions pkg/meta/redis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1427,7 +1427,7 @@ func (m *redisMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, m
}, m.inodeKey(parent), m.entryKey(parent)))
}

func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno {
func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, entry *Entry, pattr *Attr, skipCheckTrash ...bool) syscall.Errno {
var trash, inode Ino
if !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {
if st := m.checkTrash(parent, &trash); st != 0 {
Expand Down Expand Up @@ -1469,20 +1469,22 @@ func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, s
if rs[0] == nil {
return redis.Nil
}
var pattr Attr
m.parseAttr([]byte(rs[0].(string)), &pattr)
if pattr.Typ != TypeDirectory {
return syscall.ENOTDIR
}
if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {
return st
if pattr == nil {
pattr = &Attr{}
m.parseAttr([]byte(rs[0].(string)), pattr)
if pattr.Typ != TypeDirectory {
return syscall.ENOTDIR
}
if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, pattr); st != 0 {
return st
}
}
if (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {
return syscall.EPERM
}
var updateParent bool
now := time.Now()
if !isTrash(parent) && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime {
if entry == nil && !isTrash(parent) && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime {
pattr.Mtime = now.Unix()
pattr.Mtimensec = uint32(now.Nanosecond())
pattr.Ctime = now.Unix()
Expand Down Expand Up @@ -1519,7 +1521,7 @@ func (m *redisMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, s
_, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
pipe.HDel(ctx, m.entryKey(parent), name)
if updateParent {
pipe.Set(ctx, m.inodeKey(parent), m.marshal(&pattr), 0)
pipe.Set(ctx, m.inodeKey(parent), m.marshal(pattr), 0)
}
if attr.Nlink > 0 {
pipe.Set(ctx, m.inodeKey(inode), m.marshal(attr), 0)
Expand Down
79 changes: 45 additions & 34 deletions pkg/meta/sql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1608,7 +1608,7 @@ func (m *dbMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode
}))
}

func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno {
func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, entry *Entry, pattr *Attr, skipCheckTrash ...bool) syscall.Errno {
var trash Ino
if !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {
if st := m.checkTrash(parent, &trash); st != 0 {
Expand All @@ -1622,53 +1622,62 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip
opened = false
newSpace, newInode = 0, 0
var pn = node{Inode: parent}
ok, err := s.Get(&pn)
if err != nil {
return err
}
if !ok {
return syscall.ENOENT
}
if pn.Type != TypeDirectory {
return syscall.ENOTDIR
}
var pattr Attr
m.parseAttr(&pn, &pattr)
if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {
return st
if pattr == nil {
ok, err := s.Get(&pn)
if err != nil {
return err
}
if !ok {
return syscall.ENOENT
}
pattr = &Attr{}
m.parseAttr(&pn, pattr)
if pattr.Typ != TypeDirectory {
return syscall.ENOTDIR
}
if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, pattr); st != 0 {
return st
}
}
if (pn.Flags&FlagAppend) != 0 || (pn.Flags&FlagImmutable) != 0 {
if (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {
return syscall.EPERM
}
var e = edge{Parent: parent, Name: []byte(name)}
ok, err = s.Get(&e)
if err != nil {
return err
}
if !ok && m.conf.CaseInsensi {
if ee := m.resolveCase(ctx, parent, name); ee != nil {
ok = true
e.Name = ee.Name
e.Inode = ee.Inode
e.Type = ee.Attr.Typ
if entry == nil {
ok, err := s.Get(&e)
if err != nil {
return err
}
}
if !ok {
return syscall.ENOENT
if !ok && m.conf.CaseInsensi {
if ee := m.resolveCase(ctx, parent, name); ee != nil {
ok = true
e.Name = ee.Name
e.Inode = ee.Inode
e.Type = ee.Attr.Typ
}
}
if !ok {
return syscall.ENOENT
}
} else {
e.Inode = entry.Inode
e.Type = entry.Attr.Typ
}
if e.Type == TypeDirectory {
return syscall.EPERM
}

n = node{Inode: e.Inode}
ok, err = s.ForUpdate().Get(&n)
ok, err := s.ForUpdate().Get(&n)
if err != nil {
return err
}
now := time.Now().UnixNano()
if ok {
if ctx.Uid() != 0 && pn.Mode&01000 != 0 && ctx.Uid() != pn.Uid && ctx.Uid() != n.Uid {
return syscall.EACCES
if entry == nil {
if ctx.Uid() != 0 && pattr.Mode&01000 != 0 && ctx.Uid() != pattr.Uid && ctx.Uid() != n.Uid {
return syscall.EACCES
}
}
if (n.Flags&FlagAppend) != 0 || (n.Flags&FlagImmutable) != 0 {
return syscall.EPERM
Expand All @@ -1694,7 +1703,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip
defer func() { m.of.InvalidateChunk(e.Inode, invalidateAttrOnly) }()

var updateParent bool
if !isTrash(parent) && time.Duration(now-pn.getMtime()) >= m.conf.SkipDirMtime {
if entry == nil && !isTrash(parent) && time.Duration(now-pn.getMtime()) >= m.conf.SkipDirMtime {
pn.setMtime(now)
pn.setCtime(now)
updateParent = true
Expand Down Expand Up @@ -1743,9 +1752,11 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip
}
newSpace, newInode = -align4K(0), -1
}
/*
if _, err := s.Delete(&xattr{Inode: e.Inode}); err != nil {
return err
}
*/
}
if updateParent {
var _n int64
Expand All @@ -1771,7 +1782,7 @@ func (m *dbMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip
}
m.updateStats(newSpace, newInode)
}
if err == nil && attr != nil {
if entry == nil && err == nil && attr != nil {
m.parseAttr(&n, attr)
}
return errno(err)
Expand Down
22 changes: 12 additions & 10 deletions pkg/meta/tkv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,7 +1246,7 @@ func (m *kvMeta) doMknod(ctx Context, parent Ino, name string, _type uint8, mode
}, parent))
}

func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skipCheckTrash ...bool) syscall.Errno {
func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, entry *Entry, pattr *Attr, skipCheckTrash ...bool) syscall.Errno {
var trash Ino
if !(len(skipCheckTrash) == 1 && skipCheckTrash[0]) {
if st := m.checkTrash(parent, &trash); st != 0 {
Expand Down Expand Up @@ -1287,13 +1287,15 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip
if rs[0] == nil {
return syscall.ENOENT
}
var pattr Attr
m.parseAttr(rs[0], &pattr)
if pattr.Typ != TypeDirectory {
return syscall.ENOTDIR
}
if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, &pattr); st != 0 {
return st
if pattr == nil {
pattr = &Attr{}
m.parseAttr(rs[0], pattr)
if pattr.Typ != TypeDirectory {
return syscall.ENOTDIR
}
if st := m.Access(ctx, parent, MODE_MASK_W|MODE_MASK_X, pattr); st != 0 {
return st
}
}
if (pattr.Flags&FlagAppend) != 0 || (pattr.Flags&FlagImmutable) != 0 {
return syscall.EPERM
Expand Down Expand Up @@ -1328,7 +1330,7 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip

defer func() { m.of.InvalidateChunk(inode, invalidateAttrOnly) }()
var updateParent bool
if !isTrash(parent) && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {
if entry == nil && !isTrash(parent) && now.Sub(time.Unix(pattr.Mtime, int64(pattr.Mtimensec))) >= m.conf.SkipDirMtime*time.Duration(tx.retry+1) {
pattr.Mtime = now.Unix()
pattr.Mtimensec = uint32(now.Nanosecond())
pattr.Ctime = now.Unix()
Expand All @@ -1338,7 +1340,7 @@ func (m *kvMeta) doUnlink(ctx Context, parent Ino, name string, attr *Attr, skip

tx.delete(m.entryKey(parent, name))
if updateParent {
tx.set(m.inodeKey(parent), m.marshal(&pattr))
tx.set(m.inodeKey(parent), m.marshal(pattr))
}
if attr.Nlink > 0 {
tx.set(m.inodeKey(inode), m.marshal(attr))
Expand Down
6 changes: 5 additions & 1 deletion pkg/meta/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,10 @@ func updateLocks(ls []plockRecord, nl plockRecord) []plockRecord {
}

func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count *uint64, concurrent chan int) syscall.Errno {
var pattr Attr
if st := m.en.doGetAttr(ctx, inode, &pattr); st != 0 {
return st
}
for {
var entries []*Entry
if st := m.en.doReaddir(ctx, inode, 0, &entries, 10000); st != 0 && st != syscall.ENOENT {
Expand Down Expand Up @@ -303,7 +307,7 @@ func (m *baseMeta) emptyDir(ctx Context, inode Ino, skipCheckTrash bool, count *
if count != nil {
atomic.AddUint64(count, 1)
}
if st := m.Unlink(ctx, inode, string(e.Name), skipCheckTrash); st != 0 && st != syscall.ENOENT {
if st := m.Unlink2(ctx, inode, string(e.Name), e, &pattr, skipCheckTrash); st != 0 && st != syscall.ENOENT {
ctx.Cancel()
return st
}
Expand Down
Loading