Skip to content

Commit fe4a16c

Browse files
Wine93Wine93
committed
curvefs/client: fix the delayed inode not being retrieved in nocto scenario.
Signed-off-by: Wine93 <wine93.info@gmail.com>
1 parent debc07d commit fe4a16c

File tree

11 files changed

+519
-40
lines changed

11 files changed

+519
-40
lines changed

‎curvefs/src/client/filesystem/defer_sync.cpp

Lines changed: 105 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,94 @@ namespace curvefs {
3030
namespace client {
3131
namespace filesystem {
3232

33-
DeferSync::DeferSync(DeferSyncOption option)
34-
: option_(option),
33+
using ::curve::common::LockGuard;
34+
using ::curve::common::ReadLockGuard;
35+
using ::curve::common::WriteLockGuard;
36+
using ::curvefs::client::filesystem::AttrCtime;
37+
38+
#define RETURN_FALSE_IF_CTO_ON() \
39+
do { \
40+
if (cto_) { \
41+
return false; \
42+
} \
43+
} while (0)
44+
45+
DeferInodes::DeferInodes(bool cto)
46+
: cto_(cto),
47+
rwlock_(),
48+
inodes_() {}
49+
50+
bool DeferInodes::Add(const std::shared_ptr<InodeWrapper>& inode) {
51+
RETURN_FALSE_IF_CTO_ON();
52+
WriteLockGuard lk(rwlock_);
53+
Ino ino = inode->GetInodeId();
54+
auto ret = inodes_.emplace(ino, inode);
55+
auto iter = ret.first;
56+
bool yes = ret.second;
57+
if (!yes) { // already exists
58+
iter->second = inode;
59+
}
60+
return true;
61+
}
62+
63+
bool DeferInodes::Get(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
64+
RETURN_FALSE_IF_CTO_ON();
65+
ReadLockGuard lk(rwlock_);
66+
auto iter = inodes_.find(ino);
67+
if (iter == inodes_.end()) {
68+
return false;
69+
}
70+
*inode = iter->second;
71+
return true;
72+
}
73+
74+
bool DeferInodes::Remove(const std::shared_ptr<InodeWrapper>& inode) {
75+
RETURN_FALSE_IF_CTO_ON();
76+
WriteLockGuard lk(rwlock_);
77+
InodeAttr attr;
78+
inode->GetInodeAttrLocked(&attr);
79+
auto iter = inodes_.find(attr.inodeid());
80+
if (iter == inodes_.end()) {
81+
return false;
82+
}
83+
84+
InodeAttr defered;
85+
iter->second->GetInodeAttrLocked(&defered);
86+
if (AttrCtime(attr) < AttrCtime(defered)) {
87+
// it means the old defered inode already replaced by the lastest one,
88+
// so we can't remove it before it synced yet.
89+
return false;
90+
}
91+
inodes_.erase(iter);
92+
return true;
93+
}
94+
95+
size_t DeferInodes::Size() {
96+
ReadLockGuard lk(rwlock_);
97+
return inodes_.size();
98+
}
99+
100+
SyncInodeClosure::SyncInodeClosure(const std::shared_ptr<DeferInodes>& inodes,
101+
const std::shared_ptr<InodeWrapper>& inode)
102+
: inodes_(inodes), inode_(inode) {}
103+
104+
void SyncInodeClosure::Run() {
105+
std::unique_ptr<SyncInodeClosure> self_guard(this);
106+
MetaStatusCode rc = GetStatusCode();
107+
if (rc == MetaStatusCode::OK || rc == MetaStatusCode::NOT_FOUND) {
108+
inodes_->Remove(inode_);
109+
}
110+
}
111+
112+
DeferSync::DeferSync(bool cto, DeferSyncOption option)
113+
: cto_(cto),
114+
option_(option),
35115
mutex_(),
36116
running_(false),
37117
thread_(),
38118
sleeper_(),
39-
inodes_() {
40-
}
119+
pending_(),
120+
inodes_(std::make_shared<DeferInodes>(cto)) {}
41121

42122
void DeferSync::Start() {
43123
if (!running_.exchange(true)) {
@@ -55,20 +135,32 @@ void DeferSync::Stop() {
55135
}
56136
}
57137

138+
SyncInodeClosure* DeferSync::NewSyncInodeClosure(
139+
const std::shared_ptr<InodeWrapper>& inode) {
140+
// NOTE: we only store the defer inodes in nocto scenario,
141+
// which means we don't need to remove the inode from defer inodes
142+
// even if the inode already synced done in cto scenario.
143+
if (cto_) {
144+
return nullptr;
145+
}
146+
return new SyncInodeClosure(inodes_, inode);
147+
}
148+
58149
void DeferSync::SyncTask() {
59-
std::vector<std::shared_ptr<InodeWrapper>> inodes;
150+
std::vector<std::shared_ptr<InodeWrapper>> syncing;
60151
for ( ;; ) {
61152
bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
62153

63154
{
64155
LockGuard lk(mutex_);
65-
inodes.swap(inodes_);
156+
syncing.swap(pending_);
66157
}
67-
for (const auto& inode : inodes) {
158+
for (const auto& inode : syncing) {
159+
auto closure = NewSyncInodeClosure(inode);
68160
UniqueLock lk(inode->GetUniqueLock());
69-
inode->Async(nullptr, true);
161+
inode->Async(closure, true);
70162
}
71-
inodes.clear();
163+
syncing.clear();
72164

73165
if (!running) {
74166
break;
@@ -78,18 +170,12 @@ void DeferSync::SyncTask() {
78170

79171
void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
80172
LockGuard lk(mutex_);
81-
inodes_.emplace_back(inode);
173+
pending_.emplace_back(inode);
174+
inodes_->Add(inode);
82175
}
83176

84-
bool DeferSync::IsDefered(Ino ino, InodeAttr* attr) {
85-
LockGuard lk(mutex_);
86-
for (const auto& inode : inodes_) {
87-
if (inode->GetInodeId() == ino) {
88-
inode->GetInodeAttr(attr);
89-
return true;
90-
}
91-
}
92-
return false;
177+
bool DeferSync::IsDefered(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
178+
return inodes_->Get(ino, inode);
93179
}
94180

95181
} // namespace filesystem

‎curvefs/src/client/filesystem/defer_sync.h

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,42 +27,83 @@
2727
#include <vector>
2828
#include <memory>
2929

30+
#include "absl/container/btree_map.h"
3031
#include "src/common/interruptible_sleeper.h"
3132
#include "curvefs/src/client/common/config.h"
33+
#include "curvefs/src/client/rpcclient/task_excutor.h"
3234
#include "curvefs/src/client/filesystem/meta.h"
3335

3436
namespace curvefs {
3537
namespace client {
3638
namespace filesystem {
3739

38-
using ::curvefs::client::common::DeferSyncOption;
39-
40+
using ::curve::common::RWLock;
4041
using ::curve::common::Mutex;
41-
using ::curve::common::LockGuard;
4242
using ::curve::common::InterruptibleSleeper;
43+
using ::curvefs::client::common::DeferSyncOption;
44+
using ::curvefs::client::rpcclient::MetaServerClientDone;
45+
46+
// NOTE: we only store the defer inodes in nocto scenario.
47+
class DeferInodes {
48+
public:
49+
explicit DeferInodes(bool cto);
50+
51+
bool Add(const std::shared_ptr<InodeWrapper>& inode);
52+
53+
bool Get(Ino ino, std::shared_ptr<InodeWrapper>* inode);
54+
55+
bool Remove(const std::shared_ptr<InodeWrapper>& inode);
56+
57+
size_t Size();
58+
59+
private:
60+
bool cto_;
61+
RWLock rwlock_;
62+
absl::btree_map<Ino, std::shared_ptr<InodeWrapper>> inodes_;
63+
};
64+
65+
class SyncInodeClosure : public MetaServerClientDone {
66+
public:
67+
explicit SyncInodeClosure(const std::shared_ptr<DeferInodes>& inodes,
68+
const std::shared_ptr<InodeWrapper>& inode);
69+
70+
void Run() override;
71+
72+
private:
73+
std::shared_ptr<DeferInodes> inodes_;
74+
std::shared_ptr<InodeWrapper> inode_;
75+
};
4376

4477
class DeferSync {
4578
public:
46-
explicit DeferSync(DeferSyncOption option);
79+
explicit DeferSync(bool cto, DeferSyncOption option);
4780

4881
void Start();
4982

5083
void Stop();
5184

5285
void Push(const std::shared_ptr<InodeWrapper>& inode);
5386

54-
bool IsDefered(Ino ino, InodeAttr* attr);
87+
bool IsDefered(Ino ino, std::shared_ptr<InodeWrapper>* inode);
5588

5689
private:
90+
SyncInodeClosure* NewSyncInodeClosure(
91+
const std::shared_ptr<InodeWrapper>& inode);
92+
5793
void SyncTask();
5894

5995
private:
96+
friend class SyncInodeClosure;
97+
98+
private:
99+
bool cto_;
60100
DeferSyncOption option_;
61101
Mutex mutex_;
62102
std::atomic<bool> running_;
63103
std::thread thread_;
64104
InterruptibleSleeper sleeper_;
65-
std::vector<std::shared_ptr<InodeWrapper>> inodes_;
105+
std::vector<std::shared_ptr<InodeWrapper>> pending_;
106+
std::shared_ptr<DeferInodes> inodes_;
66107
};
67108

68109
} // namespace filesystem

‎curvefs/src/client/filesystem/filesystem.cpp

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ namespace filesystem {
3131

3232
FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
3333
: option_(option), member(member) {
34-
deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
34+
deferSync_ = std::make_shared<DeferSync>(option.cto,
35+
option.deferSyncOption);
3536
negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
3637
dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
3738
openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
@@ -257,11 +258,6 @@ CURVEFS_ERROR FileSystem::Lookup(Ino parent,
257258

258259
CURVEFS_ERROR FileSystem::GetAttr(Ino ino, AttrOut* attrOut) {
259260
InodeAttr attr;
260-
if (!option_.cto && deferSync_->IsDefered(ino, &attr)) {
261-
*attrOut = AttrOut(attr);
262-
return CURVEFS_ERROR::OK;
263-
}
264-
265261
auto rc = rpc_->GetAttr(ino, &attr);
266262
if (rc == CURVEFS_ERROR::OK) {
267263
*attrOut = AttrOut(attr);
@@ -319,7 +315,7 @@ CURVEFS_ERROR FileSystem::Open(Ino ino, FileInfo* fi) {
319315
bool yes = openFiles_->IsOpened(ino, &inode);
320316
if (yes) {
321317
openFiles_->Open(ino, inode);
322-
// fi->keep_cache = 1;
318+
// fi->keep_cache = 1; // FIXME(Wine93): let it works.
323319
return CURVEFS_ERROR::OK;
324320
}
325321

0 commit comments

Comments
 (0)