-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathpmt.patch
180 lines (166 loc) · 5.62 KB
/
pmt.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h
index 64b6117..1bb40bf 100644
--- a/arch/x86/include/asm/spinlock.h
+++ b/arch/x86/include/asm/spinlock.h
@@ -62,7 +62,14 @@ static inline void __ticket_unlock_kick(arch_spinlock_t *lock,
#endif /* CONFIG_PARAVIRT_SPINLOCKS */
static inline int __tickets_equal(__ticket_t one, __ticket_t two)
{
- return !((one ^ two) & ~TICKET_SLOWPATH_FLAG);
+ return one == two;
+}
+
+static inline __ticket_t __tickets_diff(__ticket_t ticket, __ticket_t head)
+{
+ // the (__ticket_t) trick handles tail wraparound case
+ // e.g. (unsigned char)(1 - 255) = 2
+ return (__ticket_t)(ticket - head);
}
static inline void __ticket_check_and_clear_slowpath(arch_spinlock_t *lock,
@@ -99,27 +106,40 @@ static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
* in the high part, because a wide xadd increment of the low part would carry
* up and contaminate the high part.
*/
+extern u64 pmtlock_timeout_shift;
+
static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
{
register struct __raw_tickets inc = { .tail = TICKET_LOCK_INC };
+ u64 timeout = 0;
+ __ticket_t current_head;
inc = xadd(&lock->tickets, inc);
- if (likely(inc.head == inc.tail))
- goto out;
+ if (likely(__tickets_equal(inc.head, inc.tail)))
+ goto spin;
+
+ timeout = __tickets_diff(inc.tail, inc.head) << pmtlock_timeout_shift;
+ do {
+ current_head = ACCESS_ONCE(lock->tickets.head);
+ if (__tickets_equal(inc.tail, current_head)) {
+ goto spin;
+ } else if (__tickets_diff(inc.tail, current_head) > NR_CPUS) {
+ goto spin;
+ } else if (!__tickets_equal(current_head, inc.head)) {
+ // dynamic timout update when ticket head is updated.
+ inc.head = current_head;
+ timeout = __tickets_diff(inc.tail, current_head) << pmtlock_timeout_shift;
+ }
+ cpu_relax();
+ } while (timeout--);
+spin:
for (;;) {
- unsigned count = SPIN_THRESHOLD;
-
- do {
- inc.head = READ_ONCE(lock->tickets.head);
- if (__tickets_equal(inc.head, inc.tail))
- goto clear_slowpath;
- cpu_relax();
- } while (--count);
- __ticket_lock_spinning(lock, inc.tail);
- }
-clear_slowpath:
- __ticket_check_and_clear_slowpath(lock, inc.head);
+ if (xchg(&lock->acquired, 1) == 0)
+ goto out;
+ cpu_relax();
+ }
+
out:
barrier(); /* make sure nothing creeps before the lock is taken */
}
@@ -128,47 +148,40 @@ static __always_inline int arch_spin_trylock(arch_spinlock_t *lock)
{
arch_spinlock_t old, new;
- old.tickets = READ_ONCE(lock->tickets);
- if (!__tickets_equal(old.tickets.head, old.tickets.tail))
+ *(__ticketfull_t *)&old = ACCESS_ONCE(*(__ticketfull_t *)lock);
+
+ if (old.tickets.head != old.tickets.tail)
+ return 0;
+ if (ACCESS_ONCE(lock->acquired) == 1)
return 0;
- new.head_tail = old.head_tail + (TICKET_LOCK_INC << TICKET_SHIFT);
- new.head_tail &= ~TICKET_SLOWPATH_FLAG;
+ new.head_tail = old.head_tail + (1 << TICKET_SHIFT);
+ new.acquired = 1;
/* cmpxchg is a full barrier, so nothing can move before it */
- return cmpxchg(&lock->head_tail, old.head_tail, new.head_tail) == old.head_tail;
+ if (cmpxchg((__ticketfull_t *)lock, *(__ticketfull_t *)&old,
+ *(__ticketfull_t *)&new) == *(__ticketfull_t *)&old) {
+ return 1;
+ } else return 0;
}
static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
{
- if (TICKET_SLOWPATH_FLAG &&
- static_key_false(¶virt_ticketlocks_enabled)) {
- __ticket_t head;
-
- BUILD_BUG_ON(((__ticket_t)NR_CPUS) != NR_CPUS);
-
- head = xadd(&lock->tickets.head, TICKET_LOCK_INC);
-
- if (unlikely(head & TICKET_SLOWPATH_FLAG)) {
- head &= ~TICKET_SLOWPATH_FLAG;
- __ticket_unlock_kick(lock, (head + TICKET_LOCK_INC));
- }
- } else
- __add(&lock->tickets.head, TICKET_LOCK_INC, UNLOCK_LOCK_PREFIX);
+ __add(&lock->tickets.head, 1, UNLOCK_LOCK_PREFIX);
+ xchg(&lock->acquired, 0);
}
static inline int arch_spin_is_locked(arch_spinlock_t *lock)
{
struct __raw_tickets tmp = READ_ONCE(lock->tickets);
- return !__tickets_equal(tmp.tail, tmp.head);
+ return !__tickets_equal(tmp.tail, tmp.head) || (ACCESS_ONCE(lock->acquired)==1);
}
static inline int arch_spin_is_contended(arch_spinlock_t *lock)
{
struct __raw_tickets tmp = READ_ONCE(lock->tickets);
- tmp.head &= ~TICKET_SLOWPATH_FLAG;
return (__ticket_t)(tmp.tail - tmp.head) > TICKET_LOCK_INC;
}
#define arch_spin_is_contended arch_spin_is_contended
diff --git a/arch/x86/include/asm/spinlock_types.h b/arch/x86/include/asm/spinlock_types.h
index 5f9d757..2af929a 100644
--- a/arch/x86/include/asm/spinlock_types.h
+++ b/arch/x86/include/asm/spinlock_types.h
@@ -14,9 +14,15 @@
#if (CONFIG_NR_CPUS < (256 / __TICKET_LOCK_INC))
typedef u8 __ticket_t;
typedef u16 __ticketpair_t;
+typedef u16 __acquired_t;
+typedef u32 __ticketfull_t;
+typedef s8 __ticket_st;
#else
typedef u16 __ticket_t;
typedef u32 __ticketpair_t;
+typedef u32 __acquired_t;
+typedef u64 __ticketfull_t;
+typedef s16 __ticket_st;
#endif
#define TICKET_LOCK_INC ((__ticket_t)__TICKET_LOCK_INC)
@@ -30,6 +36,7 @@ typedef struct arch_spinlock {
__ticket_t head, tail;
} tickets;
};
+ __acquired_t acquired;
} arch_spinlock_t;
#define __ARCH_SPIN_LOCK_UNLOCKED { { 0 } }
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index db3ccb1..1c7b1d9 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -21,6 +21,10 @@
#include <linux/debug_locks.h>
#include <linux/export.h>
+// timeout threshold for waiter N = N * 2 ^ pmtlock_timeout_shift
+u64 pmtlock_timeout_shift = 8;
+EXPORT_SYMBOL(pmtlock_timeout_shift);
+
/*
* If lockdep is enabled then we use the non-preemption spin-ops
* even on CONFIG_PREEMPT, because lockdep assumes that interrupts are