Failing at spinlock implementation

Author: hp77-creator

runtime/include/bpftime_epoll.h

@@ -0,0 +1,215 @@
+#pragma once
+/*
+ Keeping declaration for structs that are specific to linux
+ structures added from
+ <sys/epoll.h>
+ <linux/perf_event.h>
+ <linux/types.h>
+*/
+union epoll_data {
+           void     *ptr;
+           int       fd;
+           uint32_t  u32;
+           uint64_t  u64;
+       };
+
+typedef union epoll_data  epoll_data_t;
+struct epoll_event {
+           uint32_t      events;  /* Epoll events */
+           epoll_data_t  data;    /* User data variable */
+       };
+typedef uint64_t __u64;
+typedef uint64_t __aligned_u64;
+typedef uint32_t __u32;
+typedef uint16_t __u16;
+typedef uint8_t  __u8;
+
+typedef int32_t  __s32;
+typedef int64_t  __s64;
+
+struct perf_event_header {
+        __u32 type;
+        __u16 misc;
+        __u16 size;
+    };
+
+/*
+https://github.com/torvalds/linux/blob/f06ce441457d4abc4d76be7acba26868a2d02b1c/include/uapi/linux/perf_event.h#L571
+*/
+struct perf_event_mmap_page {
+
+	__u32	version;		/* version number of this structure */
+	__u32	compat_version;		/* lowest version this is compat with */
+
+	/*
+	 * Bits needed to read the hw events in user-space.
+	 *
+	 *   u32 seq, time_mult, time_shift, index, width;
+	 *   u64 count, enabled, running;
+	 *   u64 cyc, time_offset;
+	 *   s64 pmc = 0;
+	 *
+	 *   do {
+	 *     seq = pc->lock;
+	 *     barrier()
+	 *
+	 *     enabled = pc->time_enabled;
+	 *     running = pc->time_running;
+	 *
+	 *     if (pc->cap_usr_time && enabled != running) {
+	 *       cyc = rdtsc();
+	 *       time_offset = pc->time_offset;
+	 *       time_mult   = pc->time_mult;
+	 *       time_shift  = pc->time_shift;
+	 *     }
+	 *
+	 *     index = pc->index;
+	 *     count = pc->offset;
+	 *     if (pc->cap_user_rdpmc && index) {
+	 *       width = pc->pmc_width;
+	 *       pmc = rdpmc(index - 1);
+	 *     }
+	 *
+	 *     barrier();
+	 *   } while (pc->lock != seq);
+	 *
+	 * NOTE: for obvious reason this only works on self-monitoring
+	 *       processes.
+	 */
+	__u32	lock;			/* seqlock for synchronization */
+	__u32	index;			/* hardware event identifier */
+	__s64	offset;			/* add to hardware event value */
+	__u64	time_enabled;		/* time event active */
+	__u64	time_running;		/* time event on cpu */
+	union {
+		__u64	capabilities;
+		struct {
+			__u64	cap_bit0		: 1, /* Always 0, deprecated, see commit 860f085b74e9 */
+				cap_bit0_is_deprecated	: 1, /* Always 1, signals that bit 0 is zero */
+
+				cap_user_rdpmc		: 1, /* The RDPMC instruction can be used to read counts */
+				cap_user_time		: 1, /* The time_{shift,mult,offset} fields are used */
+				cap_user_time_zero	: 1, /* The time_zero field is used */
+				cap_user_time_short	: 1, /* the time_{cycle,mask} fields are used */
+				cap_____res		: 58;
+		};
+	};
+
+	/*
+	 * If cap_user_rdpmc this field provides the bit-width of the value
+	 * read using the rdpmc() or equivalent instruction. This can be used
+	 * to sign extend the result like:
+	 *
+	 *   pmc <<= 64 - width;
+	 *   pmc >>= 64 - width; // signed shift right
+	 *   count += pmc;
+	 */
+	__u16	pmc_width;
+
+	/*
+	 * If cap_usr_time the below fields can be used to compute the time
+	 * delta since time_enabled (in ns) using rdtsc or similar.
+	 *
+	 *   u64 quot, rem;
+	 *   u64 delta;
+	 *
+	 *   quot = (cyc >> time_shift);
+	 *   rem = cyc & (((u64)1 << time_shift) - 1);
+	 *   delta = time_offset + quot * time_mult +
+	 *              ((rem * time_mult) >> time_shift);
+	 *
+	 * Where time_offset,time_mult,time_shift and cyc are read in the
+	 * seqcount loop described above. This delta can then be added to
+	 * enabled and possible running (if index), improving the scaling:
+	 *
+	 *   enabled += delta;
+	 *   if (index)
+	 *     running += delta;
+	 *
+	 *   quot = count / running;
+	 *   rem  = count % running;
+	 *   count = quot * enabled + (rem * enabled) / running;
+	 */
+	__u16	time_shift;
+	__u32	time_mult;
+	__u64	time_offset;
+	/*
+	 * If cap_usr_time_zero, the hardware clock (e.g. TSC) can be calculated
+	 * from sample timestamps.
+	 *
+	 *   time = timestamp - time_zero;
+	 *   quot = time / time_mult;
+	 *   rem  = time % time_mult;
+	 *   cyc = (quot << time_shift) + (rem << time_shift) / time_mult;
+	 *
+	 * And vice versa:
+	 *
+	 *   quot = cyc >> time_shift;
+	 *   rem  = cyc & (((u64)1 << time_shift) - 1);
+	 *   timestamp = time_zero + quot * time_mult +
+	 *               ((rem * time_mult) >> time_shift);
+	 */
+	__u64	time_zero;
+
+	__u32	size;			/* Header size up to __reserved[] fields. */
+	__u32	__reserved_1;
+
+	/*
+	 * If cap_usr_time_short, the hardware clock is less than 64bit wide
+	 * and we must compute the 'cyc' value, as used by cap_usr_time, as:
+	 *
+	 *   cyc = time_cycles + ((cyc - time_cycles) & time_mask)
+	 *
+	 * NOTE: this form is explicitly chosen such that cap_usr_time_short
+	 *       is a correction on top of cap_usr_time, and code that doesn't
+	 *       know about cap_usr_time_short still works under the assumption
+	 *       the counter doesn't wrap.
+	 */
+	__u64	time_cycles;
+	__u64	time_mask;
+
+		/*
+		 * Hole for extension of the self monitor capabilities
+		 */
+
+	__u8	__reserved[116*8];	/* align to 1k. */
+
+	/*
+	 * Control data for the mmap() data buffer.
+	 *
+	 * User-space reading the @data_head value should issue an smp_rmb(),
+	 * after reading this value.
+	 *
+	 * When the mapping is PROT_WRITE the @data_tail value should be
+	 * written by userspace to reflect the last read data, after issueing
+	 * an smp_mb() to separate the data read from the ->data_tail store.
+	 * In this case the kernel will not over-write unread data.
+	 *
+	 * See perf_output_put_handle() for the data ordering.
+	 *
+	 * data_{offset,size} indicate the location and size of the perf record
+	 * buffer within the mmapped area.
+	 */
+	__u64   data_head;		/* head in the data section */
+	__u64	data_tail;		/* user-space written tail */
+	__u64	data_offset;		/* where the buffer starts */
+	__u64	data_size;		/* data buffer size */
+
+	/*
+	 * AUX area is defined by aux_{offset,size} fields that should be set
+	 * by the userspace, so that
+	 *
+	 *   aux_offset >= data_offset + data_size
+	 *
+	 * prior to mmap()ing it. Size of the mmap()ed area should be aux_size.
+	 *
+	 * Ring buffer pointers aux_{head,tail} have the same semantics as
+	 * data_{head,tail} and same ordering rules apply.
+	 */
+	__u64	aux_head;
+	__u64	aux_tail;
+	__u64	aux_offset;
+	__u64	aux_size;
+};
+
+

runtime/include/bpftime_shm.hpp

@@ -13,6 +13,8 @@
 #include <ebpf-vm.h>
 #if __linux__
 #include <sys/epoll.h>
+#elif __APPLE__
+#include "bpftime_epoll.h"
 #endif
 
 namespace bpftime

runtime/include/platform_utils.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <cstdlib>
+#include <functional>
+
+#if defined(__linux__)
+    #include <sched.h>
+#elif defined(__APPLE__) && defined(__MACH__)
+    #include <sys/sysctl.h>
+    #include <pthread.h>
+    typedef int cpu_set_t;
+
+    inline void CPU_ZERO(cpu_set_t *set) {
+        *set = 0;
+    }
+
+    inline void CPU_SET(int cpu, cpu_set_t *set) {
+        *set |= (1 << cpu);
+    }
+
+    inline int CPU_ISSET(int cpu, const cpu_set_t *set) {
+        return (*set & (1 << cpu)) != 0;
+    }
+    int sched_getaffinity(pid_t pid, size_t cpusetsize, cpu_set_t *mask);
+    int sched_setaffinity(pid_t pid, size_t cpusetsize, const cpu_set_t *mask);
+#else
+    #error "Unsupported platform"
+#endif
+
+int get_current_cpu();

runtime/include/spinlock.hpp

@@ -0,0 +1,36 @@
+#ifndef SPINLOCK_HPP
+#define SPINLOCK_HPP
+
+#include <atomic>
+
+class Spinlock {
+public:
+    std::atomic_flag lock = ATOMIC_FLAG_INIT;
+
+    // Default constructor to initialize the atomic_flag
+    Spinlock() : lock(ATOMIC_FLAG_INIT) {}
+
+    // Deleted copy constructor to prevent copying
+    Spinlock(const Spinlock&) = delete;
+
+    void spin_lock() {
+        while (lock.test_and_set(std::memory_order_acquire)) {
+            // busy-wait
+        }
+    }
+
+    void spin_unlock() {
+        lock.clear(std::memory_order_release);
+    }
+};
+
+// Global functions to match pthread_spin_lock/unlock interface
+inline void spin_lock(Spinlock* spinlock) {
+    spinlock->spin_lock();
+}
+
+inline void spin_unlock(Spinlock* spinlock) {
+    spinlock->spin_unlock();
+}
+
+#endif // SPINLOCK_HPP

runtime/include/spinlock_wrapper.hpp

@@ -0,0 +1,79 @@
+#ifndef SPINLOCK_WRAPPER_HPP
+#define SPINLOCK_WRAPPER_HPP
+
+#include <pthread.h>
+#include <atomic>
+
+// Include the custom spinlock implementation
+#include "spinlock.hpp"
+
+// Define the custom spinlock type and functions only if the standard library is not available
+#if !defined(_POSIX_SPIN_LOCKS) || _POSIX_SPIN_LOCKS <= 0
+typedef Spinlock pthread_spinlock_t; // Define custom spinlock type
+
+// Non-volatile version
+inline int pthread_spin_init(pthread_spinlock_t* lock, int pshared) {
+    (void)pshared; // suppress unused parameter warning
+    lock->lock.clear();
+    return 0;
+}
+
+inline int pthread_spin_destroy(pthread_spinlock_t* lock) {
+    (void)lock; // suppress unused parameter warning
+    return 0;
+}
+
+inline int pthread_spin_lock(pthread_spinlock_t* lock) {
+    spin_lock(lock);
+    return 0;
+}
+
+inline int pthread_spin_unlock(pthread_spinlock_t* lock) {
+    spin_unlock(lock);
+    return 0;
+}
+
+// Volatile version
+inline int pthread_spin_init(volatile pthread_spinlock_t* lock, int pshared) {
+    (void)pshared; // suppress unused parameter warning
+    const_cast<pthread_spinlock_t*>(lock)->lock.clear();
+    return 0;
+}
+
+inline int pthread_spin_destroy(volatile pthread_spinlock_t* lock) {
+    (void)lock; // suppress unused parameter warning
+    return 0;
+}
+
+inline int pthread_spin_lock(volatile pthread_spinlock_t* lock) {
+    spin_lock(const_cast<pthread_spinlock_t*>(lock));
+    return 0;
+}
+
+inline int pthread_spin_unlock(volatile pthread_spinlock_t* lock) {
+    spin_unlock(const_cast<pthread_spinlock_t*>(lock));
+    return 0;
+}
+
+#else
+// Ensure the volatile versions of the standard library functions
+
+inline int pthread_spin_init(volatile pthread_spinlock_t* lock, int pshared) {
+    return pthread_spin_init(const_cast<pthread_spinlock_t*>(lock), pshared);
+}
+
+inline int pthread_spin_destroy(volatile pthread_spinlock_t* lock) {
+    return pthread_spin_destroy(const_cast<pthread_spinlock_t*>(lock));
+}
+
+inline int pthread_spin_lock(volatile pthread_spinlock_t* lock) {
+    return pthread_spin_lock(const_cast<pthread_spinlock_t*>(lock));
+}
+
+inline int pthread_spin_unlock(volatile pthread_spinlock_t* lock) {
+    return pthread_spin_unlock(const_cast<pthread_spinlock_t*>(lock));
+}
+
+#endif // _POSIX_SPIN_LOCKS
+
+#endif // SPINLOCK_WRAPPER_HPP

runtime/src/bpf_map/map_common_def.hpp

@@ -11,7 +11,7 @@
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/containers/vector.hpp>
 #include <functional>
-#include <sched.h>
+#include "platform_utils.hpp"
 
 namespace bpftime
 {
@@ -23,14 +23,14 @@ using bytes_vec = boost::interprocess::vector<uint8_t, bytes_vec_allocator>;
 template <class T>
 static inline T ensure_on_current_cpu(std::function<T(int cpu)> func)
 {
-	return func(sched_getcpu());
+	return func(get_current_cpu());
 }
 
 template <class T>
 static inline T ensure_on_certain_cpu(int cpu, std::function<T()> func)
 {
 	static thread_local int currcpu = -1;
-	if (currcpu == sched_getcpu()) {
+	if (currcpu == get_current_cpu()) {
 		return func(currcpu);
 	}
 	cpu_set_t orig, set;