From f583f5753ef5ffc70cf7b48f57906ef6bb4cab6b Mon Sep 17 00:00:00 2001 From: Emery Berger Date: Wed, 11 Oct 2023 16:17:43 -0400 Subject: [PATCH 1/3] Send and receive thread-level memory signals. --- scalene/scalene_profiler.py | 4 ++++ src/include/sampleheap.hpp | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scalene/scalene_profiler.py b/scalene/scalene_profiler.py index 230b6c994..c697c8886 100644 --- a/scalene/scalene_profiler.py +++ b/scalene/scalene_profiler.py @@ -1393,6 +1393,7 @@ def alloc_sigqueue_processor(x: Optional[List[int]]) -> None: reported_fname, reported_lineno, bytei_str, + thread_id, ) = count_str.split(",") if int(curr_pid) != int(pid): continue @@ -1406,6 +1407,7 @@ def alloc_sigqueue_processor(x: Optional[List[int]]) -> None: Filename(reported_fname), LineNumber(int(reported_lineno)), ByteCodeIndex(int(bytei_str)), + thread_id ) ) @@ -1427,6 +1429,7 @@ def alloc_sigqueue_processor(x: Optional[List[int]]) -> None: fname, lineno, bytei, + thread_id, ) = item is_malloc = action == Scalene.MALLOC_ACTION count /= Scalene.BYTES_PER_MB @@ -1491,6 +1494,7 @@ def alloc_sigqueue_processor(x: Optional[List[int]]) -> None: fname, lineno, bytei, + thread_id ) = item is_malloc = action == Scalene.MALLOC_ACTION diff --git a/src/include/sampleheap.hpp b/src/include/sampleheap.hpp index 3102a04b4..eb8a7eb46 100644 --- a/src/include/sampleheap.hpp +++ b/src/include/sampleheap.hpp @@ -291,18 +291,28 @@ class SampleHeap : public SuperHeap { if (_pythonCount == 0) { _pythonCount = 1; // prevent 0/0 } + uint64_t thread_id; +#if defined(__APPLE__) + // Use OS X / BSD thread identifier function to get "actual" thread ID. + pthread_threadid_np(pthread_self(), &thread_id); +#else + // On Linux, we cast pthread_self and hope for the best. + thread_id = (uint64_t) pthread_self(); +#endif snprintf_( buf, sizeof(buf), #if defined(__APPLE__) - "%c,%llu,%llu,%f,%d,%p,%s,%d,%d\n\n", + "%c,%llu,%llu,%f,%d,%p,%s,%d,%d,%lu\n\n", #else - "%c,%lu,%lu,%f,%d,%p,%s,%d,%d\n\n", + "%c,%lu,%lu,%f,%d,%p,%s,%d,%d,%lu\n\n", #endif ((sig == MallocSignal) ? 'M' : ((_freedLastMallocTrigger) ? 'f' : 'F')), mallocTriggered() + freeTriggered(), count, (float)_pythonCount / (_pythonCount + _cCount), getpid(), _freedLastMallocTrigger ? _lastMallocTrigger : ptr, filename.c_str(), - lineno, bytei); + lineno, + bytei, + thread_id); // Ensure we don't report last-malloc-freed multiple times. _freedLastMallocTrigger = false; getSampleFile().writeToFile(buf); From 9dc46af0be76ce4ced890625520ec5c82f58090c Mon Sep 17 00:00:00 2001 From: Emery Berger Date: Wed, 11 Oct 2023 16:35:44 -0400 Subject: [PATCH 2/3] Use gettid() on Linux. --- src/include/sampleheap.hpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/include/sampleheap.hpp b/src/include/sampleheap.hpp index eb8a7eb46..e97d655b5 100644 --- a/src/include/sampleheap.hpp +++ b/src/include/sampleheap.hpp @@ -291,14 +291,19 @@ class SampleHeap : public SuperHeap { if (_pythonCount == 0) { _pythonCount = 1; // prevent 0/0 } + // Get the thread ID, which must match the logic used by Python. uint64_t thread_id; -#if defined(__APPLE__) - // Use OS X / BSD thread identifier function to get "actual" thread ID. +#if defined(__APPLE__) || defined(BSD) + // Use the OS X / BSD thread identifier function to get "actual" thread ID. pthread_threadid_np(pthread_self(), &thread_id); +#elif defined(__linux__) + // On Linux, use gettid(). + thread_id = (uint64_t) gettid(); #else - // On Linux, we cast pthread_self and hope for the best. + // On other systems, cast pthread_self and hope for the best. thread_id = (uint64_t) pthread_self(); #endif + snprintf_( buf, sizeof(buf), #if defined(__APPLE__) From 63a491ce72617e5ab0bdbd3aac32c1ca4ca95d6b Mon Sep 17 00:00:00 2001 From: Emery Berger Date: Wed, 11 Oct 2023 17:02:13 -0400 Subject: [PATCH 3/3] Commented out switch to main thread. --- src/source/pywhere.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/source/pywhere.cpp b/src/source/pywhere.cpp index 16ab27d59..50fe4a1e7 100644 --- a/src/source/pywhere.cpp +++ b/src/source/pywhere.cpp @@ -251,12 +251,16 @@ int whereInPython(std::string& filename, int& lineno, int& bytei) { PyPtr frame = threadState ? PyThreadState_GetFrame(threadState) : nullptr; + // EDB: Below commented out to allow correct attribution to individual threads. + // As long as the frames contain file and line number information, we are good. +#if 0 if (static_cast(frame) == nullptr) { // Various packages may create native threads; attribute what they do // to what the main thread is doing, as it's likely to have requested it. frame = findMainPythonThread_frame(); // note this may be nullptr } - +#endif + auto traceConfig = TraceConfig::getInstance(); if (!traceConfig) { return 0;