Merge pull request #275 from plasma-umass/python-memory-module

Python memory module

Author: emeryberger

.github/workflows/release-pypi-upload.yml

@@ -15,7 +15,19 @@ on:
   workflow_dispatch: # manual execution
 
 jobs:
-  pypi-upload:
+  pick-devN:
+    name: create .devN build date coordinated across all matrix jobs
+    runs-on: ubuntu-latest
+    steps:
+      - run: TZ='America/New_York' date '+%Y%m%d%H%M' > devN.txt
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: devN
+          path: devN.txt
+
+  build-and-upload:
+    needs: pick-devN
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -37,57 +49,62 @@ jobs:
       TWINE_REPOSITORY: ${{ secrets.PYPI_REPOSITORY }}
 
     steps:
-    - name: Pick .devN name if a test build
-      if: env.TWINE_REPOSITORY == 'testpypi'
-      run: |
-        echo "DEV_BUILD=$(date '+%Y%m%d%H%M')" >> $GITHUB_ENV  # for setup.py
-
-    - uses: actions/checkout@v2
-
-    - name: Set up python (script version)
-      if: matrix.container == ''
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python_version }}
-
-    - name: Set up python (container version)
-      if: matrix.container != ''
-      run: |
-        PYV=`echo "${{ matrix.python_version }}" | tr -d "."`; ls -d -1 /opt/python/cp$PYV*/bin | head -n 1 >> $GITHUB_PATH
-        cat $GITHUB_PATH
-
-    - name: Install dependencies
-      run: |
-        pip3 install setuptools wheel twine find_libpython
-
-    - name: Work around arm64 support on MacOS
-      # https://github.com/actions/virtual-environments/issues/2557
-      if: matrix.os == 'macos-latest'
-      run: sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*
-
-    - name: Build source dist
-      if: matrix.upload_source
-      run: make sdist
-
-    - name: Build binary dist
-      run: make bdist
-
-    - name: Figure out google drive upload target
-      if: env.GOOGLE_DRIVE_FOLDER != ''
-      run: |
-        echo "UPLOAD_TARGET=$(ls dist/scalene*.whl)" >> $GITHUB_ENV
-
-    - name: Upload to google drive
-      if: env.GOOGLE_DRIVE_FOLDER != ''
-      uses: Jodebu/upload-to-drive@master
-      with:
-        target: ${{ env.UPLOAD_TARGET }}
-        credentials: ${{ secrets.GOOGLE_DRIVE_CREDENTIALS }}
-        folder: ${{ env.GOOGLE_DRIVE_FOLDER }}
-
-    - name: Upload
-      if: env.GOOGLE_DRIVE_FOLDER == ''
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: twine upload dist/*
+      - name: get coordinated .devN
+        uses: actions/download-artifact@v2
+        with:
+          name: devN
+
+      - name: set up to use .devN name if a test build
+        if: env.TWINE_REPOSITORY == 'testpypi'
+        run: |
+          echo "DEV_BUILD=$(cat devN.txt)" >> $GITHUB_ENV  # for setup.py
+  
+      - uses: actions/checkout@v2
+  
+      - name: Set up python (script version)
+        if: matrix.container == ''
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python_version }}
+  
+      - name: Set up python (container version)
+        if: matrix.container != ''
+        run: |
+          PYV=`echo "${{ matrix.python_version }}" | tr -d "."`; ls -d -1 /opt/python/cp$PYV*/bin | head -n 1 >> $GITHUB_PATH
+          cat $GITHUB_PATH
+  
+      - name: Install dependencies
+        run: |
+          pip3 install setuptools wheel twine
+  
+      - name: Work around arm64 support on MacOS
+        # https://github.com/actions/virtual-environments/issues/2557
+        if: matrix.os == 'macos-latest'
+        run: sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*
+  
+      - name: Build source dist
+        if: matrix.upload_source
+        run: make sdist
+  
+      - name: Build binary dist
+        run: make bdist
+  
+      - name: Figure out google drive upload target
+        if: env.GOOGLE_DRIVE_FOLDER != ''
+        run: |
+          echo "UPLOAD_TARGET=$(ls dist/scalene*.whl)" >> $GITHUB_ENV
+  
+      - name: Upload to google drive
+        if: env.GOOGLE_DRIVE_FOLDER != ''
+        uses: Jodebu/upload-to-drive@master
+        with:
+          target: ${{ env.UPLOAD_TARGET }}
+          credentials: ${{ secrets.GOOGLE_DRIVE_CREDENTIALS }}
+          folder: ${{ env.GOOGLE_DRIVE_FOLDER }}
+  
+      - name: Upload
+        if: env.GOOGLE_DRIVE_FOLDER == ''
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: twine upload dist/*

GNUmakefile

@@ -9,13 +9,10 @@ C_SOURCES = src/source/libscalene.cpp src/source/get_line_atomic.cpp src/include
 CXXFLAGS = -std=c++17 -g -O3 -DNDEBUG -D_REENTRANT=1 -pipe -fno-builtin-malloc -fvisibility=hidden
 CXX = g++
 
-PYTHON_INCLUDE := $(shell python3 -c "from sysconfig import get_paths as gp; print(gp()['include'])")
-PYTHON_LIBRARY := $(shell python3 -m find_libpython)
-
 INCLUDES  = -Isrc -Isrc/include
 INCLUDES := $(INCLUDES) -Ivendor/Heap-Layers -Ivendor/Heap-Layers/wrappers -Ivendor/Heap-Layers/utility
 INCLUDES := $(INCLUDES) -Ivendor/printf
-INCLUDES := $(INCLUDES) -I$(PYTHON_INCLUDE)
+INCLUDES := $(INCLUDES) $(shell python3-config --includes)
 
 ifeq ($(shell uname -s),Darwin)
   LIBFILE := lib$(LIBNAME).dylib
@@ -26,7 +23,6 @@ ifeq ($(shell uname -s),Darwin)
     ARCH := -arch x86_64
   endif
   CXXFLAGS := $(CXXFLAGS) -flto -ftls-model=initial-exec -ftemplate-depth=1024 $(ARCH) -compatibility_version 1 -current_version 1 -dynamiclib
-  RPATH_FLAGS := -Wl,-rpath $(shell dirname $(PYTHON_LIBRARY))
   SED_INPLACE = -i ''
 
 else # non-Darwin
@@ -46,10 +42,10 @@ OUTDIR=scalene
 all: $(OUTDIR)/$(LIBFILE)
 
 $(OUTDIR)/$(LIBFILE): vendor/Heap-Layers $(SRC) $(C_SOURCES) GNUmakefile
-	$(CXX) $(CXXFLAGS) $(INCLUDES) $(SRC) -o $(OUTDIR)/$(LIBFILE) -ldl -lpthread $(RPATH_FLAGS) $(PYTHON_LIBRARY)
+	$(CXX) $(CXXFLAGS) $(INCLUDES) $(SRC) -o $(OUTDIR)/$(LIBFILE) -ldl -lpthread
 
 clean:
-	rm -f $(OUTDIR)/$(LIBFILE) scalene/get_line_atomic*.so
+	rm -f $(OUTDIR)/$(LIBFILE) scalene/*.so scalene/*.dylib
 	rm -rf $(OUTDIR)/$(LIBFILE).dSYM
 	rm -rf scalene.egg-info
 	rm -rf build dist *egg-info
@@ -77,9 +73,9 @@ clang-format:
 black:
 	-black -l 79 $(PYTHON_SOURCES)
 
-#ifeq ($(shell uname -s),Darwin)
-#  PYTHON_PLAT:=-p $(shell $(PYTHON) -c 'from pkg_resources import get_build_platform; p=get_build_platform(); print(p[:p.rindex("-")])')-universal2
-#endif
+ifeq ($(shell uname -s),Darwin)
+  PYTHON_PLAT:=-p $(shell $(PYTHON) -c 'from pkg_resources import get_build_platform; p=get_build_platform(); print(p[:p.rindex("-")])')-universal2
+endif
 
 PYTHON_API_VER:=$(shell $(PYTHON) -c 'from pip._vendor.packaging.tags import interpreter_name, interpreter_version; print(interpreter_name()+interpreter_version())')
 

scalene/scalene_profiler.py

@@ -1438,9 +1438,9 @@ def run_profiler(args: argparse.Namespace, left: List[str]) -> None:
                         Scalene.__program_path = program_path
                     # Grab local and global variables.
                     if not Scalene.__args.cpu_only:
-                        import register_files_to_profile
+                        from scalene import pywhere
 
-                        register_files_to_profile.register_files_to_profile(
+                        pywhere.register_files_to_profile(
                             list(Scalene.__files_to_profile.keys()),
                             Scalene.__program_path,
                             Scalene.__args.profile_all,

setup.py

@@ -19,9 +19,7 @@ def multiarch_args():
     """Returns args requesting multi-architecture support, if applicable."""
     # On MacOS we build "universal2" packages, for both x86_64 and arm64/M1
     if sys.platform == 'darwin':
-        return []
-        # multiple architectures disabled for now, because of issues linking with the Python library.
-        # return ['-arch', 'x86_64', '-arch', 'arm64']
+        return ['-arch', 'x86_64', '-arch', 'arm64']
     return []
 
 def extra_compile_args():
@@ -85,6 +83,14 @@ def build_libscalene(self):
     language="c++"
 )
 
+pywhere = Extension('scalene.pywhere',
+    include_dirs=['.', 'src', 'src/include'],
+    sources = ['src/source/pywhere.cpp'],
+    extra_compile_args=extra_compile_args(),
+    extra_link_args=multiarch_args(),
+    py_limited_api=False,
+    language="c++")
+
 # If we're testing packaging, build using a ".devN" suffix in the version number,
 # so that we can upload new files (as testpypi/pypi don't allow re-uploading files with
 # the same name as previously uploaded).
@@ -135,7 +141,7 @@ def build_libscalene(self):
         "wheel==0.36.2",
         "numpy"
     ],
-    ext_modules=[get_line_atomic],
+    ext_modules=[get_line_atomic, pywhere],
     setup_requires=['setuptools_scm'],
     include_package_data=True,
     entry_points={"console_scripts": ["scalene = scalene.__main__:main"]},

src/include/memcpysampler.hpp

@@ -2,7 +2,6 @@
 #ifndef MEMCPYSAMPLER_HPP
 #define MEMCPYSAMPLER_HPP
 
-#include <Python.h>
 #include <fcntl.h>
 #include <signal.h>
 #include <stdint.h>
@@ -268,7 +267,7 @@ class MemcpySampler {
   void incrementMemoryOps(int n) {
     _memcpyOps += n;
     auto sampleMemop = _memcpySampler.sample(n);
-    if (Py_IsInitialized() && unlikely(sampleMemop)) {
+    if (unlikely(sampleMemop)) {
       writeCount();
       _memcpyTriggered++;
       _memcpyOps = 0;

src/include/py_env.hpp

@@ -0,0 +1,18 @@
+#ifndef __PYWHERE_H
+#define __PYWHERE_H
+
+#include <string>
+#include <atomic>
+
+/**
+ * Examines the current Python stack frame and let us know where in the code we are.
+ */
+extern "C" int whereInPython(std::string& filename, int& lineno, int& bytei);
+
+
+/**
+ * Pointer to "whereInPython" for efficient linkage between pywhere and libscalene.
+ */
+extern "C"  std::atomic<decltype(whereInPython)*> p_whereInPython;
+
+#endif