Clean up pyscamp build logic in setup.py (#96)

* Stop trying to detect visual studio in setup.py.
* Increases the required cmake version for pyscamp to be 3.15 or more.
* Move autoselection of CMAKE_GENERATOR_PLATFORM to inside CMakeLists.txt
* Pass CMAKE_GENERATOR_PLATFORM on Windows when compiling pyscamp. Remove it in SCAMP's CMakeLists.txt if it is not needed.
* Change recommendations for how to specify environment variables to set compilers/generators for SCAMP and pyscamp, This should be more aligned with the normal usage of cmake.
* Update documentation.

CMakeLists.txt

@@ -1,8 +1,44 @@
-# cmake file to build the project and tests
+
 cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
 include(CheckLanguage)
 include(CheckCXXCompilerFlag)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/cmake)
+
+# Determine if we are using a VS generator where the platform is specified in the name
+# of the generator. We do not want to specify CMAKE_GENERATOR_PLATFORM in that case.
+if (CMAKE_GENERATOR MATCHES "Visual Studio")
+  if (CMAKE_GENERATOR MATCHES "Win64" OR CMAKE_GENERATOR MATCHES "ARM")
+    set(USING_PLATFORM_SPECIFIC_VS_GENERATOR "True")
+  endif()
+endif()
+
+if ((NOT CMAKE_GENERATOR) OR (NOT CMAKE_GENERATOR MATCHES "Visual Studio") OR USING_PLATFORM_SPECIFIC_VS_GENERATOR)
+  # CMAKE_GENERATOR_PLATFORM is only needed when building with Visual Studio, and the
+  # Visual Studio Generator is not configured by-name to already be building for a
+  # particular platform, this generator-name-based selection of platform is only
+  # supported on older versions of visual studio but some of these older versions are
+  # still in wide use.
+  # See: https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2015%202017.html
+
+  # Unfortunately, it is nearly impossible to determine what generator cmake will
+  # use by default before invoking it, on Windows, cmake checks the registry to
+  # figure out what generator to use.
+
+  # Since we do not want to do 32-bit builds when 64-bit is available, checking and
+  # removing the CMAKE_GENERATOR_PLATFORM variable if it does not apply allows for
+  # us to set "-A x64" as an option to cmake regardless of the generator we are using.
+  # This prevents some issues with building on Windows and makes it unnecessary for us
+  # to determine what generator cmake will use before invoking it.
+
+  # Note: this could break on other generators which make use of
+  # CMAKE_GENERATOR_PLATFORM in the future.
+  if (CMAKE_GENERATOR_PLATFORM OR ENV{CMAKE_GENERATOR_PLATFORM})
+    unset(CMAKE_GENERATOR_PLATFORM CACHE)
+    unset(ENV{CMAKE_GENERATOR_PLATFORM})
+    message(WARNING "unset CMAKE_GENERATOR_PLATFORM because it is meaningless for generator ${CMAKE_GENERATOR}")
+  endif()
+endif()
+
 # ----------------------------------------------------------------------------------------
 #                              ===== Project Setup =====
 project(SCAMPmain LANGUAGES CXX)
@@ -16,6 +52,10 @@ if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release)
 endif()
 
+if (CMAKE_SIZEOF_VOID_P EQUAL 4)
+  message(WARNING "Detected a 32-bit system. 32-bit operation is unsupported and may not work correctly.")
+endif()
+
 # Thread libraries
 find_package(Threads REQUIRED)
 

README.md

@@ -41,14 +41,14 @@ This is a much improved framework over [GPU-STOMP](https://github.com/zpzim/STOM
 SCAMP's documentation can be found at [readthedocs](https://scamp-docs.readthedocs.io/en/latest/).
 
 ## Performance
-SCAMP is extremely fast, especially on Tesla series GPUs. I belive this repository contains the fastest code in existance for computing the matrix profile. If you find a way to improve the speed of SCAMP, or compute matrix profiles any faster than SCAMP does, please let me know, I would be glad to point to your work and incorporate any improvements that can be made to SCAMP.
+SCAMP is extremely fast, especially on Tesla series GPUs. I believe this repository contains the fastest code in existance for computing the matrix profile. If you find a way to improve the speed of SCAMP, or compute matrix profiles any faster than SCAMP does, please let me know, I would be glad to point to your work and incorporate any improvements that can be made to SCAMP.
 
 More details on the performance of SCAMP can be found in the documentation.
 
 ## Python module
 A source distribution for a python3 module using pybind11 is available on pypi.org to install run:
 ~~~
-# Python 3; python 2 can work but is unsupported
+# Python 3 and a c/c++ compiler is required.
 # cmake is required (if you don't have it you can pip install cmake)
 pip install pyscamp
 ~~~
@@ -57,7 +57,7 @@ then you can use SCAMP in Python as follows:
 ~~~
 import pyscamp as mp # Uses GPU if available and CUDA was available during the build
 
-# Allows checking if pyscamp was built with CUDA and has GPU support
+# Allows checking if pyscamp was built with CUDA and has GPU support.
 has_gpu_support = mp.gpu_supported()
 
 # Self join

docs/source/cli.rst

@@ -158,31 +158,27 @@ Build Configuration Options
 Specifying a Compliler
 ************************************
 
-On Linux or Mac, if you need to specify a specific compiler or cuda toolkit if you have multiple installed, you can use the following defines. By default cmake will look for cuda at the /usr/local/cuda symlink on linux::
+You can set the environment variables ``CUDACXX=/path/to/nvcc``, ``CXX=/path/to/cpp/compiler``, and ``CC=/path/to/c/compiler`` to manually specify a compiler. By default cmake will look for cuda at the /usr/local/cuda symlink on linux and mac.
 
-  cmake -D CMAKE_CUDA_COMPILER=/path/to/nvcc \
-        -D CMAKE_CXX_COMPILER=/path/to/cpp/compiler \
-        -D CMAKE_C_COMPILER=/path/to/c/compiler ..
-
-On Windows this is slightly different as you need to specify the generator to cmake::
+On Windows you may also need to specify the generator to cmake::
 
   # Build with Visual Studio 2015 tools
   cmake -G "Visual Studio 14 2015" ..
   # Build with Ninja (requires ninja)
-  cmake -G "Ninja" -DCMAKE_CXX_COMPILER=/path/to/compiler
+  cmake -G "Ninja"
 
-Windows CUDA builds will only work using visual studio tools (and the CUDA visual studio extensions). This is due to the fact that the visual studio toolchain is the only suppored toolchain for compiling cuda on windows, changing the C++ compiler will cause nvcc to fail. Therefore you can only use other generators for C++ only builds.
+Windows CUDA builds will only work using Visual Studio tools (and the CUDA visual studio extensions). This is due to the fact that the visual studio toolchain is the only suppored toolchain for compiling cuda on windows, changing the C++ compiler will cause nvcc to fail. Therefore you can only use other generators for C++ only builds.
 
 Forcing CUDA (or No CUDA)
 ************************************
 
 If you desire explicit CUDA support, you can make the build fail using the flag FORCE_CUDA=1 if cuda is not found::
   
-  cmake -D FORCE_CUDA=1 ..
+  cmake -DFORCE_CUDA=1 ..
 
 The same is true if you want to disable CUDA support using FORCE_NO_CUDA=1, this will cause CUDA not to be used, even if it is found on the system::
 
-  cmake -D FORCE_NO_CUDA=1 ..
+  cmake -DFORCE_NO_CUDA=1 ..
 
 
 

docs/source/environment.rst

@@ -4,11 +4,12 @@ Environment
 Currently builds under Windows/Mac/Linux using msvc/gcc/clang and nvcc (if CUDA is available) with cmake (3.8+ for cuda support)
 
 Base dependancies (required for all builds of SCAMP):
-  * cmake 3.8 or greater
+  * cmake 3.8 or greater (3.15 for pyscamp)
 
     * This version is not available directly from all package managers so you may need to install it manually, the easist way to do this is with python via ``pip install cmake`` or you can download it manually from `here <https://cmake.org/download/>`_
-
 
+  * C/C++ compiler (e.g. gcc/clang/Visual Studio Build tools)
+
 For GPU support (required for any SCAMP build which will use a GPU):
   * cuda toolkit v9.0 or greater
 
@@ -17,16 +18,14 @@ For GPU support (required for any SCAMP build which will use a GPU):
   * NVIDIA GPU with CUDA (compute capability 3.0+) support.
 
     * You can find a list of CUDA compatible GPUs `here <https://developer.nvidia.com/cuda-gpus>`_
-    * Currently Supports Kepler-Turing, but Ampere and beyond will likely work as well, just add the -gencode flag for your specific architecture in CMakeLists.txt
     * Highly recommend using a Pascal/Volta GPU as they are much better (V100 is ~10x faster than a K80 for SCAMP, V100 is ~2-3x faster than a P100)
 
 
 For python support:
-  * Only python 3 is supported
-  * Python 2 can work, but will not be supported if things break
+  * Only Python 3 is supported.
 
 Recommended Compiler:
- * If you are using CPUs, using clang v6.0 or above is highly recomended as gcc may not properly autovectorize the CPU kernels.
+ * If you are using CPUs, using a newer version of clang is recommended as it tends to have better performance.
 
 
 Notes on GPU Support

docs/source/pyscamp/intro.rst

@@ -6,7 +6,7 @@ pyscamp is a python module which uses the SCAMP CUDA/C++ library to compute the
 
 Installation
 ------------
-A source distribution for pyscamp is available on pypi.org. The python module supports python 3 and requires that cmake is installed in order to build properly, you can install cmake using ``pip install cmake``
+A source distribution for pyscamp is available on pypi.org. The python module supports python 3 and requires that cmake 3.15 or greater is installed in order to build properly, you can install cmake using ``pip install cmake``
 
 If you want GPU support for pyscamp, you must have CUDA installed and available to the default cmake compiler on your system. 
 
@@ -18,17 +18,19 @@ pyscamp allows you to specify certain build options to pip using environment var
 
   * ``FORCE_CUDA=1`` will allow you to force pyscamp to build with cuda (or fail if it can't find CUDA)
   * ``FORCE_NO_CUDA=1`` will force pyscamp to build without cuda, even if it is found on the system.
-  * ``CMAKE_CXX_COMPILER=<compiler>`` will force pyscamp to use a compiler other than the default compiler detected by cmake, this works only for builds not using visual studio tools
-  * ``CMAKE_CUDA_COMPILER=<nvcc location>`` will point pyscamp to the specified cuda compiler to build with, this works only for builds not using visual studio tools
+  * ``CC=<compiler>`` will force pyscamp to use a c compiler other than the default compiler detected by cmake, this works only for builds not using visual studio tools.
+  * ``CXX=<compiler>`` will force pyscamp to use a c++ compiler other than the default compiler detected by cmake, this works only for builds not using visual studio tools
+  * ``CUDACXX=<nvcc location>`` will point pyscamp to the specified cuda compiler to build with, useful if your cuda compiler is not detected. This works only for builds not using visual studio tools
   * ``CMAKE_GENERATOR=<generator tag ("Ninja", etc.)>`` useful for windows builds to specify a generator other than visual studio tools
   * ``CMAKE_GENERATOR_TOOLSET=<toolset tag ("llvm", etc.)>`` for specifying a non-default compiler toolchain when using visual studio tools
+  * ``CMAKE_GENERATOR_PLATFORM=<platform tag (x64)>`` for specifying a non-default platform arch when using visual studio tools.
 
 .. highlight:: console
 
 Example::
 
   # Force a cuda build using clang++ (on the system path) as the cpp compiler
-  FORCE_CUDA=1 CMAKE_CXX_COMPILER=clang++ pip install pyscamp
+  FORCE_CUDA=1 CC=clang CXX=clang++ pip install pyscamp
 
 **Installation Notes**: If you do not specify any of the above options, cmake will use the default compilers available to it, will try to find cuda and install GPU support if it is found.
 
@@ -44,18 +46,16 @@ This is a new feature and still has some kinks to work out. If you have problems
 
   * If you installed pyscamp previously and you have since installed cuda, make sure to add the ``-I`` and ``--no-cache-dir`` flags to pip install just to make sure you are reinstalling correctly.
   * Use ``pip install -v`` to get more information about the build configuration and make sure it is using the compilers and cuda like you expect.
-  * On Mac/Linux make sure nvcc (the CUDA compiler, usually located at /usr/local/cuda/bin), is in your PATH. You can also specify a cuda compiler using ``CMAKE_CUDA_COMPILER=/path/to/cuda/compiler pip install pyscamp``
+  * On Mac/Linux make sure nvcc (the CUDA compiler, usually located at /usr/local/cuda/bin), is in your PATH. You can also specify a cuda compiler using ``CUDACXX=/path/to/cuda/compiler pip install pyscamp``
   * On Windows, CUDA will only work using using the visual studio toochains **with the appropriate visual studio plugins installed** so make sure cuda is installed with these plugins. (see :doc:`GPU support </environment>` for more information and links to the cuda installation guide)
 
-**Using a newer compiler for faster CPU Code**:
+**Using a different compiler**:
 
-  * On Mac/Linux: You can install clang v6 or greater and point pyscamp to it using ``CMAKE_CXX_COMPILER=path/to/compiler pip install pyscamp``
-  * On Windows: You can use Ninja to build with ``CMAKE_GENERATOR=Ninja CMAKE_CXX_COMPILER=path/to/compiler FORCE_NO_CUDA=1 pip install pyscamp``
+  * On Mac/Linux: You can install clang v6 or greater and point pyscamp to it using ``CXX=path/to/compiler pip install pyscamp``
+  * On Windows: You can use Ninja (or another generator) to build with ``CMAKE_GENERATOR=Ninja CXX=path/to/compiler FORCE_NO_CUDA=1 pip install pyscamp``
 
     * CUDA **will not** work on Windows with a custom toolchain, MSVC + Visual Studio plugins for CUDA must be installed, so using the above method will not have CUDA support.
 
-
-
 Python Example
 --------------
 

setup.py

@@ -17,53 +17,49 @@ def __init__(self, name, sourcedir=''):
 class CMakeBuild(build_ext):
     def run(self):
         try:
-            out = subprocess.check_output(['cmake', '--version'])
+          out = subprocess.check_output(['cmake', '--version'])
         except OSError:
-            raise RuntimeError("CMake must be installed to build the following extensions: " +
-                               ", ".join(e.name for e in self.extensions))
+          raise RuntimeError("CMake must be installed to build the following extensions: " +
+                             ", ".join(e.name for e in self.extensions))
 
         cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
-        if cmake_version < LooseVersion('3.12.0'):
-            raise RuntimeError("CMake >= 3.12.0 is required")
-
-        cmake_help = ''
-        try:
-            cmake_help = subprocess.check_output(['cmake', '--help'])
-        except OSError:
-            raise RuntimeError("Cmake could not be queried for its default generator")
-
-        # Check if visual studio is the default cmake generator
-        if '* Visual Studio' in cmake_help.decode():
-          self.cmake_vs_default_generator = True
-        else:
-          self.cmake_vs_default_generator = False
+        if cmake_version < LooseVersion('3.15.0'):
+          raise RuntimeError("CMake >= 3.15.0 is required")
 
         try:
             out = subprocess.check_output(['nvcc', '--version'])
         except OSError:
-           print('CUDA was not found on the system, to build with CUDA, verify nvcc can be found in the PATH')
-
+          print('WARNING: CUDA was not found on the system, to build with CUDA, verify nvcc can be found in the PATH')
+
+        if sys.maxsize <= 2**32:
+          print('WARNING: building/using pyscamp on a 32 bit platform is unsupported.')
+
         for ext in self.extensions:
-            self.build_extension(ext)
+          self.build_extension(ext)
 
     def build_extension(self, ext):
+        env = os.environ.copy()
+        env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''),
+                                                              self.distribution.get_version())
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
-        cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
-                      '-DPYTHON_EXECUTABLE=' + sys.executable]
+        cmake_args = ['-DPYTHON_EXECUTABLE=' + sys.executable]
+        cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir]
 
         force_cuda = os.environ.get("FORCE_CUDA", "")
         force_no_cuda = os.environ.get("FORCE_NO_CUDA", "")
-        cmake_cpp_compiler = os.environ.get("CMAKE_CXX_COMPILER", "")
-        cmake_cuda_compiler = os.environ.get("CMAKE_CUDA_COMPILER", "")
-        cmake_generator = os.environ.get("CMAKE_GENERATOR", "")     
-        cmake_toolset = os.environ.get("CMAKE_GENERATOR_TOOLSET", "")        
 
-        build_type = os.environ.get("BUILD_TYPE", "Release")
-        build_args = ['--config', build_type]
+        # This environment variable is a way to opt out of platform auto-selection on windows.
+        # It may be useful if build errors occur on windows related to setting CMAKE_GENERATOR_PLATFORM.
+        do_not_auto_select_cmake_platform = os.environ.get("PYSCAMP_NO_PLATFORM_AUTOSELECT", "")
+
+        # Default to release build.
+        build_type = os.environ.get("PYSCAMP_BUILD_TYPE", "Release")
 
         # Pile all .so in one place and use $ORIGIN as RPATH
         cmake_args += ["-DCMAKE_BUILD_WITH_INSTALL_RPATH=TRUE"]
         cmake_args += ["-DCMAKE_INSTALL_RPATH={}".format("$ORIGIN")]
+
+        # Build pyscamp module.
         cmake_args += ["-DBUILD_PYTHON_MODULE=TRUE"]
 
         if force_cuda:
@@ -72,54 +68,25 @@ def build_extension(self, ext):
         if force_no_cuda:
           cmake_args += ["-DFORCE_NO_CUDA={}".format(force_no_cuda)]
 
-        if cmake_cpp_compiler:
-          cmake_args += ["-DCMAKE_CXX_COMPILER={}".format(cmake_cpp_compiler)]
-
-        if cmake_cuda_compiler:
-          cmake_args += ["-DCMAKE_CUDA_COMPILER={}".format(cmake_cuda_compiler)]      
-
-        if cmake_generator:
-          cmake_args += ["-DCMAKE_GENERATOR={}".format(cmake_generator)]
-
-        if cmake_toolset:
-          cmake_args += ["-DCMAKE_GENERATOR_TOOLSET={}".format(cmake_toolset)]
-
-
-
         if platform.system() == "Windows":
-            generator_is_vs = False
-            # If the user specified a visual studio generator OR the default generator is visual studio
-            # then we need to specify the correct options for the visual studio generator
-            if 'Visual Studio' in cmake_generator:
-              generator_is_vs = True
-            elif not cmake_generator and self.cmake_vs_default_generator:
-              generator_is_vs = True
-
-            if generator_is_vs:
-              cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(build_type.upper(), extdir)]
-              build_args += ['--', '/m']
-            else:
-              cmake_args += ['-DCMAKE_BUILD_TYPE=' + build_type]
-              build_args += ['--', '-j4']
-
-            if sys.maxsize > 2**32 and generator_is_vs:
-                cmake_args += ['-A', 'x64']
-
-        else:
-            cmake_args += ['-DCMAKE_BUILD_TYPE=' + build_type]
-            build_args += ['--', '-j4']
+          # Make sure the libraries get placed in the extdir on Windows VS builds.
+          cmake_args += ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(build_type.upper(), extdir)]
+          # On older versions of Visual Studio, we may need to specify the generator platform manually
+          # as it defaults to 32-bit compilation. Note that we can set this here regardless of the
+          # generator used because SCAMP's CMakeLists.txt will remove the setting if it is unused.
+          cmake_generator_platform = os.environ.get("CMAKE_GENERATOR_PLATFORM", "")
+          if not cmake_generator_platform and sys.maxsize > 2**32 and not do_not_auto_select_cmake_platform:
+            env['CMAKE_GENERATOR_PLATFORM'] = 'x64'
 
-        env = os.environ.copy()
-        env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''),
-                                                              self.distribution.get_version())
         if not os.path.exists(self.build_temp):
             os.makedirs(self.build_temp)
+
         subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)
         subprocess.check_call(['cmake',
                                '--build', '.',
-                               '--target', ext.name
-                               ] + build_args,
-                              cwd=self.build_temp)
+                               '--target', ext.name,
+                               '--config', build_type,
+                               '--parallel', '4'], cwd=self.build_temp)
 
 setup(
     name='pyscamp',