Create python package with setup.py. Add unittests for installed scri…

…pts invoked from command line. Build successfully with python3.6 on travis. Build for scrutinizer and build coverage data. Update README with installation instructions and provide build_artm.sh script to download, build and install BigARTM as well as generate a suitable python wheel.

patmtk/.coveragerc → .coveragerc

@@ -6,10 +6,7 @@ source =
 [run]
 branch = true
 source =
-    patm
-    processors
-    reporting
-    results
+    topic_modeling_toolkit
     tests
 parallel = true
 
@@ -27,3 +24,4 @@ exclude_lines =
     abstractproperty
     abstractclassmethod
     warnings.warn
+    download_callback\(url,\ directory,

.gitignore

@@ -1,10 +1,16 @@
 *\.pyc
 \.eggs
 *\.egg-info
+*bigartm*
+*__pycache__
+snippets\.txt
 .idea
+main\.py
 \.fuse*
 *\.bak
-*bigartm*
+bigartm\.*log\.*
+bigartm\.INFO
+bigartm\.WARNING
+legacy
 \.cache*
 \.coverage
-*__pycache__

.scrutinizer.yml

@@ -0,0 +1,87 @@
+checks:
+    python:
+        code_rating: true
+        duplicate_code: true
+
+build:
+#    environment:
+#        variables:
+#            BIGARTM_PARENT_DIR:   $(echo $PWD)
+#            COLLECTIONS_DIR: "$(echo $PWD)/collections"
+#        apt_packages:
+#          - make
+#          - cmake
+#          - build-essential
+#          - libboost-all-dev
+#          - gfortran
+#          - libblas-dev
+#          - liblapack-dev
+        # Languages
+    nodes:
+        analysis:
+            project_setup:
+                override:
+                    - 'true'
+            tests:
+                override:
+                    - py-scrutinizer-run
+                    -
+                        command: pylint-run
+                        use_website_config: true
+        tests:
+            tests:
+                override:
+                    - 'true'
+#            environment:
+#                python:
+#                    version: "3.6.0"
+##        variables:
+##            BIGARTM_PARENT_DIR:   $(echo $PWD)
+##            COLLECTIONS_DIR: "$(echo $PWD)/collections"
+#                apt_packages:
+#                  - make
+#                  - cmake
+#                  - build-essential
+#                  - libboost-all-dev
+#                  - gfortran
+#                  - libblas-dev
+#                  - liblapack-dev
+##            requires:
+#                # will only run when my-first-test passes
+##                - node: my-first-test
+#                # will only run when no failure conditions were met in the analysis
+##                - analysis
+#            dependencies:
+#                before:
+#                  - pip install tox
+#                  - export BIGARTM_PARENT_DIR=$(echo $PWD)
+#                  - export COLLECTIONS_DIR="$(echo $PWD)/collections"
+#                  - echo EDW $BIGARTM_PARENT_DIR
+#                  - echo EDW $COLLECTIONS_DIR
+#                  - mkdir $COLLECTIONS_DIR
+#                  - pip install -U setuptools>=40.0.0
+#                  - git clone https://github.com/bigartm/bigartm.git
+#                  - chmod +x .travis/build-n-wheel.sh
+#                  - chmod +x .travis/select-wheel.sh
+#                  - .travis/build-n-wheel.sh 3 6
+#                  - export BIGARTM_WHEEL=$(.travis/select-wheel.sh 3 6)
+#                  - ls $BIGARTM_WHEEL
+#            tests:
+#                before:
+#                    - pip install coverage
+#                override:
+#                    - tox -e clean,check,py36
+#                    -
+#                        command: coverage
+#                        coverage:
+#                            file: .coverage
+#                            config_file: '.coveragerc'
+#                            format: py-cc
+
+
+filter:
+    excluded_paths:
+        - '*/test/*'
+        - '*/build/*'
+    dependency_paths:
+        - 'lib/*'

.travis.yml

@@ -0,0 +1,86 @@
+os: linux
+language: python
+
+
+env:
+  global:
+    - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
+    - SEGFAULT_SIGNALS=all
+    - TOX_SKIP_MISSING_INTERPRETERS="False"
+    - BIGARTM_PARENT_DIR=$(echo $PWD)
+    - COLLECTIONS_DIR="$(echo $PWD)/collections"
+
+before_install:
+  - mkdir $COLLECTIONS_DIR
+  - python --version
+  - pip --version
+  - uname -a
+  - lsb_release -a
+  - sudo apt-get --yes update
+  - sudo apt-get --yes install git make cmake build-essential libboost-all-dev gfortran libblas-dev liblapack-dev
+  - sudo python -m pip install -U pip
+  - sudo python -m pip install -U setuptools>=40.0.0
+  - python -m pip install -U setuptools>=40.0.0
+  - virtualenv --version
+  - git clone https://github.com/bigartm/bigartm.git
+  - chmod +x .travis/build-n-wheel.sh
+  - chmod +x .travis/select-wheel.sh
+  - python -m pip install tox
+
+
+cache: pip
+script: tox -v
+
+jobs:
+  fail_fast: true
+  include:
+#  - stage: Check
+#    env: TOXENV=clean,check
+  - stage: Build_n_test
+    before_install:
+      - python --version
+      - python -m pip install tox
+    env: TOXENV=clean,check
+  - stage: Build_n_test
+    python: '3.6'
+    env: TOXENV=py36,codecov
+    install:
+      - .travis/build-n-wheel.sh 3 6
+      - export BIGARTM_WHEEL=$(.travis/select-wheel.sh 3 6)
+      - export ARTM_SHARED_LIBRARY=/usr/local/lib/libartm.so
+  - stage: Build_n_test
+    python: '2.7'
+    env: TOXENV=py27
+    install:
+      - .travis/build-n-wheel.sh 2 7
+      - export BIGARTM_WHEEL=$(.travis/select-wheel.sh 2 7)
+      - export ARTM_SHARED_LIBRARY=/usr/local/lib/libartm.so
+  - stage: Build_n_test
+    python: '3.6'
+    before_install:
+      - python --version
+      - python -m pip install tox
+    env: TOXENV=quality
+
+  allow_failures:
+    - env: TOXENV=py27
+    - env: TOXENV=quality
+#  allow_failures:
+#    - env: TOXENV=quality
+
+after_failure:
+  - more .tox/log/* | cat
+  - more .tox/*/log/* | cat
+
+#
+#deploy:
+#  provider: script
+#  script: .travis/deploy.sh
+#  on:
+#    all_branches: true
+
+
+#notifications:
+#  email:
+#    on_success: never
+#    on_failure: never

.travis/build-n-wheel.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -e
+
+python_version=$1
+python_subversion=$2
+
+if [[ "$python_version" == "2" ]]; then
+    python_subversion=7
+elif [[ "$python_subversion" == "" ]]; then
+    echo "Please supply a 2nd argument to indicate specific python3 subversion (ie 6, 7)"
+    exit 1
+fi
+
+python -m pip install "setuptools>=40.0.0"
+python -m pip install -U setuptools
+python -m pip install protobuf tqdm wheel
+
+# sudo -H python -m pip install wheel
+
+sudo apt-get --yes install python-setuptools python-wheel
+sudo apt-get --yes install python3-setuptools python3-wheel
+
+python -m pip list
+
+current_dir=$(echo $PWD)
+mkdir $BIGARTM_PARENT_DIR/bigartm/build
+cd $BIGARTM_PARENT_DIR/bigartm/build
+
+#sudo apt-get update --yes
+sudo apt-get --yes install git make cmake build-essential libboost-all-dev gfortran libblas-dev liblapack-dev
+#sudo apt-get upgrade --yes
+
+# installs by default under /usr/local. To manipulate this use -DCMAKE_INSTALL_PREFIX=xxx flag in cmake
+if [[ "$python_version" == "2" ]]; then
+    cmake ..
+elif [[ "$python_version" == "3" ]]; then
+    cmake -DPYTHON=python3 ..
+fi
+echo DONE cmake
+
+
+make
+echo DONE make
+
+sudo make install
+echo DONE make install
+export ARTM_SHARED_LIBRARY=/usr/local/lib/libartm.so
+
+echo SETUPTOOLS
+python -m pip list
+
+# now the 'bigartm' executable should be accessible
+which bigartm
+if [[ $? -ne 0 ]]; then
+    echo "Failed to find bigartm executable. Maybe not in path?"
+    exit 1
+fi
+
+
+cd $current_dir

.travis/clone-bigartm.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+current_dir=$(echo $PWD)
+cd $1
+git clone https://github.com/bigartm/bigartm.git
+cd $current_dir

.travis/requirements.txt

@@ -0,0 +1,26 @@
+numpy>=1.16.5
+regex==2019.6.8
+attrs==19.1.0
+tqdm==4.32.2
+Click==7.0
+scipy>=1.2.2
+six==1.12.0
+
+matplotlib>=2.2.4
+
+future==0.17.1
+
+in-place==0.4.0
+
+nltk==3.4.4
+
+pandas>=0.24.2
+
+Pattern==3.6
+gensim==3.8.0
+protobuf==3.9.0
+prompt-toolkit==1.0.14
+PyInquirer==1.0.3
+
+# EasyPlot==1.0.0
+# configparser

.travis/select-wheel.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+python_version=$1
+python_subversion=$2
+
+if [[ "$python_version" == "2" ]]; then
+    python_subversion=7
+elif [[ "$python_subversion" == "" ]]; then
+    echo "Please supply a 2nd argument to indicate specific python3 subversion (ie 6, 7)"
+    exit 1
+fi
+
+reg="bigartm.*cp$python_version$python_subversion.*.whl"
+
+for filename in $BIGARTM_PARENT_DIR/bigartm/build/python/bigartm*.whl; do
+    if [[ $filename =~ $reg ]]; then
+        echo $filename
+#  	    echo "Found wheel for python$python_version.$python_subversion"
+#	    export BIGARTM_WHEEL=$filename
+	    break
+	fi
+done

MANIFEST.in

@@ -0,0 +1,34 @@
+include README.rst
+include LICENSE.txt
+include requirements.txt
+
+include tests/data/*
+
+graft src
+graft tests
+graft data
+
+include pipeline.cfg
+include regularizers.cfg
+include train.cfg
+
+include pyproject.toml
+include tox.ini
+
+include build_artm.sh
+
+global-exclude *.py[cod] __pycache__ *.so *.dylib
+
+# include .travis.yml
+include .scrutinizer.yml
+# include appveyor.yml
+
+# include .bandit.yml
+
+include .coveragerc
+
+graft .travis
+
+
+# recursive-include ci *.cmd
+# recursive-include ci *.py