Mentos integration

.gitignore

@@ -2,7 +2,8 @@
 __pycache__/
 *.py[cod]
 *$py.class
-
+.idea/
+.vscode/
 # C extensions
 *.so
 

.travis.yml

@@ -0,0 +1,61 @@
+language: python
+dist: trusty
+sudo: required
+notifications:
+  email: false
+services:
+- docker
+matrix:
+  include:
+  - python: '2.7'
+    env: OS=ubuntu-14.04
+  - python: '3.5'
+    env: OS=ubuntu-14.04
+  - python: '3.6'
+    env: OS=ubuntu-14.04
+env:
+  global:
+  - DOCKER_COMPOSE_VERSION=1.6.2
+before_install:
+- pwd
+- bash ci/scripts/travis_docker_setup.sh
+- docker version
+- docker-compose version
+- export DOCKER_IP=127.0.0.1
+- docker-compose up -d
+- docker ps -a
+- docker images
+- sudo apt-get install pandoc
+- pip install pypandoc
+install:
+- which python
+script:
+- python setup.py test
+after_success:
+- pip install coveralls
+- coveralls
+before_deploy:
+- pip install pypandoc
+deploy:
+  - provider: pypi
+    distributions: "sdist bdist_wheel"
+    server: https://testpypi.python.org/pypi
+    user: daskos
+    password:
+      secure: K7Ic/pSDVhqY63iSatOpXig77WnMhIiRgU+zZKhAJHxwGFAbq4b4QfIxD8xKR4GwLm/vGpX285cKp1nuleAwLRj4GpOmVU2o1exZHO0HNFTpSsb9fb+lYWQIFB9WFdueiv/V0WmxR6sR2+iDGCy2itTg5pNuu2POr0olJga3ixsoS1EaxRQW4DOXciblTHBRZIPNQVMi823iacq7ovvJ/ZmSBDf6KOEbouqGSMYsbgywFRpqnTKDMYgU60Mov/ATkXNMkUye6ffIKJbr/UhhVgpnkhcRBAa0Hx2/BfA23CokiBFAusH3p8Ko8BMfUCnkZ8pJw2SJ/kXqeMn+SLMX6YqsAIYQSRE33mDtDJkK5XHP/MY3/soO3M+fPGtz3vIUmPB6ycH8INw+R0FUKnblGgRoNbmGF+MbrfnRXe5Qo09dUil859ySbIYfkYkadBbtn7MNEtbp6anaAJgYnBAQkWimT+uslEKM1AVqKQ/MvW+xBzmOkz3oZIFQsqkQS4O0vT/SlH2V1GMDfXG/R8heozMYNWSCLZM4PfocZY0GZIL1qEJV+z0Q3DwwqghHZ/2v+MR35r7iTPPd0Wf+LfK762IplM4/Va+nQ8DCbI5ZF7fO5OUUCgpZpaEGKLb8yHJqfiwZ42qzVtkJkobeyBkzyQmBzxe/zBE9US8cD7R0O6E=
+    on:
+      branch: master
+      tags: false
+      condition: $TRAVIS_PYTHON_VERSION = "2.7"
+  - provider: pypi
+    distributions: "sdist bdist_wheel"
+    user: daskos
+    password:
+      secure: K7Ic/pSDVhqY63iSatOpXig77WnMhIiRgU+zZKhAJHxwGFAbq4b4QfIxD8xKR4GwLm/vGpX285cKp1nuleAwLRj4GpOmVU2o1exZHO0HNFTpSsb9fb+lYWQIFB9WFdueiv/V0WmxR6sR2+iDGCy2itTg5pNuu2POr0olJga3ixsoS1EaxRQW4DOXciblTHBRZIPNQVMi823iacq7ovvJ/ZmSBDf6KOEbouqGSMYsbgywFRpqnTKDMYgU60Mov/ATkXNMkUye6ffIKJbr/UhhVgpnkhcRBAa0Hx2/BfA23CokiBFAusH3p8Ko8BMfUCnkZ8pJw2SJ/kXqeMn+SLMX6YqsAIYQSRE33mDtDJkK5XHP/MY3/soO3M+fPGtz3vIUmPB6ycH8INw+R0FUKnblGgRoNbmGF+MbrfnRXe5Qo09dUil859ySbIYfkYkadBbtn7MNEtbp6anaAJgYnBAQkWimT+uslEKM1AVqKQ/MvW+xBzmOkz3oZIFQsqkQS4O0vT/SlH2V1GMDfXG/R8heozMYNWSCLZM4PfocZY0GZIL1qEJV+z0Q3DwwqghHZ/2v+MR35r7iTPPd0Wf+LfK762IplM4/Va+nQ8DCbI5ZF7fO5OUUCgpZpaEGKLb8yHJqfiwZ42qzVtkJkobeyBkzyQmBzxe/zBE9US8cD7R0O6E=
+    on:
+      tags: true
+      repo: daskos/mentor
+      condition: $TRAVIS_PYTHON_VERSION = "2.7"
+
+
+

Dockerfile

@@ -1,5 +1,5 @@
-FROM lensa/mesos-python:0.27.0-py2.7-python-2.7
+FROM python:3.5.2-alpine
 
-ADD . /satyr
-WORKDIR /satyr
-RUN pip install .
+ADD . /mentor
+WORKDIR /mentor
+RUN pip install .

ci/scripts/travis_docker_setup.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+# Update package for ubuntu
+sudo apt-get -qq update
+sudo apt-get install -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" docker-engine
+
+# Start service
+sudo service docker stop
+sudo service docker start
+
+# Setup/update docker compose
+curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+chmod +x docker-compose
+sudo mv docker-compose /usr/local/bin

docker-compose.yml

@@ -0,0 +1,70 @@
+version: '2'
+services:
+  zookeeper:
+    network_mode: "host"
+    image: bobrik/zookeeper:latest
+    environment:
+      - ZK_CONFIG=tickTime=2000,initLimit=10,syncLimit=5,maxClientCnxns=128,forceSync=no,clientPort=2181
+      - ZK_ID=1
+  mesos_master_0:
+    network_mode: "host"
+    image: mesosphere/mesos-master:1.1.0-2.0.107.ubuntu1404
+    environment:
+      - MESOS_LOGGING_LEVEL=ERROR
+      - MESOS_IP=127.0.0.1
+      - MESOS_HOSTNAME=localhost
+      - MESOS_ZK=zk://127.0.0.1:2181/mesos
+      - MESOS_QUORUM=1
+      - MESOS_CLUSTER=test
+      - MESOS_REGISTRY=in_memory
+    restart: "always"
+  mesos_master_1:
+    network_mode: "host"
+    image: mesosphere/mesos-master:1.1.0-2.0.107.ubuntu1404
+    environment:
+      - MESOS_LOGGING_LEVEL=ERROR
+      - MESOS_IP=127.0.0.1
+      - MESOS_HOSTNAME=localhost
+      - MESOS_ZK=zk://127.0.0.1:2181/mesos
+      - MESOS_QUORUM=1
+      - MESOS_CLUSTER=test
+      - MESOS_REGISTRY=in_memory
+      - MESOS_PORT=6060
+    restart: "always"
+  mesos_slave_0:
+    network_mode: "host"
+    image: mesosphere/mesos-slave:1.1.0-2.0.107.ubuntu1404
+    environment:
+      - MESOS_LOGGING_LEVEL=ERROR
+      - MESOS_IP=127.0.0.1
+      - MESOS_MASTER=zk://127.0.0.1:2181/mesos
+      - MESOS_CONTAINERIZERS=mesos
+      - MESOS_IMAGE_PROVIDERS=docker
+      - MESOS_ISOLATION=cgroups/cpu,cgroups/mem,docker/runtime
+      # - MESOS_DOCKER_REGISTRY=http://localhost:5000
+      - MESOS_PORT=5052
+      - MESOS_RESOURCES=cpus(*):0.5;mem(*):1024;ports(*):[11000-11999]
+      - MESOS_WORK_DIR=/tmp/mesos1
+      - MESOS_ATTRIBUTES=mentos:true
+      ## To speed up provisioning images, default is copy
+      # MESOS_IMAGE_PROVISIONER_BACKEND=aufs
+      ## To enable docker containerizer too`
+      - MESOS_CONTAINERIZERS=mesos,docker
+      - MESOS_SWITCH_USER=0
+    volumes:
+      - /sys/fs/cgroup:/sys/fs/cgroup
+    #   ## To cache docker downloaded images
+      - /tmp/mesos/store/docker:/tmp/mesos/store/docker
+    #   ## To enable docker containerizer
+      - /usr/bin/docker:/usr/local/bin/docker
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /cgroup:/cgroup
+      - /sys:/sys
+
+    privileged: true
+#  marathon:
+#    network_mode: "host"
+#    image: mesosphere/marathon:v1.3.7
+#    command: --master zk://localhost:2181/mesos --zk zk://localhost:2181/marathon
+#    restart: "always"
+#

docs/conf.py

@@ -257,7 +257,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'satyr', u'Satyr Documentation',
+    (master_doc, 'mentor', u'Satyr Documentation',
      [author], 1)
 ]
 

examples/test.py

@@ -0,0 +1,41 @@
+
+from mentor.scheduler import Framework
+from mentos.scheduler import SchedulerDriver
+import os
+import getpass
+from mentor.messages.base import TaskID, Disk, Cpus, Mem, TaskInfo, CommandInfo, Environment
+
+sched = Framework()
+driver = SchedulerDriver(sched, "Queue", getpass.getuser())
+
+driver.start()
+
+from mentor.messages.satyr import PythonTask
+import sys
+executor = {
+    "executor_id": {
+        "value": "MinimalExecutor"
+    },
+    "name": "MinimalExecutor",
+    "command": {
+        "value": '%s %s' % (
+            sys.executable, "~/workdir/mesos/malefico/malefico/executor.py"
+        )
+
+    }
+
+}
+
+
+task  = TaskInfo(name='command-task', command=CommandInfo(value='echo $HOME'), resources=[Cpus(0.1), Mem(128), Disk(0)])
+# task = PythonTask(task_id=TaskID(value='test-task-id'), executor=executor,
+#                   fn=sum, args=[range(5)],
+#                   resources=[Cpus(0.1), Mem(128), Disk(0)])
+sched.submit(task)
+sched.wait()
+print("Clean Exit, I guess")
+
+
+task_infos = [{'task_id': {'value': '39e762f5-9767-4d7d-b187-67029df5b151'}, 'agent_id': {'value': '7037ef03-cbef-4e11-aac6-d662a5ef4179-S1'}, 'name': 'task 39e762f5-9767-4d7d-b187-67029df5b151', 'executor': {'executor_id': {'value': 'MinimalExecutor'}, 'name': 'MinimalExecutor', 'command': {'value': '/opt/anaconda3/envs/mentos/bin/python /home/arti/workdir/mesos/dmentos/examples/executor.py'}, 'resources': [{'name': 'mem', 'type': 'SCALAR', 'scalar': {'value': 32}}, {'name': 'cpus', 'type': 'SCALAR', 'scalar': {'value': 0.1}}]}, 'data': 'SGVsbG8gZnJvbSB0YXNrIDM5ZTc2MmY1LTk3NjctNGQ3ZC1iMTg3LTY3MDI5ZGY1YjE1MSE=', 'resources': [{'name': 'cpus', 'type': 'SCALAR', 'scalar': {'value': 0.2}}, {'name': 'mem', 'type': 'SCALAR', 'scalar': {'value': 128}}]}]
+offers = [{'agent_id': {'value': '7037ef03-cbef-4e11-aac6-d662a5ef4179-S0'}, 'framework_id': {'value': '4dca24e0-63bf-4505-8023-f544de0842d6-0001'}, 'hostname': 'malefico.io', 'id': {'value': '4dca24e0-63bf-4505-8023-f544de0842d6-O2'}, 'resources': [{'name': 'ports', 'ranges': {'range': [{'begin': 1025, 'end': 60000}]}, 'role': '*', 'type': 'RANGES'}, {'name': 'cpus', 'role': '*', 'scalar': {'value': 7.9}, 'type': 'SCALAR'}, {'name': 'mem', 'role': '*', 'scalar': {'value': 14895.0}, 'type': 'SCALAR'}, {'name': 'disk', 'role': '*', 'scalar': {'value': 145941.0}, 'type': 'SCALAR'}], 'url': {'address': {'hostname': 'malefico.io', 'ip': '127.0.0.1', 'port': 5051}, 'path': '/slave(1)', 'scheme': 'http'}}, {'agent_id': {'value': '7037ef03-cbef-4e11-aac6-d662a5ef4179-S1'}, 'attributes': [{'name': 'mentos', 'text': {'value': 'true'}, 'type': 'TEXT'}], 'framework_id': {'value': '4dca24e0-63bf-4505-8023-f544de0842d6-0001'}, 'hostname': 'malefico.io', 'id': {'value': '4dca24e0-63bf-4505-8023-f544de0842d6-O3'}, 'resources': [{'name': 'cpus', 'role': '*', 'scalar': {'value': 0.5}, 'type': 'SCALAR'}, {'name': 'mem', 'role': '*', 'scalar': {'value': 1024.0}, 'type': 'SCALAR'}, {'name': 'ports', 'ranges': {'range': [{'begin': 11000, 'end': 11999}]}, 'role': '*', 'type': 'RANGES'}, {'name': 'disk', 'role': '*', 'scalar': {'value': 5114.0}, 'type': 'SCALAR'}], 'url': {'address': {'hostname': 'malefico.io', 'ip': '127.0.0.1', 'port': 5052}, 'path': '/slave(1)', 'scheme': 'http'}}]
+

mentor/__init__.py

@@ -2,14 +2,16 @@
 
 import pkg_resources as _pkg_resources
 
-from .scheduler import QueueScheduler
-from .executor import OneOffExecutor
-from .messages import PythonTask, PythonTaskStatus  # important to register classes
-
+from .scheduler import QueueScheduler, SchedulerDriver
+from .executor import ThreadExecutor, ProcessExecutor, ExecutorDriver
+from .messages import PythonTask, PythonTaskStatus
 
 __version__ = _pkg_resources.get_distribution('mentor').version
 
 __all__ = ('QueueScheduler',
-           'OneOffExecutor',
+           'SchedulerDriver',
+           'ExecutorDriver',
+           'ThreadExecutor',
+           'ProcessExecutor',
            'PythonTask',
            'PythonTaskStatus')