PB-417 💎 Release v0.5.0

CHANGELOG.md

@@ -20,6 +20,30 @@ For reference, the possible headings are:
 
 ## [Unreleased]
 
+## [0.5.0] - 2020-02-12
+
+Fix minor issues, update documentation.
+
+- PB-402 Add more logs (#281) [Robert Steiner]
+- DO-76 :whale: non alpine image (#287) [Daniel Kravetz]
+- PB-401 Add console renderer (#280) [Robert Steiner]
+- DO-80 :ambulance: Update dev Dockerfile to build gRPC (#286) [Daniel Kravetz]
+- DO-78 :sparkles: add grafana (#284) [Daniel Kravetz]
+- DO-66 :sparkles: Add keycloak (#283) [Daniel Kravetz]
+- PB-400 increment epoch base (#282) [janpetschexain]
+- PB-397 Simplify write metrics function (#279) [Robert Steiner]
+- PB-385 Fix xain-sdk test (#278) [Robert Steiner]
+- PB-352 Add sdk config (#272) [Robert Steiner]
+- Merge pull request #277 from xainag/master. [Daniel Kravetz]
+- Hotfix: update ci. [Daniel Kravetz]
+- DO-72 :art: Make CI name and feature consistent with other repos. [Daniel Kravetz]
+- DO-47 :newspaper: Build test package on release branch. [Daniel Kravetz]
+- PB-269: enable reading participants weights from S3 (#254) [Corentin Henry]
+- PB-363 Start new development cycle (#271) [Anastasiia Tymoshchuk]
+- PB-119 enable isort diff (#262) [janpetschexain]
+- PB-363 :gem: Release v0.4.0. [Daniel Kravetz]
+- DO-73 :green_heart: Disable continue_on_failure for CI jobs. Fix mypy. [Daniel Kravetz]
+
 ## [0.4.0] - 2020-02-04
 
 Flatten model weights instead of using lists.

Dockerfile

@@ -1,13 +1,13 @@
-FROM python:3.6-alpine
+FROM python:3.6-slim
 
 ENV USER="xain"
 ENV HOST="0.0.0.0"
 ENV PORT="50051"
 ENV PATH="/home/${USER}/.local/bin:${PATH}"
 ENV CONFIG_FILE="/app/xain-fl.toml"
 
-RUN addgroup -S ${USER} && adduser -S ${USER} -G ${USER}
-RUN apk update && apk add python3-dev build-base git
+RUN groupadd ${USER} && useradd -g ${USER} ${USER}
+RUN apt update -y && apt install -y python3-dev git
 
 WORKDIR /app
 

Dockerfile.dev

@@ -1,6 +1,6 @@
-FROM python:3.6-alpine
+FROM python:3.6-slim
 
-RUN apk update && apk add python3-dev build-base git
+RUN apt update -y && apt install -y python3-dev git
 
 WORKDIR /app
 

configs/example-config.toml

@@ -1,4 +1,4 @@
-# An example configfiguration file. 
+# An example configfiguration file.
 # The values that are used for the optional keys correspond to their default values.
 
 # (Optional)
@@ -10,11 +10,11 @@ port = 50051
 
 # (Optional)
 [server.grpc_options]
-# More information about all available gRPC options can be found here: 
+# More information about all available gRPC options can be found here:
 # https://grpc.github.io/grpc/cpp/group__grpc__arg__keys.html#ga813f94f9ac3174571dd712c96cdbbdc1
 
 # Example
-# Maximum message length that the channel can receive. -1 means unlimited. 
+# Maximum message length that the channel can receive. -1 means unlimited.
 # "grpc.max_receive_message_length" = -1
 
 # (Required)
@@ -34,8 +34,10 @@ fraction_participants = 1.0
 [storage]
 # (Required) URL to the storage service to use
 endpoint = "http://minio-dev:9000"
-# (Required) Name of the bucket for storing the aggregated models
-bucket = "xain-fl-aggregated-weights"
+# (Required) Name of the bucket for storing the global model weights
+global_weights_bucket = "xain-fl-aggregated-weights"
+# (Required) Name of the bucket for retrieving the local model weights
+local_weights_bucket = "xain-fl-participants-weights"
 # (Required) AWS access key ID to use to authenticate to the storage service
 access_key_id = "minio"
 # (Required) AWS secret access to use to authenticate to the storage service
@@ -46,6 +48,10 @@ secret_access_key = "minio123"
 # (Optional) The log level can be one of "notset", "debug", "info", "warning",
 # "error" and "critical".
 level = "info"
+# (Optional) Enable ConsoleRenderer (useful for development)
+console = false
+# (Optional) Enable logging of third party libraries like boto
+third_party = false
 
 # (Optional)
 [metrics]

configs/xain-fl.toml

@@ -7,12 +7,12 @@ host = "0.0.0.0"
 port = 50051
 
 [server.grpc_options]
-# More information about all available gRPC options can be found here: 
+# More information about all available gRPC options can be found here:
 # https://grpc.github.io/grpc/cpp/group__grpc__arg__keys.html#ga813f94f9ac3174571dd712c96cdbbdc1
 
-# Maximum message length that the channel can receive. -1 means unlimited. 
+# Maximum message length that the channel can receive. -1 means unlimited.
 "grpc.max_receive_message_length" = -1
-# Maximum message length that the channel can send. -1 means unlimited. 
+# Maximum message length that the channel can send. -1 means unlimited.
 "grpc.max_send_message_length" = -1
 
 [ai]
@@ -30,8 +30,10 @@ fraction_participants = 1.0
 [storage]
 # URL to the storage service to use
 endpoint = "http://minio-dev:9000"
-# Name of the bucket for storing the aggregated models
-bucket = "xain-fl-aggregated-weights"
+# Name of the bucket for storing the global model weights
+global_weights_bucket = "xain-fl-aggregated-weights"
+# Name of the bucket for retrieving the local model weights
+local_weights_bucket = "xain-fl-participants-weights"
 # AWS access key ID to use to authenticate to the storage service
 access_key_id = "minio"
 # AWS secret access to use to authenticate to the storage service
@@ -40,8 +42,9 @@ secret_access_key = "minio123"
 [logging]
 # The log level can be one of "notset", "debug", "info", "warning",
 # "error" and "critical". It defaults to "info".
-level = "info"
-
+level = "debug"
+# Enable ConsoleRenderer (useful for development)
+console = true
 
 [metrics]
 enable = true

docker-compose-dev.yml

@@ -70,6 +70,43 @@ services:
     ports:
       - "8086:8086"
 
+  keycloak:
+    image: jboss/keycloak:8.0.2
+    hostname: keycloak
+    environment:
+      KEYCLOAK_USER: admin
+      KEYCLOAK_PASSWORD: adminadmin
+      DB_VENDOR: h2
+      # Can't properly provision a login theme yet
+#    volumes:
+#      - ${PWD}/xain-theme-keycloak:/opt/jboss/keycloak/themes
+#      - ${PWD}/xain-theme-keycloak/standalone.xml:/opt/jboss/keycloak/standalone/configuration/standalone.xml
+    networks:
+      - xain-fl-dev
+    ports:
+      - "8080:8080"
+
+  grafana:
+    image: grafana/grafana
+    depends_on:
+      - keycloak
+    hostname: grafana
+    ports:
+      - "3000:3000"
+    networks:
+      - xain-fl-dev
+    environment:
+      GF_AUTH_GENERIC_OAUTH_ENABLED: "true"
+      GF_AUTH_GENERIC_OAUTH_NAME: "OAuth"
+      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "grafana"
+      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "c3fbd14d-dfe9-4dc3-9894-8a97bc6c6575"
+      GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: "true"
+      GF_AUTH_GENERIC_OAUTH_SCOPES: "openid profile email"
+      GF_AUTH_GENERIC_OAUTH_AUTH_URL: "http://localhost:8080/auth/realms/master/protocol/openid-connect/auth"
+      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "http://keycloak:8080/auth/realms/master/protocol/openid-connect/token"
+      GF_AUTH_GENERIC_OAUTH_API_URL: "http://keycloak:8080/auth/realms/master/protocol/openid-connect/userinfo"
+      GF_SERVER_ENFORCE_DOMAIN: "false"
+
   xain-fl-dev:
     build:
       context: .

docker/entrypoint.sh

@@ -1,8 +1,6 @@
 #!/usr/bin/env sh
 
-set -o errexit
-set -o pipefail
-set -o nounset
+set -Eeuxo pipefail
 # set -o xtrace
 
 if [ $# -eq 0 ]; then

scripts/test.sh

@@ -5,7 +5,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 cd $DIR/../
 
 # sort import
-isort --check-only --settings-path=.isort.cfg -rc setup.py xain_fl tests && echo "===> isort says: well done <===" &&
+isort --check-only --diff --settings-path=.isort.cfg -rc setup.py xain_fl tests && echo "===> isort says: well done <===" &&
 
 # format code
 black --check setup.py xain_fl tests && echo "===> black says: well done <===" &&

setup.py

@@ -27,12 +27,13 @@
     "numpy==1.15",  # BSD
     "grpcio==1.23",  # Apache License 2.0
     "structlog==19.2.0",  # Apache License 2.0
-    "xain-proto==0.4.0",  # Apache License 2.0
+    "xain-proto==0.5.0",  # Apache License 2.0
     "boto3==1.10.48",  # Apache License 2.0
     "toml==0.10.0",  # MIT
     "schema~=0.7",  # MIT
     "idna==2.8",  # BSD
     "influxdb==5.2.3",  # MIT
+    "jsonschema==3.2.0",  # MIT
 ]
 
 dev_require = [
@@ -51,7 +52,7 @@
     "pytest==5.3.2",  # MIT license
     "pytest-cov==2.8.1",  # MIT
     "pytest-watch==4.2.0",  # MIT
-    "xain-sdk==0.4.0",  # Apache License 2.0
+    "xain-sdk==0.5.0",  # Apache License 2.0
 ]
 
 docs_require = [

tests/conftest.py

@@ -1,6 +1,7 @@
 """XAIN FL conftest for cproto"""
 
 from concurrent import futures
+import json
 import threading
 
 import grpc
@@ -16,6 +17,27 @@
 from .port_forwarding import ConnectionManager
 
 
+@pytest.fixture()
+def metrics_sample():
+    """Return a valid metric object."""
+    return json.dumps(
+        [
+            {
+                "measurement": "CPU utilization",
+                "time": 1234326435,
+                "tags": {"id": "127.0.0.1:1345"},
+                "fields": {"CPU_1": 90.8, "CPU_2": 90, "CPU_3": "23", "CPU_4": 0.00,},
+            },
+            {
+                "measurement": "CPU utilization",
+                "time": 3542626236,
+                "tags": {"id": "127.0.0.1:1345"},
+                "fields": {"CPU_1": 90.8, "CPU_2": 90, "CPU_3": "23", "CPU_4": 0.00,},
+            },
+        ]
+    )
+
+
 @pytest.fixture
 def coordinator_service():
     """[summary]

tests/store.py

@@ -8,7 +8,7 @@
 import numpy as np
 
 from xain_fl.config import StorageConfig
-from xain_fl.coordinator.store import S3Store
+from xain_fl.coordinator.store import S3GlobalWeightsWriter
 
 
 class MockS3Resource:
@@ -62,9 +62,10 @@ def download_fileobj(self, key: str, buf: typing.IO):
         self.reads[key] += 1
 
 
-class MockS3Store(S3Store):
-    """A partial mock of the ``xain-fl.coordinator.store.S3Store`` class
-    that does not perform any IO. Instead, data is stored in memory.
+class MockS3Writer(S3GlobalWeightsWriter):
+    """A partial mock of the
+    ``xain-fl.coordinator.store.S3GlobalWeightsWriter`` class that
+    does not perform any IO. Instead, data is stored in memory.
 
     """
 
@@ -77,7 +78,8 @@ def __init__(self):
             endpoint="endpoint",
             access_key_id="access_key_id",
             secret_access_key="secret_access_key",
-            bucket="bucket",
+            global_weights_bucket="bucket",
+            local_weights_bucket="bucket",
         )
         self.s3 = MockS3Resource()