From 5283105a4a2ecec576bc3f6011766f26ce0ff44d Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 20 Dec 2022 14:46:27 -0500 Subject: [PATCH 1/9] Add initial Arm Virtual Hardware (AVH) workflow with end to end test --- .github/.wordlist.txt | 1 + .github/workflows/avh.yml | 96 +++++++ requirements.txt | 2 + scripts/tests/avh/README.md | 38 +++ .../avh/helpers/avh_chiptool_instance.py | 78 ++++++ scripts/tests/avh/helpers/avh_client.py | 89 +++++++ scripts/tests/avh/helpers/avh_instance.py | 240 ++++++++++++++++++ .../avh/helpers/avh_lighting_app_instance.py | 93 +++++++ scripts/tests/avh/requirements.txt | 3 + scripts/tests/avh/test_lighting_app.py | 165 ++++++++++++ 10 files changed, 805 insertions(+) create mode 100644 .github/workflows/avh.yml create mode 100644 requirements.txt create mode 100644 scripts/tests/avh/README.md create mode 100644 scripts/tests/avh/helpers/avh_chiptool_instance.py create mode 100644 scripts/tests/avh/helpers/avh_client.py create mode 100644 scripts/tests/avh/helpers/avh_instance.py create mode 100644 scripts/tests/avh/helpers/avh_lighting_app_instance.py create mode 100644 scripts/tests/avh/requirements.txt create mode 100644 scripts/tests/avh/test_lighting_app.py diff --git a/.github/.wordlist.txt b/.github/.wordlist.txt index cb8e31e28b2d23..8eea5371dd9e26 100644 --- a/.github/.wordlist.txt +++ b/.github/.wordlist.txt @@ -116,6 +116,7 @@ autogenerated automake autotools avahi +AVH avL AwaitNextAction AXXXF diff --git a/.github/workflows/avh.yml b/.github/workflows/avh.yml new file mode 100644 index 00000000000000..78fb50385a6060 --- /dev/null +++ b/.github/workflows/avh.yml @@ -0,0 +1,96 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Arm Virtual Hardware + +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }}-${{ (github.event_name == 'pull_request' && github.event.number) || (github.event_name == 'workflow_dispatch' && github.run_number) || github.sha }} + cancel-in-progress: false + +jobs: + arm_crosscompile: + name: Linux ARM Cross compile + timeout-minutes: 70 + + runs-on: ubuntu-latest + if: github.actor != 'restyled-io[bot]' + + container: + image: ghcr.io/project-chip/chip-build-crosscompile:1 + volumes: + - "/tmp/bloat_reports:/tmp/bloat_reports" + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Checkout submodules & Bootstrap + uses: ./.github/actions/checkout-submodules-and-bootstrap + with: + platform: linux + + - name: Build Samples + timeout-minutes: 45 + run: | + ./scripts/run_in_build_env.sh \ + "./scripts/build/build_examples.py \ + --target linux-arm64-chip-tool-ipv6only-mbedtls-clang-minmdns-verbose \ + --target linux-arm64-light-ipv6only-mbedtls-clang-minmdns-verbose \ + build \ + " + + - name: Upload built samples + uses: actions/upload-artifact@v3 + with: + name: arm_crosscompiled_samples + path: | + out/linux-arm64-chip-tool-ipv6only-mbedtls-clang-minmdns-verbose/chip-tool + out/linux-arm64-light-ipv6only-mbedtls-clang-minmdns-verbose/chip-lighting-app + + arm_e2e_tests: + name: Arm Virtual Hardware End to end tests + timeout-minutes: 10 + + runs-on: ubuntu-latest + + env: + AVH_API_TOKEN: ${{ secrets.AVH_API_TOKEN }} + AVH_API_ENDPOINT: https://csa.app.avh.arm.com/api + AVH_PROJECT_NAME: "${{ github.workflow }} #${{ github.run_number }} - End to end tests" + + needs: arm_crosscompile + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Downloads Cross-compiled samples + uses: actions/download-artifact@v3 + with: + name: arm_crosscompiled_samples + path: scripts/tests/avh/out + + - name: Install Python dependencies + run: | + pip3 install -r scripts/tests/avh/requirements.txt + + - name: Run end to end test + run: | + cd scripts/tests/avh + python3 -u -m unittest test_lighting_app.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000000000..a001fb8dec23db --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +avh-api +paramiko diff --git a/scripts/tests/avh/README.md b/scripts/tests/avh/README.md new file mode 100644 index 00000000000000..c2d9dd83bc6771 --- /dev/null +++ b/scripts/tests/avh/README.md @@ -0,0 +1,38 @@ +# Arm Virtual Hardware (AVH) based tests + +This folder contains end to end tests that use the +[Arm Virtual Hardware (AVH)](https://www.arm.com/products/development-tools/simulation/virtual-hardware) +service. + +The tests require the `AVH_API_TOKEN` environment variable is set with the value +from `AVH -> Profile -> API -> API Token`. + +## Current tests + +- [`test_lighting_app.py`](test_lighting_app.py) + - This test uses two virtual Raspberry Pi Model 4 boards running Ubuntu + Server 22.04 and pre-built `chip-tool` and `chip-lighting-app` binaries + (`linux-arm64`), and tests commissioning and control over BLE and Wi-Fi + using the virtual Bluetooth and Wi-Fi network features of AVH. + +## Running the tests + +1. Install dependencies + +``` +pip3 install -r requirements.txt +``` + +2. Set AVH_API_TOKEN` environment variable + +``` +export AVH_API_TOKEN= +``` + +3. Place cross-compiled `chip-tool` and `lighting-app` binaries in `out` folder + +4. Run + +``` +python3 -u -m unittest test_lighting_app.py +``` diff --git a/scripts/tests/avh/helpers/avh_chiptool_instance.py b/scripts/tests/avh/helpers/avh_chiptool_instance.py new file mode 100644 index 00000000000000..ca13a9c4f8dcf3 --- /dev/null +++ b/scripts/tests/avh/helpers/avh_chiptool_instance.py @@ -0,0 +1,78 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime + +from .avh_instance import AvhInstance + +APPLICATION_BINARY = "chip-tool" + + +class AvhChiptoolInstance(AvhInstance): + def __init__(self, avh_client, name, application_binary_path): + super().__init__(avh_client, name) + + self.application_binary_path = application_binary_path + + def upload_application_binary(self): + super().upload_application_binary( + self.application_binary_path, APPLICATION_BINARY + ) + + def configure_system(self): + self.log_in_to_console() + + # set current date and time + self.console_exec_command("sudo timedatectl set-ntp false", timeout=300) + self.console_exec_command( + f"sudo timedatectl set-time '{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}'", + timeout=300, + ) + self.console_exec_command("sudo timedatectl set-ntp true", timeout=300) + + # install network manager + self.console_exec_command("sudo apt-get update", timeout=300) + self.console_exec_command( + "sudo apt-get -y install network-manager", timeout=300 + ) + + # connect Wi-Fi to the Arm ssid + self.console_exec_command("sudo nmcli r wifi on") + self.console_exec_command("sudo nmcli d wifi connect Arm password password") + + # disable eth0 + self.console_exec_command("sudo nmcli dev set eth0 managed no") + self.console_exec_command("sudo ip link set dev eth0 down") + + def pairing_ble_wifi(self, node_id, ssid, password, pin_code, discriminator): + output = self.console_exec_command( + f"./{APPLICATION_BINARY} pairing ble-wifi {node_id} {ssid} {password} {pin_code} {discriminator}", + timeout=120.0, + ) + + return output + + def on(self, node_id): + output = self.console_exec_command( + f"./{APPLICATION_BINARY} onoff on {node_id} 1", timeout=30.0 + ) + + return output + + def off(self, node_id): + output = self.console_exec_command( + f"./{APPLICATION_BINARY} onoff off {node_id} 1", timeout=30.0 + ) + + return output diff --git a/scripts/tests/avh/helpers/avh_client.py b/scripts/tests/avh/helpers/avh_client.py new file mode 100644 index 00000000000000..e51b419b75dcfa --- /dev/null +++ b/scripts/tests/avh/helpers/avh_client.py @@ -0,0 +1,89 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +from avh_api import ApiClient as AvhApiClient +from avh_api import Configuration as AvhApiConfiguration +from avh_api.api.arm_api import ArmApi as AvhApi +from avh_api.model.project_key import ProjectKey as AvhProjectKey + + +class AvhClient: + def __init__(self, api_token, api_endpoint=None): + avh_api_config = AvhApiConfiguration(host=api_endpoint) + self.avh_api_client = AvhApiClient(avh_api_config) + + self.avh_api = AvhApi(self.avh_api_client) + + avh_api_config.access_token = self.avh_api.v1_auth_login( + {"api_token": api_token} + ).token + + self.default_project_id = self.avh_api.v1_get_projects()[0]["id"] + + def create_project(self, name, num_cores): + return self.avh_api.v1_create_project( + { + "id": str(uuid.uuid4()), + "name": name, + "settings": { + "internet_access": True, + "dhcp": True, + }, + "quotas": {"cores": num_cores}, + } + )["id"] + + def delete_project(self, id): + self.avh_api.v1_delete_project(id) + + def create_instance(self, name, flavor, os, osbuild): + return self.avh_api.v1_create_instance( + { + "name": name, + "project": self.default_project_id, + "flavor": flavor, + "os": os, + "osbuild": osbuild, + } + )["id"] + + def instance_state(self, instance_id): + return str(self.avh_api.v1_get_instance_state(instance_id)) + + def instance_console_log(self, instance_id): + return self.avh_api.v1_get_instance_console_log(instance_id) + + def instance_quick_connect_command(self, instance_id): + return self.avh_api.v1_get_instance_quick_connect_command(instance_id) + + def create_ssh_project_key(self, label, key): + return self.avh_api.v1_add_project_key( + self.default_project_id, + AvhProjectKey(kind="ssh", key=key, label=label), + )["identifier"] + + def instance_console_url(self, instance_id): + return self.avh_api.v1_get_instance_console(instance_id).url + + def delete_ssh_project_key(self, key_id): + self.avh_api.v1_remove_project_key(self.default_project_id, key_id) + + def delete_instance(self, instance_id): + self.avh_api.v1_delete_instance(instance_id) + + def close(self): + self.avh_api_client.rest_client.pool_manager.clear() + self.avh_api_client.close() diff --git a/scripts/tests/avh/helpers/avh_instance.py b/scripts/tests/avh/helpers/avh_instance.py new file mode 100644 index 00000000000000..07747daee000aa --- /dev/null +++ b/scripts/tests/avh/helpers/avh_instance.py @@ -0,0 +1,240 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +import avh_api +import paramiko +import websocket + +DEFAULT_INSTANCE_FLAVOR = "rpi4b" +DEFAULT_INSTANCE_OS = "Ubuntu Server" +DEFAULT_INSTANCE_OS_VERSION = "22.04.1" + +DEFAULT_OS_BOOTED_OUTPUT = "-----END SSH HOST KEY KEYS-----" + +DEFAULT_SSH_USERNAME = "pi" +DEFAULT_SSH_PASSWORD = "raspberry" + + +class AvhInstance: + def __init__( + self, + avh_client, + name, + flavor=DEFAULT_INSTANCE_FLAVOR, + os=DEFAULT_INSTANCE_OS, + os_version=DEFAULT_INSTANCE_OS_VERSION, + username=DEFAULT_SSH_USERNAME, + password=DEFAULT_SSH_PASSWORD, + ): + self.avh_client = avh_client + self.name = name + self.flavor = flavor + self.os = os + self.os_version = os_version + self.username = username + self.password = password + self.instance_id = None + self.console = None + self.ssh_pkey = None + self.ssh_key_id = None + + def create(self): + self.instance_id = self.avh_client.create_instance( + name=self.name, flavor=self.flavor, os=self.os_version, osbuild=self.os + ) + + def wait_for_state_on(self, timeout=240): + start_time = time.monotonic() + + while True: + instance_state = self.avh_client.instance_state(self.instance_id) + + if instance_state == "on": + break + elif instance_state == "error": + raise Exception("VM entered error state") + elif (time.monotonic() - start_time) > timeout: + raise Exception( + f"Timed out waiting for state 'on' for instance id {self.instance_id}, current state is '{instance_state}'" + ) + + time.sleep(1.0) + + self.console = websocket.create_connection( + self.avh_client.instance_console_url(self.instance_id) + ) + + def wait_for_os_boot(self, booted_output=DEFAULT_OS_BOOTED_OUTPUT, timeout=240): + start_time = time.monotonic() + + while True: + console_log = self.avh_client.instance_console_log(self.instance_id) + + if booted_output in console_log: + break + elif (time.monotonic() - start_time) > timeout: + raise Exception( + f"Timed out waiting for OS to boot for instance id {self.instance_id}", + f"Did not find {booted_output} in {console_log}", + ) + + time.sleep(1.0) + + def ssh_client(self, timeout=30): + if self.ssh_pkey is None: + self.ssh_pkey = paramiko.ecdsakey.ECDSAKey.generate() + + self.ssh_key_id = self.avh_client.create_ssh_project_key( + self.name, f"{self.ssh_pkey.get_name()} {self.ssh_pkey.get_base64()}" + ) + + instance_quick_connect_command = self.avh_client.instance_quick_connect_command( + self.instance_id + ) + + split_instance_quick_connect_command = instance_quick_connect_command.split() + proxy_username = split_instance_quick_connect_command[-2].split("@")[-2] + proxy_hostname = split_instance_quick_connect_command[-2].split("@")[-1] + instance_ip = split_instance_quick_connect_command[-1].split("@")[-1] + + ssh_proxy_client = paramiko.SSHClient() + ssh_proxy_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + ssh_client = paramiko.SSHClient() + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + + ssh_proxy_client.connect( + hostname=proxy_hostname, + username=proxy_username, + pkey=self.ssh_pkey, + look_for_keys=False, + timeout=timeout, + ) + + try: + proxy_sock = ssh_proxy_client.get_transport().open_channel( + kind="direct-tcpip", + dest_addr=(instance_ip, 22), + src_addr=("", 0), + timeout=timeout, + ) + + ssh_client.connect( + hostname=instance_ip, + username=self.username, + password=self.password, + sock=proxy_sock, + timeout=timeout, + look_for_keys=False, + ) + except Exception: + raise Exception( + f"Failled to connect to {instance_ip} via SSH proxy {proxy_username}@{proxy_hostname}" + ) + + return ssh_client + + def delete(self): + if self.console is not None: + self.console.close() + + self.console = None + + if self.ssh_key_id is not None: + self.avh_client.delete_ssh_project_key(self.ssh_key_id) + + self.ssh_key_id = None + + if self.instance_id is not None: + self.avh_client.delete_instance(self.instance_id) + + def wait_for_state_deleted(self, timeout=60): + if self.instance_id is None: + return + + start_time = time.monotonic() + + while True: + try: + self.avh_client.instance_state(self.instance_id) + except avh_api.exceptions.NotFoundException: + break + + if (time.monotonic() - start_time) > timeout: + raise Exception( + f"Timedout waiting for instance id {self.instance_id} to be deleted" + ) + + time.sleep(1.0) + + self.instance_id = None + + def upload_application_binary(self, local_path, remote_path): + ssh_client = self.ssh_client() + + stfp_client = ssh_client.open_sftp() + stfp_client.put(local_path, remote_path) + stfp_client.close() + + ssh_client.exec_command(f"chmod +x {remote_path}") + ssh_client.close() + + def wait_for_console_output(self, expected_output, timeout=10.0): + self.console.settimeout(1.0) + + start_time = time.monotonic() + + output = b"" + while True: + if (time.monotonic() - start_time) > timeout: + raise Exception( + f"Timed out waiting for {expected_output} in console output" + f"Current output is {output}" + ) + + try: + output += self.console.recv() + except websocket.WebSocketTimeoutException: + pass + + if expected_output in output: + break + + return output + + def wait_for_console_prompt(self, timeout=10.0): + return self.wait_for_console_output(b"$ ", timeout) + + def log_in_to_console(self): + self.console.recv() # flush input + self.console.send("\n") + + self.wait_for_console_output(b"login: ") + self.console.send(f"{self.username}\n") + + self.wait_for_console_output(b"Password: ") + self.console.send(f"{self.password}\n") + + self.wait_for_console_prompt() + + def console_exec_command(self, command, timeout=10.0): + self.console.send("\03") # CTRL-C + self.wait_for_console_prompt() + self.console.send(f"{command}\n") + + output = self.wait_for_console_prompt(timeout) + + return output diff --git a/scripts/tests/avh/helpers/avh_lighting_app_instance.py b/scripts/tests/avh/helpers/avh_lighting_app_instance.py new file mode 100644 index 00000000000000..b31c9e444919c0 --- /dev/null +++ b/scripts/tests/avh/helpers/avh_lighting_app_instance.py @@ -0,0 +1,93 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime +import time + +import websocket + +from .avh_instance import AvhInstance + +APPLICATION_BINARY = "chip-lighting-app" + + +class AvhLightingAppInstance(AvhInstance): + def __init__(self, avh_client, name, application_binary_path): + super().__init__(avh_client, name) + + self.application_binary_path = application_binary_path + + def upload_application_binary(self): + super().upload_application_binary( + self.application_binary_path, APPLICATION_BINARY + ) + + def configure_system(self): + self.log_in_to_console() + + # set current date and time + self.console_exec_command("sudo timedatectl set-ntp false", timeout=300) + self.console_exec_command( + f"sudo timedatectl set-time '{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}'", + timeout=300, + ) + self.console_exec_command("sudo timedatectl set-ntp true", timeout=300) + + # install network manager + self.console_exec_command("sudo apt-get update", timeout=300) + self.console_exec_command( + "sudo apt-get -y install network-manager", timeout=300 + ) + + # remove the Wi-Fi configuration and disable network manager on the Wi-Fi interface + self.console_exec_command("sudo nmcli connection delete Arm") + self.console_exec_command("sudo nmcli dev set wlan0 managed no") + + # set wlan0 ipv6 to have generated address based on EUI64 + self.console_exec_command("sudo sysctl net.ipv6.conf.wlan0.addr_gen_mode=0") + + # patch and restart wpa_supplication DBus + self.console_exec_command( + 'sudo sed -i "s/wpa_supplicant -u -s -O/wpa_supplicant -u -s -i wlan0 -O/i" /etc/systemd/system/dbus-fi.w1.wpa_supplicant1.service' + ) + self.console_exec_command("sudo systemctl restart wpa_supplicant.service") + self.console_exec_command("sudo systemctl daemon-reload") + + # disable eth0 + self.console_exec_command("sudo nmcli dev set eth0 managed no") + self.console_exec_command("sudo ip link set dev eth0 down") + + def start_application(self): + self.console.send(f"./{APPLICATION_BINARY} --wifi\n") + + def stop_application(self): + self.console.send("\03") # CTRL-C + super().wait_for_console_prompt() + + def get_application_output(self, timeout=5.0): + self.console.settimeout(1.0) + + start_time = time.monotonic() + output = b"" + + while True: + if (time.monotonic() - start_time) > timeout: + break + + try: + output += self.console.recv() + except websocket.WebSocketTimeoutException: + pass + + return output diff --git a/scripts/tests/avh/requirements.txt b/scripts/tests/avh/requirements.txt new file mode 100644 index 00000000000000..d20fee8f1bebc2 --- /dev/null +++ b/scripts/tests/avh/requirements.txt @@ -0,0 +1,3 @@ +avh-api==1.0.5 +paramiko==3.1.0 +websocket-client==1.5.1 diff --git a/scripts/tests/avh/test_lighting_app.py b/scripts/tests/avh/test_lighting_app.py new file mode 100644 index 00000000000000..68d75d2dae0a67 --- /dev/null +++ b/scripts/tests/avh/test_lighting_app.py @@ -0,0 +1,165 @@ +# Copyright (c) 2022 Project CHIP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +import unittest + +from helpers.avh_chiptool_instance import AvhChiptoolInstance +from helpers.avh_client import AvhClient +from helpers.avh_lighting_app_instance import AvhLightingAppInstance + +INSTANCE_NAME_PREFIX = "matter-test-" + +TEST_NODE_ID = 17 +TEST_WIFI_SSID = "Arm" +TEST_WIFI_PASSWORD = "password" +TEST_PIN_CODE = 20202021 +TEST_DISCRIMINATOR = 3840 + +if "AVH_API_TOKEN" not in os.environ: + raise Exception("Please set AVH_API_TOKEN environment variable value") + + +class TestLightingApp(unittest.TestCase): + def setUp(self): + self.logger = logging.getLogger(__name__) + self.logger.setLevel(logging.INFO) + stdout_logger_handler = logging.StreamHandler(sys.stdout) + stdout_logger_handler.setFormatter( + logging.Formatter( + fmt="%(asctime)s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + self.logger.addHandler(stdout_logger_handler) + + self.avh_client = AvhClient( + api_token=os.environ["AVH_API_TOKEN"], + api_endpoint=os.environ["AVH_API_ENDPOINT"] + if "AVH_API_ENDPOINT" in os.environ + else None, + ) + + if "AVH_PROJECT_NAME" in os.environ: + self.logger.info(f"creating project '{os.environ['AVH_PROJECT_NAME']}' ...") + project_id = self.avh_client.create_project( + name=os.environ["AVH_PROJECT_NAME"], num_cores=8 + ) + self.avh_client.default_project_id = project_id + + self.chip_tool_instance = AvhChiptoolInstance( + self.avh_client, + name=INSTANCE_NAME_PREFIX + "chip-tool", + application_binary_path="out/linux-arm64-chip-tool-ipv6only-mbedtls-clang-minmdns-verbose/chip-tool", + ) + + self.lighting_app_instance = AvhLightingAppInstance( + self.avh_client, + name=INSTANCE_NAME_PREFIX + "lighting-app", + application_binary_path="out/linux-arm64-light-ipv6only-mbedtls-clang-minmdns-verbose/chip-lighting-app", + ) + + self.logger.info("creating instances ...") + self.addCleanup(self.cleanupInstances) + + self.chip_tool_instance.create() + self.lighting_app_instance.create() + + self.logger.info( + f"\tchip-tool instance id = {self.chip_tool_instance.instance_id}" + ) + self.logger.info( + f"\tlighting-app instance id = {self.lighting_app_instance.instance_id}" + ) + + self.chip_tool_instance.wait_for_state_on() + self.lighting_app_instance.wait_for_state_on() + + self.logger.info("waiting for OS to boot ...") + self.chip_tool_instance.wait_for_os_boot() + self.lighting_app_instance.wait_for_os_boot() + + self.logger.info("uploading application binaries ...") + self.chip_tool_instance.upload_application_binary() + self.lighting_app_instance.upload_application_binary() + + self.logger.info("configuring systems ...") + self.chip_tool_instance.configure_system() + self.lighting_app_instance.configure_system() + + def test_commissioning_and_control(self): + self.logger.info("starting chip-lighting-app ...") + self.lighting_app_instance.start_application() + + lighting_app_start_output = self.lighting_app_instance.get_application_output() + self.assertIn(b"Server Listening...", lighting_app_start_output) + + self.logger.info("commissioning with chip-tool using BLE pairing ...") + chip_tool_commissioning_output = self.chip_tool_instance.pairing_ble_wifi( + TEST_NODE_ID, + TEST_WIFI_SSID, + TEST_WIFI_PASSWORD, + TEST_PIN_CODE, + TEST_DISCRIMINATOR, + ) + + self.assertIn( + b"Device commissioning completed with success", + chip_tool_commissioning_output, + ) + + lighting_app_commissioning_output = ( + self.lighting_app_instance.get_application_output() + ) + + self.assertIn( + b"Commissioning completed successfully", lighting_app_commissioning_output + ) + + self.logger.info("turning light on with chip-tool ...") + chip_tool_on_output = self.chip_tool_instance.on(TEST_NODE_ID) + self.assertIn(b"Received Command Response Status for", chip_tool_on_output) + + lighting_app_on_output = self.lighting_app_instance.get_application_output() + self.assertIn(b"Toggle ep1 on/off from state 0 to 1", lighting_app_on_output) + + self.logger.info("turning light off with chip-tool ...") + chip_tool_off_output = self.chip_tool_instance.off(TEST_NODE_ID) + self.assertIn(b"Received Command Response Status for", chip_tool_off_output) + + lighting_app_off_output = self.lighting_app_instance.get_application_output() + self.assertIn(b"Toggle ep1 on/off from state 1 to 0", lighting_app_off_output) + + self.logger.info("stopping chip-lighting-app ...") + self.lighting_app_instance.stop_application() + + def cleanupInstances(self): + self.logger.info("deleting instances ...") + self.chip_tool_instance.delete() + self.lighting_app_instance.delete() + + self.chip_tool_instance.wait_for_state_deleted() + self.lighting_app_instance.wait_for_state_deleted() + + if "AVH_PROJECT_NAME" in os.environ: + self.logger.info("deleting project ...") + self.avh_client.delete_project(self.avh_client.default_project_id) + + self.avh_client.close() + + +if __name__ == "__main__": + unittest.main() From 59aad4674681efe1b85c678b569e43eb8abd8c23 Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 14 Nov 2023 12:47:49 -0500 Subject: [PATCH 2/9] Remove leftover top level requirements.txt --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a001fb8dec23db..00000000000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -avh-api -paramiko From d0853488bf8f2f0298608848d9c3de2d525e1065 Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 14 Nov 2023 12:49:22 -0500 Subject: [PATCH 3/9] Set copyright year to 2003 --- .github/workflows/avh.yml | 2 +- scripts/tests/avh/helpers/avh_chiptool_instance.py | 2 +- scripts/tests/avh/helpers/avh_client.py | 2 +- scripts/tests/avh/helpers/avh_instance.py | 2 +- scripts/tests/avh/helpers/avh_lighting_app_instance.py | 2 +- scripts/tests/avh/test_lighting_app.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/avh.yml b/.github/workflows/avh.yml index 78fb50385a6060..9ffb2a8595edfa 100644 --- a/.github/workflows/avh.yml +++ b/.github/workflows/avh.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Project CHIP Authors +# Copyright (c) 2023 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/tests/avh/helpers/avh_chiptool_instance.py b/scripts/tests/avh/helpers/avh_chiptool_instance.py index ca13a9c4f8dcf3..7a19062d20db09 100644 --- a/scripts/tests/avh/helpers/avh_chiptool_instance.py +++ b/scripts/tests/avh/helpers/avh_chiptool_instance.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Project CHIP Authors +# Copyright (c) 2023 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/tests/avh/helpers/avh_client.py b/scripts/tests/avh/helpers/avh_client.py index e51b419b75dcfa..e0efeab519bdc8 100644 --- a/scripts/tests/avh/helpers/avh_client.py +++ b/scripts/tests/avh/helpers/avh_client.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Project CHIP Authors +# Copyright (c) 2023 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/tests/avh/helpers/avh_instance.py b/scripts/tests/avh/helpers/avh_instance.py index 07747daee000aa..2457a06c36e407 100644 --- a/scripts/tests/avh/helpers/avh_instance.py +++ b/scripts/tests/avh/helpers/avh_instance.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Project CHIP Authors +# Copyright (c) 2023 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/tests/avh/helpers/avh_lighting_app_instance.py b/scripts/tests/avh/helpers/avh_lighting_app_instance.py index b31c9e444919c0..374434b09dafea 100644 --- a/scripts/tests/avh/helpers/avh_lighting_app_instance.py +++ b/scripts/tests/avh/helpers/avh_lighting_app_instance.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Project CHIP Authors +# Copyright (c) 2023 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/scripts/tests/avh/test_lighting_app.py b/scripts/tests/avh/test_lighting_app.py index 68d75d2dae0a67..c44f840696a161 100644 --- a/scripts/tests/avh/test_lighting_app.py +++ b/scripts/tests/avh/test_lighting_app.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 Project CHIP Authors +# Copyright (c) 2023 Project CHIP Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From c79dd77d0b1cb0fe8e937e5fe7e9f6b72602e513 Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 14 Nov 2023 13:05:17 -0500 Subject: [PATCH 4/9] Add _s suffix to timeout variable names --- .../avh/helpers/avh_chiptool_instance.py | 16 +++++----- scripts/tests/avh/helpers/avh_instance.py | 32 +++++++++---------- .../avh/helpers/avh_lighting_app_instance.py | 14 ++++---- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/scripts/tests/avh/helpers/avh_chiptool_instance.py b/scripts/tests/avh/helpers/avh_chiptool_instance.py index 7a19062d20db09..a7c55fd4ba41ea 100644 --- a/scripts/tests/avh/helpers/avh_chiptool_instance.py +++ b/scripts/tests/avh/helpers/avh_chiptool_instance.py @@ -34,17 +34,17 @@ def configure_system(self): self.log_in_to_console() # set current date and time - self.console_exec_command("sudo timedatectl set-ntp false", timeout=300) + self.console_exec_command("sudo timedatectl set-ntp false", timeout_s=300) self.console_exec_command( f"sudo timedatectl set-time '{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}'", - timeout=300, + timeout_s=300, ) - self.console_exec_command("sudo timedatectl set-ntp true", timeout=300) + self.console_exec_command("sudo timedatectl set-ntp true", timeout_s=300) # install network manager - self.console_exec_command("sudo apt-get update", timeout=300) + self.console_exec_command("sudo apt-get update", timeout_s=300) self.console_exec_command( - "sudo apt-get -y install network-manager", timeout=300 + "sudo apt-get -y install network-manager", timeout_s=300 ) # connect Wi-Fi to the Arm ssid @@ -58,21 +58,21 @@ def configure_system(self): def pairing_ble_wifi(self, node_id, ssid, password, pin_code, discriminator): output = self.console_exec_command( f"./{APPLICATION_BINARY} pairing ble-wifi {node_id} {ssid} {password} {pin_code} {discriminator}", - timeout=120.0, + timeout_s=120.0, ) return output def on(self, node_id): output = self.console_exec_command( - f"./{APPLICATION_BINARY} onoff on {node_id} 1", timeout=30.0 + f"./{APPLICATION_BINARY} onoff on {node_id} 1", timeout_s=30.0 ) return output def off(self, node_id): output = self.console_exec_command( - f"./{APPLICATION_BINARY} onoff off {node_id} 1", timeout=30.0 + f"./{APPLICATION_BINARY} onoff off {node_id} 1", timeout_s=30.0 ) return output diff --git a/scripts/tests/avh/helpers/avh_instance.py b/scripts/tests/avh/helpers/avh_instance.py index 2457a06c36e407..1b8556d9e0a573 100644 --- a/scripts/tests/avh/helpers/avh_instance.py +++ b/scripts/tests/avh/helpers/avh_instance.py @@ -56,7 +56,7 @@ def create(self): name=self.name, flavor=self.flavor, os=self.os_version, osbuild=self.os ) - def wait_for_state_on(self, timeout=240): + def wait_for_state_on(self, timeout_s=240): start_time = time.monotonic() while True: @@ -66,7 +66,7 @@ def wait_for_state_on(self, timeout=240): break elif instance_state == "error": raise Exception("VM entered error state") - elif (time.monotonic() - start_time) > timeout: + elif (time.monotonic() - start_time) > timeout_s: raise Exception( f"Timed out waiting for state 'on' for instance id {self.instance_id}, current state is '{instance_state}'" ) @@ -77,7 +77,7 @@ def wait_for_state_on(self, timeout=240): self.avh_client.instance_console_url(self.instance_id) ) - def wait_for_os_boot(self, booted_output=DEFAULT_OS_BOOTED_OUTPUT, timeout=240): + def wait_for_os_boot(self, booted_output=DEFAULT_OS_BOOTED_OUTPUT, timeout_s=240): start_time = time.monotonic() while True: @@ -85,7 +85,7 @@ def wait_for_os_boot(self, booted_output=DEFAULT_OS_BOOTED_OUTPUT, timeout=240): if booted_output in console_log: break - elif (time.monotonic() - start_time) > timeout: + elif (time.monotonic() - start_time) > timeout_s: raise Exception( f"Timed out waiting for OS to boot for instance id {self.instance_id}", f"Did not find {booted_output} in {console_log}", @@ -93,7 +93,7 @@ def wait_for_os_boot(self, booted_output=DEFAULT_OS_BOOTED_OUTPUT, timeout=240): time.sleep(1.0) - def ssh_client(self, timeout=30): + def ssh_client(self, timeout_s=30): if self.ssh_pkey is None: self.ssh_pkey = paramiko.ecdsakey.ECDSAKey.generate() @@ -121,7 +121,7 @@ def ssh_client(self, timeout=30): username=proxy_username, pkey=self.ssh_pkey, look_for_keys=False, - timeout=timeout, + timeout=timeout_s, ) try: @@ -129,7 +129,7 @@ def ssh_client(self, timeout=30): kind="direct-tcpip", dest_addr=(instance_ip, 22), src_addr=("", 0), - timeout=timeout, + timeout=timeout_s, ) ssh_client.connect( @@ -137,7 +137,7 @@ def ssh_client(self, timeout=30): username=self.username, password=self.password, sock=proxy_sock, - timeout=timeout, + timeout=timeout_s, look_for_keys=False, ) except Exception: @@ -161,7 +161,7 @@ def delete(self): if self.instance_id is not None: self.avh_client.delete_instance(self.instance_id) - def wait_for_state_deleted(self, timeout=60): + def wait_for_state_deleted(self, timeout_s=60): if self.instance_id is None: return @@ -173,7 +173,7 @@ def wait_for_state_deleted(self, timeout=60): except avh_api.exceptions.NotFoundException: break - if (time.monotonic() - start_time) > timeout: + if (time.monotonic() - start_time) > timeout_s: raise Exception( f"Timedout waiting for instance id {self.instance_id} to be deleted" ) @@ -192,14 +192,14 @@ def upload_application_binary(self, local_path, remote_path): ssh_client.exec_command(f"chmod +x {remote_path}") ssh_client.close() - def wait_for_console_output(self, expected_output, timeout=10.0): + def wait_for_console_output(self, expected_output, timeout_s=10.0): self.console.settimeout(1.0) start_time = time.monotonic() output = b"" while True: - if (time.monotonic() - start_time) > timeout: + if (time.monotonic() - start_time) > timeout_s: raise Exception( f"Timed out waiting for {expected_output} in console output" f"Current output is {output}" @@ -215,8 +215,8 @@ def wait_for_console_output(self, expected_output, timeout=10.0): return output - def wait_for_console_prompt(self, timeout=10.0): - return self.wait_for_console_output(b"$ ", timeout) + def wait_for_console_prompt(self, timeout_s=10.0): + return self.wait_for_console_output(b"$ ", timeout_s) def log_in_to_console(self): self.console.recv() # flush input @@ -230,11 +230,11 @@ def log_in_to_console(self): self.wait_for_console_prompt() - def console_exec_command(self, command, timeout=10.0): + def console_exec_command(self, command, timeout_s=10.0): self.console.send("\03") # CTRL-C self.wait_for_console_prompt() self.console.send(f"{command}\n") - output = self.wait_for_console_prompt(timeout) + output = self.wait_for_console_prompt(timeout_s) return output diff --git a/scripts/tests/avh/helpers/avh_lighting_app_instance.py b/scripts/tests/avh/helpers/avh_lighting_app_instance.py index 374434b09dafea..cc3468bb132039 100644 --- a/scripts/tests/avh/helpers/avh_lighting_app_instance.py +++ b/scripts/tests/avh/helpers/avh_lighting_app_instance.py @@ -37,17 +37,17 @@ def configure_system(self): self.log_in_to_console() # set current date and time - self.console_exec_command("sudo timedatectl set-ntp false", timeout=300) + self.console_exec_command("sudo timedatectl set-ntp false", timeout_s=300) self.console_exec_command( f"sudo timedatectl set-time '{datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}'", - timeout=300, + timeout_s=300, ) - self.console_exec_command("sudo timedatectl set-ntp true", timeout=300) + self.console_exec_command("sudo timedatectl set-ntp true", timeout_s=300) # install network manager - self.console_exec_command("sudo apt-get update", timeout=300) + self.console_exec_command("sudo apt-get update", timeout_s=300) self.console_exec_command( - "sudo apt-get -y install network-manager", timeout=300 + "sudo apt-get -y install network-manager", timeout_s=300 ) # remove the Wi-Fi configuration and disable network manager on the Wi-Fi interface @@ -75,14 +75,14 @@ def stop_application(self): self.console.send("\03") # CTRL-C super().wait_for_console_prompt() - def get_application_output(self, timeout=5.0): + def get_application_output(self, timeout_s=5.0): self.console.settimeout(1.0) start_time = time.monotonic() output = b"" while True: - if (time.monotonic() - start_time) > timeout: + if (time.monotonic() - start_time) > timeout_s: break try: From 30cbdf95f77df5293d36c46c523946d90ca799b3 Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 14 Nov 2023 13:06:28 -0500 Subject: [PATCH 5/9] Add comment on break --- scripts/tests/avh/helpers/avh_instance.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tests/avh/helpers/avh_instance.py b/scripts/tests/avh/helpers/avh_instance.py index 1b8556d9e0a573..8ba62a87e6336a 100644 --- a/scripts/tests/avh/helpers/avh_instance.py +++ b/scripts/tests/avh/helpers/avh_instance.py @@ -171,6 +171,7 @@ def wait_for_state_deleted(self, timeout_s=60): try: self.avh_client.instance_state(self.instance_id) except avh_api.exceptions.NotFoundException: + # Not Found implies deleted break if (time.monotonic() - start_time) > timeout_s: From 4fcb0d03a9de64ce5e94227a7e3fa16e91ff0a28 Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 14 Nov 2023 13:16:26 -0500 Subject: [PATCH 6/9] Add comment on timeout exception --- scripts/tests/avh/helpers/avh_instance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/tests/avh/helpers/avh_instance.py b/scripts/tests/avh/helpers/avh_instance.py index 8ba62a87e6336a..30b827bcc4484f 100644 --- a/scripts/tests/avh/helpers/avh_instance.py +++ b/scripts/tests/avh/helpers/avh_instance.py @@ -209,6 +209,8 @@ def wait_for_console_output(self, expected_output, timeout_s=10.0): try: output += self.console.recv() except websocket.WebSocketTimeoutException: + # ignore timeout exceptions as the AVH instance might not produce console output + # while processing commands, the timeout will be managed by the while loop pass if expected_output in output: From 116cfbc67c3f9e8dc04b685c5f5a89a97089472d Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Tue, 14 Nov 2023 13:18:54 -0500 Subject: [PATCH 7/9] Use Python virtual environment --- .github/workflows/avh.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/avh.yml b/.github/workflows/avh.yml index 9ffb2a8595edfa..4a8ccc81e9a7d4 100644 --- a/.github/workflows/avh.yml +++ b/.github/workflows/avh.yml @@ -88,9 +88,12 @@ jobs: - name: Install Python dependencies run: | + python3 -m venv venv + source venv/bin/activate pip3 install -r scripts/tests/avh/requirements.txt - name: Run end to end test run: | + source venv/bin/activate cd scripts/tests/avh python3 -u -m unittest test_lighting_app.py From 4d516e4274df564624ec13314b1d42d29b62cf5f Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Wed, 15 Nov 2023 14:06:31 -0500 Subject: [PATCH 8/9] Apply restyler patch --- scripts/tests/avh/helpers/avh_lighting_app_instance.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/avh/helpers/avh_lighting_app_instance.py b/scripts/tests/avh/helpers/avh_lighting_app_instance.py index cc3468bb132039..07a46effd94b1b 100644 --- a/scripts/tests/avh/helpers/avh_lighting_app_instance.py +++ b/scripts/tests/avh/helpers/avh_lighting_app_instance.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from datetime import datetime import time +from datetime import datetime import websocket From 710e19a3046e6ed9e03b98e68703561349fb8a64 Mon Sep 17 00:00:00 2001 From: Sandeep Mistry Date: Fri, 17 Nov 2023 13:03:03 -0500 Subject: [PATCH 9/9] Add check for empty AVH_API_TOKEN env. var. --- scripts/tests/avh/test_lighting_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/avh/test_lighting_app.py b/scripts/tests/avh/test_lighting_app.py index c44f840696a161..a37c692f39bdec 100644 --- a/scripts/tests/avh/test_lighting_app.py +++ b/scripts/tests/avh/test_lighting_app.py @@ -29,7 +29,7 @@ TEST_PIN_CODE = 20202021 TEST_DISCRIMINATOR = 3840 -if "AVH_API_TOKEN" not in os.environ: +if "AVH_API_TOKEN" not in os.environ or len(os.environ["AVH_API_TOKEN"]) == 0: raise Exception("Please set AVH_API_TOKEN environment variable value")