IS-12 Command Handling Fix

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Install Node.js
         uses: actions/setup-node@v1
         with:
-          node-version: '12.x'
+          node-version: '16.x'
       - name: Setup for python lint
         run: pip install flake8
       - name: Lint python

Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:bionic
+FROM ubuntu:focal
 
 WORKDIR /home/nmos-testing
 ADD . .
@@ -7,9 +7,9 @@ ADD .git .git
 RUN apt-get update \
     && export DEBIAN_FRONTEND=noninteractive \
     && apt-get install -y wget \
-    && wget https://deb.nodesource.com/setup_14.x \
-    && chmod 755 setup_14.x \
-    && /home/nmos-testing/setup_14.x \
+    && wget https://deb.nodesource.com/setup_16.x \
+    && chmod 755 setup_16.x \
+    && /home/nmos-testing/setup_16.x \
     && apt-get install -y --no-install-recommends \
     gcc openssl libssl-dev wget ca-certificates avahi-daemon avahi-utils libnss-mdns libavahi-compat-libdnssd-dev \
     python3 python3-pip python3-dev nodejs \
@@ -26,7 +26,7 @@ RUN apt-get update \
     && rm v3.0.7.tar.gz \
     && npm config set unsafe-perm true \
     && npm install -g AMWA-TV/sdpoker#v0.3.0 \
-    && rm /home/nmos-testing/setup_14.x \
+    && rm /home/nmos-testing/setup_16.x \
     && apt-get remove -y wget \
     && apt-get clean -y --no-install-recommends \
     && apt-get autoclean -y --no-install-recommends \

docs/1.1. Installation - Local.md

@@ -4,7 +4,7 @@
 
 Please ensure that the following dependencies are installed on your system first.
 
-- Python 3.6 or higher, including the 'pip' package manager
+- Python 3.8 or higher, including the 'pip' package manager
 - Git
 - [testssl.sh](https://testssl.sh) (required for BCP-003-01 testing, see our [README](../testssl/README.md) for instructions)
 - [OpenSSL](https://www.openssl.org/) (required for BCP-003-01 OCSP testing)

nmostesting/Config.py

@@ -19,6 +19,9 @@
 # Please consult the documentation for instructions on how to adjust these values for common testing setups including
 # unicast DNS-SD and HTTPS testing.
 
+# Number of seconds to wait after starting the mock DNS server, authorization server, etc. before running tests.
+# This gives the API or client under test a chance to use these services before any test case is run.
+MOCK_SERVICES_WARM_UP_DELAY = 0
 
 # Enable or disable DNS-SD advertisements. Browsing is always permitted.
 # The IS-04 Node tests create a mock registry on the network unless the `ENABLE_DNS_SD` parameter is set to `False`.

nmostesting/IS12Utils.py

@@ -151,6 +151,7 @@ def __init__(self, url, spec_path, spec_branch):
         self.ROOT_BLOCK_OID = 1
         self.ncp_websocket = None
         self.command_handle = 0
+        self.expect_notifications = False
         self.notifications = []
 
     def load_is12_schemas(self, spec_path):
@@ -230,47 +231,50 @@ def send_command(self, test, command_json):
 
         self.ncp_websocket.send(json.dumps(command_json))
 
-        # Wait for server to respond
+        results = []
         start_time = time.time()
         while time.time() < start_time + WS_MESSAGE_TIMEOUT:
-            if self.ncp_websocket.is_messages_received():
+            if not self.ncp_websocket.is_messages_received():
+                time.sleep(0.2)
+                continue
+
+            # find the response to our request
+            for message in self.ncp_websocket.get_messages():
+                parsed_message = json.loads(message)
+
+                if self.message_type_to_schema_name(parsed_message.get("messageType")):
+                    self.validate_is12_schema(
+                        test,
+                        parsed_message,
+                        self.message_type_to_schema_name(parsed_message["messageType"]),
+                        context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ")
+                else:
+                    raise NMOSTestException(test.FAIL("Unrecognised message type: " + parsed_message.get("messageType"),
+                                                      "https://specs.amwa.tv/is-12/branches/{}"
+                                                      "/docs/Protocol_messaging.html#command-message-type"
+                                                      .format(self.spec_branch)))
+
+                if parsed_message["messageType"] == MessageTypes.CommandResponse:
+                    responses = parsed_message["responses"]
+                    for response in responses:
+                        if response["handle"] == command_handle:
+                            if response["result"]["status"] != NcMethodStatus.OK:
+                                raise NMOSTestException(test.FAIL(response["result"]))
+                            results.append(response)
+                if parsed_message["messageType"] == MessageTypes.SubscriptionResponse:
+                    results.append(parsed_message["subscriptions"])
+                if parsed_message["messageType"] == MessageTypes.Notification:
+                    self.notifications += parsed_message["notifications"]
+                if parsed_message["messageType"] == MessageTypes.Error:
+                    raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}"
+                                                      "/docs/Protocol_messaging.html#error-messages"
+                                                      .format(self.spec_branch)))
+
+            if not self.expect_notifications and len(results) != 0:
+                break
+            if self.expect_notifications and len(results) != 0 and len(self.notifications) != 0:
                 break
-            time.sleep(0.2)
-
-        messages = self.ncp_websocket.get_messages()
 
-        results = []
-        # find the response to our request
-        for message in messages:
-            parsed_message = json.loads(message)
-
-            if self.message_type_to_schema_name(parsed_message["messageType"]):
-                self.validate_is12_schema(
-                    test,
-                    parsed_message,
-                    self.message_type_to_schema_name(parsed_message["messageType"]),
-                    context=self.message_type_to_schema_name(parsed_message["messageType"]) + ": ")
-            else:
-                raise NMOSTestException(test.FAIL("Unrecognised message type: " + parsed_message["messageType"],
-                                                  "https://specs.amwa.tv/is-12/branches/{}"
-                                                  "/docs/Protocol_messaging.html#command-message-type"
-                                                  .format(self.spec_branch)))
-
-            if parsed_message["messageType"] == MessageTypes.CommandResponse:
-                responses = parsed_message["responses"]
-                for response in responses:
-                    if response["handle"] == command_handle:
-                        if response["result"]["status"] != NcMethodStatus.OK:
-                            raise NMOSTestException(test.FAIL(response["result"]))
-                        results.append(response)
-            if parsed_message["messageType"] == MessageTypes.SubscriptionResponse:
-                results.append(parsed_message["subscriptions"])
-            if parsed_message["messageType"] == MessageTypes.Notification:
-                self.notifications += parsed_message["notifications"]
-            if parsed_message["messageType"] == MessageTypes.Error:
-                raise NMOSTestException(test.FAIL(parsed_message, "https://specs.amwa.tv/is-12/branches/{}"
-                                                  "/docs/Protocol_messaging.html#error-messages"
-                                                  .format(self.spec_branch)))
         if len(results) == 0:
             raise NMOSTestException(test.FAIL("No Message Response received.",
                                               "https://specs.amwa.tv/is-12/branches/{}"
@@ -285,9 +289,13 @@ def send_command(self, test, command_json):
     def get_notifications(self):
         return self.notifications
 
-    def reset_notifications(self):
+    def start_logging_notifications(self):
+        self.expect_notifications = True
         self.notifications = []
 
+    def stop_logging_notifications(self):
+        self.expect_notifications = False
+
     def create_command_JSON(self, oid, method_id, arguments):
         """for sending over websocket"""
         self.command_handle += 1

nmostesting/NMOSTesting.py

@@ -1189,6 +1189,13 @@ def main(args):
     print(" * Testing tool running on 'http://{}:{}'. Version '{}'"
           .format(get_default_ip(), core_app.config['PORT'], TOOL_VERSION))
 
+    # Give an API or client that is already running a chance to use the mock services
+    # before running any test cases
+    if CONFIG.MOCK_SERVICES_WARM_UP_DELAY:
+        print(" * Waiting for {} seconds to allow discovery of mock services"
+              .format(CONFIG.MOCK_SERVICES_WARM_UP_DELAY))
+        time.sleep(CONFIG.MOCK_SERVICES_WARM_UP_DELAY)
+
     exit_code = 0
     if "suite" not in vars(CMD_ARGS):
         # Interactive testing mode. Await user input.

nmostesting/suites/BCP00301Test.py

@@ -56,7 +56,9 @@ def perform_test_ssl(self, test, args=None):
                                       "--openssl-timeout",
                                       str(CONFIG.HTTP_TIMEOUT),
                                       "--add-ca",
-                                      CONFIG.CERT_TRUST_ROOT_CA
+                                      CONFIG.CERT_TRUST_ROOT_CA,
+                                      "--ip",
+                                      self.apis[SECURE_API_KEY]["ip"]
                                       ] + args + ["{}:{}".format(self.apis[SECURE_API_KEY]["hostname"],
                                                                  self.apis[SECURE_API_KEY]["port"])]
                                      )

nmostesting/suites/IS0401Test.py

@@ -32,10 +32,6 @@
 from ..IS04Utils import IS04Utils
 from ..TestHelper import get_default_ip, is_ip_address, load_resolved_schema, check_content_type
 
-# monkey patch zeroconf to allow us to advertise "_nmos-registration._tcp"
-from zeroconf import service_type_name
-service_type_name.__kwdefaults__['strict'] = False
-
 NODE_API_KEY = "node"
 RECEIVER_CAPS_KEY = "receiver-caps"
 CAPS_REGISTER_KEY = "caps-register"
@@ -94,6 +90,10 @@ def tear_down_tests(self):
         if self.dns_server:
             self.dns_server.reset()
 
+    def _strict_service_name(self, info):
+        # avoid zeroconf._exceptions.BadTypeInNameException: Service name (nmos-registration) must be <= 15 bytes
+        return len(info.type[1:info.type.find('.')]) <= 15
+
     def _mdns_info(self, port, service_type, txt={}, api_ver=None, api_proto=None, api_auth=None, ip=None):
         """Get an mDNS ServiceInfo object in order to create an advertisement"""
         if api_ver is None:
@@ -192,9 +192,9 @@ def do_registry_basics_prereqs(self):
         if CONFIG.DNS_SD_MODE == "multicast":
             # Advertise the primary registry and invalid ones at pri 0, and allow the Node to do a basic registration
             if self.is04_utils.compare_api_version(self.apis[NODE_API_KEY]["version"], "v1.0") != 0:
-                self.zc.register_service(registry_mdns[0])
-                self.zc.register_service(registry_mdns[1])
-            self.zc.register_service(registry_mdns[2])
+                self.zc.register_service(registry_mdns[0], strict=self._strict_service_name(registry_mdns[0]))
+                self.zc.register_service(registry_mdns[1], strict=self._strict_service_name(registry_mdns[1]))
+            self.zc.register_service(registry_mdns[2], strict=self._strict_service_name(registry_mdns[2]))
 
         # Wait for n seconds after advertising the service for the first POST from a Node
         start_time = time.time()
@@ -226,7 +226,7 @@ def do_registry_basics_prereqs(self):
 
             if CONFIG.DNS_SD_MODE == "multicast":
                 for info in registry_mdns[3:]:
-                    self.zc.register_service(info)
+                    self.zc.register_service(info, strict=self._strict_service_name(info))
 
             # Kill registries one by one to collect data around failover
             self.invalid_registry.disable()
@@ -1461,7 +1461,7 @@ def test_21(self, test):
 
         if CONFIG.DNS_SD_MODE == "multicast":
             # Advertise a registry at pri 0 and allow the Node to do a basic registration
-            self.zc.register_service(registry_info)
+            self.zc.register_service(registry_info, strict=self._strict_service_name(registry_info))
 
         # Wait for n seconds after advertising the service for the first POST and then DELETE from a Node
         self.primary_registry.wait_for_registration(CONFIG.DNS_SD_ADVERT_TIMEOUT)
@@ -2166,7 +2166,7 @@ def collect_mdns_announcements(self):
 
         if CONFIG.DNS_SD_MODE == "multicast":
             # Advertise a registry at pri 0 and allow the Node to do a basic registration
-            self.zc.register_service(registry_info)
+            self.zc.register_service(registry_info, strict=self._strict_service_name(registry_info))
 
         # Wait for n seconds after advertising the service for the first POST from a Node
         self.primary_registry.wait_for_registration(CONFIG.DNS_SD_ADVERT_TIMEOUT)

nmostesting/suites/IS1201Test.py

@@ -1385,8 +1385,9 @@ def test_35(self, test):
             context = "oid: " + str(oid) + ", "
 
             for label in [new_user_label, old_user_label]:
-                self.is12_utils.reset_notifications()
+                self.is12_utils.start_logging_notifications()
                 self.is12_utils.set_property(test, oid, NcObjectProperties.USER_LABEL.value, label)
+                self.is12_utils.stop_logging_notifications()
 
                 if len(self.is12_utils.get_notifications()) == 0:
                     error = True

requirements.txt

@@ -1,7 +1,7 @@
 flask>=2.0.0
 wtforms
 jsonschema
-zeroconf>=0.32.0
+zeroconf>=0.75.0
 requests
 netifaces
 gitpython