diff --git a/README.md b/README.md index a9f3e13a..679404ac 100644 --- a/README.md +++ b/README.md @@ -285,12 +285,28 @@ Contributions welcome: c.CylcUIServer.ui_build_dir = os.path.expanduser('~/cylc-ui/dist') ``` -Note about testing: unlike cylc-flow, cylc-uiserver uses the +**Note about testing:** + +Unlike cylc-flow, cylc-uiserver uses the [pytest-tornasync](https://github.com/eukaryote/pytest-tornasync/) plugin instead of [pytest-asyncio](https://github.com/pytest-dev/pytest-asyncio). This means you should not decorate async test functions with `@pytest.mark.asyncio`. +**Profiling:** + +There are some built-in profilers in the server, activate them using the +`profile` trait, e.g: + +``` +cylc gui --CylcUIServer.profile=track_data_store +``` + + +See the +[config docs](https://cylc.github.io/cylc-doc/stable/html/reference/config/ui-server.html#cylc.uiserver.app.CylcUIServer.profile) +for more details. + ## Copyright and Terms of Use Copyright (C) 2019-2025 NIWA & British Crown (Met Office) & Contributors. diff --git a/cylc/uiserver/app.py b/cylc/uiserver/app.py index 13761776..97947b49 100644 --- a/cylc/uiserver/app.py +++ b/cylc/uiserver/app.py @@ -62,7 +62,6 @@ ) import sys from textwrap import dedent -from types import SimpleNamespace from typing import ( List, Optional, @@ -74,14 +73,13 @@ from tornado import ioloop from tornado.web import RedirectHandler from traitlets import ( - Bool, Dict, Float, Int, + Unicode, TraitError, TraitType, Undefined, - Unicode, default, validate, ) @@ -90,7 +88,6 @@ from cylc.flow.network.graphql import ( CylcExecutionContext, IgnoreFieldMiddleware ) -from cylc.flow.profiler import Profiler from cylc.uiserver import __file__ as uis_pkg from cylc.uiserver.authorise import ( Authorization, @@ -109,6 +106,7 @@ UIServerGraphQLHandler, UserProfileHandler, ) +from cylc.uiserver.profilers import get_profiler from cylc.uiserver.resolvers import Resolvers from cylc.uiserver.schema import schema from cylc.uiserver.graphql.tornado_ws import TornadoSubscriptionServer @@ -336,15 +334,33 @@ class CylcUIServer(ExtensionApp): ''', default_value=100, ) - profile = Bool( + profile = Unicode( config=True, help=''' - Turn on Python profiling. + Developer extension: Turn on the specified profiler. + + The default (empty string) does not invoke a profiler. + + Only one profiler may be run at a time. + + Options: + cprofile: + Profile Python code execution time with cprofile. - The profile results will be saved to ~/.cylc/uiserver/profile.prof - in cprofile format. + Results will be saved to ~/.cylc/uiserver/profile.prof + in cprofile format. + track_objects: + Track attributes of the CylcUIServer class. + + Results will be saved to + ~/.cylc/uiserver/cylc.flow.main_loop.log_memory.pdf. + track_data_store + Track attributes of the DataStoreMgr class. + + Results will be saved to + ~/.cylc/uiserver/cylc.flow.main_loop.log_memory.pdf. ''', - default_value=False, + default_value='', ) log_timeout = Float( @@ -477,14 +493,13 @@ def initialize_settings(self): ) ) - # start profiling - self.profiler = Profiler( - # the profiler is designed to attach to a Cylc scheduler - schd=SimpleNamespace(workflow_log_dir=USER_CONF_ROOT), - # profiling is turned on via the "profile" traitlet - enabled=self.profile, - ) - self.profiler.start() + profiler_cls = get_profiler(self.profile) + if profiler_cls: + self.profiler = profiler_cls(self) + ioloop.PeriodicCallback( + self.profiler.periodic, + 1000, # PT1S + ).start() # start the async scan task running (do this on server start not init) ioloop.IOLoop.current().add_callback( @@ -633,4 +648,7 @@ async def stop_extension(self): # Destroy ZeroMQ context of all sockets self.workflows_mgr.context.destroy() - self.profiler.stop() + + # stop the profiler + if getattr(self, 'profiler', None): + self.profiler.shutdown() diff --git a/cylc/uiserver/profilers.py b/cylc/uiserver/profilers.py new file mode 100644 index 00000000..34fc1989 --- /dev/null +++ b/cylc/uiserver/profilers.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Profilers for the ServerApp instance. + +This is effectively a cut-down version of the cylc.flow.main_loop plugin +system. It's only intended for developer use. + +NOTE: All profiler specific imports are handled in their `__init__` methods +to avoid importing profiler code when not requested. +""" + +from time import time +from types import SimpleNamespace + +from cylc.uiserver.config_util import USER_CONF_ROOT + + +class Profiler: + def __init__(self, app): + self.app = app + self.app.log.warning(f'Starting profiler: {self.__class__.__name__}') + + def periodic(self): + pass + + def shutdown(self): + pass + + +class CProfiler(Profiler): + """Invoke cprofile via the cylc.flow.profiler interface.""" + + def __init__(self, app): + Profiler.__init__(self, app) + + from cylc.flow.profiler import Profiler as CylcCProfiler + + self.cprofiler = CylcCProfiler( + # the profiler is designed to attach to a Cylc scheduler + schd=SimpleNamespace(workflow_log_dir=USER_CONF_ROOT), + enabled=True, + ) + + self.cprofiler.start() + + def periodic(self): + pass + + def shutdown(self): + self.cprofiler.stop() + + +class TrackObjects(Profiler): + """Invoke pympler.asized via the cylc.main_loop.log_memory interface.""" + + def __init__(self, app): + Profiler.__init__(self, app) + + from cylc.flow.main_loop.log_memory import ( + _compute_sizes, + _plot, + _transpose, + ) + + self._compute_sizes = _compute_sizes + self._transpose = _transpose + self._plot = _plot + self.data = [] + self.min_size = 100 + self.obj = app + + def periodic(self): + self.data.append( + ( + time(), + self._compute_sizes(self.obj, min_size=self.min_size), + ) + ) + + def shutdown(self): + self.periodic() + fields, times = self._transpose(self.data) + self._plot( + fields, + times, + USER_CONF_ROOT, + f'{self.obj} attrs > {self.min_size / 1000}kb', + ) + + +class TrackDataStore(TrackObjects): + """Like TrackObjects but for the Data Store.""" + + def __init__(self, app): + TrackObjects.__init__(self, app) + self.obj = self.app.data_store_mgr + + +PROFILERS = { + 'cprofile': CProfiler, + 'track_objects': TrackObjects, + 'track_data_store': TrackDataStore, +} + + +def get_profiler(profiler: str): + if not profiler: + return None + try: + return PROFILERS[profiler] + except KeyError: + raise Exception( + f'Unknown profiler: {profiler}' + f'\nValid options: {", ".join(PROFILERS)}' + )