diff --git a/lib/galaxy/managers/hdas.py b/lib/galaxy/managers/hdas.py index 48e69eb356b5..8eefb8434753 100644 --- a/lib/galaxy/managers/hdas.py +++ b/lib/galaxy/managers/hdas.py @@ -625,18 +625,19 @@ def serialize_display_apps(self, item, key, trans=None, **context): """ hda = item display_apps: List[Dict[str, Any]] = [] - for display_app in hda.get_display_applications(trans).values(): - app_links = [] - for link_app in display_app.links.values(): - app_links.append( - { - "target": link_app.url.get("target_frame", "_blank"), - "href": link_app.get_display_url(hda, trans), - "text": gettext.gettext(link_app.name), - } - ) - if app_links: - display_apps.append(dict(label=display_app.name, links=app_links)) + if hda.state == model.HistoryDatasetAssociation.states.OK and not hda.deleted: + for display_app in hda.get_display_applications(trans).values(): + app_links = [] + for link_app in display_app.links.values(): + app_links.append( + { + "target": link_app.url.get("target_frame", "_blank"), + "href": link_app.get_display_url(hda, trans), + "text": gettext.gettext(link_app.name), + } + ) + if app_links: + display_apps.append(dict(label=display_app.name, links=app_links)) return display_apps @@ -646,28 +647,30 @@ def serialize_old_display_applications(self, item, key, trans=None, **context): """ hda = item display_apps: List[Dict[str, Any]] = [] - if not self.app.config.enable_old_display_applications: - return display_apps - - display_link_fn = hda.datatype.get_display_links - for display_app in hda.datatype.get_display_types(): - target_frame, display_links = display_link_fn( - hda, - display_app, - self.app, - trans.request.base, - ) + if ( + self.app.config.enable_old_display_applications + and hda.state == model.HistoryDatasetAssociation.states.OK + and not hda.deleted + ): + display_link_fn = hda.datatype.get_display_links + for display_app in hda.datatype.get_display_types(): + target_frame, display_links = display_link_fn( + hda, + display_app, + self.app, + trans.request.base, + ) - if len(display_links) > 0: - display_label = hda.datatype.get_display_label(display_app) + if len(display_links) > 0: + display_label = hda.datatype.get_display_label(display_app) - app_links = [] - for display_name, display_link in display_links: - app_links.append( - {"target": target_frame, "href": display_link, "text": gettext.gettext(display_name)} - ) - if app_links: - display_apps.append(dict(label=display_label, links=app_links)) + app_links = [] + for display_name, display_link in display_links: + app_links.append( + {"target": target_frame, "href": display_link, "text": gettext.gettext(display_name)} + ) + if app_links: + display_apps.append(dict(label=display_label, links=app_links)) return display_apps diff --git a/lib/galaxy/web/framework/middleware/error.py b/lib/galaxy/web/framework/middleware/error.py index 2d63a1158327..df5ad74efe03 100644 --- a/lib/galaxy/web/framework/middleware/error.py +++ b/lib/galaxy/web/framework/middleware/error.py @@ -9,6 +9,7 @@ When an exception is thrown from the wrapper application, this logs the exception and displays an error page. """ +import logging import sys import traceback from io import StringIO @@ -25,6 +26,8 @@ reporter, ) +log = logging.getLogger(__name__) + __all__ = ("ErrorMiddleware", "handle_exception") @@ -169,10 +172,11 @@ def __call__(self, environ, start_response): for expect in environ.get("paste.expected_exceptions", []): if isinstance(exc_info[1], expect): raise + log.exception("Uncaught Exception") start_response("500 Internal Server Error", [("content-type", "text/html")], exc_info) # @@: it would be nice to deal with bad content types here response = self.exception_handler(exc_info, environ) - return [response] + return [response.encode(errors="ignore")] finally: # clean up locals... exc_info = None @@ -183,7 +187,7 @@ def make_catching_iter(self, app_iter, environ, sr_checker): return app_iter return CatchingIter(app_iter, environ, sr_checker, self) - def exception_handler(self, exc_info, environ): + def exception_handler(self, exc_info, environ) -> str: simple_html_error = False if self.xmlhttp_key: get_vars = wsgilib.parse_querystring(environ) @@ -192,7 +196,6 @@ def exception_handler(self, exc_info, environ): return handle_exception( exc_info, environ["wsgi.errors"], - html=True, debug_mode=self.debug_mode, error_email=self.error_email, error_log=self.error_log, @@ -341,7 +344,6 @@ def extraData(self): def handle_exception( exc_info, error_stream, - html=True, debug_mode=False, error_email=None, error_log=None, @@ -355,7 +357,7 @@ def handle_exception( error_message=None, simple_html_error=False, environ=None, -): +) -> str: """ For exception handling outside of a web context @@ -386,54 +388,51 @@ def handle_exception( smtp_use_tls=smtp_use_tls, subject_prefix=error_subject_prefix, ) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True if error_log: rep = reporter.LogReporter(filename=error_log) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True if show_exceptions_in_wsgi_errors: rep = reporter.FileReporter(file=error_stream) - rep_err = send_report(rep, exc_data, html=html) + rep_err = send_report(rep, exc_data, html=True) if rep_err: extra_data += rep_err else: reported = True else: error_stream.write(f"Error - {exc_data.exception_type}: {exc_data.exception_value}\n") - if html: - if debug_mode and simple_html_error: - return_error = formatter.format_html( - exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False - ) - reported = True - elif debug_mode and not simple_html_error: - error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False) - head_html = formatter.error_css + formatter.hide_display_js - return_error = error_template(head_html, error_html, extra_data) - extra_data = "" - reported = True - else: - msg = ( - error_message - or """ - An error occurred. - """ - ) - extra = "

The error has been logged to our team." - if "sentry_event_id" in environ: - extra += " If you want to contact us about this error, please reference the following

" - extra += f"GURU MEDITATION: #{environ['sentry_event_id']}" - extra += "

" - return_error = error_template("", msg, extra) + if debug_mode and simple_html_error: + return_error = formatter.format_html( + exc_data, include_hidden_frames=False, include_reusable=False, show_extra_data=False + ) + reported = True + elif debug_mode and not simple_html_error: + error_html = formatter.format_html(exc_data, include_hidden_frames=True, include_reusable=False) + head_html = formatter.error_css + formatter.hide_display_js + return_error = error_template(head_html, error_html, extra_data) + extra_data = "" + reported = True else: - return_error = None + msg = ( + error_message + or """ + An error occurred. + """ + ) + extra = "

The error has been logged to our team." + if "sentry_event_id" in environ: + extra += " If you want to contact us about this error, please reference the following

" + extra += f"GURU MEDITATION: #{environ['sentry_event_id']}" + extra += "

" + return_error = error_template("", msg, extra) if not reported and error_stream: err_report = formatter.format_text(exc_data, show_hidden_frames=True) err_report += f"\n{'-' * 60}\n" diff --git a/lib/tool_shed/managers/repositories.py b/lib/tool_shed/managers/repositories.py index 42bb6c03c223..58f2158c5cd3 100644 --- a/lib/tool_shed/managers/repositories.py +++ b/lib/tool_shed/managers/repositories.py @@ -471,7 +471,7 @@ def create_repository(trans: ProvidesUserContext, request: CreateRepositoryReque type=request.type_, description=request.synopsis, long_description=request.description, - user_id=user.id, + user=user, category_ids=category_ids, remote_repository_url=request.remote_repository_url, homepage_url=request.homepage_url, diff --git a/lib/tool_shed/util/hg_util.py b/lib/tool_shed/util/hg_util.py index a0225c0d1a05..4f2a03155c2e 100644 --- a/lib/tool_shed/util/hg_util.py +++ b/lib/tool_shed/util/hg_util.py @@ -71,14 +71,13 @@ def get_hgrc_path(repo_path): return os.path.join(repo_path, ".hg", "hgrc") -def create_hgrc_file(app, repository): +def create_hgrc_file(app, repository, repo_path): # Since we support both http and https, we set `push_ssl` to False to # override the default (which is True) in the Mercurial API. # The hg purge extension purges all files and directories not being tracked # by Mercurial in the current repository. It will remove unknown files and # empty directories. This is not currently used because it is not supported # in the Mercurial API. - repo_path = repository.repo_path(app) hgrc_path = get_hgrc_path(repo_path) with open(hgrc_path, "w") as fp: fp.write("[paths]\n") diff --git a/lib/tool_shed/util/hgweb_config.py b/lib/tool_shed/util/hgweb_config.py index 8c1656406b0f..2efc1a06422e 100644 --- a/lib/tool_shed/util/hgweb_config.py +++ b/lib/tool_shed/util/hgweb_config.py @@ -5,8 +5,6 @@ import threading from datetime import date -from galaxy.util import unicodify - log = logging.getLogger(__name__) new_hgweb_config_template = """ @@ -20,6 +18,7 @@ def __init__(self): self.hgweb_config_dir = None self.in_memory_config = None self.lock = threading.Lock() + self.hgweb_repo_prefix = None def add_entry(self, lhs, rhs): """Add an entry in the hgweb.config file for a new repository.""" @@ -35,8 +34,8 @@ def add_entry(self, lhs, rhs): self.in_memory_config.set("paths", lhs, rhs) # Persist our in-memory configuration. self.write_config() - except Exception as e: - log.debug("Exception in HgWebConfigManager.add_entry(): %s", unicodify(e)) + except Exception: + log.exception("Exception in HgWebConfigManager.add_entry()") finally: self.lock.release() @@ -51,8 +50,8 @@ def change_entry(self, old_lhs, new_lhs, new_rhs): self.in_memory_config.set("paths", new_lhs, new_rhs) # Persist our in-memory configuration. self.write_config() - except Exception as e: - log.debug("Exception in HgWebConfigManager.change_entry(): %s", unicodify(e)) + except Exception: + log.exception("Exception in HgWebConfigManager.change_entry()") finally: self.lock.release() diff --git a/lib/tool_shed/util/repository_util.py b/lib/tool_shed/util/repository_util.py index 4acfd2bb7330..11c739a70f8d 100644 --- a/lib/tool_shed/util/repository_util.py +++ b/lib/tool_shed/util/repository_util.py @@ -2,6 +2,7 @@ import logging import os import re +import tempfile from typing import ( List, Optional, @@ -164,23 +165,18 @@ def create_repo_info_dict( def create_repository_admin_role(app: "ToolShedApp", repository: "Repository"): """ Create a new role with name-spaced name based on the repository name and its owner's public user - name. This will ensure that the tole name is unique. + name. This will ensure that the role name is unique. """ sa_session = app.model.session name = get_repository_admin_role_name(str(repository.name), str(repository.user.username)) description = "A user or group member with this role can administer this repository." role = app.model.Role(name=name, description=description, type=app.model.Role.types.SYSTEM) sa_session.add(role) - session = sa_session() - with transaction(session): - session.commit() # Associate the role with the repository owner. app.model.UserRoleAssociation(repository.user, role) # Associate the role with the repository. rra = app.model.RepositoryRoleAssociation(repository, role) sa_session.add(rra) - with transaction(session): - session.commit() return role @@ -190,7 +186,7 @@ def create_repository( type: str, description, long_description, - user_id, + user, category_ids: Optional[List[str]] = None, remote_repository_url=None, homepage_url=None, @@ -206,43 +202,40 @@ def create_repository( homepage_url=homepage_url, description=description, long_description=long_description, - user_id=user_id, + user=user, ) - # Flush to get the id. sa_session.add(repository) - session = sa_session() - with transaction(session): - session.commit() - # Create an admin role for the repository. - create_repository_admin_role(app, repository) - # Determine the repository's repo_path on disk. - dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) - # Create directory if it does not exist. - if not os.path.exists(dir): - os.makedirs(dir) - # Define repo name inside hashed directory. - repository_path = os.path.join(dir, "repo_%d" % repository.id) - # Create local repository directory. - if not os.path.exists(repository_path): - os.makedirs(repository_path) - # Create the local repository. - init_repository(repo_path=repository_path) - # Add an entry in the hgweb.config file for the local repository. - lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" - app.hgweb_config_manager.add_entry(lhs, repository_path) - # Create a .hg/hgrc file for the local repository. - create_hgrc_file(app, repository) - flush_needed = False if category_ids: # Create category associations for category_id in category_ids: category = sa_session.get(app.model.Category, app.security.decode_id(category_id)) rca = app.model.RepositoryCategoryAssociation(repository, category) sa_session.add(rca) - flush_needed = True - if flush_needed: - with transaction(session): - session.commit() + # Create an admin role for the repository. + create_repository_admin_role(app, repository) + # Create a temporary repo_path on disk. + repository_path = tempfile.mkdtemp( + dir=app.config.file_path, + prefix=f"{repository.user.username}-{repository.name}", + ) + # Create the local repository. + init_repository(repo_path=repository_path) + # Create a .hg/hgrc file for the local repository. + create_hgrc_file(app, repository, repo_path=repository_path) + # Add an entry in the hgweb.config file for the local repository. + lhs = f"{app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + # Flush to get the id. + session = sa_session() + with transaction(session): + session.commit() + dir = os.path.join(app.config.file_path, *util.directory_hash_id(repository.id)) + # Define repo name inside hashed directory. + final_repository_path = os.path.join(dir, "repo_%d" % repository.id) + # Create final repository directory. + if not os.path.exists(final_repository_path): + os.makedirs(final_repository_path) + os.rename(repository_path, final_repository_path) + app.hgweb_config_manager.add_entry(lhs, final_repository_path) # Update the repository registry. app.repository_registry.add_entry(repository) message = f"Repository {escape(str(repository.name))} has been created." @@ -486,8 +479,8 @@ def update_repository(trans: "ProvidesUserContext", id: str, **kwds) -> Tuple[Op repo_dir = repository.repo_path(app) # Change the entry in the hgweb.config file for the repository. - old_lhs = f"repos/{repository.user.username}/{repository.name}" - new_lhs = f"repos/{repository.user.username}/{kwds['name']}" + old_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{repository.name}" + new_lhs = f"{trans.app.config.hgweb_repo_prefix}{repository.user.username}/{kwds['name']}" trans.app.hgweb_config_manager.change_entry(old_lhs, new_lhs, repo_dir) # Change the entry in the repository's hgrc file. diff --git a/lib/tool_shed/util/shed_index.py b/lib/tool_shed/util/shed_index.py index bd823b6cb094..ee94d4c1a785 100644 --- a/lib/tool_shed/util/shed_index.py +++ b/lib/tool_shed/util/shed_index.py @@ -37,7 +37,7 @@ def _get_or_create_index(whoosh_index_dir): return get_or_create_index(whoosh_index_dir, repo_schema), get_or_create_index(tool_index_dir, tool_schema) -def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): +def build_index(whoosh_index_dir, file_path, hgweb_config_dir, hgweb_repo_prefix, dburi, **kwargs): """ Build two search indexes simultaneously One is for repositories and the other for tools. @@ -55,7 +55,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): execution_timer = ExecutionTimer() with repo_index.searcher() as searcher: - for repo in get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): + for repo in get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs): tools_list = repo.pop("tools_list") repo_id = repo["id"] indexed_document = searcher.document(id=repo_id) @@ -89,7 +89,7 @@ def build_index(whoosh_index_dir, file_path, hgweb_config_dir, dburi, **kwargs): return repos_indexed, tools_indexed -def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): +def get_repos(sa_session, file_path, hgweb_config_dir, hgweb_repo_prefix, **kwargs): """ Load repos from DB and included tools from .xml configs. """ @@ -120,7 +120,7 @@ def get_repos(sa_session, file_path, hgweb_config_dir, **kwargs): # Load all changesets of the repo for lineage. repo_path = os.path.join( - hgweb_config_dir, hgwcm.get_entry(os.path.join("repos", repo.user.username, repo.name)) + hgweb_config_dir, hgwcm.get_entry(os.path.join(hgweb_repo_prefix, repo.user.username, repo.name)) ) hg_repo = hg.repository(ui.ui(), repo_path.encode("utf-8")) lineage = [] diff --git a/lib/tool_shed/webapp/api/tools.py b/lib/tool_shed/webapp/api/tools.py index 33099f5e2ada..99b676bbdc32 100644 --- a/lib/tool_shed/webapp/api/tools.py +++ b/lib/tool_shed/webapp/api/tools.py @@ -33,6 +33,7 @@ def build_search_index(self, trans, **kwd): trans.app.config.whoosh_index_dir, trans.app.config.file_path, trans.app.config.hgweb_config_dir, + trans.app.config.hgweb_repo_prefix, trans.app.config.database_connection, ) return { diff --git a/lib/tool_shed/webapp/api2/tools.py b/lib/tool_shed/webapp/api2/tools.py index 0d8c2f2d5524..bd48db71ce01 100644 --- a/lib/tool_shed/webapp/api2/tools.py +++ b/lib/tool_shed/webapp/api2/tools.py @@ -74,6 +74,7 @@ def build_search_index(self) -> BuildSearchIndexResponse: config.whoosh_index_dir, config.file_path, config.hgweb_config_dir, + config.hgweb_repo_prefix, config.database_connection, ) return BuildSearchIndexResponse( diff --git a/lib/tool_shed/webapp/app.py b/lib/tool_shed/webapp/app.py index c71ad3938c68..58ccf206596c 100644 --- a/lib/tool_shed/webapp/app.py +++ b/lib/tool_shed/webapp/app.py @@ -104,6 +104,7 @@ def __init__(self, **kwd) -> None: # Let the Tool Shed's HgwebConfigManager know where the hgweb.config file is located. self.hgweb_config_manager = hgweb_config_manager self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir + self.hgweb_config_manager.hgweb_repo_prefix = self.config.hgweb_repo_prefix # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry(self) # Configure Sentry client if configured diff --git a/lib/tool_shed/webapp/controllers/repository.py b/lib/tool_shed/webapp/controllers/repository.py index c6a39a05a496..2fb4015ce9f4 100644 --- a/lib/tool_shed/webapp/controllers/repository.py +++ b/lib/tool_shed/webapp/controllers/repository.py @@ -737,7 +737,7 @@ def create_repository(self, trans, **kwd): repository_type, description, long_description, - user_id=trans.user.id, + user=trans.user, category_ids=category_ids, remote_repository_url=remote_repository_url, homepage_url=homepage_url, diff --git a/lib/tool_shed/webapp/model/__init__.py b/lib/tool_shed/webapp/model/__init__.py index 445427d65c63..859b8fe2b096 100644 --- a/lib/tool_shed/webapp/model/__init__.py +++ b/lib/tool_shed/webapp/model/__init__.py @@ -148,6 +148,11 @@ def __init__(self, email=None, password=None): self.purged = False self.new_repo_alert = False + @property + def current_galaxy_session(self): + if self.galaxy_sessions: + return self.galaxy_sessions[0] + def all_roles(self): roles = [ura.role for ura in self.roles] for group in [uga.group for uga in self.groups]: @@ -423,12 +428,13 @@ class Repository(Base, Dictifiable): ] file_states = Bunch(NORMAL="n", NEEDS_MERGING="m", MARKED_FOR_REMOVAL="r", MARKED_FOR_ADDITION="a", NOT_TRACKED="?") - def __init__(self, private=False, times_downloaded=0, deprecated=False, **kwd): + def __init__(self, private=False, times_downloaded=0, deprecated=False, user=None, **kwd): super().__init__(**kwd) self.private = private self.times_downloaded = times_downloaded self.deprecated = deprecated self.name = self.name or "Unnamed repository" + self.user = user @property def hg_repo(self): @@ -514,7 +520,9 @@ def is_new(self): def repo_path(self, app=None): # Keep app argument for compatibility with tool_shed_install Repository model - return hgweb_config_manager.get_entry(os.path.join("repos", self.user.username, self.name)) + return hgweb_config_manager.get_entry( + os.path.join(hgweb_config_manager.hgweb_repo_prefix, self.user.username, self.name) + ) def revision(self): repo = self.hg_repo diff --git a/scripts/tool_shed/build_ts_whoosh_index.py b/scripts/tool_shed/build_ts_whoosh_index.py index d74cd99d80cc..d74420f17ad7 100644 --- a/scripts/tool_shed/build_ts_whoosh_index.py +++ b/scripts/tool_shed/build_ts_whoosh_index.py @@ -40,6 +40,7 @@ def parse_arguments(): config = ts_config.ToolShedAppConfiguration(**app_properties) args.dburi = config.database_connection args.hgweb_config_dir = config.hgweb_config_dir + args.hgweb_repo_prefix = config.hgweb_repo_prefix args.whoosh_index_dir = config.whoosh_index_dir args.file_path = config.file_path if args.debug: diff --git a/test/unit/tool_shed/_util.py b/test/unit/tool_shed/_util.py index cf4f82255d0c..df50c5270ef2 100644 --- a/test/unit/tool_shed/_util.py +++ b/test/unit/tool_shed/_util.py @@ -76,6 +76,7 @@ def __init__(self, temp_directory=None): hgweb_config_dir = os.path.join(temp_directory, "hgweb") safe_makedirs(hgweb_config_dir) self.hgweb_config_manager.hgweb_config_dir = hgweb_config_dir + self.hgweb_config_manager.hgweb_repo_prefix = "repos/" self.config = TestToolShedConfig(temp_directory) self.security = IdEncodingHelper(id_secret=self.config.id_secret) self.repository_registry = tool_shed.repository_registry.Registry(self) @@ -133,7 +134,7 @@ def repository_fixture(app: ToolShedApp, user: User, name: str, category: Option type, description, long_description, - user.id, + user, category_ids=category_ids, remote_repository_url=None, homepage_url=None, diff --git a/test/unit/tool_shed/test_shed_index.py b/test/unit/tool_shed/test_shed_index.py index ea5d5ef1bba4..bc929dcddfb1 100644 --- a/test/unit/tool_shed/test_shed_index.py +++ b/test/unit/tool_shed/test_shed_index.py @@ -36,10 +36,11 @@ def community_file_dir(): @pytest.fixture() def community_file_structure(community_file_dir): - community = namedtuple("community", "file_path hgweb_config_dir dburi") + community = namedtuple("community", "file_path hgweb_config_dir hgweb_repo_prefix dburi") return community( file_path=os.path.join(community_file_dir, "database", "community_files"), hgweb_config_dir=community_file_dir, + hgweb_repo_prefix="repos/", dburi="sqlite:///{}".format(os.path.join(community_file_dir, "database", "community.sqlite")), ) @@ -49,6 +50,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 1 @@ -59,6 +61,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 0 @@ -74,6 +77,7 @@ def test_build_index(whoosh_index_dir, community_file_structure): whoosh_index_dir, community_file_structure.file_path, community_file_structure.hgweb_config_dir, + community_file_structure.hgweb_repo_prefix, community_file_structure.dburi, ) assert repos_indexed == 1