Merge pull request galaxyproject#16825 from nsoranzo/toolshed_wf_fixes

Tool Shed 2.0 fixes

.github/workflows/toolshed.yaml

@@ -19,24 +19,11 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        include:
-        - test-install-client: 'galaxy_api'
-          python-version: '3.7'
-          shed-api: 'v1'
-          shed-browser: 'twill'
-        - test-install-client: 'standalone'
-          python-version: '3.8'
-          shed-api: 'v1'
-          shed-browser: 'twill'
-        - test-install-client: 'galaxy_api'
-          python-version: '3.9'
-          shed-api: 'v2'
-          shed-browser: 'playwright'
-        - test-install-client: 'standalone'
-          python-version: '3.10'
-          shed-api: 'v2'
-          shed-browser: 'playwright'
+        python-version: ['3.7', '3.11']
+        shed-api: ['v1', 'v2']
+        test-install-client: ['galaxy_api', 'standalone']
     services:
       postgres:
         image: postgres:13
@@ -50,6 +37,11 @@ jobs:
       - uses: actions/checkout@v3
         with:
           path: 'galaxy root'
+      - uses: actions/setup-node@v3
+        with:
+          node-version: '18.12.1'
+          cache: 'yarn'
+          cache-dependency-path: 'galaxy root/client/yarn.lock'
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
@@ -68,25 +60,30 @@ jobs:
         with:
           path: 'galaxy root/.venv'
           key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-toolshed
-          key: gxy-venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy_root/requirements.txt') }}-toolshed
       - name: Install dependencies
         run: ./scripts/common_startup.sh --skip-client-build
         working-directory: 'galaxy root'
       - name: Build Frontend
-        run: '. .venv/bin/activate && cd lib/tool_shed/webapp/frontend && yarn && make client'
+        run: |
+          . .venv/bin/activate
+          cd lib/tool_shed/webapp/frontend
+          yarn
+          make client
         working-directory: 'galaxy root'
       - name: Install playwright
-        run: '. .venv/bin/activate && playwright install'
+        run: |
+          . .venv/bin/activate
+          playwright install
         working-directory: 'galaxy root'
       - name: Run tests
-        run: './run_tests.sh -toolshed'
+        run: ./run_tests.sh -toolshed
         env:
           TOOL_SHED_TEST_INSTALL_CLIENT: ${{ matrix.test-install-client }}
           TOOL_SHED_API_VERSION: ${{ matrix.shed-api }}
-          TOOL_SHED_TEST_BROWSER: ${{ matrix.shed-browser }}
+          TOOL_SHED_TEST_BROWSER: ${{ matrix.shed-api == 'v1' && 'twill' || 'playwright' }}
         working-directory: 'galaxy root'
       - uses: actions/upload-artifact@v3
         if: failure()
         with:
-          name: Toolshed test results (${{ matrix.python-version }}, ${{ matrix.test-install-client }})
+          name: Toolshed test results (${{ matrix.python-version }}, ${{ matrix.shed-api }}, ${{ matrix.test-install-client }})
           path: 'galaxy root/run_toolshed_tests.html'

lib/galaxy_test/driver/driver_util.py

@@ -227,7 +227,6 @@ def setup_galaxy_config(
         template_cache_path=template_cache_path,
         tool_config_file=tool_config_file,
         tool_data_table_config_path=tool_data_table_config_path,
-        tool_parse_help=False,
         tool_path=tool_path,
         update_integrated_tool_panel=update_integrated_tool_panel,
         use_tasked_jobs=True,

lib/tool_shed/managers/categories.py

@@ -26,6 +26,9 @@ class CategoryManager:
     def __init__(self, app: ToolShedApp):
         self.app = app
 
+    def get(self, encoded_category_id: str) -> Category:
+        return suc.get_category(self.app, encoded_category_id)
+
     def create(self, trans: ProvidesUserContext, category_request: CreateCategoryRequest) -> Category:
         name = category_request.name
         description = category_request.description or name
@@ -71,6 +74,7 @@ def to_model(self, category: Category) -> CategoryResponse:
             name=as_dict["name"],
             description=as_dict["description"],
             repositories=as_dict["repositories"],
+            deleted=as_dict["deleted"],
         )
 
 

lib/tool_shed/managers/repositories.py

@@ -20,6 +20,7 @@
     false,
     select,
 )
+from sqlalchemy.orm import scoped_session
 
 from galaxy import web
 from galaxy.exceptions import (
@@ -254,7 +255,9 @@ def index_tool_ids(app: ToolShedApp, tool_ids: List[str]) -> Dict[str, Any]:
 
 
 def index_repositories(app: ToolShedApp, name: Optional[str], owner: Optional[str], deleted: bool):
-    return list(_get_repository_by_name_and_owner_and_deleted(app.model.context, name, owner, deleted, app.model.User))
+    return list(
+        _get_repositories_by_name_and_owner_and_deleted(app.model.context, name, owner, deleted, app.model.User)
+    )
 
 
 def can_manage_repo(trans: ProvidesUserContext, repository: Repository) -> bool:
@@ -579,7 +582,7 @@ def upload_tar_and_set_metadata(
     return message
 
 
-def _get_repository_by_name_and_owner(session, name, owner, user_model):
+def _get_repository_by_name_and_owner(session: scoped_session, name: str, owner: str, user_model):
     stmt = (
         select(Repository)
         .where(Repository.deprecated == false())
@@ -592,12 +595,14 @@ def _get_repository_by_name_and_owner(session, name, owner, user_model):
     return session.scalars(stmt).first()
 
 
-def _get_repository_by_name_and_owner_and_deleted(session, name, owner, deleted, user_model):
+def _get_repositories_by_name_and_owner_and_deleted(
+    session: scoped_session, name: Optional[str], owner: Optional[str], deleted: bool, user_model
+):
     stmt = select(Repository).where(Repository.deprecated == false()).where(Repository.deleted == deleted)
     if owner is not None:
         stmt = stmt.where(user_model.username == owner)
         stmt = stmt.where(Repository.user_id == user_model.id)
     if name is not None:
         stmt = stmt.where(Repository.name == name)
-    stmt = stmt.order_by(Repository.name).limit(1)
-    return session.scalars(stmt).first()
+    stmt = stmt.order_by(Repository.name)
+    return session.scalars(stmt)

lib/tool_shed/metadata/repository_metadata_manager.py

@@ -1097,7 +1097,7 @@ def get_current_repositories(session, order=False):
 
 
 def get_filtered_repositories(session, repo_ids, order):
-    stmt = select(Repository).where(Repository.in_(repo_ids))
+    stmt = select(Repository).where(Repository.id.in_(repo_ids))
     if order:
         stmt = stmt.order_by(Repository.name, Repository.user_id)
     return session.scalars(stmt)

lib/tool_shed/test/base/driver.py

@@ -80,11 +80,11 @@ def _setup_local(self):
             os.environ["GALAXY_TEST_TOOL_DATA_PATH"] = tool_data_path
         galaxy_db_path = driver_util.database_files_path(tool_shed_test_tmp_dir)
         shed_file_path = os.path.join(shed_db_path, "files")
-        hgweb_config_file_path = tempfile.mkdtemp(dir=tool_shed_test_tmp_dir)
+        whoosh_index_dir = os.path.join(shed_db_path, "toolshed_whoosh_indexes")
+        hgweb_config_dir = tempfile.mkdtemp(dir=tool_shed_test_tmp_dir)
         new_repos_path = tempfile.mkdtemp(dir=tool_shed_test_tmp_dir)
         galaxy_shed_tool_path = tempfile.mkdtemp(dir=tool_shed_test_tmp_dir)
         galaxy_migrated_tool_path = tempfile.mkdtemp(dir=tool_shed_test_tmp_dir)
-        hgweb_config_dir = hgweb_config_file_path
         os.environ["TEST_HG_WEB_CONFIG_DIR"] = hgweb_config_dir
         print("Directory location for hgweb.config:", hgweb_config_dir)
         toolshed_database_conf = driver_util.database_conf(shed_db_path, prefix="TOOL_SHED")
@@ -103,8 +103,8 @@ def _setup_local(self):
             shed_tool_data_table_config=shed_tool_data_table_conf_file,
             smtp_server="smtp.dummy.string.tld",
             email_from="functional@localhost",
-            tool_parse_help=False,
             use_heartbeat=False,
+            whoosh_index_dir=whoosh_index_dir,
         )
         kwargs.update(toolshed_database_conf)
         # Generate the tool_data_table_conf.xml file.

lib/tool_shed/test/base/populators.py

@@ -266,9 +266,12 @@ def get_categories(self) -> List[Category]:
         response.raise_for_status()
         return [Category(**c) for c in response.json()]
 
-    def get_category_with_name(self, name: str) -> Optional[Category]:
-        response = self._api_interactor.get("categories")
+    def get_category_with_id(self, category_id: str) -> Category:
+        response = self._api_interactor.get(f"categories/{category_id}")
         response.raise_for_status()
+        return Category(**response.json())
+
+    def get_category_with_name(self, name: str) -> Optional[Category]:
         categories = [c for c in self.get_categories() if c.name == name]
         return categories[0] if categories else None
 

lib/tool_shed/test/base/twilltestcase.py

@@ -764,7 +764,7 @@ def login(
         )
         # v2 doesn't log you in on account creation... so force a login here
         if previously_created or self.is_v2:
-            # The acount has previously been created, so just login.
+            # The account has previously been created, so just login.
             # HACK: don't use panels because late_javascripts() messes up the twill browser and it
             # can't find form fields (and hence user can't be logged in).
             params = {"use_panels": False}
@@ -1040,7 +1040,7 @@ def create_category(self, **kwd) -> Category:
 
     def create_repository_dependency(
         self,
-        repository: Optional[Repository] = None,
+        repository: Repository,
         repository_tuples=None,
         filepath=None,
         prior_installation_required=False,
@@ -1050,7 +1050,6 @@ def create_repository_dependency(
         strings_displayed=None,
         strings_not_displayed=None,
     ):
-        assert repository
         repository_tuples = repository_tuples or []
         repository_names = []
         if complex:
@@ -1402,21 +1401,20 @@ def get_repositories_category_api(
             self.check_for_strings(strings_displayed, strings_not_displayed)
 
     def get_or_create_repository(
-        self, category: Category, owner=None, strings_displayed=None, strings_not_displayed=None, **kwd
+        self, category: Category, owner: str, name: str, strings_displayed=None, strings_not_displayed=None, **kwd
     ) -> Optional[Repository]:
         # If not checking for a specific string, it should be safe to assume that
         # we expect repository creation to be successful.
         if strings_displayed is None:
-            strings_displayed = ["Repository", kwd["name"], "has been created"]
+            strings_displayed = ["Repository", name, "has been created"]
         if strings_not_displayed is None:
             strings_not_displayed = []
-        name = kwd["name"]
         repository = self.populator.get_repository_for(owner, name)
-        category_id = category.id
-        assert category_id
         if repository is None:
+            category_id = category.id
+            assert category_id
             self.visit_url("/repository/create_repository")
-            self.submit_form(button="create_repository_button", category_id=category_id, **kwd)
+            self.submit_form(button="create_repository_button", name=name, category_id=category_id, **kwd)
             self.check_for_strings(strings_displayed, strings_not_displayed)
             repository = self.populator.get_repository_for(owner, name)
         return repository
@@ -2093,7 +2091,7 @@ def get_installed_repository(session, name, owner, changeset):
     if owner is not None:
         stmt = stmt.where(ToolShedRepository.owner == owner)
     if changeset is not None:
-        stmt = stmt.wehre(ToolShedRepository.changeset_revision == changeset)
+        stmt = stmt.where(ToolShedRepository.changeset_revision == changeset)
     stmt = stmt.where(ToolShedRepository.deleted == false())
     stmt = stmt.where(ToolShedRepository.uninstalled == false())
     return session.scalars(stmt).one_or_none()

lib/tool_shed/test/functional/conftest.py

@@ -10,13 +10,12 @@
     Browser,
     BrowserContext,
 )
-from typing_extensions import Literal
 
 from ..base.browser import ShedBrowser
 from ..base.playwrightbrowser import PlaywrightShedBrowser
 from ..base.twillbrowser import TwillShedBrowser
 
-DEFAULT_BROWSER: Literal["twill", "playwright"] = "playwright"
+DEFAULT_BROWSER = "playwright"
 
 
 def twill_browser() -> Generator[ShedBrowser, None, None]:
@@ -28,10 +27,13 @@ def playwright_browser(class_context: BrowserContext) -> Generator[ShedBrowser,
     yield PlaywrightShedBrowser(page)
 
 
-if os.environ.get("TOOL_SHED_TEST_BROWSER", DEFAULT_BROWSER) == "twill":
+test_browser = os.environ.get("TOOL_SHED_TEST_BROWSER", DEFAULT_BROWSER)
+if test_browser == "twill":
     shed_browser = pytest.fixture(scope="class")(twill_browser)
-else:
+elif test_browser == "playwright":
     shed_browser = pytest.fixture(scope="class")(playwright_browser)
+else:
+    raise ValueError(f"Unrecognized value for TOOL_SHED_TEST_BROWSER: {test_browser}")
 
 
 @pytest.fixture(scope="class")