Merge pull request #547 from fetchai/fix/package-path

fix package path and gui test issues.

aea/cli/common.py

@@ -31,7 +31,6 @@
 import jsonschema  # type: ignore
 from dotenv import load_dotenv
 
-import aea
 from aea.cli.loggers import default_logging_config
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE, AgentConfig, SkillConfig, ConnectionConfig, ProtocolConfig, \
     DEFAULT_PROTOCOL_CONFIG_FILE, DEFAULT_CONNECTION_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, Dependencies
@@ -40,7 +39,7 @@
 logger = logging.getLogger("aea")
 logger = default_logging_config(logger)
 
-DEFAULT_REGISTRY_PATH = str(Path(aea.AEA_DIR, "..", "packages").resolve())
+DEFAULT_REGISTRY_PATH = str(Path("..", "packages"))
 DEFAULT_CONNECTION = "stub"
 DEFAULT_SKILL = "error"
 

aea/cli/search.py

@@ -18,15 +18,18 @@
 # ------------------------------------------------------------------------------
 
 """Implementation of the 'aea search' subcommand."""
+import os
 from pathlib import Path
 from typing import cast, List, Dict
+
 import click
-import os
 
 from aea import AEA_DIR
-from aea.cli.common import Context, pass_ctx, DEFAULT_REGISTRY_PATH, logger, retrieve_details, ConfigLoader, format_items, format_skills
-from aea.configurations.base import DEFAULT_CONNECTION_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, DEFAULT_PROTOCOL_CONFIG_FILE
+from aea.cli.common import Context, pass_ctx, DEFAULT_REGISTRY_PATH, logger, retrieve_details, ConfigLoader, \
+    format_items, format_skills
 from aea.cli.registry.utils import request_api
+from aea.configurations.base import DEFAULT_CONNECTION_CONFIG_FILE, DEFAULT_SKILL_CONFIG_FILE, \
+    DEFAULT_PROTOCOL_CONFIG_FILE
 
 
 @click.group()
@@ -42,14 +45,7 @@ def search(ctx: Context, registry):
     if registry:
         ctx.set_config("is_registry", True)
     else:
-        # if the search is executed from an agent directory:
-        if hasattr(ctx, "agent_config"):
-            # take the path defined by the user, and distinguish between absolute and relative path.
-            path = Path(ctx.agent_config.registry_path)
-            registry = path if path.is_absolute() else Path(ctx.cwd) / path
-        else:
-            # use the default registry path. This works only for source installations.
-            registry = DEFAULT_REGISTRY_PATH
+        registry = os.path.join(ctx.cwd, DEFAULT_REGISTRY_PATH)
         ctx.set_config("registry", registry)
         logger.debug("Using registry {}".format(registry))
 

aea/cli_gui/__init__.py

@@ -132,14 +132,14 @@ def _sync_extract_items_from_tty(pid: subprocess.Popen):
 def get_registered_items(item_type: str):
     """Create a new AEA project."""
     # need to place ourselves one directory down so the searcher can find the packages
-    pid = _call_aea_async([sys.executable, "-m", "aea.cli", "search", item_type + "s"], app_context.agents_dir)
+    pid = _call_aea_async(["aea", "search", item_type + "s"], os.path.join(app_context.agents_dir, "aea"))
     return _sync_extract_items_from_tty(pid)
 
 
 def search_registered_items(item_type: str, search_term: str):
     """Create a new AEA project."""
     # need to place ourselves one directory down so the searcher can find the packages
-    pid = _call_aea_async([sys.executable, "-m", "aea.cli", "search", item_type + "s", "--query", search_term], app_context.agents_dir)
+    pid = _call_aea_async(["aea", "search", item_type + "s", "--query", search_term], os.path.join(app_context.agents_dir, "aea"))
     ret = _sync_extract_items_from_tty(pid)
     search_result, status = ret
     response = {"search_result": search_result, "item_type": item_type, "search_term": search_term}
@@ -165,7 +165,7 @@ def delete_agent(agent_id: str):
 def add_item(agent_id: str, item_type: str, item_id: str):
     """Add a protocol, skill or connection to the register to a local agent."""
     agent_dir = os.path.join(app_context.agents_dir, agent_id)
-    if _call_aea([sys.executable, "-m", "aea.cli", "add", item_type, item_id], agent_dir) == 0:
+    if _call_aea(["aea", "add", item_type, item_id], agent_dir) == 0:
         return agent_id, 201  # 200 (OK)
     else:
         return {"detail": "Failed to add {} {} to agent {}".format(item_type, item_id, agent_id)}, 400  # 400 Bad request
@@ -204,6 +204,7 @@ def _call_aea(param_list: List[str], dir_arg: str) -> int:
         old_cwd = os.getcwd()
         os.chdir(dir_arg)
         ret = subprocess.call(param_list)
+        subprocess.Popen(param_list, stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()
         os.chdir(old_cwd)
     return ret
 

tests/test_cli/test_add/test_connection.py

@@ -31,7 +31,7 @@
 import aea.cli.common
 import aea.configurations.base
 from aea.cli import cli
-from tests.conftest import CLI_LOG_OPTION
+from ...conftest import CLI_LOG_OPTION, CUR_PATH
 
 
 class TestAddConnectionFailsWhenConnectionAlreadyExists:
@@ -48,6 +48,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -116,6 +119,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -158,6 +164,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -206,6 +215,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0

tests/test_cli/test_add/test_protocol.py

@@ -31,7 +31,7 @@
 import aea.cli.common
 import aea.configurations.base
 from aea.cli import cli
-from tests.conftest import CLI_LOG_OPTION
+from ...conftest import CLI_LOG_OPTION, CUR_PATH
 
 
 class TestAddProtocolFailsWhenProtocolAlreadyExists:
@@ -48,6 +48,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -114,6 +117,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -156,6 +162,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -204,6 +213,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0

tests/test_cli/test_add/test_skill.py

@@ -32,7 +32,7 @@
 import aea.cli.common
 from aea.cli import cli
 from aea.configurations.base import AgentConfig, DEFAULT_AEA_CONFIG_FILE
-from tests.conftest import ROOT_DIR, CLI_LOG_OPTION
+from ...conftest import ROOT_DIR, CLI_LOG_OPTION, CUR_PATH
 
 
 class TestAddSkillFailsWhenSkillAlreadyExists:
@@ -49,17 +49,15 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         # this also by default adds the oef connection and error skill
         assert result.exit_code == 0
         os.chdir(cls.agent_name)
 
-        # change default registry path
-        config = AgentConfig.from_json(yaml.safe_load(open(DEFAULT_AEA_CONFIG_FILE)))
-        config.registry_path = os.path.join(ROOT_DIR, "packages")
-        yaml.safe_dump(config.json, open(DEFAULT_AEA_CONFIG_FILE, "w"))
-
         # add the error skill again
         cls.result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "add", "skill", cls.skill_name], standalone_mode=False)
 
@@ -121,6 +119,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -163,6 +164,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0
@@ -216,6 +220,9 @@ def setup_class(cls):
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
 
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
+
         os.chdir(cls.t)
         result = cls.runner.invoke(cli, [*CLI_LOG_OPTION, "create", cls.agent_name], standalone_mode=False)
         assert result.exit_code == 0

tests/test_cli/test_add_key.py

@@ -33,7 +33,7 @@
 from aea.crypto.ethereum import ETHEREUM
 from aea.crypto.fetchai import FetchAICrypto, FETCHAI
 from aea.crypto.helpers import DEFAULT_PRIVATE_KEY_FILE, FETCHAI_PRIVATE_KEY_FILE, ETHEREUM_PRIVATE_KEY_FILE
-from tests.conftest import CLI_LOG_OPTION
+from ..conftest import CLI_LOG_OPTION
 from ..common.click_testing import CliRunner
 
 

tests/test_cli/test_config.py

@@ -26,7 +26,7 @@
 
 import aea.cli.common
 from aea.cli import cli
-from tests.conftest import CLI_LOG_OPTION, CUR_PATH
+from ..conftest import CLI_LOG_OPTION, CUR_PATH
 from ..common.click_testing import CliRunner
 
 

tests/test_cli/test_create.py

@@ -39,7 +39,7 @@
 from aea.cli import cli
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE
 from aea.configurations.loader import ConfigLoader
-from tests.conftest import AGENT_CONFIGURATION_SCHEMA, ROOT_DIR, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION
+from ..conftest import AGENT_CONFIGURATION_SCHEMA, ROOT_DIR, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION
 
 
 class TestCreate:

tests/test_cli/test_delete.py

@@ -29,7 +29,7 @@
 import aea
 import aea.cli.common
 from aea.cli import cli
-from tests.conftest import CLI_LOG_OPTION
+from ..conftest import CLI_LOG_OPTION
 
 
 class TestDelete:

tests/test_cli/test_freeze.py

@@ -26,7 +26,7 @@
 from jsonschema import Draft4Validator
 
 from aea.cli import cli
-from tests.conftest import AGENT_CONFIGURATION_SCHEMA, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION, CUR_PATH
+from ..conftest import AGENT_CONFIGURATION_SCHEMA, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION, CUR_PATH
 from ..common.click_testing import CliRunner
 
 

tests/test_cli/test_generate_key.py

@@ -28,7 +28,7 @@
 from aea.crypto.ethereum import EthereumCrypto
 from aea.crypto.fetchai import FetchAICrypto
 from aea.crypto.helpers import DEFAULT_PRIVATE_KEY_FILE, FETCHAI_PRIVATE_KEY_FILE, ETHEREUM_PRIVATE_KEY_FILE
-from tests.conftest import CLI_LOG_OPTION
+from ..conftest import CLI_LOG_OPTION
 from ..common.click_testing import CliRunner
 
 

tests/test_cli/test_gui.py

@@ -30,7 +30,7 @@
 import pytest
 from jsonschema import Draft4Validator
 
-from tests.conftest import AGENT_CONFIGURATION_SCHEMA, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION, tcpping
+from ..conftest import AGENT_CONFIGURATION_SCHEMA, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION, tcpping
 
 
 class TestGui:

tests/test_cli/test_install.py

@@ -29,7 +29,7 @@
 import aea.cli.common
 from aea.cli import cli
 from aea.configurations.base import DEFAULT_PROTOCOL_CONFIG_FILE
-from tests.conftest import CLI_LOG_OPTION, CUR_PATH
+from ..conftest import CLI_LOG_OPTION, CUR_PATH
 
 
 class TestInstall:

tests/test_cli/test_list.py

@@ -28,7 +28,7 @@
 from jsonschema import Draft4Validator
 
 from aea.cli import cli
-from tests.conftest import AGENT_CONFIGURATION_SCHEMA, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION, CUR_PATH
+from ..conftest import AGENT_CONFIGURATION_SCHEMA, CONFIGURATION_SCHEMA_DIR, CLI_LOG_OPTION, CUR_PATH
 from tests.test_cli.constants import FORMAT_ITEMS_SAMPLE_OUTPUT
 
 

tests/test_cli/test_login.py

@@ -22,7 +22,7 @@
 from aea.cli import cli
 
 from tests.common.click_testing import CliRunner
-from tests.conftest import CLI_LOG_OPTION
+from ..conftest import CLI_LOG_OPTION
 
 
 @mock.patch('aea.cli.login.registry_login', return_value='token')

tests/test_cli/test_remove/test_connection.py

@@ -32,7 +32,7 @@
 import aea.configurations.base
 from aea.cli import cli
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE
-from tests.conftest import CLI_LOG_OPTION
+from ...conftest import CLI_LOG_OPTION, CUR_PATH
 
 
 class TestRemoveConnection:
@@ -45,6 +45,8 @@ def setup_class(cls):
         cls.agent_name = "myagent"
         cls.cwd = os.getcwd()
         cls.t = tempfile.mkdtemp()
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
         cls.connection_name = "local"
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
@@ -133,6 +135,8 @@ def setup_class(cls):
         cls.agent_name = "myagent"
         cls.cwd = os.getcwd()
         cls.t = tempfile.mkdtemp()
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
         cls.connection_name = "local"
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()

tests/test_cli/test_remove/test_protocol.py

@@ -32,7 +32,7 @@
 import aea.configurations.base
 from aea.cli import cli
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE
-from tests.conftest import CLI_LOG_OPTION
+from ...conftest import CLI_LOG_OPTION, CUR_PATH
 
 
 class TestRemoveProtocol:
@@ -45,6 +45,8 @@ def setup_class(cls):
         cls.agent_name = "myagent"
         cls.cwd = os.getcwd()
         cls.t = tempfile.mkdtemp()
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
         cls.protocol_name = "gym"
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()
@@ -133,6 +135,8 @@ def setup_class(cls):
         cls.agent_name = "myagent"
         cls.cwd = os.getcwd()
         cls.t = tempfile.mkdtemp()
+        # copy the 'packages' directory in the parent of the agent folder.
+        shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(cls.t, "packages"))
         cls.protocol_name = "gym"
         cls.patch = unittest.mock.patch.object(aea.cli.common.logger, 'error')
         cls.mocked_logger_error = cls.patch.__enter__()

tests/test_cli/test_remove/test_skill.py

@@ -32,7 +32,7 @@
 import aea.configurations.base
 from aea.cli import cli
 from aea.configurations.base import DEFAULT_AEA_CONFIG_FILE, AgentConfig
-from tests.conftest import ROOT_DIR, CLI_LOG_OPTION
+from ...conftest import ROOT_DIR, CLI_LOG_OPTION
 
 
 class TestRemoveSkill: