Merge pull request #60 from CapCap/add_mw_drw_proxy

[mw][e2e] Add miniwallet<->drw proxy

backend/Pipfile

@@ -77,7 +77,7 @@ python-dotenv = "*"
 dataclasses-json = "*"
 jwcrypto = "*"
 sqlalchemy-paginator = "*"
-diem = "*"
+diem = {extras = ["all"], version = "*"}
 
 [pipenv]
 allow_prereleases = true

backend/README.md

@@ -69,6 +69,15 @@ To test:
     $ ./format.sh  # runs black
     $ ./test.sh  # runs unit tests and type-checker
 
+
+### Running the Diem Mini Wallet (DMW) TestSuite
+1. Ensure that you have DRW running `../scripts/lrw.sh develop 8080`
+2. Run the mw_drw proxy: `DRW_URL_PREFIX=http://localhost:8080 MW_DRW_PROXY_PORT=3130 pipenv run python3 ./tests/mw_drw_proxy/proxy.py` . 
+   There is additional documentation on this command in the `tests/mw_drw_proxy` folder.
+3. Run the test suite: `pipenv run dmw -- test --verbose=true --target http://127.0.0.1:3130`.
+  To run a specific test, you can add `--pytest-args '-ktest_payment_under_threshold_succeed[sender-999999]'` to the test command
+
+
 ### Git setup
 
 It is recommended to install our git pre-commit hook. This hook runs all needed validations before committing:

backend/tests/e2e_tests/__init__.py

@@ -5,6 +5,9 @@
 import random
 import requests
 from dataclasses import dataclass
+
+from requests import HTTPError
+
 from diem_utils.types.currencies import DiemCurrency, FiatCurrency
 
 LRW_WEB_1 = os.getenv("LRW_WEB_1")
@@ -21,7 +24,7 @@
 print(VASP_ADDR_1)
 print(VASP_ADDR_2)
 
-sherlock_info = {
+SHERLOCK_INFO = {
     "address_1": "221B Baker Street",
     "address_2": "",
     "city": "London",
@@ -37,16 +40,31 @@
 }
 
 
+def default_log_fn(*args, **kwargs):
+    print(*args, **kwargs)
+
+
 @dataclass
 class UserClient:
     name: str
     backend: str
     token: str
     payment_method: str
+    log_fn: typing.Callable
 
     @staticmethod
-    def create(backend: str, name: str) -> "UserClient":
-        return create_test_user(backend, f"{name}_{random.randint(0, 1000)}")
+    def create(
+        backend: str,
+        name: str,
+        kyc_info=None,
+        log_fn=default_log_fn,
+    ) -> "UserClient":
+        return create_test_user(
+            backend,
+            name,
+            log_fn=log_fn,
+            kyc_info=kyc_info,
+        )
 
     def auth_headers(self) -> typing.Dict[str, str]:
         ret = {"Authorization": f"Bearer {self.token}"}
@@ -67,26 +85,42 @@ def wait_for_balance(
             f"wait for {self.name}'s {currency} balance timeout ({timeout} secs), found balance: {new_balance}, expect {target_balance}"
         )
 
+    def update_user(self, user_data: dict) -> dict:
+        res = requests.put(
+            f"{self.backend}/api/user", headers=self.auth_headers(), json=user_data
+        )
+        res.raise_for_status()
+        return res.json()
+
+    def get_payment_methods(self) -> list:
+        res = requests.get(
+            f"{self.backend}/api/user/payment-methods", headers=self.auth_headers()
+        )
+        res.raise_for_status()
+        return res.json().get("payment_methods")
+
     def buy(self, amount: int, currency: str, by_currency: str):
+        self.log_fn(f"Buying {amount} {currency} for {self.name} ")
         err = None
         for i in range(8):
             try:
                 self._buy(amount, currency, by_currency)
                 return
             except Exception as e:
+                if isinstance(e, HTTPError):
+                    raise
                 err = e
-                pass
 
         raise RuntimeError(f"failed after retry, last error: {err}")
 
     def _buy(self, amount: int, currency: str, by_currency: str):
         before_balance = self.get_balance(currency)
-
+        self.log_fn(f"Current balance for {currency}: {amount} for '{self.name}'")
         pair = f"{currency}_{by_currency}"
         quote_id = get_test_quote(
             self.backend, self.auth_headers(), amount, "buy", pair
         )
-        print(f"quote_id: {quote_id}")
+        self.log_fn(f"quote_id: {quote_id}")
         # pay with the first payment method added to wallet 1
         payload = {"payment_method": self.payment_method}
         res = requests.post(
@@ -95,7 +129,7 @@ def _buy(self, amount: int, currency: str, by_currency: str):
             json=payload,
         )
         res.raise_for_status()
-        self.wait_for_balance(currency, before_balance + amount, 20)
+        self.wait_for_balance(currency, before_balance + amount, 50)
 
     def get_recv_addr(self) -> str:
         """Get the receiving subaddr for a test user"""
@@ -106,18 +140,20 @@ def get_recv_addr(self) -> str:
         res.raise_for_status()
         return res.json().get("address")
 
-    def transfer(self, addr: str, amount: int, currency) -> None:
+    def transfer(self, addr: str, amount: int, currency: str) -> dict:
         payload = {
             "amount": amount,
             "currency": currency,
             "receiver_address": addr,
         }
+        self.log_fn(f"Transferring money '{payload}' for {self.name}")
         res = requests.post(
             f"{self.backend}/api/account/transactions",
             headers=self.auth_headers(),
             json=payload,
         )
         res.raise_for_status()
+        return res.json()
 
     def get_transactions(self) -> list:
         params = {"limit": 10}
@@ -129,53 +165,74 @@ def get_transactions(self) -> list:
         res.raise_for_status()
         return res.json().get("transaction_list")
 
-    def get_balance(self, currency: str) -> list:
+    def get_balances(self) -> typing.List[typing.Dict]:
         res = requests.get(f"{self.backend}/api/account", headers=self.auth_headers())
         res.raise_for_status()
-        balances = res.json().get("balances")
+        return res.json().get("balances")
+
+    def get_balance(self, currency: str) -> int:
+        self.log_fn(f"Getting balance for {currency} for '{self.name}'")
+        balances = self.get_balances()
+        self.log_fn(f"got balances: {balances} for '{self.name}'")
         for b in balances:
             if b.get("currency") == currency:
                 return b.get("balance")
 
         raise ValueError(f"no balance for the currency {currency} in {balances}")
 
 
-def invoke_kyc_check(backend, headers) -> None:
-    """Invoke a KYC check by updating user info for the first time"""
+def randomize_username(name):
+    suffix = "@test.com"
+    if "@" in name:
+        name, suffix = name.split("@")
+    return f"{name}_{random.randint(0, 100000000)}{suffix}"
 
-    res = requests.put(f"{backend}/api/user", headers=headers, json=sherlock_info)
+
+def invoke_kyc_check(backend, headers, kyc_info=None, log_fn=default_log_fn) -> None:
+    """Invoke a KYC check by updating user info for the first time"""
+    kyc_info = kyc_info or SHERLOCK_INFO
+    log_fn(f"Updating user info with KYC for '{kyc_info}'")
+    res = requests.put(f"{backend}/api/user", headers=headers, json=kyc_info)
     res.raise_for_status()
 
 
-def create_test_user(backend, username) -> UserClient:
+def create_test_user(
+    backend, username, log_fn=default_log_fn, kyc_info=None
+) -> UserClient:
     """Create a test user"""
-
+    username = randomize_username(username)
     payload = {"username": username, "password": "fakepassword"}
+    log_fn(f"Creating user '{username}', with password 'fakepassword'")
     res = requests.post(f"{backend}/api/user", json=payload)
     res.raise_for_status()
 
     token = res.text.strip('"')
     headers = {"Authorization": f"Bearer {token}"}
 
-    invoke_kyc_check(backend, headers)
-    payment_method = create_test_payment_method(backend, headers)
+    invoke_kyc_check(backend, headers, kyc_info=kyc_info, log_fn=log_fn)
+    payment_method = create_test_payment_method(backend, headers, log_fn=log_fn)
 
     return UserClient(
-        name=username, backend=backend, token=token, payment_method=payment_method
+        name=username,
+        backend=backend,
+        token=token,
+        payment_method=payment_method,
+        log_fn=log_fn,
     )
 
 
-def create_test_payment_method(backend, headers) -> str:
+def create_test_payment_method(backend, headers, log_fn=default_log_fn) -> str:
     """
     Create a test payment method and return its id as a string
     NOTE: assumes that this is the only payment method being added for the user
     """
-    payment_token = f"paymenttoken{random.randint(0, 1000)}"
+    payment_token = f"paymenttoken{random.randint(0, 1000000)}"
     payment_payload = {
         "name": "credit",
         "provider": "CreditCard",
         "token": payment_token,
     }
+    log_fn(f"Creating payment method '{payment_payload}'")
     res = requests.post(
         f"{backend}/api/user/payment-methods", headers=headers, json=payment_payload
     )
@@ -185,7 +242,10 @@ def create_test_payment_method(backend, headers) -> str:
     res.raise_for_status()
 
     assert len(res.json().get("payment_methods")) > 0
-    return str(res.json().get("payment_methods")[0].get("id"))
+
+    result_id = str(res.json().get("payment_methods")[0].get("id"))
+    log_fn(f"Created payment method '{result_id}'")
+    return result_id
 
 
 def get_test_quote(backend, headers, amount, buy_sell="buy", pair="XUS_USD") -> str:

backend/tests/mw_drw_proxy/README.md

@@ -0,0 +1,16 @@
+# MiniWallet <=> Reference Wallet Proxy
+
+This will run a proxy which allows the DMW test suite to communicate with DRW.
+
+For example, when the DRW requests to set up a user, the proxy registers a user, updates their information with KYC, 
+and deposits any initial test balances that the DMW required, before returning an identifier for that user which DMW 
+will reuse in future calls.
+
+
+### Usage:
+(from inside the venv) 
+```
+DRW_URL_PREFIX=http://reference_wallet:8080 MW_DRW_PROXY_PORT=3130 pipenv run python3 tests/mw_drw_proxy/proxy.py
+```
+This will launch the proxy server on `http://localhost:3130`, and will proxy the requests to an instance of 
+DRW running at `http://reference_wallet:8080`

backend/tests/mw_drw_proxy/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) The Diem Core Contributors
+# SPDX-License-Identifier: Apache-2.0

backend/tests/mw_drw_proxy/helpers.py

@@ -0,0 +1,78 @@
+# Copyright (c) The Diem Core Contributors
+# SPDX-License-Identifier: Apache-2.0
+import logging
+import traceback
+import random
+
+import typing
+from requests import HTTPError
+
+DEFAULT_KYC_INFO = {
+    "first_name": "Default",
+    "last_name": "KYC",
+    "dob": "1956-07-09",
+    "phone": "1 202-456-1414",
+    "address_1": "1840 Century Park East",
+    "address_2": "",
+    "city": "Los Angeles",
+    "state": "California",
+    "zip": "90067",
+    "country": "US",
+    "selected_fiat_currency": "USD",
+    "selected_language": "en",
+}
+
+
+def convert_user_info_to_offchain(user_info: dict) -> dict:
+    return {
+        "payload_version": 1,
+        "type": "individual",
+        "given_name": user_info.get("first_name"),
+        "surname": user_info.get("last_name"),
+        "dob": user_info.get("dob"),
+        "address": {
+            "line1": user_info.get("address_1"),
+            "line2": user_info.get("address_2"),
+            "city": user_info.get("city"),
+            "state": user_info.get("state"),
+            "postal_code": user_info.get("zip"),
+            "country": user_info.get("country"),
+        },
+    }
+
+
+def convert_offchain_to_user_info(offchain: dict) -> dict:
+    address = offchain.get("address") or {}
+    return {
+        **DEFAULT_KYC_INFO,
+        "first_name": offchain.get("given_name"),
+        "last_name": offchain.get("surname"),
+        "dob": offchain.get("dob"),
+        "address_1": address.get("line1"),
+        "address_2": address.get("line2"),
+        "city": address.get("city"),
+        "state": address.get("state"),
+        "zip": address.get("postal_code"),
+        "country": address.get("country"),
+        "selected_fiat_currency": "USD",
+        "selected_language": "en",
+    }
+
+
+def error_response(logger: logging.Logger, e: Exception) -> typing.Tuple[dict, int]:
+    # Use status code 418 (I'm a teapot) to indicate actual proxy server failures vs upstream issues
+    status_code = 418
+    if isinstance(e, HTTPError):
+        status_code = e.response.status_code
+        logger.warning(
+            f"Got error: {e}. {e.request.method} - {e.request.path_url} - {e.request.body}"
+        )
+    else:
+        backtrace = "\n".join(traceback.format_tb(e.__traceback__))
+        logger.warning(f"Got error: {e}. \n{backtrace}")
+
+    response = {
+        "error": str(e),
+        "stacktrace": traceback.format_tb(e.__traceback__),
+    }
+    return response, status_code