Merge branch 'main' into custom_cache_page

README.md

@@ -200,6 +200,9 @@ https://docs.safe.global/api-supported-networks
 ### What means banned field in SafeContract model?
 The `banned` field in the `SafeContract` model is used to prevent indexing of certain Safes that have an unsupported `MasterCopy` or unverified proxies that have issues during indexing. This field does not remove the banned Safe and indexing can be resumed once the issue has been resolved.
 
+### Why is my ERC20 token not indexed?
+For an ERC20 token to be indexed it needs to have `name`, `symbol`, `decimals` and `balanceOf()`, otherwise the service will ignore it and add it to the `TokenNotValid` model.
+
 ## Troubleshooting
 
 ### Issues installing grpc on an Apple silicon system

config/settings/base.py

@@ -585,6 +585,9 @@
 ETH_REORG_BLOCKS = env.int(
     "ETH_REORG_BLOCKS", default=200 if ETH_L2_NETWORK else 10
 )  # Number of blocks from the current block number needed to consider a block valid/stable
+ETH_ERC20_LOAD_ADDRESSES_CHUNK_SIZE = env.int(
+    "ETH_ERC20_LOAD_ADDRESSES_CHUNK_SIZE", default=500_000
+)  # Load Safe addresses for the ERC20 indexer with a database iterator with the defined `chunk_size`
 
 # Events processing
 # ------------------------------------------------------------------------------

requirements-test.txt

@@ -1,5 +1,5 @@
 -r requirements.txt
-coverage==7.6.1
+coverage==7.6.4
 django-stubs==5.1.0
 django-test-migrations==1.4.0
 factory-boy==3.3.1
@@ -8,6 +8,6 @@ mypy==1.11.2
 pytest==8.3.3
 pytest-celery==1.1.3
 pytest-django==4.8.0
-pytest-env==1.1.3
+pytest-env==1.1.5
 pytest-rerunfailures==14.0
 pytest-sugar==1.0.0

requirements.txt

@@ -4,7 +4,7 @@ cachetools==5.5.0
 celery==5.4.0
 django==5.0.9
 django-cache-memoize==0.2.0
-django-celery-beat==2.6.0
+django-celery-beat==2.7.0
 django-cors-headers==4.5.0
 django-db-geventpool==4.0.6
 django-debug-toolbar
@@ -28,7 +28,7 @@ hexbytes==0.3.1
 hiredis==3.0.0
 packaging>=21.0
 pika==1.3.2
-pillow==10.4.0
+pillow==11.0.0
 psycogreen==1.0.2
 psycopg2==2.9.9
 redis==5.0.8

safe_transaction_service/account_abstraction/filters.py

@@ -0,0 +1,47 @@
+import django_filters
+from django_filters import rest_framework as filters
+from safe_eth.eth.django.filters import Keccak256Filter
+
+from safe_transaction_service.utils.filters import filter_overrides
+
+from .models import SafeOperation
+
+
+class SafeOperationFilter(filters.FilterSet):
+    executed = django_filters.BooleanFilter(method="filter_executed")
+    has_confirmations = django_filters.BooleanFilter(method="filter_confirmations")
+    execution_date__gte = django_filters.IsoDateTimeFilter(
+        field_name="user_operation__ethereum_tx__block__timestamp", lookup_expr="gte"
+    )
+    execution_date__lte = django_filters.IsoDateTimeFilter(
+        field_name="user_operation__ethereum_tx__block__timestamp", lookup_expr="lte"
+    )
+    submission_date__gte = django_filters.IsoDateTimeFilter(
+        field_name="created", lookup_expr="gte"
+    )
+    submission_date__lte = django_filters.IsoDateTimeFilter(
+        field_name="created", lookup_expr="lte"
+    )
+    transaction_hash = Keccak256Filter(field_name="user_operation__ethereum_tx_id")
+
+    def filter_confirmations(self, queryset, _name: str, value: bool):
+        if value:
+            return queryset.with_confirmations()
+        else:
+            return queryset.without_confirmations()
+
+    def filter_executed(self, queryset, _name: str, value: bool):
+        if value:
+            return queryset.executed()
+        else:
+            return queryset.not_executed()
+
+    class Meta:
+        model = SafeOperation
+        fields = {
+            "modified": ["lt", "gt", "lte", "gte"],
+            "valid_after": ["lt", "gt", "lte", "gte"],
+            "valid_until": ["lt", "gt", "lte", "gte"],
+            "module_address": ["exact"],
+        }
+        filter_overrides = filter_overrides

safe_transaction_service/account_abstraction/models.py

@@ -126,7 +126,22 @@ def __str__(self) -> str:
         return f"{HexBytes(self.user_operation_id).hex()} UserOperationReceipt"
 
 
+class SafeOperationQuerySet(models.QuerySet):
+    def executed(self):
+        return self.exclude(user_operation__ethereum_tx=None)
+
+    def not_executed(self):
+        return self.filter(user_operation__ethereum_tx=None)
+
+    def with_confirmations(self):
+        return self.exclude(confirmations__isnull=True)
+
+    def without_confirmations(self):
+        return self.filter(confirmations__isnull=True)
+
+
 class SafeOperation(TimeStampedModel):
+    objects = SafeOperationQuerySet.as_manager()
     hash = Keccak256Field(primary_key=True)  # safeOperationHash
     user_operation = models.OneToOneField(
         UserOperation, on_delete=models.CASCADE, related_name="safe_operation"

safe_transaction_service/account_abstraction/tests/test_views.py

@@ -175,6 +175,16 @@ def test_safe_operations_view(self):
             {"count": 1, "next": None, "previous": None, "results": [expected]},
         )
 
+        # Check confirmations flag
+        response = self.client.get(
+            reverse("v1:account_abstraction:safe-operations", args=(safe_address,))
+            + "?has_confirmations=True"
+        )
+        self.assertDictEqual(
+            response.json(),
+            {"count": 0, "next": None, "previous": None, "results": []},
+        )
+
         # Add a confirmation
         safe_operation_confirmation = factories.SafeOperationConfirmationFactory(
             safe_operation=safe_operation
@@ -204,6 +214,47 @@ def test_safe_operations_view(self):
             {"count": 1, "next": None, "previous": None, "results": [expected]},
         )
 
+        # Check executed flag
+        response = self.client.get(
+            reverse("v1:account_abstraction:safe-operations", args=(safe_address,))
+            + "?executed=False"
+        )
+        self.assertDictEqual(
+            response.json(),
+            {"count": 0, "next": None, "previous": None, "results": []},
+        )
+
+        response = self.client.get(
+            reverse("v1:account_abstraction:safe-operations", args=(safe_address,))
+            + "?executed=True"
+        )
+        self.assertDictEqual(
+            response.json(),
+            {"count": 1, "next": None, "previous": None, "results": [expected]},
+        )
+
+        # Set transaction as not executed and check again
+        safe_operation.user_operation.ethereum_tx = None
+        safe_operation.user_operation.save(update_fields=["ethereum_tx"])
+        response = self.client.get(
+            reverse("v1:account_abstraction:safe-operations", args=(safe_address,))
+            + "?executed=True"
+        )
+        self.assertDictEqual(
+            response.json(),
+            {"count": 0, "next": None, "previous": None, "results": []},
+        )
+
+        response = self.client.get(
+            reverse("v1:account_abstraction:safe-operations", args=(safe_address,))
+            + "?executed=False"
+        )
+        expected["userOperation"]["ethereumTxHash"] = None
+        self.assertDictEqual(
+            response.json(),
+            {"count": 1, "next": None, "previous": None, "results": [expected]},
+        )
+
     @mock.patch.object(
         SafeOperationSerializer,
         "_get_owners",

safe_transaction_service/account_abstraction/views.py

@@ -6,7 +6,7 @@
 from rest_framework.response import Response
 from safe_eth.eth.utils import fast_is_checksum_address
 
-from . import pagination, serializers
+from . import filters, pagination, serializers
 from .models import SafeOperation, SafeOperationConfirmation, UserOperation
 
 
@@ -29,11 +29,15 @@ class SafeOperationsView(ListCreateAPIView):
         django_filters.rest_framework.DjangoFilterBackend,
         OrderingFilter,
     ]
+    filterset_class = filters.SafeOperationFilter
     ordering = ["-user_operation__nonce", "-created"]
     ordering_fields = ["user_operation__nonce", "created"]
     pagination_class = pagination.DefaultPagination
 
     def get_queryset(self):
+        if getattr(self, "swagger_fake_view", False):
+            return SafeOperation.objects.none()
+
         safe = self.kwargs["address"]
         return (
             SafeOperation.objects.filter(user_operation__sender=safe)
@@ -43,9 +47,6 @@ def get_queryset(self):
 
     def get_serializer_context(self):
         context = super().get_serializer_context()
-        if getattr(self, "swagger_fake_view", False):
-            return context
-
         context["safe_address"] = self.kwargs["address"]
         return context
 
@@ -100,6 +101,9 @@ class SafeOperationConfirmationsView(ListCreateAPIView):
     pagination_class = pagination.DefaultPagination
 
     def get_queryset(self):
+        if getattr(self, "swagger_fake_view", False):
+            return SafeOperationConfirmation.objects.none()
+
         return SafeOperationConfirmation.objects.filter(
             safe_operation__hash=self.kwargs["safe_operation_hash"]
         )
@@ -171,6 +175,9 @@ class UserOperationsView(ListAPIView):
     serializer_class = serializers.UserOperationWithSafeOperationResponseSerializer
 
     def get_queryset(self):
+        if getattr(self, "swagger_fake_view", False):
+            return UserOperation.objects.none()
+
         safe = self.kwargs["address"]
         return (
             UserOperation.objects.filter(sender=safe)
@@ -180,9 +187,6 @@ def get_queryset(self):
 
     def get_serializer_context(self):
         context = super().get_serializer_context()
-        if getattr(self, "swagger_fake_view", False):
-            return context
-
         context["safe_address"] = self.kwargs["address"]
         return context
 

safe_transaction_service/history/filters.py

@@ -4,19 +4,10 @@
 from django_filters import rest_framework as filters
 from rest_framework.exceptions import ValidationError
 from safe_eth.eth.django.filters import EthereumAddressFilter, Keccak256Filter
-from safe_eth.eth.django.models import (
-    EthereumAddressBinaryField,
-    Keccak256Field,
-    Uint256Field,
-)
 
-from .models import ModuleTransaction, MultisigTransaction
+from safe_transaction_service.utils.filters import filter_overrides
 
-filter_overrides = {
-    Uint256Field: {"filter_class": django_filters.NumberFilter},
-    Keccak256Field: {"filter_class": Keccak256Filter},
-    EthereumAddressBinaryField: {"filter_class": EthereumAddressFilter},
-}
+from .models import ModuleTransaction, MultisigTransaction
 
 
 class DelegateListFilter(filters.FilterSet):

safe_transaction_service/history/indexers/erc20_events_indexer.py

@@ -40,7 +40,10 @@ def __new__(cls):
     def get_new_instance(cls) -> "Erc20EventsIndexer":
         from django.conf import settings
 
-        return Erc20EventsIndexer(EthereumClient(settings.ETHEREUM_NODE_URL))
+        return Erc20EventsIndexer(
+            EthereumClient(settings.ETHEREUM_NODE_URL),
+            eth_erc20_load_addresses_chunk_size=settings.ETH_ERC20_LOAD_ADDRESSES_CHUNK_SIZE,
+        )
 
     @classmethod
     def del_singleton(cls):
@@ -65,6 +68,9 @@ def __init__(self, *args, **kwargs):
 
         self._processed_element_cache = FixedSizeDict(maxlen=40_000)  # Around 3MiB
         self.addresses_cache: Optional[AddressesCache] = None
+        self.eth_erc20_load_addresses_chunk_size = kwargs.get(
+            "eth_erc20_load_addresses_chunk_size", 500_000
+        )
 
     @property
     def contract_events(self) -> List[ContractEvent]:
@@ -252,15 +258,40 @@ def get_almost_updated_addresses(
             addresses = set()
             last_checked = None
 
-        for created, address in query.values_list("created", "address").order_by(
-            "created"
+        """
+        Chunk size optimization
+        -----------------------
+        Testing with 3M Safes
+        2k - 90 seconds
+        100k - 73 seconds
+        500k - 60 seconds
+        1M - 60 seconds
+        3M - 60 seconds
+
+        Testing with 15M Safes
+        50k - 854 seconds
+        500k - 460 seconds
+        750k - 415 seconds
+        1M - 430 seconds
+        2M - 398 seconds
+        3M - 407 seconds
+
+        500k sounds like a good compromise memory/speed wise
+        """
+        created: Optional[datetime.datetime] = None
+        for i, (created, address) in enumerate(
+            query.values_list("created", "address")
+            .order_by("created")
+            .iterator(chunk_size=self.eth_erc20_load_addresses_chunk_size)
         ):
             addresses.add(address)
 
-        try:
+            # Store addresses in cache every chunk, just in case task is interrupted during address loading
+            if i % self.eth_erc20_load_addresses_chunk_size == 0:
+                self.addresses_cache = AddressesCache(addresses, created)
+
+        if created:
             last_checked = created
-        except NameError:  # database query empty, `created` not defined
-            pass
 
         if last_checked:
             # Don't use caching if list is empty

safe_transaction_service/history/indexers/ethereum_indexer.py

@@ -40,6 +40,7 @@ def __init__(
         updated_blocks_behind: int = 20,
         query_chunk_size: Optional[int] = 1_000,
         block_auto_process_limit: bool = True,
+        **kwargs,
     ):
         """
         :param ethereum_client:

safe_transaction_service/history/services/balance_service.py

@@ -85,7 +85,7 @@ def __init__(self, ethereum_client: EthereumClient, redis: Redis):
             maxsize=4096, ttl=60 * 30
         )  # 2 hours of caching
 
-    def _filter_addresses(
+    def _filter_tokens(
         self,
         erc20_addresses: Sequence[ChecksumAddress],
         only_trusted: bool,
@@ -200,7 +200,7 @@ def _get_page_erc20_balances(
         for address in all_erc20_addresses:
             # Store tokens in database if not present
             self.get_token_info(address)  # This is cached
-        erc20_addresses = self._filter_addresses(
+        erc20_addresses = self._filter_tokens(
             all_erc20_addresses, only_trusted, exclude_spam
         )
         # Total count should take into account the request filters

safe_transaction_service/history/services/safe_service.py

@@ -22,7 +22,13 @@
 from safe_transaction_service.utils.abis.gelato import gelato_relay_1_balance_v2_abi
 
 from ..exceptions import NodeConnectionException
-from ..models import EthereumTx, InternalTx, SafeLastStatus, SafeMasterCopy
+from ..models import (
+    EthereumTx,
+    InternalTx,
+    InternalTxType,
+    SafeLastStatus,
+    SafeMasterCopy,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -116,7 +122,8 @@ def get_safe_creation_info(
             # Get first the actual creation transaction for the safe
             creation_internal_tx = (
                 InternalTx.objects.filter(
-                    ethereum_tx__status=1  # Ignore Internal Transactions for failed Transactions
+                    ethereum_tx__status=1,  # Ignore Internal Transactions for failed Transactions
+                    tx_type=InternalTxType.CREATE.value,
                 )
                 .select_related("ethereum_tx__block")
                 .get(contract_address=safe_address)