Merge pull request #1582 from BoostryJP/feature/#1580

Optimize row count query performance
BoostryJP · Dec 9, 2024 · f7a6d0d · f7a6d0d
2 parents 414c48e + 8dd489c
commit f7a6d0d
Show file tree

Hide file tree

Showing 10 changed files with 92 additions and 59 deletions.
diff --git a/app/api/routers/bc_explorer.py b/app/api/routers/bc_explorer.py
@@ -125,7 +125,7 @@ async def list_block_data(
         stmt = stmt.where(IDXBlockData.number <= to_block_number)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).select_from(IDXBlockData).order_by(None)
     )
 
     # Sort
@@ -134,18 +134,22 @@ async def list_block_data(
     else:
         stmt = stmt.order_by(desc(IDXBlockData.number))
 
+    if (
+        await async_session.scalar(
+            stmt.with_only_columns(func.count())
+            .select_from(IDXBlockData)
+            .order_by(None)
+        )
+        > BLOCK_RESPONSE_LIMIT
+    ):
+        raise ResponseLimitExceededError("Search results exceed the limit")
+
     # Pagination
     if limit is not None:
         stmt = stmt.limit(limit)
     if offset is not None:
         stmt = stmt.offset(offset)
 
-    if (
-        await async_session.scalar(select(func.count()).select_from(stmt.subquery()))
-        > BLOCK_RESPONSE_LIMIT
-    ):
-        raise ResponseLimitExceededError("Search results exceed the limit")
-
     block_data_tmp: Sequence[IDXBlockData] = (await async_session.scalars(stmt)).all()
 
     block_data = [
@@ -269,24 +273,26 @@ async def list_tx_data(
         stmt = stmt.where(IDXTxData.to_address == to_checksum_address(to_address))
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).select_from(IDXTxData).order_by(None)
     )
 
     # Sort
     stmt = stmt.order_by(desc(IDXTxData.created))
 
+    if (
+        await async_session.scalar(
+            stmt.with_only_columns(func.count()).select_from(IDXTxData).order_by(None)
+        )
+        > TX_RESPONSE_LIMIT
+    ):
+        raise ResponseLimitExceededError("Search results exceed the limit")
+
     # Pagination
     if limit is not None:
         stmt = stmt.limit(limit)
     if offset is not None:
         stmt = stmt.offset(offset)
 
-    if (
-        await async_session.scalar(select(func.count()).select_from(stmt.subquery()))
-        > TX_RESPONSE_LIMIT
-    ):
-        raise ResponseLimitExceededError("Search results exceed the limit")
-
     tx_data_tmp: Sequence[IDXTxData] = (await async_session.scalars(stmt)).all()
 
     tx_data = [

diff --git a/app/api/routers/notification.py b/app/api/routers/notification.py
@@ -71,7 +71,7 @@ async def list_all_notifications(
 
     stmt = select(Notification)
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).select_from(Notification).order_by(None)
     )
 
     # Search Filter
@@ -82,7 +82,7 @@ async def list_all_notifications(
     if priority is not None:
         stmt = stmt.where(Notification.priority == priority)
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).select_from(Notification).order_by(None)
     )
 
     # Sort

diff --git a/app/api/routers/position.py b/app/api/routers/position.py
@@ -190,9 +190,10 @@ async def get_list_from_index(
             )
             .order_by(Listing.id)
         )
-
         total = await async_session.scalar(
-            select(func.count()).select_from(stmt.subquery())
+            select(func.count()).select_from(
+                stmt.with_only_columns(1).order_by(None).subquery()
+            )
         )
         count = total
         if limit is not None:
@@ -786,9 +787,10 @@ async def get_list_from_index(
             )
             .order_by(Listing.id)
         )
-
         total = await async_session.scalar(
-            select(func.count()).select_from(stmt.subquery())
+            select(func.count()).select_from(
+                stmt.with_only_columns(1).order_by(None).subquery()
+            )
         )
         count = total
         if limit is not None:
@@ -906,8 +908,11 @@ async def get_list_from_index(
         )
 
         total = await async_session.scalar(
-            select(func.count()).select_from(stmt.subquery())
+            select(func.count()).select_from(
+                stmt.with_only_columns(1).order_by(None).subquery()
+            )
         )
+
         count = total
         if limit is not None:
             stmt = stmt.limit(limit)
@@ -1479,8 +1484,11 @@ async def list_all_token_position(
     )
 
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        select(func.count()).select_from(
+            stmt.with_only_columns(1).order_by(None).subquery()
+        )
     )
+
     count = total
 
     if limit is not None:

diff --git a/app/api/routers/position_lock.py b/app/api/routers/position_lock.py
@@ -119,14 +119,18 @@ async def __call__(
         )
 
         total = await async_session.scalar(
-            select(func.count()).select_from(stmt.subquery())
+            stmt.with_only_columns(func.count())
+            .select_from(IDXLockedPosition)
+            .order_by(None)
         )
 
         if lock_address is not None:
             stmt = stmt.where(IDXLockedPosition.lock_address == lock_address)
 
         count = await async_session.scalar(
-            select(func.count()).select_from(stmt.subquery())
+            stmt.with_only_columns(func.count())
+            .select_from(IDXLockedPosition)
+            .order_by(None)
         )
 
         sort_attr = getattr(IDXLockedPosition, sort_item, None)
@@ -243,9 +247,13 @@ async def __call__(
             )
 
         total = await async_session.scalar(
-            select(func.count()).select_from(stmt_lock.subquery())
+            stmt_lock.with_only_columns(func.count())
+            .select_from(IDXLock)
+            .order_by(None)
         ) + await async_session.scalar(
-            select(func.count()).select_from(stmt_unlock.subquery())
+            stmt_unlock.with_only_columns(func.count())
+            .select_from(IDXUnlock)
+            .order_by(None)
         )
 
         match category:
@@ -279,7 +287,7 @@ async def __call__(
                 cast(column("data"), String).like("%" + request_query.data + "%")
             )
         count = await async_session.scalar(
-            select(func.count()).select_from(stmt.subquery())
+            stmt.with_only_columns(func.count()).order_by(None)
         )
 
         # Sort

diff --git a/app/api/routers/token.py b/app/api/routers/token.py
@@ -206,7 +206,9 @@ async def get_token_holders(
             )
         )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        select(func.count()).select_from(
+            stmt.with_only_columns(1).order_by(None).subquery()
+        )
     )
 
     if request_query.exclude_owner is True:
@@ -280,7 +282,9 @@ async def get_token_holders(
                 stmt = stmt.where(IDXLockedPosition.value <= request_query.locked)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        select(func.count()).select_from(
+            stmt.with_only_columns(1).order_by(None).subquery()
+        )
     )
 
     # Pagination
@@ -386,7 +390,9 @@ async def search_token_holders(
             )
         )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        select(func.count()).select_from(
+            stmt.with_only_columns(1).order_by(None).subquery()
+        )
     )
 
     if data.exclude_owner is True:
@@ -442,7 +448,9 @@ async def search_token_holders(
                 stmt = stmt.where(IDXLockedPosition.value <= data.locked)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        select(func.count()).select_from(
+            stmt.with_only_columns(1).order_by(None).subquery()
+        )
     )
 
     # Sort
@@ -590,7 +598,9 @@ async def get_token_holders_count(
         stmt = stmt.where(IDXPosition.account_address != listed_token.owner_address)
 
     _count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        select(func.count()).select_from(
+            stmt.with_only_columns(1).order_by(None).subquery()
+        )
     )
 
     resp_body = {"count": _count}
@@ -796,8 +806,9 @@ async def list_all_transfer_histories(
                 to_address_tag.account_tag == request_query.account_tag,
             )
         )
+
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if request_query.source_event is not None:
@@ -834,7 +845,7 @@ async def list_all_transfer_histories(
                 stmt = stmt.where(IDXTransfer.value <= request_query.value)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     # Pagination
@@ -892,7 +903,7 @@ async def search_transfer_histories(
             )
         )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if data.source_event is not None:
@@ -925,7 +936,7 @@ async def search_transfer_histories(
                 stmt = stmt.where(IDXTransfer.value <= data.value)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     def _order(_order):
@@ -1042,7 +1053,7 @@ async def list_all_transfer_approval_histories(
             )
         )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if request_query.from_address is not None:
@@ -1065,7 +1076,7 @@ async def list_all_transfer_approval_histories(
                 stmt = stmt.where(IDXTransferApproval.value <= request_query.value)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     # Pagination
@@ -1134,7 +1145,7 @@ async def search_transfer_approval_histories(
             )
         )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if data.application_datetime_from is not None:
@@ -1195,7 +1206,7 @@ async def search_transfer_approval_histories(
                 stmt = stmt.where(IDXTransferApproval.value <= data.value)
 
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     def _order(_order):

diff --git a/app/api/routers/token_bond.py b/app/api/routers/token_bond.py
@@ -87,7 +87,7 @@ async def list_all_straight_bond_tokens(
     if len(request_query.address_list):
         stmt = stmt.where(IDXBondToken.token_address.in_(request_query.address_list))
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     # Search Filter
@@ -128,7 +128,7 @@ async def list_all_straight_bond_tokens(
     if request_query.is_redeemed is not None:
         stmt = stmt.where(IDXBondToken.is_redeemed == request_query.is_redeemed)
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if sort_item == "created":
@@ -197,7 +197,7 @@ async def list_all_straight_bond_token_addresses(
         .where(Listing.is_public == True)
     )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     # Search Filter
@@ -238,7 +238,7 @@ async def list_all_straight_bond_token_addresses(
     if request_query.is_redeemed is not None:
         stmt = stmt.where(IDXBondToken.is_redeemed == request_query.is_redeemed)
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if sort_item == "created":

diff --git a/app/api/routers/token_coupon.py b/app/api/routers/token_coupon.py
@@ -96,7 +96,7 @@ async def list_all_coupon_tokens(
     if len(request_query.address_list):
         stmt = stmt.where(IDXCouponToken.token_address.in_(request_query.address_list))
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     # Search Filter
@@ -119,7 +119,7 @@ async def list_all_coupon_tokens(
             IDXCouponToken.initial_offering_status == initial_offering_status
         )
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if sort_item == "created":
@@ -197,7 +197,7 @@ async def list_all_coupon_token_addresses(
         .where(Listing.is_public == True)
     )
     total = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     # Search Filter
@@ -220,7 +220,7 @@ async def list_all_coupon_token_addresses(
             IDXCouponToken.initial_offering_status == initial_offering_status
         )
     count = await async_session.scalar(
-        select(func.count()).select_from(stmt.subquery())
+        stmt.with_only_columns(func.count()).order_by(None)
     )
 
     if sort_item == "created":