From c47c4e51feb2bad4f9f5063013e9e1bc8cf4a7ca Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:09:26 -0500 Subject: [PATCH 1/8] collectibles table migration --- .../ddl/migrations/0118_add_collectibles.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql diff --git a/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql b/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql new file mode 100644 index 00000000000..54e552c7de3 --- /dev/null +++ b/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql @@ -0,0 +1,17 @@ +begin; + +create table if not exists collectibles_data ( + user_id integer not null, + data jsonb not null, + constraint pk_user_id primary key (user_id) +); + + +INSERT INTO collectibles_data (user_id, data) +SELECT u.user_id, cid.data->'collectibles' AS data +FROM users u +LEFT JOIN cid_data cid ON u.metadata_multihash = cid.cid +WHERE u.has_collectibles = TRUE +ON CONFLICT (user_id) DO NOTHING; + +commit; \ No newline at end of file From cab6e858d7e381d6ae20263058f64eff251b7b2e Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Wed, 5 Feb 2025 17:03:31 -0500 Subject: [PATCH 2/8] add table and endpoint for getting collectibles --- .../ddl/migrations/0118_add_collectibles.sql | 24 ++++-- .../src/api/v1/models/users.py | 16 ++++ .../discovery-provider/src/api/v1/users.py | 28 +++++++ .../src/models/users/collectibles_data.py | 24 ++++++ .../src/queries/get_collectibles_data.py | 29 ++++++++ .../default/.openapi-generator/FILES | 2 + .../api/generated/default/apis/UsersApi.ts | 38 ++++++++++ .../default/models/CollectiblesData.ts | 66 +++++++++++++++++ .../default/models/CollectiblesResponse.ts | 73 +++++++++++++++++++ .../sdk/api/generated/default/models/index.ts | 2 + 10 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 packages/discovery-provider/src/models/users/collectibles_data.py create mode 100644 packages/discovery-provider/src/queries/get_collectibles_data.py create mode 100644 packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts create mode 100644 packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts diff --git a/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql b/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql index 54e552c7de3..e50b596cf16 100644 --- a/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql +++ b/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql @@ -1,14 +1,28 @@ begin; create table if not exists collectibles_data ( - user_id integer not null, - data jsonb not null, - constraint pk_user_id primary key (user_id) + user_id INTEGER NOT NULL, + data JSONB NOT NULL, + blockhash TEXT NOT NULL, + blocknumber INTEGER NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT now(), + CONSTRAINT pk_user_id PRIMARY KEY (user_id), + FOREIGN KEY (blocknumber) REFERENCES blocks(number) ON DELETE CASCADE ); +COMMENT ON TABLE collectibles_data IS 'Stores collectibles data for users'; +COMMENT ON COLUMN collectibles_data.user_id IS 'User ID of the person who owns the collectibles'; +COMMENT ON COLUMN collectibles_data.data IS 'Data about the collectibles'; +COMMENT ON COLUMN collectibles_data.blockhash IS 'Blockhash of the most recent block that changed the collectibles data'; +COMMENT ON COLUMN collectibles_data.blocknumber IS 'Block number of the most recent block that changed the collectibles data'; -INSERT INTO collectibles_data (user_id, data) -SELECT u.user_id, cid.data->'collectibles' AS data +INSERT INTO collectibles_data (user_id, data, blockhash, blocknumber) +SELECT + u.user_id, + cid.data->'collectibles' AS data, + u.blockhash, + u.blocknumber FROM users u LEFT JOIN cid_data cid ON u.metadata_multihash = cid.cid WHERE u.has_collectibles = TRUE diff --git a/packages/discovery-provider/src/api/v1/models/users.py b/packages/discovery-provider/src/api/v1/models/users.py index abb8d0688ff..680f875455d 100644 --- a/packages/discovery-provider/src/api/v1/models/users.py +++ b/packages/discovery-provider/src/api/v1/models/users.py @@ -156,6 +156,15 @@ }, ) +collectibles_data = ns.model( + "collectibles_data", + { + "data": fields.Raw( + description="Raw collectibles JSON structure generated by client" + ), + }, +) + challenge_response = ns.model( "challenge_response", { @@ -310,3 +319,10 @@ "updated_at": fields.String(required=True), }, ) + +collectibles_data = ns.model( + "collectibles_data", + { + "data": fields.Raw(description="Raw collectibles data from the blockchain"), + }, +) diff --git a/packages/discovery-provider/src/api/v1/users.py b/packages/discovery-provider/src/api/v1/users.py index 5cbda110e18..5b59de2db16 100644 --- a/packages/discovery-provider/src/api/v1/users.py +++ b/packages/discovery-provider/src/api/v1/users.py @@ -85,6 +85,7 @@ account_full, associated_wallets, challenge_response, + collectibles_data, connected_wallets, decoded_user_token, email_access, @@ -113,6 +114,10 @@ from src.queries.get_associated_user_wallet import get_associated_user_wallet from src.queries.get_authorization import is_authorized_request from src.queries.get_challenges import get_challenges +from src.queries.get_collectibles_data import ( + GetCollectiblesDataArgs, + get_collectibles_data, +) from src.queries.get_collection_library import ( CollectionType, GetCollectionLibraryArgs, @@ -1958,6 +1963,29 @@ def get(self, id): ) +collectibles_response = make_response( + "collectibles_response", ns, fields.Nested(collectibles_data, allow_null=True) +) + + +@ns.route("//collectibles") +class UserCollectibles(Resource): + @ns.doc( + id="""Get User Collectibles""", + description="""Get the User's indexed collectibles data""", + params={"id": "A User ID"}, + responses={200: "Success", 400: "Bad request", 500: "Server error"}, + ) + @ns.marshal_with(collectibles_response) + @cache(ttl_sec=10) + def get(self, id): + decoded_id = decode_with_abort(id, full_ns) + collectibles = get_collectibles_data( + GetCollectiblesDataArgs(user_id=decoded_id) + ) + return success_response({"data": collectibles} if collectibles else None) + + get_challenges_route_parser = reqparse.RequestParser(argument_class=DescriptiveArgument) get_challenges_route_parser.add_argument( "show_historical", diff --git a/packages/discovery-provider/src/models/users/collectibles_data.py b/packages/discovery-provider/src/models/users/collectibles_data.py new file mode 100644 index 00000000000..de4514aeba0 --- /dev/null +++ b/packages/discovery-provider/src/models/users/collectibles_data.py @@ -0,0 +1,24 @@ +import enum + +from sqlalchemy import Column, DateTime, Integer, String +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.sql import func + +from src.models.base import Base +from src.models.model_utils import RepresentableMixin + + +class CollectiblesData(Base, RepresentableMixin): + __tablename__ = "collectibles_data" + + user_id = Column( + Integer, + primary_key=True, + ) + data = Column(JSONB, nullable=False) + blockhash = Column(String, nullable=False) + blocknumber = Column(Integer, nullable=False) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) diff --git a/packages/discovery-provider/src/queries/get_collectibles_data.py b/packages/discovery-provider/src/queries/get_collectibles_data.py new file mode 100644 index 00000000000..a72963db3f9 --- /dev/null +++ b/packages/discovery-provider/src/queries/get_collectibles_data.py @@ -0,0 +1,29 @@ +from typing import Dict, Optional, TypedDict + +from src.models.users.collectibles_data import CollectiblesData +from src.utils.db_session import get_db_read_replica + + +class GetCollectiblesDataArgs(TypedDict): + user_id: int + + +def get_collectibles_data(args: GetCollectiblesDataArgs) -> Optional[Dict]: + """Gets the collectibles data for a user. + + Args: + args: GetCollectiblesDataArgs containing user_id + + Returns: + Dict containing collectibles data if found, None otherwise + """ + db = get_db_read_replica() + with db.scoped_session() as session: + collectibles = ( + session.query(CollectiblesData) + .filter(CollectiblesData.user_id == args["user_id"]) + .first() + ) + if collectibles: + return collectibles.data + return None diff --git a/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES b/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES index 7f21a0e083f..b98182f114d 100644 --- a/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES +++ b/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES @@ -19,6 +19,8 @@ models/AuthorizedApp.ts models/AuthorizedApps.ts models/BlobInfo.ts models/ChallengeResponse.ts +models/CollectiblesData.ts +models/CollectiblesResponse.ts models/CollectionActivity.ts models/Comment.ts models/CommentMention.ts diff --git a/packages/sdk/src/sdk/api/generated/default/apis/UsersApi.ts b/packages/sdk/src/sdk/api/generated/default/apis/UsersApi.ts index f51b7caf1f2..9b625ec011d 100644 --- a/packages/sdk/src/sdk/api/generated/default/apis/UsersApi.ts +++ b/packages/sdk/src/sdk/api/generated/default/apis/UsersApi.ts @@ -18,6 +18,7 @@ import * as runtime from '../runtime'; import type { AlbumsResponse, AuthorizedApps, + CollectiblesResponse, ConnectedWalletsResponse, DeveloperApps, EmailAccessResponse, @@ -51,6 +52,8 @@ import { AlbumsResponseToJSON, AuthorizedAppsFromJSON, AuthorizedAppsToJSON, + CollectiblesResponseFromJSON, + CollectiblesResponseToJSON, ConnectedWalletsResponseFromJSON, ConnectedWalletsResponseToJSON, DeveloperAppsFromJSON, @@ -309,6 +312,10 @@ export interface GetUserChallengesRequest { showHistorical?: boolean; } +export interface GetUserCollectiblesRequest { + id: string; +} + export interface GetUserEmailKeyRequest { receivingUserId: string; grantorUserId: string; @@ -1589,6 +1596,37 @@ export class UsersApi extends runtime.BaseAPI { return await response.value(); } + /** + * @hidden + * Get the User\'s indexed collectibles data + */ + async getUserCollectiblesRaw(params: GetUserCollectiblesRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise> { + if (params.id === null || params.id === undefined) { + throw new runtime.RequiredError('id','Required parameter params.id was null or undefined when calling getUserCollectibles.'); + } + + const queryParameters: any = {}; + + const headerParameters: runtime.HTTPHeaders = {}; + + const response = await this.request({ + path: `/users/{id}/collectibles`.replace(`{${"id"}}`, encodeURIComponent(String(params.id))), + method: 'GET', + headers: headerParameters, + query: queryParameters, + }, initOverrides); + + return new runtime.JSONApiResponse(response, (jsonValue) => CollectiblesResponseFromJSON(jsonValue)); + } + + /** + * Get the User\'s indexed collectibles data + */ + async getUserCollectibles(params: GetUserCollectiblesRequest, initOverrides?: RequestInit | runtime.InitOverrideFunction): Promise { + const response = await this.getUserCollectiblesRaw(params, initOverrides); + return await response.value(); + } + /** * @hidden * Gets the encrypted key for email access between the receiving user and granting user. diff --git a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts new file mode 100644 index 00000000000..c561b649058 --- /dev/null +++ b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts @@ -0,0 +1,66 @@ +/* tslint:disable */ +/* eslint-disable */ +// @ts-nocheck +/** + * API + * Audius V1 API + * + * The version of the OpenAPI document: 1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +/** + * + * @export + * @interface CollectiblesData + */ +export interface CollectiblesData { + /** + * Raw collectibles data from the blockchain + * @type {object} + * @memberof CollectiblesData + */ + data?: object; +} + +/** + * Check if a given object implements the CollectiblesData interface. + */ +export function instanceOfCollectiblesData(value: object): value is CollectiblesData { + let isInstance = true; + + return isInstance; +} + +export function CollectiblesDataFromJSON(json: any): CollectiblesData { + return CollectiblesDataFromJSONTyped(json, false); +} + +export function CollectiblesDataFromJSONTyped(json: any, ignoreDiscriminator: boolean): CollectiblesData { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'data': !exists(json, 'data') ? undefined : json['data'], + }; +} + +export function CollectiblesDataToJSON(value?: CollectiblesData | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'data': value.data, + }; +} + diff --git a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts new file mode 100644 index 00000000000..d2bf77e3419 --- /dev/null +++ b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts @@ -0,0 +1,73 @@ +/* tslint:disable */ +/* eslint-disable */ +// @ts-nocheck +/** + * API + * Audius V1 API + * + * The version of the OpenAPI document: 1.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { exists, mapValues } from '../runtime'; +import type { CollectiblesData } from './CollectiblesData'; +import { + CollectiblesDataFromJSON, + CollectiblesDataFromJSONTyped, + CollectiblesDataToJSON, +} from './CollectiblesData'; + +/** + * + * @export + * @interface CollectiblesResponse + */ +export interface CollectiblesResponse { + /** + * + * @type {CollectiblesData} + * @memberof CollectiblesResponse + */ + data?: CollectiblesData; +} + +/** + * Check if a given object implements the CollectiblesResponse interface. + */ +export function instanceOfCollectiblesResponse(value: object): value is CollectiblesResponse { + let isInstance = true; + + return isInstance; +} + +export function CollectiblesResponseFromJSON(json: any): CollectiblesResponse { + return CollectiblesResponseFromJSONTyped(json, false); +} + +export function CollectiblesResponseFromJSONTyped(json: any, ignoreDiscriminator: boolean): CollectiblesResponse { + if ((json === undefined) || (json === null)) { + return json; + } + return { + + 'data': !exists(json, 'data') ? undefined : CollectiblesDataFromJSON(json['data']), + }; +} + +export function CollectiblesResponseToJSON(value?: CollectiblesResponse | null): any { + if (value === undefined) { + return undefined; + } + if (value === null) { + return null; + } + return { + + 'data': CollectiblesDataToJSON(value.data), + }; +} + diff --git a/packages/sdk/src/sdk/api/generated/default/models/index.ts b/packages/sdk/src/sdk/api/generated/default/models/index.ts index d3905d5aaf8..dcc6a201a33 100644 --- a/packages/sdk/src/sdk/api/generated/default/models/index.ts +++ b/packages/sdk/src/sdk/api/generated/default/models/index.ts @@ -9,6 +9,8 @@ export * from './AuthorizedApp'; export * from './AuthorizedApps'; export * from './BlobInfo'; export * from './ChallengeResponse'; +export * from './CollectiblesData'; +export * from './CollectiblesResponse'; export * from './CollectionActivity'; export * from './Comment'; export * from './CommentMention'; From fdfd82911706dfeacf92be36f140ea62ed6ebaa9 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Thu, 6 Feb 2025 15:21:01 -0500 Subject: [PATCH 3/8] add collectible data indexing --- .../test_user_entity_manager.py | 308 ++++++++++++++++++ .../integration_tests/utils.py | 13 + .../src/models/users/collectibles_data.py | 8 +- .../src/tasks/entity_manager/entities/user.py | 52 +++ .../tasks/entity_manager/entity_manager.py | 30 ++ .../src/tasks/entity_manager/utils.py | 8 + .../discovery-provider/src/tasks/metadata.py | 4 + 7 files changed, 419 insertions(+), 4 deletions(-) diff --git a/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py b/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py index 4ae0172f5fb..63c7f1ae63a 100644 --- a/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py +++ b/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py @@ -14,6 +14,7 @@ from src.challenges.challenge_event import ChallengeEvent from src.models.indexing.cid_data import CIDData from src.models.users.associated_wallet import AssociatedWallet +from src.models.users.collectibles_data import CollectiblesData from src.models.users.user import User from src.queries.get_balances import IMMEDIATE_REFRESH_REDIS_PREFIX from src.solana.solana_client_manager import SolanaClientManager @@ -1951,3 +1952,310 @@ def get_events_side_effect(_, tx_receipt): mock.call(IMMEDIATE_REFRESH_REDIS_PREFIX, 1), ] ) + + +def test_add_user_collectibles(app, mocker): + """Tests adding user collectibles data""" + bus_mock = set_patches(mocker) + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(web3, bus_mock) + + # Test data for valid collectibles + valid_collectibles = { + "collectibles": { + "order": ["collection1"], + "collection1": {}, + } + } + + # Test data for invalid collectibles (not a dict) + invalid_collectibles = {"collectibles": "not a dict"} + + tx_receipts = { + "AddCollectiblesTx": [ + { + "args": AttributeDict( + { + "_entityId": "", + "_entityType": "CollectiblesData", + "_userId": 1, + "_action": "Create", + "_metadata": json.dumps( + {"cid": "", "data": valid_collectibles} + ), + "_signer": "user1wallet", + } + ) + }, + ], + "InvalidCollectiblesTx": [ + { + "args": AttributeDict( + { + "_entityId": "", + "_entityType": "CollectiblesData", + "_userId": 2, + "_action": "Create", + "_metadata": json.dumps( + {"cid": "", "data": invalid_collectibles} + ), + "_signer": "user2wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.to_bytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt["transactionHash"].decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + + # Create user + entities = { + "users": [ + { + "user_id": 1, + "handle": "user-1", + "wallet": "user1wallet", + }, + { + "user_id": 2, + "handle": "user-2", + "wallet": "user2wallet", + }, + ], + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # Test adding new collectibles + total_changes, _ = entity_manager_update( + update_task, + session, + [entity_manager_txs[0]], # AddCollectiblesTx + block_number=0, + block_timestamp=BLOCK_DATETIME.timestamp(), + block_hash=hex(0), + ) + + # Verify collectibles were added + collectibles_data = ( + session.query(CollectiblesData) + .filter(CollectiblesData.user_id == 1) + .first() + ) + assert collectibles_data is not None + assert collectibles_data.data == valid_collectibles["collectibles"] + assert total_changes == 1 + + # Ensure collectibles flag was set to True + user = session.query(User).filter(User.user_id == 1).first() + assert user.has_collectibles + + # Test invalid collectibles data + total_changes, _ = entity_manager_update( + update_task, + session, + [entity_manager_txs[1]], # InvalidCollectiblesTx + block_number=0, + block_timestamp=BLOCK_DATETIME.timestamp(), + block_hash=hex(0), + ) + + # Verify no collectibles were added + assert total_changes == 0 + current_data = ( + session.query(CollectiblesData) + .filter(CollectiblesData.user_id == 2) + .first() + ) + assert current_data is None + + +def test_update_user_collectibles(app, mocker): + """Tests updating user collectibles data""" + bus_mock = set_patches(mocker) + + # setup db and mocked txs + with app.app_context(): + db = get_db() + web3 = Web3() + update_task = UpdateTask(web3, bus_mock) + + updated_collectibles = { + "collectibles": { + "order": ["collection1"], + "collection1": {}, + "collection2": {}, + } + } + + tx_receipts = { + "UpdateCollectiblesTx": [ + { + "args": AttributeDict( + { + "_entityId": "", + "_entityType": "CollectiblesData", + "_userId": 1, + "_action": "Update", + "_metadata": json.dumps( + {"cid": "", "data": updated_collectibles} + ), + "_signer": "user1wallet", + } + ) + }, + ], + "InvalidCollectiblesTx": [ + { + "args": AttributeDict( + { + "_entityId": "", + "_entityType": "CollectiblesData", + "_userId": 1, + "_action": "Update", + "_metadata": json.dumps( + {"cid": "", "data": {"collectibles": "not a dict"}} + ), + "_signer": "user1wallet", + } + ) + }, + ], + "EmptyCollectiblesTx": [ + { + "args": AttributeDict( + { + "_entityId": "", + "_entityType": "CollectiblesData", + "_userId": 2, + "_action": "Update", + "_metadata": json.dumps( + {"cid": "", "data": {"collectibles": {}}} + ), + "_signer": "user2wallet", + } + ) + }, + ], + } + + entity_manager_txs = [ + AttributeDict({"transactionHash": update_task.web3.to_bytes(text=tx_receipt)}) + for tx_receipt in tx_receipts + ] + + def get_events_side_effect(_, tx_receipt): + return tx_receipts[tx_receipt["transactionHash"].decode("utf-8")] + + mocker.patch( + "src.tasks.entity_manager.entity_manager.get_entity_manager_events_tx", + side_effect=get_events_side_effect, + autospec=True, + ) + + # Create user + entities = { + "users": [ + { + "user_id": 1, + "handle": "user-1", + "wallet": "user1wallet", + "has_collectibles": True, + }, + { + "user_id": 2, + "handle": "user-2", + "wallet": "user2wallet", + "has_collectibles": True, + }, + ], + "collectibles_data": [ + { + "user_id": 1, + "data": {"order": ["collection1"], "collection1": {}}, + }, + { + "user_id": 2, + "data": {"order": ["collection1"], "collection1": {}}, + }, + ], + } + populate_mock_db(db, entities) + + with db.scoped_session() as session: + # Test updating existing collectibles + total_changes, _ = entity_manager_update( + update_task, + session, + [entity_manager_txs[0]], # UpdateCollectiblesTx + block_number=0, + block_timestamp=BLOCK_DATETIME.timestamp(), + block_hash=hex(0), + ) + + # Verify collectibles were updated + updated_data = ( + session.query(CollectiblesData) + .filter(CollectiblesData.user_id == 1) + .first() + ) + assert updated_data is not None + assert updated_data.data == updated_collectibles["collectibles"] + assert total_changes == 1 + + # Test invalid collectibles data + total_changes, _ = entity_manager_update( + update_task, + session, + [entity_manager_txs[1]], # InvalidCollectiblesTx + block_number=0, + block_timestamp=BLOCK_DATETIME.timestamp(), + block_hash=hex(0), + ) + assert total_changes == 0 + current_data = ( + session.query(CollectiblesData) + .filter(CollectiblesData.user_id == 1) + .first() + ) + assert current_data is not None + assert current_data.data == updated_collectibles["collectibles"] + + # Test empty collectibles data + total_changes, _ = entity_manager_update( + update_task, + session, + [entity_manager_txs[2]], # EmptyCollectiblesTx + block_number=1, + block_timestamp=BLOCK_DATETIME.timestamp(), + block_hash=hex(1), + ) + assert total_changes == 1 + + current_data = ( + session.query(CollectiblesData) + .filter(CollectiblesData.user_id == 2) + .first() + ) + assert current_data is not None + assert current_data.data == {} + + # Verify collectibles flag was set to False + user = session.query(User).filter(User.user_id == 2).first() + assert not user.has_collectibles diff --git a/packages/discovery-provider/integration_tests/utils.py b/packages/discovery-provider/integration_tests/utils.py index 79ab5f7fc75..a62859d0bd8 100644 --- a/packages/discovery-provider/integration_tests/utils.py +++ b/packages/discovery-provider/integration_tests/utils.py @@ -43,6 +43,7 @@ from src.models.tracks.track_route import TrackRoute from src.models.users.aggregate_user import AggregateUser from src.models.users.associated_wallet import AssociatedWallet, WalletChain +from src.models.users.collectibles_data import CollectiblesData from src.models.users.email import EmailAccess, EncryptedEmail from src.models.users.supporter_rank_up import SupporterRankUp from src.models.users.usdc_purchase import PurchaseAccessType, USDCPurchase @@ -182,6 +183,7 @@ def populate_mock_db(db, entities, block_offset=None): user_payout_wallet_history = entities.get("user_payout_wallet_history", []) encrypted_emails = entities.get("encrypted_emails", []) email_access = entities.get("email_access", []) + collectibles_data = entities.get("collectibles_data", []) num_blocks = max( len(tracks), @@ -203,6 +205,7 @@ def populate_mock_db(db, entities, block_offset=None): len(track_price_history), len(album_price_history), len(user_payout_wallet_history), + len(collectibles_data), ) for i in range(block_offset, block_offset + num_blocks): max_block = session.query(Block).filter(Block.number == i).first() @@ -918,5 +921,15 @@ def populate_mock_db(db, entities, block_offset=None): updated_at=email_access_meta.get("updated_at", datetime.now()), ) session.add(email_access) + for i, collectible_data in enumerate(collectibles_data): + collectible_data_record = CollectiblesData( + user_id=collectible_data.get("user_id", i), + data=collectible_data.get("data", {}), + blockhash=collectible_data.get("blockhash", str(i + block_offset)), + blocknumber=collectible_data.get("blocknumber", i + block_offset), + created_at=collectible_data.get("created_at", datetime.now()), + updated_at=collectible_data.get("updated_at", datetime.now()), + ) + session.add(collectible_data_record) session.commit() diff --git a/packages/discovery-provider/src/models/users/collectibles_data.py b/packages/discovery-provider/src/models/users/collectibles_data.py index de4514aeba0..2dea48cab31 100644 --- a/packages/discovery-provider/src/models/users/collectibles_data.py +++ b/packages/discovery-provider/src/models/users/collectibles_data.py @@ -1,6 +1,4 @@ -import enum - -from sqlalchemy import Column, DateTime, Integer, String +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.sql import func @@ -17,7 +15,9 @@ class CollectiblesData(Base, RepresentableMixin): ) data = Column(JSONB, nullable=False) blockhash = Column(String, nullable=False) - blocknumber = Column(Integer, nullable=False) + blocknumber = Column( + Integer, ForeignKey("blocks.number"), index=True, nullable=False + ) created_at = Column(DateTime(timezone=True), server_default=func.now()) updated_at = Column( DateTime(timezone=True), server_default=func.now(), onupdate=func.now() diff --git a/packages/discovery-provider/src/tasks/entity_manager/entities/user.py b/packages/discovery-provider/src/tasks/entity_manager/entities/user.py index e78d8858f68..44bd9dee0ab 100644 --- a/packages/discovery-provider/src/tasks/entity_manager/entities/user.py +++ b/packages/discovery-provider/src/tasks/entity_manager/entities/user.py @@ -17,6 +17,7 @@ from src.models.indexing.cid_data import CIDData from src.models.tracks.track import Track from src.models.users.associated_wallet import AssociatedWallet +from src.models.users.collectibles_data import CollectiblesData from src.models.users.user import User from src.models.users.user_events import UserEvent from src.models.users.user_payout_wallet_history import UserPayoutWalletHistory @@ -335,6 +336,22 @@ def update_user_metadata( user_record.tiktok_handle = user_record.handle if "collectibles" in metadata: + # Dual-write for collectibles data to support legacy indexing + # TODO: Remove after clients updated to use new transactions + # https://linear.app/audius/issue/PAY-3894/remove-collection-and-other-cid-metadata-indexing + collectibles_data = CollectiblesData( + user_id=user_record.user_id, + data=metadata["collectibles"], + blockhash=params.event_blockhash, + blocknumber=params.block_number, + ) + + # We can just add_record here. Outer EM logic will take care + # of deleting previous record if it exists + params.add_record( + user_record.user_id, collectibles_data, EntityType.COLLECTIBLES_DATA + ) + if ( metadata["collectibles"] and isinstance(metadata["collectibles"], dict) @@ -650,6 +667,41 @@ def remove_associated_wallet(params: ManageEntityParameters): raise e +def update_user_collectibles(params: ManageEntityParameters): + """Updates the user's collectibles data""" + validate_signer(params) + user_id = params.user_id + metadata = params.metadata + existing_user = params.existing_records["User"][user_id] + try: + if not isinstance(metadata.get("collectibles"), dict): + # If invalid format, don't update + raise IndexingValidationError("Invalid collectibles data format") + + collectibles_data = CollectiblesData( + user_id=user_id, + data=metadata["collectibles"], + blockhash=params.event_blockhash, + blocknumber=params.block_number, + ) + + # We can just add_record here. Outer EM logic will take care + # of deleting previous record if it exists + params.add_record(user_id, collectibles_data, EntityType.COLLECTIBLES_DATA) + + if metadata["collectibles"].items(): + existing_user.has_collectibles = True + else: + existing_user.has_collectibles = False + + except Exception as e: + logger.error( + f"index.py | users.py | Fatal error updating user collectibles {e}", + exc_info=True, + ) + raise e + + def validate_signature( chain: str, web3, user_id: int, associated_wallet: str, signature: str ): diff --git a/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py b/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py index 69a75b39093..55832f94f98 100644 --- a/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py +++ b/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py @@ -32,6 +32,7 @@ from src.models.tracks.track import Track from src.models.tracks.track_route import TrackRoute from src.models.users.associated_wallet import AssociatedWallet +from src.models.users.collectibles_data import CollectiblesData from src.models.users.email import EmailAccess, EncryptedEmail from src.models.users.user import User from src.models.users.user_events import UserEvent @@ -103,6 +104,7 @@ create_user, remove_associated_wallet, update_user, + update_user_collectibles, verify_user, ) from src.tasks.entity_manager.utils import ( @@ -138,6 +140,7 @@ "Track": Track.__tablename__, "User": User.__tablename__, "AssociatedWallet": AssociatedWallet.__tablename__, + "CollectiblesData": CollectiblesData.__tablename__, "UserEvent": UserEvent.__tablename__, "TrackRoute": TrackRoute.__tablename__, "PlaylistRoute": PlaylistRoute.__tablename__, @@ -454,6 +457,10 @@ def entity_manager_update( and params.entity_type == EntityType.ASSOCIATED_WALLET ): remove_associated_wallet(params) + elif ( + params.action == Action.CREATE or params.action == Action.UPDATE + ) and params.entity_type == EntityType.COLLECTIBLES_DATA: + update_user_collectibles(params) logger.debug("process transaction") # log event context except IndexingValidationError as e: @@ -619,6 +626,8 @@ def collect_entities_to_fetch(update_task, entity_manager_txs): if entity_type == EntityType.USER: entities_to_fetch[EntityType.USER_EVENT].add(user_id) entities_to_fetch[EntityType.ASSOCIATED_WALLET].add(user_id) + if action == Action.UPDATE: + entities_to_fetch[EntityType.COLLECTIBLES_DATA].add(user_id) if action == Action.MUTE or action == Action.UNMUTE: entities_to_fetch[EntityType.MUTED_USER].add((user_id, entity_id)) entities_to_fetch[EntityType.USER].add(entity_id) @@ -848,6 +857,9 @@ def collect_entities_to_fetch(update_task, entity_manager_txs): ) if entity_type == EntityType.ASSOCIATED_WALLET: entities_to_fetch[EntityType.ASSOCIATED_WALLET].add(user_id) + if entity_type == EntityType.COLLECTIBLES_DATA: + entities_to_fetch[EntityType.COLLECTIBLES_DATA].add(user_id) + entities_to_fetch[EntityType.USER].add(user_id) return entities_to_fetch @@ -987,6 +999,24 @@ def fetch_existing_entities(session: Session, entities_to_fetch: EntitiesToFetch for _, wallet_json in associated_wallets } + if entities_to_fetch["CollectiblesData"]: + collectibles_data_results: List[Tuple[CollectiblesData, dict]] = ( + session.query( + CollectiblesData, + literal_column(f"row_to_json({CollectiblesData.__tablename__})"), + ) + .filter(CollectiblesData.user_id.in_(entities_to_fetch["CollectiblesData"])) + .all() + ) + existing_entities[EntityType.COLLECTIBLES_DATA] = { + collectibles_data.user_id: collectibles_data + for collectibles_data, _ in collectibles_data_results + } + existing_entities_in_json[EntityType.COLLECTIBLES_DATA] = { + collectible_json["user_id"]: collectible_json + for _, collectible_json in collectibles_data_results + } + # FOLLOWS if entities_to_fetch["Follow"]: follow_ops_to_fetch: Set[Tuple] = entities_to_fetch["Follow"] diff --git a/packages/discovery-provider/src/tasks/entity_manager/utils.py b/packages/discovery-provider/src/tasks/entity_manager/utils.py index 90cd3f37031..9114d9c6f76 100644 --- a/packages/discovery-provider/src/tasks/entity_manager/utils.py +++ b/packages/discovery-provider/src/tasks/entity_manager/utils.py @@ -35,10 +35,12 @@ from src.models.tracks.track import Track from src.models.tracks.track_route import TrackRoute from src.models.users.associated_wallet import AssociatedWallet +from src.models.users.collectibles_data import CollectiblesData from src.models.users.user import User from src.solana.solana_client_manager import SolanaClientManager from src.tasks.metadata import ( add_associated_wallet_metadata_format, + collectibles_data_metadata_format, comment_metadata_format, encrypted_email_metadata_format, playlist_metadata_format, @@ -127,6 +129,7 @@ class EntityType(str, Enum): COMMENT_NOTIFICATION_SETTING = "CommentNotificationSetting" ENCRYPTED_EMAIL = "EncryptedEmail" EMAIL_ACCESS = "EmailAccess" + COLLECTIBLES_DATA = "CollectiblesData" def __str__(self) -> str: return str.__str__(self) @@ -168,6 +171,7 @@ class RecordDict(TypedDict): class ExistingRecordDict(TypedDict): AssociatedWallet: Dict[str, AssociatedWallet] + CollectiblesData: Dict[int, CollectiblesData] Playlist: Dict[int, Playlist] Track: Dict[int, Track] UserWallet: Dict[str, User] @@ -206,6 +210,7 @@ class EntitiesToFetchDict(TypedDict): PlaylistRoute: Set[int] UserEvent: Set[int] AssociatedWallet: Set[int] + CollectiblesData: Set[int] UserWallet: Set[str] Comment: Set[int] CommentReaction: Set[Tuple] @@ -384,6 +389,9 @@ def get_metadata_type_and_format(entity_type, action=None): if action == Action.CREATE else remove_associated_wallet_metadata_format ) + elif entity_type == EntityType.COLLECTIBLES_DATA: + metadata_type = "collectibles_data" + metadata_format = collectibles_data_metadata_format else: raise IndexingValidationError(f"Unknown metadata type ${entity_type}") return metadata_type, metadata_format diff --git a/packages/discovery-provider/src/tasks/metadata.py b/packages/discovery-provider/src/tasks/metadata.py index ef7425b4b62..a5e872d35f4 100644 --- a/packages/discovery-provider/src/tasks/metadata.py +++ b/packages/discovery-provider/src/tasks/metadata.py @@ -272,6 +272,10 @@ class TrackMetadata(TypedDict): "chain": None, } +collectibles_data_metadata_format = { + "collectibles": None, +} + class PlaylistMetadata(TypedDict): playlist_contents: Optional[Any] From c3da3f0aac5f6b7b364d646d283bae52d072ab9e Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Thu, 6 Feb 2025 15:29:42 -0500 Subject: [PATCH 4/8] changeset --- .changeset/fresh-poems-visit.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fresh-poems-visit.md diff --git a/.changeset/fresh-poems-visit.md b/.changeset/fresh-poems-visit.md new file mode 100644 index 00000000000..8269c17f91c --- /dev/null +++ b/.changeset/fresh-poems-visit.md @@ -0,0 +1,5 @@ +--- +"@audius/sdk": minor +--- + +Add support for fetching collectibles From 4e0c9d093259255ab67ee4868bdc8480391b7989 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Thu, 6 Feb 2025 17:05:50 -0500 Subject: [PATCH 5/8] remove extra data definition --- packages/discovery-provider/src/api/v1/models/users.py | 7 ------- .../sdk/api/generated/default/models/CollectiblesData.ts | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/discovery-provider/src/api/v1/models/users.py b/packages/discovery-provider/src/api/v1/models/users.py index 680f875455d..745b412672c 100644 --- a/packages/discovery-provider/src/api/v1/models/users.py +++ b/packages/discovery-provider/src/api/v1/models/users.py @@ -319,10 +319,3 @@ "updated_at": fields.String(required=True), }, ) - -collectibles_data = ns.model( - "collectibles_data", - { - "data": fields.Raw(description="Raw collectibles data from the blockchain"), - }, -) diff --git a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts index c561b649058..7e4dca2a800 100644 --- a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts +++ b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts @@ -21,7 +21,7 @@ import { exists, mapValues } from '../runtime'; */ export interface CollectiblesData { /** - * Raw collectibles data from the blockchain + * Raw collectibles JSON structure generated by client * @type {object} * @memberof CollectiblesData */ From b8eeff451e8604daba3330f3e457b72c2667a26d Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 10 Feb 2025 12:28:30 -0500 Subject: [PATCH 6/8] rename collectibles_data -> collectibles --- .../ddl/migrations/0118_add_collectibles.sql | 14 ++++---- .../integration_tests/utils.py | 10 +++--- .../src/api/v1/models/users.py | 4 +-- .../discovery-provider/src/api/v1/users.py | 13 +++----- .../{collectibles_data.py => collectibles.py} | 4 +-- ...llectibles_data.py => get_collectibles.py} | 12 +++---- .../src/tasks/entity_manager/entities/user.py | 12 +++---- .../tasks/entity_manager/entity_manager.py | 32 +++++++++---------- .../src/tasks/entity_manager/utils.py | 16 +++++----- .../discovery-provider/src/tasks/metadata.py | 2 +- 10 files changed, 56 insertions(+), 63 deletions(-) rename packages/discovery-provider/src/models/users/{collectibles_data.py => collectibles.py} (88%) rename packages/discovery-provider/src/queries/{get_collectibles_data.py => get_collectibles.py} (57%) diff --git a/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql b/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql index e50b596cf16..e5827657666 100644 --- a/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql +++ b/packages/discovery-provider/ddl/migrations/0118_add_collectibles.sql @@ -1,6 +1,6 @@ begin; -create table if not exists collectibles_data ( +create table if not exists collectibles ( user_id INTEGER NOT NULL, data JSONB NOT NULL, blockhash TEXT NOT NULL, @@ -11,13 +11,13 @@ create table if not exists collectibles_data ( FOREIGN KEY (blocknumber) REFERENCES blocks(number) ON DELETE CASCADE ); -COMMENT ON TABLE collectibles_data IS 'Stores collectibles data for users'; -COMMENT ON COLUMN collectibles_data.user_id IS 'User ID of the person who owns the collectibles'; -COMMENT ON COLUMN collectibles_data.data IS 'Data about the collectibles'; -COMMENT ON COLUMN collectibles_data.blockhash IS 'Blockhash of the most recent block that changed the collectibles data'; -COMMENT ON COLUMN collectibles_data.blocknumber IS 'Block number of the most recent block that changed the collectibles data'; +COMMENT ON TABLE collectibles IS 'Stores collectibles data for users'; +COMMENT ON COLUMN collectibles.user_id IS 'User ID of the person who owns the collectibles'; +COMMENT ON COLUMN collectibles.data IS 'Data about the collectibles'; +COMMENT ON COLUMN collectibles.blockhash IS 'Blockhash of the most recent block that changed the collectibles data'; +COMMENT ON COLUMN collectibles.blocknumber IS 'Block number of the most recent block that changed the collectibles data'; -INSERT INTO collectibles_data (user_id, data, blockhash, blocknumber) +INSERT INTO collectibles (user_id, data, blockhash, blocknumber) SELECT u.user_id, cid.data->'collectibles' AS data, diff --git a/packages/discovery-provider/integration_tests/utils.py b/packages/discovery-provider/integration_tests/utils.py index a62859d0bd8..680ce63255e 100644 --- a/packages/discovery-provider/integration_tests/utils.py +++ b/packages/discovery-provider/integration_tests/utils.py @@ -43,7 +43,7 @@ from src.models.tracks.track_route import TrackRoute from src.models.users.aggregate_user import AggregateUser from src.models.users.associated_wallet import AssociatedWallet, WalletChain -from src.models.users.collectibles_data import CollectiblesData +from src.models.users.collectibles import Collectibles from src.models.users.email import EmailAccess, EncryptedEmail from src.models.users.supporter_rank_up import SupporterRankUp from src.models.users.usdc_purchase import PurchaseAccessType, USDCPurchase @@ -183,7 +183,7 @@ def populate_mock_db(db, entities, block_offset=None): user_payout_wallet_history = entities.get("user_payout_wallet_history", []) encrypted_emails = entities.get("encrypted_emails", []) email_access = entities.get("email_access", []) - collectibles_data = entities.get("collectibles_data", []) + collectibles = entities.get("collectibles", []) num_blocks = max( len(tracks), @@ -205,7 +205,7 @@ def populate_mock_db(db, entities, block_offset=None): len(track_price_history), len(album_price_history), len(user_payout_wallet_history), - len(collectibles_data), + len(collectibles), ) for i in range(block_offset, block_offset + num_blocks): max_block = session.query(Block).filter(Block.number == i).first() @@ -921,8 +921,8 @@ def populate_mock_db(db, entities, block_offset=None): updated_at=email_access_meta.get("updated_at", datetime.now()), ) session.add(email_access) - for i, collectible_data in enumerate(collectibles_data): - collectible_data_record = CollectiblesData( + for i, collectible_data in enumerate(collectibles): + collectible_data_record = Collectibles( user_id=collectible_data.get("user_id", i), data=collectible_data.get("data", {}), blockhash=collectible_data.get("blockhash", str(i + block_offset)), diff --git a/packages/discovery-provider/src/api/v1/models/users.py b/packages/discovery-provider/src/api/v1/models/users.py index 745b412672c..2a737393244 100644 --- a/packages/discovery-provider/src/api/v1/models/users.py +++ b/packages/discovery-provider/src/api/v1/models/users.py @@ -156,8 +156,8 @@ }, ) -collectibles_data = ns.model( - "collectibles_data", +collectibles = ns.model( + "collectibles", { "data": fields.Raw( description="Raw collectibles JSON structure generated by client" diff --git a/packages/discovery-provider/src/api/v1/users.py b/packages/discovery-provider/src/api/v1/users.py index 5b59de2db16..7c72b35955e 100644 --- a/packages/discovery-provider/src/api/v1/users.py +++ b/packages/discovery-provider/src/api/v1/users.py @@ -85,7 +85,7 @@ account_full, associated_wallets, challenge_response, - collectibles_data, + collectibles, connected_wallets, decoded_user_token, email_access, @@ -114,10 +114,7 @@ from src.queries.get_associated_user_wallet import get_associated_user_wallet from src.queries.get_authorization import is_authorized_request from src.queries.get_challenges import get_challenges -from src.queries.get_collectibles_data import ( - GetCollectiblesDataArgs, - get_collectibles_data, -) +from src.queries.get_collectibles import GetCollectiblesArgs, get_collectibles from src.queries.get_collection_library import ( CollectionType, GetCollectionLibraryArgs, @@ -1964,7 +1961,7 @@ def get(self, id): collectibles_response = make_response( - "collectibles_response", ns, fields.Nested(collectibles_data, allow_null=True) + "collectibles_response", ns, fields.Nested(collectibles, allow_null=True) ) @@ -1980,9 +1977,7 @@ class UserCollectibles(Resource): @cache(ttl_sec=10) def get(self, id): decoded_id = decode_with_abort(id, full_ns) - collectibles = get_collectibles_data( - GetCollectiblesDataArgs(user_id=decoded_id) - ) + collectibles = get_collectibles(GetCollectiblesArgs(user_id=decoded_id)) return success_response({"data": collectibles} if collectibles else None) diff --git a/packages/discovery-provider/src/models/users/collectibles_data.py b/packages/discovery-provider/src/models/users/collectibles.py similarity index 88% rename from packages/discovery-provider/src/models/users/collectibles_data.py rename to packages/discovery-provider/src/models/users/collectibles.py index 2dea48cab31..42487e8acd4 100644 --- a/packages/discovery-provider/src/models/users/collectibles_data.py +++ b/packages/discovery-provider/src/models/users/collectibles.py @@ -6,8 +6,8 @@ from src.models.model_utils import RepresentableMixin -class CollectiblesData(Base, RepresentableMixin): - __tablename__ = "collectibles_data" +class Collectibles(Base, RepresentableMixin): + __tablename__ = "collectibles" user_id = Column( Integer, diff --git a/packages/discovery-provider/src/queries/get_collectibles_data.py b/packages/discovery-provider/src/queries/get_collectibles.py similarity index 57% rename from packages/discovery-provider/src/queries/get_collectibles_data.py rename to packages/discovery-provider/src/queries/get_collectibles.py index a72963db3f9..ca5dbd65a05 100644 --- a/packages/discovery-provider/src/queries/get_collectibles_data.py +++ b/packages/discovery-provider/src/queries/get_collectibles.py @@ -1,18 +1,18 @@ from typing import Dict, Optional, TypedDict -from src.models.users.collectibles_data import CollectiblesData +from src.models.users.collectibles import Collectibles from src.utils.db_session import get_db_read_replica -class GetCollectiblesDataArgs(TypedDict): +class GetCollectiblesArgs(TypedDict): user_id: int -def get_collectibles_data(args: GetCollectiblesDataArgs) -> Optional[Dict]: +def get_collectibles(args: GetCollectiblesArgs) -> Optional[Dict]: """Gets the collectibles data for a user. Args: - args: GetCollectiblesDataArgs containing user_id + args: GetCollectiblesArgs containing user_id Returns: Dict containing collectibles data if found, None otherwise @@ -20,8 +20,8 @@ def get_collectibles_data(args: GetCollectiblesDataArgs) -> Optional[Dict]: db = get_db_read_replica() with db.scoped_session() as session: collectibles = ( - session.query(CollectiblesData) - .filter(CollectiblesData.user_id == args["user_id"]) + session.query(Collectibles) + .filter(Collectibles.user_id == args["user_id"]) .first() ) if collectibles: diff --git a/packages/discovery-provider/src/tasks/entity_manager/entities/user.py b/packages/discovery-provider/src/tasks/entity_manager/entities/user.py index 44bd9dee0ab..ae5fbbf8e1c 100644 --- a/packages/discovery-provider/src/tasks/entity_manager/entities/user.py +++ b/packages/discovery-provider/src/tasks/entity_manager/entities/user.py @@ -17,7 +17,7 @@ from src.models.indexing.cid_data import CIDData from src.models.tracks.track import Track from src.models.users.associated_wallet import AssociatedWallet -from src.models.users.collectibles_data import CollectiblesData +from src.models.users.collectibles import Collectibles from src.models.users.user import User from src.models.users.user_events import UserEvent from src.models.users.user_payout_wallet_history import UserPayoutWalletHistory @@ -339,7 +339,7 @@ def update_user_metadata( # Dual-write for collectibles data to support legacy indexing # TODO: Remove after clients updated to use new transactions # https://linear.app/audius/issue/PAY-3894/remove-collection-and-other-cid-metadata-indexing - collectibles_data = CollectiblesData( + collectibles = Collectibles( user_id=user_record.user_id, data=metadata["collectibles"], blockhash=params.event_blockhash, @@ -348,9 +348,7 @@ def update_user_metadata( # We can just add_record here. Outer EM logic will take care # of deleting previous record if it exists - params.add_record( - user_record.user_id, collectibles_data, EntityType.COLLECTIBLES_DATA - ) + params.add_record(user_record.user_id, collectibles, EntityType.COLLECTIBLES) if ( metadata["collectibles"] @@ -678,7 +676,7 @@ def update_user_collectibles(params: ManageEntityParameters): # If invalid format, don't update raise IndexingValidationError("Invalid collectibles data format") - collectibles_data = CollectiblesData( + collectibles = Collectibles( user_id=user_id, data=metadata["collectibles"], blockhash=params.event_blockhash, @@ -687,7 +685,7 @@ def update_user_collectibles(params: ManageEntityParameters): # We can just add_record here. Outer EM logic will take care # of deleting previous record if it exists - params.add_record(user_id, collectibles_data, EntityType.COLLECTIBLES_DATA) + params.add_record(user_id, collectibles, EntityType.COLLECTIBLES) if metadata["collectibles"].items(): existing_user.has_collectibles = True diff --git a/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py b/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py index 55832f94f98..e0c0d02c66a 100644 --- a/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py +++ b/packages/discovery-provider/src/tasks/entity_manager/entity_manager.py @@ -32,7 +32,7 @@ from src.models.tracks.track import Track from src.models.tracks.track_route import TrackRoute from src.models.users.associated_wallet import AssociatedWallet -from src.models.users.collectibles_data import CollectiblesData +from src.models.users.collectibles import Collectibles from src.models.users.email import EmailAccess, EncryptedEmail from src.models.users.user import User from src.models.users.user_events import UserEvent @@ -140,7 +140,7 @@ "Track": Track.__tablename__, "User": User.__tablename__, "AssociatedWallet": AssociatedWallet.__tablename__, - "CollectiblesData": CollectiblesData.__tablename__, + "Collectibles": Collectibles.__tablename__, "UserEvent": UserEvent.__tablename__, "TrackRoute": TrackRoute.__tablename__, "PlaylistRoute": PlaylistRoute.__tablename__, @@ -459,7 +459,7 @@ def entity_manager_update( remove_associated_wallet(params) elif ( params.action == Action.CREATE or params.action == Action.UPDATE - ) and params.entity_type == EntityType.COLLECTIBLES_DATA: + ) and params.entity_type == EntityType.COLLECTIBLES: update_user_collectibles(params) logger.debug("process transaction") # log event context @@ -627,7 +627,7 @@ def collect_entities_to_fetch(update_task, entity_manager_txs): entities_to_fetch[EntityType.USER_EVENT].add(user_id) entities_to_fetch[EntityType.ASSOCIATED_WALLET].add(user_id) if action == Action.UPDATE: - entities_to_fetch[EntityType.COLLECTIBLES_DATA].add(user_id) + entities_to_fetch[EntityType.COLLECTIBLES].add(user_id) if action == Action.MUTE or action == Action.UNMUTE: entities_to_fetch[EntityType.MUTED_USER].add((user_id, entity_id)) entities_to_fetch[EntityType.USER].add(entity_id) @@ -857,8 +857,8 @@ def collect_entities_to_fetch(update_task, entity_manager_txs): ) if entity_type == EntityType.ASSOCIATED_WALLET: entities_to_fetch[EntityType.ASSOCIATED_WALLET].add(user_id) - if entity_type == EntityType.COLLECTIBLES_DATA: - entities_to_fetch[EntityType.COLLECTIBLES_DATA].add(user_id) + if entity_type == EntityType.COLLECTIBLES: + entities_to_fetch[EntityType.COLLECTIBLES].add(user_id) entities_to_fetch[EntityType.USER].add(user_id) return entities_to_fetch @@ -999,22 +999,22 @@ def fetch_existing_entities(session: Session, entities_to_fetch: EntitiesToFetch for _, wallet_json in associated_wallets } - if entities_to_fetch["CollectiblesData"]: - collectibles_data_results: List[Tuple[CollectiblesData, dict]] = ( + if entities_to_fetch["Collectibles"]: + collectibles_results: List[Tuple[Collectibles, dict]] = ( session.query( - CollectiblesData, - literal_column(f"row_to_json({CollectiblesData.__tablename__})"), + Collectibles, + literal_column(f"row_to_json({Collectibles.__tablename__})"), ) - .filter(CollectiblesData.user_id.in_(entities_to_fetch["CollectiblesData"])) + .filter(Collectibles.user_id.in_(entities_to_fetch["Collectibles"])) .all() ) - existing_entities[EntityType.COLLECTIBLES_DATA] = { - collectibles_data.user_id: collectibles_data - for collectibles_data, _ in collectibles_data_results + existing_entities[EntityType.COLLECTIBLES] = { + collectibles.user_id: collectibles + for collectibles, _ in collectibles_results } - existing_entities_in_json[EntityType.COLLECTIBLES_DATA] = { + existing_entities_in_json[EntityType.COLLECTIBLES] = { collectible_json["user_id"]: collectible_json - for _, collectible_json in collectibles_data_results + for _, collectible_json in collectibles_results } # FOLLOWS diff --git a/packages/discovery-provider/src/tasks/entity_manager/utils.py b/packages/discovery-provider/src/tasks/entity_manager/utils.py index 9114d9c6f76..fbf8e960cba 100644 --- a/packages/discovery-provider/src/tasks/entity_manager/utils.py +++ b/packages/discovery-provider/src/tasks/entity_manager/utils.py @@ -35,12 +35,12 @@ from src.models.tracks.track import Track from src.models.tracks.track_route import TrackRoute from src.models.users.associated_wallet import AssociatedWallet -from src.models.users.collectibles_data import CollectiblesData +from src.models.users.collectibles import Collectibles from src.models.users.user import User from src.solana.solana_client_manager import SolanaClientManager from src.tasks.metadata import ( add_associated_wallet_metadata_format, - collectibles_data_metadata_format, + collectibles_metadata_format, comment_metadata_format, encrypted_email_metadata_format, playlist_metadata_format, @@ -129,7 +129,7 @@ class EntityType(str, Enum): COMMENT_NOTIFICATION_SETTING = "CommentNotificationSetting" ENCRYPTED_EMAIL = "EncryptedEmail" EMAIL_ACCESS = "EmailAccess" - COLLECTIBLES_DATA = "CollectiblesData" + COLLECTIBLES = "Collectibles" def __str__(self) -> str: return str.__str__(self) @@ -171,7 +171,7 @@ class RecordDict(TypedDict): class ExistingRecordDict(TypedDict): AssociatedWallet: Dict[str, AssociatedWallet] - CollectiblesData: Dict[int, CollectiblesData] + Collectibles: Dict[int, Collectibles] Playlist: Dict[int, Playlist] Track: Dict[int, Track] UserWallet: Dict[str, User] @@ -210,7 +210,7 @@ class EntitiesToFetchDict(TypedDict): PlaylistRoute: Set[int] UserEvent: Set[int] AssociatedWallet: Set[int] - CollectiblesData: Set[int] + Collectibles: Set[int] UserWallet: Set[str] Comment: Set[int] CommentReaction: Set[Tuple] @@ -389,9 +389,9 @@ def get_metadata_type_and_format(entity_type, action=None): if action == Action.CREATE else remove_associated_wallet_metadata_format ) - elif entity_type == EntityType.COLLECTIBLES_DATA: - metadata_type = "collectibles_data" - metadata_format = collectibles_data_metadata_format + elif entity_type == EntityType.COLLECTIBLES: + metadata_type = "collectibles" + metadata_format = collectibles_metadata_format else: raise IndexingValidationError(f"Unknown metadata type ${entity_type}") return metadata_type, metadata_format diff --git a/packages/discovery-provider/src/tasks/metadata.py b/packages/discovery-provider/src/tasks/metadata.py index a5e872d35f4..767fef61868 100644 --- a/packages/discovery-provider/src/tasks/metadata.py +++ b/packages/discovery-provider/src/tasks/metadata.py @@ -272,7 +272,7 @@ class TrackMetadata(TypedDict): "chain": None, } -collectibles_data_metadata_format = { +collectibles_metadata_format = { "collectibles": None, } From 68060f96169240139ce32911b5a61f075812b32d Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 10 Feb 2025 13:09:54 -0500 Subject: [PATCH 7/8] update tests --- .../test_user_entity_manager.py | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py b/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py index 63c7f1ae63a..5906124b9f5 100644 --- a/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py +++ b/packages/discovery-provider/integration_tests/tasks/entity_manager/test_user_entity_manager.py @@ -14,7 +14,7 @@ from src.challenges.challenge_event import ChallengeEvent from src.models.indexing.cid_data import CIDData from src.models.users.associated_wallet import AssociatedWallet -from src.models.users.collectibles_data import CollectiblesData +from src.models.users.collectibles import Collectibles from src.models.users.user import User from src.queries.get_balances import IMMEDIATE_REFRESH_REDIS_PREFIX from src.solana.solana_client_manager import SolanaClientManager @@ -1981,7 +1981,7 @@ def test_add_user_collectibles(app, mocker): "args": AttributeDict( { "_entityId": "", - "_entityType": "CollectiblesData", + "_entityType": "Collectibles", "_userId": 1, "_action": "Create", "_metadata": json.dumps( @@ -1997,7 +1997,7 @@ def test_add_user_collectibles(app, mocker): "args": AttributeDict( { "_entityId": "", - "_entityType": "CollectiblesData", + "_entityType": "Collectibles", "_userId": 2, "_action": "Create", "_metadata": json.dumps( @@ -2054,9 +2054,7 @@ def get_events_side_effect(_, tx_receipt): # Verify collectibles were added collectibles_data = ( - session.query(CollectiblesData) - .filter(CollectiblesData.user_id == 1) - .first() + session.query(Collectibles).filter(Collectibles.user_id == 1).first() ) assert collectibles_data is not None assert collectibles_data.data == valid_collectibles["collectibles"] @@ -2079,9 +2077,7 @@ def get_events_side_effect(_, tx_receipt): # Verify no collectibles were added assert total_changes == 0 current_data = ( - session.query(CollectiblesData) - .filter(CollectiblesData.user_id == 2) - .first() + session.query(Collectibles).filter(Collectibles.user_id == 2).first() ) assert current_data is None @@ -2110,7 +2106,7 @@ def test_update_user_collectibles(app, mocker): "args": AttributeDict( { "_entityId": "", - "_entityType": "CollectiblesData", + "_entityType": "Collectibles", "_userId": 1, "_action": "Update", "_metadata": json.dumps( @@ -2126,7 +2122,7 @@ def test_update_user_collectibles(app, mocker): "args": AttributeDict( { "_entityId": "", - "_entityType": "CollectiblesData", + "_entityType": "Collectibles", "_userId": 1, "_action": "Update", "_metadata": json.dumps( @@ -2142,7 +2138,7 @@ def test_update_user_collectibles(app, mocker): "args": AttributeDict( { "_entityId": "", - "_entityType": "CollectiblesData", + "_entityType": "Collectibles", "_userId": 2, "_action": "Update", "_metadata": json.dumps( @@ -2211,9 +2207,7 @@ def get_events_side_effect(_, tx_receipt): # Verify collectibles were updated updated_data = ( - session.query(CollectiblesData) - .filter(CollectiblesData.user_id == 1) - .first() + session.query(Collectibles).filter(Collectibles.user_id == 1).first() ) assert updated_data is not None assert updated_data.data == updated_collectibles["collectibles"] @@ -2230,9 +2224,7 @@ def get_events_side_effect(_, tx_receipt): ) assert total_changes == 0 current_data = ( - session.query(CollectiblesData) - .filter(CollectiblesData.user_id == 1) - .first() + session.query(Collectibles).filter(Collectibles.user_id == 1).first() ) assert current_data is not None assert current_data.data == updated_collectibles["collectibles"] @@ -2249,9 +2241,7 @@ def get_events_side_effect(_, tx_receipt): assert total_changes == 1 current_data = ( - session.query(CollectiblesData) - .filter(CollectiblesData.user_id == 2) - .first() + session.query(Collectibles).filter(Collectibles.user_id == 2).first() ) assert current_data is not None assert current_data.data == {} From 0138d3586504670569de259124b15b8e6d0ae351 Mon Sep 17 00:00:00 2001 From: Randy Schott <1815175+schottra@users.noreply.github.com> Date: Mon, 10 Feb 2025 13:10:34 -0500 Subject: [PATCH 8/8] update sdk --- .../generated/default/.openapi-generator/FILES | 2 +- .../{CollectiblesData.ts => Collectibles.ts} | 18 +++++++++--------- .../default/models/CollectiblesResponse.ts | 18 +++++++++--------- .../sdk/api/generated/default/models/index.ts | 2 +- 4 files changed, 20 insertions(+), 20 deletions(-) rename packages/sdk/src/sdk/api/generated/default/models/{CollectiblesData.ts => Collectibles.ts} (61%) diff --git a/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES b/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES index b98182f114d..b7058e79737 100644 --- a/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES +++ b/packages/sdk/src/sdk/api/generated/default/.openapi-generator/FILES @@ -19,7 +19,7 @@ models/AuthorizedApp.ts models/AuthorizedApps.ts models/BlobInfo.ts models/ChallengeResponse.ts -models/CollectiblesData.ts +models/Collectibles.ts models/CollectiblesResponse.ts models/CollectionActivity.ts models/Comment.ts diff --git a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts b/packages/sdk/src/sdk/api/generated/default/models/Collectibles.ts similarity index 61% rename from packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts rename to packages/sdk/src/sdk/api/generated/default/models/Collectibles.ts index 7e4dca2a800..0d1f130dc48 100644 --- a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesData.ts +++ b/packages/sdk/src/sdk/api/generated/default/models/Collectibles.ts @@ -17,31 +17,31 @@ import { exists, mapValues } from '../runtime'; /** * * @export - * @interface CollectiblesData + * @interface Collectibles */ -export interface CollectiblesData { +export interface Collectibles { /** * Raw collectibles JSON structure generated by client * @type {object} - * @memberof CollectiblesData + * @memberof Collectibles */ data?: object; } /** - * Check if a given object implements the CollectiblesData interface. + * Check if a given object implements the Collectibles interface. */ -export function instanceOfCollectiblesData(value: object): value is CollectiblesData { +export function instanceOfCollectibles(value: object): value is Collectibles { let isInstance = true; return isInstance; } -export function CollectiblesDataFromJSON(json: any): CollectiblesData { - return CollectiblesDataFromJSONTyped(json, false); +export function CollectiblesFromJSON(json: any): Collectibles { + return CollectiblesFromJSONTyped(json, false); } -export function CollectiblesDataFromJSONTyped(json: any, ignoreDiscriminator: boolean): CollectiblesData { +export function CollectiblesFromJSONTyped(json: any, ignoreDiscriminator: boolean): Collectibles { if ((json === undefined) || (json === null)) { return json; } @@ -51,7 +51,7 @@ export function CollectiblesDataFromJSONTyped(json: any, ignoreDiscriminator: bo }; } -export function CollectiblesDataToJSON(value?: CollectiblesData | null): any { +export function CollectiblesToJSON(value?: Collectibles | null): any { if (value === undefined) { return undefined; } diff --git a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts index d2bf77e3419..7540b544590 100644 --- a/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts +++ b/packages/sdk/src/sdk/api/generated/default/models/CollectiblesResponse.ts @@ -14,12 +14,12 @@ */ import { exists, mapValues } from '../runtime'; -import type { CollectiblesData } from './CollectiblesData'; +import type { Collectibles } from './Collectibles'; import { - CollectiblesDataFromJSON, - CollectiblesDataFromJSONTyped, - CollectiblesDataToJSON, -} from './CollectiblesData'; + CollectiblesFromJSON, + CollectiblesFromJSONTyped, + CollectiblesToJSON, +} from './Collectibles'; /** * @@ -29,10 +29,10 @@ import { export interface CollectiblesResponse { /** * - * @type {CollectiblesData} + * @type {Collectibles} * @memberof CollectiblesResponse */ - data?: CollectiblesData; + data?: Collectibles; } /** @@ -54,7 +54,7 @@ export function CollectiblesResponseFromJSONTyped(json: any, ignoreDiscriminator } return { - 'data': !exists(json, 'data') ? undefined : CollectiblesDataFromJSON(json['data']), + 'data': !exists(json, 'data') ? undefined : CollectiblesFromJSON(json['data']), }; } @@ -67,7 +67,7 @@ export function CollectiblesResponseToJSON(value?: CollectiblesResponse | null): } return { - 'data': CollectiblesDataToJSON(value.data), + 'data': CollectiblesToJSON(value.data), }; } diff --git a/packages/sdk/src/sdk/api/generated/default/models/index.ts b/packages/sdk/src/sdk/api/generated/default/models/index.ts index dcc6a201a33..5769563137a 100644 --- a/packages/sdk/src/sdk/api/generated/default/models/index.ts +++ b/packages/sdk/src/sdk/api/generated/default/models/index.ts @@ -9,7 +9,7 @@ export * from './AuthorizedApp'; export * from './AuthorizedApps'; export * from './BlobInfo'; export * from './ChallengeResponse'; -export * from './CollectiblesData'; +export * from './Collectibles'; export * from './CollectiblesResponse'; export * from './CollectionActivity'; export * from './Comment';