Token properties and attributes support

docker/polkastats-backend/docker-compose.hasura.yml

@@ -0,0 +1,20 @@
+version: '3.7'
+
+services:
+  #
+  # Hasura
+  #
+  graphql-engine:
+    image: hasura/graphql-engine:v2.2.0
+    ports:
+      - '8080:8080'
+    environment:
+      HASURA_GRAPHQL_DATABASE_URL: '${GRAPHQL_DATABASE_URL}' # postgres://polkastats:polkastats@host.docker.internal:5432/polkastats
+      HASURA_GRAPHQL_ENABLE_CONSOLE: '${GRAPHQL_ENABLE_CONSOLE}' # set to "false" to disable console
+      HASURA_GRAPHQL_ENABLED_LOG_TYPES: startup, http-log, webhook-log, websocket-log, query-log
+      HASURA_GRAPHQL_DISABLE_CORS: '${GRAPHQL_DISABLE_CORS}'
+      HASURA_GRAPHQL_UNAUTHORIZED_ROLE: 'anonymous'
+      HASURA_GRAPHQL_JWT_SECRET: '${GRAPHQL_JWT_SECRET}'
+      ## uncomment next line to set an admin secret
+      HASURA_GRAPHQL_ADMIN_SECRET: '${GRAPHQL_ADMIN_SECRET}'
+      HASURA_GRAPHQL_PG_CONNECTIONS: '${GRAPHQL_PG_CONNECTIONS}'

hasura/hasura_metadata.json

@@ -1 +1 @@
-{"version":3,"sources":[{"name":"default","kind":"postgres","tables":[{"table":{"schema":"public","name":"SequelizeMeta"}},{"table":{"schema":"public","name":"account"},"select_permissions":[{"role":"anonymous","permission":{"columns":["account_id","balances","available_balance","free_balance","locked_balance","nonce","timestamp","block_height","account_id_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","session_length","timestamp","need_rescan","new_accounts","num_transfers","spec_version","total_events","block_hash","extrinsics_root","parent_hash","spec_name","state_root","total_issuance"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"collections"},"array_relationships":[{"name":"tokens","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"tokens"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes_schema","collection_cover","collection_id","const_chain_schema","date_of_creation","description","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","properties","schema_version","sponsorship","token_limit","token_prefix","variable_on_chain_schema"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"event"}},{"table":{"schema":"public","name":"extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","args","block_index","block_number","extrinsic_index","fee","hash","is_signed","method","section","signer","signer_normalized","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"system"}},{"table":{"schema":"public","name":"tokens"},"object_relationships":[{"name":"collection","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"collections"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["date_of_creation","id","owner","token_id","data","owner_normalized","collection_id","parent_id"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"total"},"select_permissions":[{"role":"anonymous","permission":{"columns":["count","name"],"filter":{}}}]},{"table":{"schema":"public","name":"view_collections"},"select_permissions":[{"role":"anonymous","permission":{"columns":["actions_count","collection_cover","collection_id","const_chain_schema","date_of_creation","description","holders_count","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","schema_version","sponsorship","token_limit","token_prefix","tokens_count","type"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","block_index","block_number","fee","from_owner","from_owner_normalized","hash","method","section","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_holders"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_id","count","owner","owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_last_block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","event_count","extrinsic_count","timestamp"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_tokens"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_cover","collection_description","collection_id","collection_name","collection_owner","collection_owner_normalized","data","date_of_creation","image_path","is_sold","owner","owner_normalized","token_id","token_name","token_prefix"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_transfer"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_index","section","method","data"],"filter":{},"allow_aggregations":true}}]}],"configuration":{"connection_info":{"use_prepared_statements":true,"database_url":{"from_env":"HASURA_GRAPHQL_DATABASE_URL"},"isolation_level":"read-committed","pool_settings":{"connection_lifetime":600,"retries":1,"idle_timeout":180,"max_connections":2}}}}]}
+{"version":3,"sources":[{"name":"default","kind":"postgres","tables":[{"table":{"schema":"public","name":"SequelizeMeta"}},{"table":{"schema":"public","name":"account"},"select_permissions":[{"role":"anonymous","permission":{"columns":["account_id","balances","available_balance","free_balance","locked_balance","nonce","timestamp","block_height","account_id_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","session_length","timestamp","need_rescan","new_accounts","num_transfers","spec_version","total_events","block_hash","extrinsics_root","parent_hash","spec_name","state_root","total_issuance"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"collections"},"array_relationships":[{"name":"tokens","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"tokens"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes_schema","collection_cover","collection_id","const_chain_schema","date_of_creation","description","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","properties","schema_version","sponsorship","token_limit","token_prefix","variable_on_chain_schema"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"event"}},{"table":{"schema":"public","name":"extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","args","block_index","block_number","extrinsic_index","fee","hash","is_signed","method","section","signer","signer_normalized","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"system"}},{"table":{"schema":"public","name":"tokens"},"object_relationships":[{"name":"collection","using":{"manual_configuration":{"remote_table":{"schema":"public","name":"collections"},"insertion_order":null,"column_mapping":{"collection_id":"collection_id"}}}}],"select_permissions":[{"role":"anonymous","permission":{"columns":["attributes","collection_id","data","date_of_creation","id","owner","owner_normalized","parent_id","properties","token_id"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"total"},"select_permissions":[{"role":"anonymous","permission":{"columns":["count","name"],"filter":{}}}]},{"table":{"schema":"public","name":"view_collections"},"select_permissions":[{"role":"anonymous","permission":{"columns":["actions_count","collection_cover","collection_id","const_chain_schema","date_of_creation","description","holders_count","limits_account_ownership","limits_sponsore_data_rate","limits_sponsore_data_size","mint_mode","name","offchain_schema","owner","owner_can_destroy","owner_can_transfer","owner_normalized","schema_version","sponsorship","token_limit","token_prefix","tokens_count","type"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_extrinsic"},"select_permissions":[{"role":"anonymous","permission":{"columns":["amount","block_index","block_number","fee","from_owner","from_owner_normalized","hash","method","section","success","timestamp","to_owner","to_owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_holders"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_id","count","owner","owner_normalized"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_last_block"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_number","event_count","extrinsic_count","timestamp"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_tokens"},"select_permissions":[{"role":"anonymous","permission":{"columns":["collection_cover","collection_description","collection_id","collection_name","collection_owner","collection_owner_normalized","data","date_of_creation","image_path","is_sold","owner","owner_normalized","token_id","token_name","token_prefix"],"filter":{},"allow_aggregations":true}}]},{"table":{"schema":"public","name":"view_transfer"},"select_permissions":[{"role":"anonymous","permission":{"columns":["block_index","section","method","data"],"filter":{},"allow_aggregations":true}}]}],"configuration":{"connection_info":{"use_prepared_statements":true,"database_url":{"from_env":"HASURA_GRAPHQL_DATABASE_URL"},"isolation_level":"read-committed","pool_settings":{"connection_lifetime":600,"retries":1,"idle_timeout":180,"max_connections":2}}}}]}

lib/providerAPI/bridgeProviderAPI/concreate/opalAPI.ts

@@ -1,5 +1,9 @@
-import { CollectionInfoWithSchema } from '@unique-nft/sdk/tokens';
-import { UpDataStructsCollectionLimits, UpDataStructsRpcCollection } from '@unique-nft/unique-mainnet-types';
+import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens';
+import {
+  UpDataStructsCollectionLimits,
+  UpDataStructsRpcCollection,
+  UpDataStructsTokenData,
+} from '@unique-nft/unique-mainnet-types';
 import { ImplementOpalAPI } from '../implement/implementOpalAPI';
 import AbstractAPI from './abstractAPI';
 
@@ -26,9 +30,22 @@ export class OpalAPI extends AbstractAPI {
     };
   }
 
-  async getToken(collectionId, tokenId) {
-    const token = await this.impl.impGetToken(collectionId, tokenId);
-    return token || null;
+  async getToken(collectionId, tokenId): Promise<{
+    rawToken: UpDataStructsTokenData | null,
+    tokenDecoded: UniqueTokenDecoded | null,
+    tokenProperties: TokenPropertiesResult | null
+  }> {
+    const [rawToken, tokenDecoded, tokenProperties] = await Promise.all([
+      this.impl.impGetToken(collectionId, tokenId),
+      this.impl.impGetTokenSdk(collectionId, tokenId),
+      this.impl.impGetTokenPropertiesSdk(collectionId, tokenId)
+    ]);
+
+    return {
+      rawToken,
+      tokenDecoded,
+      tokenProperties
+    };
   }
 
   getCollectionCount() {

lib/providerAPI/bridgeProviderAPI/implement/implementOpalAPI.ts

@@ -2,10 +2,10 @@
 import {
   UpDataStructsCollectionLimits,
   UpDataStructsRpcCollection,
-  UpDataStructsTokenData,
+  UpDataStructsTokenData
 } from '@unique-nft/unique-mainnet-types';
-import { CollectionInfoWithSchema } from '@unique-nft/sdk/tokens';
-import '@unique-nft/sdk/tokens'; // need this to get sdk.collections
+import { CollectionInfoWithSchema, TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens';
+import '@unique-nft/sdk/tokens'; // need this to get sdk.collections and sdk.tokens declarations
 import ImplementorAPI from './implementorAPI';
 
 export class ImplementOpalAPI extends ImplementorAPI {
@@ -20,22 +20,28 @@ export class ImplementOpalAPI extends ImplementorAPI {
   }
 
   async impGetCollectionSdk(collectionId): Promise<CollectionInfoWithSchema | null> {
-    const result = await this.sdk.collections.get_new({ collectionId });
-
-    return result;
+    return this.sdk.collections.get_new({ collectionId });
   }
 
   async impGetCollectionCount() {
     const collectionStats = await this.api.rpc.unique.collectionStats();
     return collectionStats?.created.toNumber();
   }
 
-  async impGetToken(collectionId, tokenId): Promise<UpDataStructsTokenData> {
+  async impGetToken(collectionId, tokenId): Promise<UpDataStructsTokenData | null> {
     const tokenData = await this.api.rpc.unique.tokenData(collectionId, tokenId);
     return tokenData || null;
   }
 
+  async impGetTokenSdk(collectionId, tokenId): Promise<UniqueTokenDecoded | null> {
+    return this.sdk.tokens.get_new({ collectionId, tokenId });
+  }
+
   async impGetTokenCount(collectionId) {
     return (await this.api.rpc.unique.lastTokenId(collectionId)).toNumber();
   }
+
+  async impGetTokenPropertiesSdk(collectionId, tokenId): Promise<TokenPropertiesResult | null> {
+    return this.sdk.tokens.properties({ collectionId, tokenId });
+  }
 }

lib/token/tokenData.ts

@@ -1,7 +1,12 @@
+/* eslint-disable @typescript-eslint/naming-convention */
 /* eslint-disable no-underscore-dangle */
-import { ICollectionSchemaInfo } from 'crawlers/crawlers.interfaces';
 import { UpDataStructsTokenData } from '@unique-nft/unique-mainnet-types';
-import { normalizeSubstrateAddress } from '../../utils/utils';
+import { ICollectionSchemaInfo } from 'crawlers/crawlers.interfaces';
+import { TokenPropertiesResult, UniqueTokenDecoded } from '@unique-nft/sdk/tokens';
+import {
+  normalizeSubstrateAddress,
+  sanitizePropertiesValues
+} from '../../utils/utils';
 import protobuf from '../../utils/protobuf';
 import { ITokenDbEntity } from './tokenDbEntity.interface';
 import { OpalAPI } from '../providerAPI/bridgeProviderAPI/concreate/opalAPI';
@@ -51,13 +56,11 @@ function processConstData(constData, schema) {
   return getDeserializeConstData(statement);
 }
 
-function processProperties(schema: any, rawToken: UpDataStructsTokenData)
-  : { data: Object, properties: Object } {
+function processOldProperties(schema: any, rawToken: UpDataStructsTokenData)
+  : { data: Object } {
   const rawProperties = rawToken.properties;
 
-  const properties : {
-    _old_constData?: Object | string,
-  } = {};
+  let oldConstData: Object | string | null = null;
 
   rawProperties.forEach(({ key, value }) => {
     const strKey = key.toUtf8();
@@ -66,38 +69,71 @@ function processProperties(schema: any, rawToken: UpDataStructsTokenData)
 
     if (['_old_constData'].includes(strKey)) {
       try { processedValue = value.toHex(); } catch (err) { /* */ }
-    }
 
-    properties[strKey] = processedValue || strValue;
+      oldConstData = processedValue || strValue;
+    }
   });
 
   return {
-    data: processConstData(properties._old_constData, schema),
-    properties,
+    data: processConstData(oldConstData, schema),
   };
 }
 
-function formatTokenData(tokenId: number, collectionInfo: ICollectionSchemaInfo, rawToken: UpDataStructsTokenData)
-  : ITokenDbEntity {
-  const { collectionId, schema } = collectionInfo;
-
-  const rawOwnerJson = rawToken.owner.toJSON() as { substrate?: string, ethereum?: string };
-
-  const owner = rawOwnerJson?.substrate || rawOwnerJson?.ethereum;
+function formatTokenData({
+  rawToken,
+  tokenDecoded,
+  tokenProperties,
+  collectionInfo
+}: {
+  rawToken: UpDataStructsTokenData,
+  tokenDecoded: UniqueTokenDecoded,
+  tokenProperties: TokenPropertiesResult,
+  collectionInfo: ICollectionSchemaInfo
+}) : ITokenDbEntity {
+  const { schema } = collectionInfo;
+
+  const {
+    tokenId: token_id,
+    collectionId: collection_id,
+    attributes,
+    nestingParentToken,
+  } = tokenDecoded;
+
+  const {
+    owner: rawOwner,
+  }: { owner: { Ethereum?: string; Substrate?: string } } = tokenDecoded;
+
+  const owner = rawOwner?.Ethereum || rawOwner?.Substrate;
+
+  let parentId = null;
+  if (nestingParentToken) {
+    const { collectionId, tokenId } = nestingParentToken as { collectionId: number; tokenId: number };
+    parentId = `${collectionId}_${tokenId}`;
+  }
 
   return {
-    token_id: tokenId,
-    collection_id: collectionId,
+    token_id,
+    collection_id,
     owner,
     owner_normalized: normalizeSubstrateAddress(owner),
-    ...processProperties(schema, rawToken),
+    attributes: JSON.stringify(attributes),
+    properties: tokenProperties
+      ? JSON.stringify(sanitizePropertiesValues(tokenProperties))
+      : '[]',
+    parent_id: parentId,
+    ...processOldProperties(schema, rawToken),
   };
 }
 
 export async function getFormattedToken(tokenId: number, collectionInfo: ICollectionSchemaInfo, bridgeAPI: OpalAPI)
   : Promise<ITokenDbEntity | null> {
   const { collectionId } = collectionInfo;
-  const rawToken = await bridgeAPI.getToken(collectionId, tokenId);
-
-  return rawToken ? formatTokenData(tokenId, collectionInfo, rawToken) : null;
+  const { rawToken, tokenDecoded, tokenProperties } = await bridgeAPI.getToken(collectionId, tokenId);
+
+  return tokenDecoded ? formatTokenData({
+    rawToken,
+    tokenDecoded,
+    tokenProperties,
+    collectionInfo
+  }) : null;
 }

lib/token/tokenDb.ts

@@ -17,6 +17,8 @@ const TOKEN_FIELDS = [
   'data',
   'date_of_creation',
   'parent_id',
+  'properties',
+  'attributes'
 ];
 
 function prepareQueryReplacements(token: ITokenDbEntity) {

lib/token/tokenDbEntity.interface.ts

@@ -6,4 +6,7 @@ export interface ITokenDbEntity {
   owner_normalized: string,
   data: Object,
   date_of_creation?: number,
+  properties: string,
+  attributes: string,
+  parent_id: string | null
 }

sequelize/migrations/20220804094054-token-properties-and-attributes-fields.js

@@ -0,0 +1,23 @@
+'use strict';
+
+module.exports = {
+  async up (queryInterface, Sequelize) {
+    return Promise.all([
+      queryInterface.addColumn('tokens', 'properties', {
+        type: Sequelize.DataTypes.JSONB,
+        defaultValue: []  
+      }),
+      queryInterface.addColumn('tokens', 'attributes', {
+        type: Sequelize.DataTypes.JSONB,
+        defaultValue: {}  
+      }),
+    ]) 
+  },
+
+  async down (queryInterface, Sequelize) {
+    return Promise.all([
+      await queryInterface.removeColumn('tokens', 'properties'),
+      await queryInterface.removeColumn('tokens', 'attributes'),
+    ]);
+  }
+};

utils/utils.js

@@ -187,7 +187,15 @@ function getTokenIdFromNestingAddress(address) {
 }
 
 function sanitizeUnicodeString(str) {
-  return str.replace(/\\u0000/g, '');
+  // eslint-disable-next-line no-control-regex
+  return str.replace(/\\u0000|\x00/g, '');
+}
+
+function sanitizePropertiesValues(propertiesArr) {
+  return propertiesArr.map(({ key, value }) => ({
+    key,
+    value: sanitizeUnicodeString(value),
+  }));
 }
 
 module.exports = {
@@ -210,5 +218,6 @@ module.exports = {
   isNestingAddress,
   getCollectionIdFromNestingAddress,
   getTokenIdFromNestingAddress,
-  sanitizeUnicodeString
+  sanitizeUnicodeString,
+  sanitizePropertiesValues
 };