Skip to content
This repository has been archived by the owner on Jun 11, 2024. It is now read-only.

Commit

Permalink
Merge pull request #1977 from LiskHQ/1974-index-valid-generator-keys-…
Browse files Browse the repository at this point in the history 
…from-genesis-assets

Index valid generatorKeys from genesis assets
  • Loading branch information
@sameersubudhi
sameersubudhi authored Dec 7, 2023
2 parents 23b2767 + cd9149f commit 92f92e8
Show file tree
Hide file tree
Showing 7 changed files with 108 additions and 33 deletions.
2 changes: 1 addition & 1 deletion docs/api/websocket_subscribe_api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ Updates about index status changes.
"chainLength": 18780,
"numBlocksIndexed": 13956,
"percentageIndexed": 74.31,
"isIndexingInProgress": false
"isIndexingInProgress": true
},
"meta": {
"lastUpdate": 1700848735
Expand Down
18 changes: 9 additions & 9 deletions ecosystem.config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -75,7 +75,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -107,7 +107,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '1000M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -150,7 +150,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '1000M',
max_memory_restart: '500M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -209,7 +209,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -238,7 +238,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -270,7 +270,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -302,7 +302,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand Down Expand Up @@ -340,7 +340,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
SERVICE_BROKER: 'redis://127.0.0.1:6379/0',
Expand Down
59 changes: 48 additions & 11 deletions ecosystem.jenkins.config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
PORT: 9901,
Expand Down Expand Up @@ -57,7 +57,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand All @@ -76,7 +76,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '1000M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand All @@ -98,7 +98,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '1000M',
max_memory_restart: '500M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand All @@ -123,7 +123,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand All @@ -141,7 +141,7 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
Expand All @@ -161,14 +161,14 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
// --- Remember to set the properties below
SERVICE_BROKER: 'redis://127.0.0.1:6379/0',
SERVICE_STATISTICS_REDIS: 'redis://127.0.0.1:6379/1',
SERVICE_STATISTICS_MYSQL: 'mysql://lisk:password@127.0.0.1:3306/lisk',
TRANSACTION_STATS_HISTORY_LENGTH_DAYS: 366,
// TRANSACTION_STATS_HISTORY_LENGTH_DAYS: 366,
},
},
{
Expand All @@ -181,15 +181,52 @@ module.exports = {
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '500M',
max_memory_restart: '250M',
autorestart: true,
env: {
SERVICE_BROKER: 'redis://127.0.0.1:6379/0',
SERVICE_MARKET_REDIS: 'redis://127.0.0.1:6379/2',
SERVICE_MARKET_FIAT_CURRENCIES: 'EUR,USD,CHF,GBP,RUB',
SERVICE_MARKET_TARGET_PAIRS: 'LSK_BTC,LSK_EUR,LSK_USD,LSK_CHF,BTC_EUR,BTC_USD,BTC_CHF',
// SERVICE_MARKET_FIAT_CURRENCIES: 'EUR,USD,CHF,GBP,RUB',
// SERVICE_MARKET_TARGET_PAIRS: 'LSK_BTC,LSK_EUR,LSK_USD,LSK_CHF,BTC_EUR,BTC_USD,BTC_CHF',
// EXCHANGERATESAPI_IO_API_KEY: ''
},
},
{
name: 'lisk-service-export',
script: 'app.js',
cwd: './services/export',
pid_file: './pids/service_export.pid',
out_file: './logs/service_export.log',
error_file: './logs/service_export.err',
log_date_format: 'YYYY-MM-DD HH:mm:ss SSS',
watch: false,
kill_timeout: 10000,
max_memory_restart: '250M',
autorestart: true,
env: {
SERVICE_BROKER: 'redis://127.0.0.1:6379/0',
SERVICE_EXPORT_REDIS: 'redis://127.0.0.1:6379/3',
SERVICE_EXPORT_REDIS_VOLATILE: 'redis://127.0.0.1:6379/4',
// SERVICE_EXPORT_PARTIALS: './data/partials',
// EXPORT_S3_BUCKET_NAME_PARTIALS: 'partials',
// SERVICE_EXPORT_STATIC: './data/static',
// EXPORT_S3_BUCKET_NAME_EXPORTS: 'exports',
// SERVICE_BROKER_TIMEOUT: 10,
// SERVICE_LOG_LEVEL: 'info',
// SERVICE_LOG_CONSOLE: false,
// SERVICE_LOG_STDOUT: true,
// SERVICE_LOG_GELF: false,
// SERVICE_LOG_FILE: false,
// DOCKER_HOST: 'local',
// EXPORT_S3_ENDPOINT: 's3.amazonaws.com',
// EXPORT_S3_ACCESS_KEY: '',
// EXPORT_S3_SECRET_KEY: '',
// EXPORT_S3_SESSION_TOKEN: '',
// EXPORT_S3_REGION: 'eu-central-1',
// EXPORT_S3_BUCKET_NAME: 'export',
// JOB_INTERVAL_CACHE_PURGE: 0,
// JOB_SCHEDULE_CACHE_PURGE: '45 4 * * *',
},
},
],
};
7 changes: 6 additions & 1 deletion services/blockchain-connector/shared/sdk/cache.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ const cacheBlocksFromWaitlist = async () => {
const blocksCache = await getBlocksCache();
const trxIDToBlockIDCache = await getTrxIDtoBlockIDCache();

let numErrors = 0;
while (blockCacheWaitlist.length) {
const block = blockCacheWaitlist.shift();
try {
Expand All @@ -78,7 +79,11 @@ const cacheBlocksFromWaitlist = async () => {
);
logger.debug(err.stack);
blockCacheWaitlist.splice(0, 0, block);
await delay(10000); // Delay loop to facilitate reads from cache when writes are failing due to DB locks

// Skip caching if it causes errors 3 times in a row
if (numErrors++ > 3) break;

await delay(5 * 1000); // Delay loop to facilitate reads from cache when writes are failing due to DB locks
}
}

Expand Down
8 changes: 4 additions & 4 deletions services/blockchain-indexer/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,12 @@ config.queue = {
scheduledJobsMaxCount: Number(process.env.INDEX_BLOCKS_QUEUE_SCHEDULED_JOB_MAX_COUNT) || 100000,
},
indexAccountPublicKey: {
name: 'PendingAccountPublicKeyUpdates',
concurrency: 64,
name: 'PendingPublickeyUpdates',
concurrency: 512,
},
indexAccountAddress: {
name: 'PendingAccountAddressUpdates',
concurrency: 64,
name: 'PendingAddressUpdates',
concurrency: 512,
},
};

Expand Down
2 changes: 2 additions & 0 deletions services/blockchain-indexer/shared/constants.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ const COMMAND = Object.freeze({
REGISTER_MAINCHAIN: 'registerMainchain',
});

const INVALID_ED25519_KEY = 'ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'; // LIP-0063
const LENGTH_CHAIN_ID = 4 * 2; // Each byte is represented with 2 nibbles
const LENGTH_TOKEN_LOCAL_ID = 4 * 2; // Each byte is represented with 2 nibbles
const PATTERN_ANY_TOKEN_ID = '*';
Expand Down Expand Up @@ -230,6 +231,7 @@ module.exports = {
getAllRegisteredEndpoints,
initNodeConstants,

INVALID_ED25519_KEY,
LENGTH_CHAIN_ID,
PATTERN_ANY_TOKEN_ID,
PATTERN_ANY_CHAIN_TOKEN_ID,
Expand Down
45 changes: 38 additions & 7 deletions services/blockchain-indexer/shared/indexer/genesisBlock.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
* Removal or modification of this copyright notice is prohibited.
*
*/
const BluebirdPromise = require('bluebird');

const {
DB: {
MySQL: { getTableInstance },
Expand All @@ -23,21 +25,27 @@ const {

const { MODULE, MODULE_SUB_STORE, getGenesisHeight } = require('../constants');
const { updateTotalStake, updateTotalSelfStake } = require('./transactionProcessor/pos/stake');
const { requestConnector } = require('../utils/request');
const { updateAccountBalances } = require('./accountBalanceIndex');
const { indexAccountPublicKey, triggerAccountUpdates } = require('./accountIndex');
const { updateTotalLockedAmounts } = require('./utils/blockchainIndex');
const { getIndexStats } = require('./indexStatus');

const requestAll = require('../utils/requestAll');
const config = require('../../config');
const accountsTableSchema = require('../database/schema/accounts');
const stakesTableSchema = require('../database/schema/stakes');
const commissionsTableSchema = require('../database/schema/commissions');
const { getIndexStats } = require('./indexStatus');

const { getLisk32AddressFromPublicKey } = require('../utils/account');
const { requestConnector } = require('../utils/request');
const { INVALID_ED25519_KEY } = require('../constants');

const logger = Logger();

const MYSQL_ENDPOINT = config.endpoints.mysql;

const getStakesTable = () => getTableInstance(stakesTableSchema, MYSQL_ENDPOINT);
const getAccountsTable = () => getTableInstance(accountsTableSchema, MYSQL_ENDPOINT);
const getCommissionsTable = () => getTableInstance(commissionsTableSchema, MYSQL_ENDPOINT);

const allAccountsAddresses = [];
Expand Down Expand Up @@ -82,6 +90,8 @@ const indexTokenModuleAssets = async dbTrx => {
logger.info('Finished indexing all the genesis assets from the Token module.');
};

const isGeneratorKeyValid = generatorKey => generatorKey !== INVALID_ED25519_KEY;

const indexPosValidatorsInfo = async (numValidators, dbTrx) => {
logger.debug(
'Starting to index the PoS Validators information from the genesis PoS module assets.',
Expand All @@ -99,11 +109,30 @@ const indexPosValidatorsInfo = async (numValidators, dbTrx) => {
const validators = posModuleData[MODULE_SUB_STORE.POS.VALIDATORS];
const genesisHeight = await getGenesisHeight();

const commissionEntries = validators.map(validator => ({
address: validator.address,
commission: validator.commission,
height: genesisHeight,
}));
const commissionEntries = await BluebirdPromise.map(
validators,
async validator => {
// Index all valid public keys
if (isGeneratorKeyValid(validator.generatorKey)) {
const account = {
address: getLisk32AddressFromPublicKey(validator.generatorKey),
publicKey: validator.generatorKey,
};

const accountsTable = await getAccountsTable();
await accountsTable
.upsert(account)
.catch(() => indexAccountPublicKey(validator.generatorKey));
}

return {
address: validator.address,
commission: validator.commission,
height: genesisHeight,
};
},
{ concurrency: validators.length },
);

await commissionsTable.upsert(commissionEntries, dbTrx);
}
Expand Down Expand Up @@ -180,6 +209,8 @@ const indexGenesisBlockAssets = async dbTrx => {
);
await indexTokenModuleAssets(dbTrx);
await indexPosModuleAssets(dbTrx);

await triggerAccountUpdates();
clearInterval(intervalTimeout);
logger.info('Finished indexing all the genesis assets.');
};
Expand Down

0 comments on commit 92f92e8

Please sign in to comment.