From 4cfae42f7a1230746909de49115ebcab7c6a0ad6 Mon Sep 17 00:00:00 2001 From: ancelmo Date: Thu, 28 Mar 2019 19:00:43 +0800 Subject: [PATCH] pass all unittest --- libblockchain/BlockChainImp.cpp | 9 +- libblockchain/BlockChainImp.h | 7 +- libblockverifier/ExecutiveContext.cpp | 2 +- libblockverifier/ExecutiveContext.h | 8 +- libblockverifier/ExecutiveContextFactory.cpp | 6 +- libblockverifier/ExecutiveContextFactory.h | 5 + libconsensus/pbft/PBFTEngine.h | 1 + libexecutive/StateFactoryInterface.h | 2 +- libledger/DBInitializer.cpp | 21 +- libledger/DBInitializer.h | 8 +- libledger/Ledger.cpp | 1 + libmptstate/MPTStateFactory.cpp | 2 +- libmptstate/MPTStateFactory.h | 2 +- libstorage/LevelDBStorage.cpp | 319 ++++++++++----- libstorage/LevelDBStorage.h | 8 +- libstorage/LevelDBStorage2.cpp | 228 +++++++++++ libstorage/LevelDBStorage2.h | 60 +++ libstorage/MemoryTable.h | 338 ++++++++++++++-- libstorage/MemoryTable2.cpp | 367 ++++++++++++++++++ libstorage/MemoryTable2.h | 267 +++++++++++++ libstorage/MemoryTableFactory.cpp | 28 +- libstorage/MemoryTableFactory.h | 15 +- libstorage/MemoryTableFactory2.cpp | 352 +++++++++++++++++ libstorage/MemoryTableFactory2.h | 84 ++++ libstorage/MemoryTableFactoryFactory.h | 54 +++ libstorage/MemoryTableFactoryFactory2.h | 54 +++ libstorage/Storage.h | 2 +- libstorage/Table.h | 46 ++- libstorage/TableFactoryPrecompiled.cpp | 19 +- libstorage/TableFactoryPrecompiled.h | 6 +- libstoragestate/StorageState.cpp | 4 +- libstoragestate/StorageState.h | 4 +- libstoragestate/StorageStateFactory.cpp | 2 +- libstoragestate/StorageStateFactory.h | 2 +- .../unittests/libblockchain/BlockChainImp.cpp | 9 +- test/unittests/libevm/VMTest.cpp | 1 + test/unittests/libexecutive/ExecuteVMTest.cpp | 3 +- test/unittests/libstorage/MemoryStorage.h | 177 +-------- test/unittests/libstorage/MemoryStorage2.h | 230 +++++++++++ .../libstorage/test_AuthorityPrecompiled.cpp | 5 +- .../libstorage/test_CNSPrecompiled.cpp | 7 +- .../libstorage/test_CRUDPrecompiled.cpp | 5 +- .../libstorage/test_ConsensusPrecompiled.cpp | 5 +- .../test_DagTransferPrecompiled.cpp | 7 +- .../libstorage/test_HelloWorldPrecompiled.cpp | 7 +- .../libstorage/test_LevelDBStateStorage.cpp | 23 +- .../libstorage/test_LevelDBStateStorage2.cpp | 266 +++++++++++++ .../libstorage/test_MemoryTableFactory.cpp | 16 +- .../libstorage/test_MemoryTableFactory2.cpp | 252 ++++++++++++ .../test_ParallelConfigPrecompiled.cpp | 7 +- .../test_SystemConfigPrecompiled.cpp | 5 +- .../libstorage/test_TablePrecompiled.cpp | 3 +- .../libstorage/test_TablePrecompiled2.cpp | 186 +++++++++ 53 files changed, 3141 insertions(+), 406 deletions(-) create mode 100644 libstorage/LevelDBStorage2.cpp create mode 100644 libstorage/LevelDBStorage2.h create mode 100644 libstorage/MemoryTable2.cpp create mode 100644 libstorage/MemoryTable2.h create mode 100644 libstorage/MemoryTableFactory2.cpp create mode 100644 libstorage/MemoryTableFactory2.h create mode 100644 libstorage/MemoryTableFactoryFactory.h create mode 100644 libstorage/MemoryTableFactoryFactory2.h create mode 100644 test/unittests/libstorage/MemoryStorage2.h create mode 100644 test/unittests/libstorage/test_LevelDBStateStorage2.cpp create mode 100644 test/unittests/libstorage/test_MemoryTableFactory2.cpp create mode 100644 test/unittests/libstorage/test_TablePrecompiled2.cpp diff --git a/libblockchain/BlockChainImp.cpp b/libblockchain/BlockChainImp.cpp index c3baa49923..5d1ec4d5ae 100644 --- a/libblockchain/BlockChainImp.cpp +++ b/libblockchain/BlockChainImp.cpp @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -108,11 +107,15 @@ void BlockChainImp::setStateFactory(StateFactoryInterface::Ptr _stateFactory) m_stateFactory = _stateFactory; } -shared_ptr BlockChainImp::getMemoryTableFactory() +shared_ptr BlockChainImp::getMemoryTableFactory() { +#if 0 dev::storage::MemoryTableFactory::Ptr memoryTableFactory = std::make_shared(); memoryTableFactory->setStateStorage(m_stateStorage); +#endif + + auto memoryTableFactory = m_tableFactoryFactory->newTableFactory(dev::h256(), 0); return memoryTableFactory; } @@ -425,7 +428,7 @@ bool BlockChainImp::checkAndBuildGenesisBlock(GenesisBlockParam& initParam) /// modification 2019.3.20: set timestamp to block header block->setEmptyBlock(initParam.timeStamp); block->header().appendExtraDataArray(asBytes(initParam.groupMark)); - shared_ptr mtb = getMemoryTableFactory(); + shared_ptr mtb = getMemoryTableFactory(); Table::Ptr tb = mtb->openTable(SYS_NUMBER_2_HASH, false); if (tb) { diff --git a/libblockchain/BlockChainImp.h b/libblockchain/BlockChainImp.h index 60cdf641ff..94210aac75 100644 --- a/libblockchain/BlockChainImp.h +++ b/libblockchain/BlockChainImp.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -93,7 +94,7 @@ class BlockChainImp : public BlockChainInterface std::shared_ptr context) override; virtual void setStateStorage(dev::storage::Storage::Ptr stateStorage); virtual void setStateFactory(dev::executive::StateFactoryInterface::Ptr _stateFactory); - virtual std::shared_ptr getMemoryTableFactory(); + virtual std::shared_ptr getMemoryTableFactory(); bool checkAndBuildGenesisBlock(GenesisBlockParam& initParam) override; virtual std::pair totalTransactionCount() override; dev::bytes getCode(dev::Address _address) override; @@ -104,6 +105,8 @@ class BlockChainImp : public BlockChainInterface void getNonces( std::vector& _nonceVector, int64_t _blockNumber) override; + void setTableFactoryFactory(dev::storage::TableFactoryFactory::Ptr tableFactoryFactory) { m_tableFactoryFactory = tableFactoryFactory; } + private: std::shared_ptr getBlock(int64_t _i); std::shared_ptr getBlock(dev::h256 const& _blockHash); @@ -152,6 +155,8 @@ class BlockChainImp : public BlockChainInterface /// cache the block number mutable SharedMutex m_blockNumberMutex; int64_t m_blockNumber = -1; + + dev::storage::TableFactoryFactory::Ptr m_tableFactoryFactory; }; } // namespace blockchain } // namespace dev diff --git a/libblockverifier/ExecutiveContext.cpp b/libblockverifier/ExecutiveContext.cpp index e693777f03..b664702069 100644 --- a/libblockverifier/ExecutiveContext.cpp +++ b/libblockverifier/ExecutiveContext.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include using namespace dev::executive; using namespace dev::eth; diff --git a/libblockverifier/ExecutiveContext.h b/libblockverifier/ExecutiveContext.h index 1ca6030ec8..7e33fbd930 100644 --- a/libblockverifier/ExecutiveContext.h +++ b/libblockverifier/ExecutiveContext.h @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include namespace dev @@ -86,12 +86,12 @@ class ExecutiveContext : public std::enable_shared_from_this void dbCommit(dev::eth::Block& block); - void setMemoryTableFactory(std::shared_ptr memoryTableFactory) + void setMemoryTableFactory(std::shared_ptr memoryTableFactory) { m_memoryTableFactory = memoryTableFactory; } - std::shared_ptr getMemoryTableFactory() + std::shared_ptr getMemoryTableFactory() { return m_memoryTableFactory; } @@ -108,7 +108,7 @@ class ExecutiveContext : public std::enable_shared_from_this BlockInfo m_blockInfo; std::shared_ptr m_stateFace; std::unordered_map m_precompiledContract; - std::shared_ptr m_memoryTableFactory; + std::shared_ptr m_memoryTableFactory; uint64_t m_txGasLimit = 300000000; }; diff --git a/libblockverifier/ExecutiveContextFactory.cpp b/libblockverifier/ExecutiveContextFactory.cpp index e885aeef7e..e7c422c954 100644 --- a/libblockverifier/ExecutiveContextFactory.cpp +++ b/libblockverifier/ExecutiveContextFactory.cpp @@ -39,12 +39,15 @@ using namespace dev::precompiled; void ExecutiveContextFactory::initExecutiveContext( BlockInfo blockInfo, h256 stateRoot, ExecutiveContext::Ptr context) { +#if 0 // DBFactoryPrecompiled dev::storage::MemoryTableFactory::Ptr memoryTableFactory = std::make_shared(); memoryTableFactory->setStateStorage(m_stateStorage); memoryTableFactory->setBlockHash(blockInfo.hash); memoryTableFactory->setBlockNum(blockInfo.number); +#endif + auto memoryTableFactory = m_tableFactoryFactory->newTableFactory(blockInfo.hash, blockInfo.number); auto tableFactoryPrecompiled = std::make_shared(); tableFactoryPrecompiled->setMemoryTableFactory(memoryTableFactory); @@ -86,8 +89,6 @@ void ExecutiveContextFactory::setStateFactory( void ExecutiveContextFactory::setTxGasLimitToContext(ExecutiveContext::Ptr context) { - (void)context; -#if 0 // get value from db try { @@ -132,5 +133,4 @@ void ExecutiveContextFactory::setTxGasLimitToContext(ExecutiveContext::Ptr conte EXECUTIVECONTEXT_LOG(ERROR) << LOG_DESC("[#setTxGasLimitToContext]Failed") << LOG_KV("EINFO", boost::diagnostic_information(e)); } -#endif } diff --git a/libblockverifier/ExecutiveContextFactory.h b/libblockverifier/ExecutiveContextFactory.h index 082efcf727..d2fd1ca4af 100644 --- a/libblockverifier/ExecutiveContextFactory.h +++ b/libblockverifier/ExecutiveContextFactory.h @@ -24,6 +24,8 @@ #include #include #include +#include + namespace dev { namespace blockverifier @@ -57,7 +59,10 @@ class ExecutiveContextFactory : public std::enable_shared_from_this stateFactoryInterface); + virtual void setTableFactoryFactory(dev::storage::TableFactoryFactory::Ptr tableFactoryFactory) { m_tableFactoryFactory = tableFactoryFactory; } + private: + dev::storage::TableFactoryFactory::Ptr m_tableFactoryFactory; dev::storage::Storage::Ptr m_stateStorage; std::shared_ptr m_stateFactoryInterface; std::unordered_map m_precompiledContract; diff --git a/libconsensus/pbft/PBFTEngine.h b/libconsensus/pbft/PBFTEngine.h index 5d89a9c7bb..8a801b1953 100644 --- a/libconsensus/pbft/PBFTEngine.h +++ b/libconsensus/pbft/PBFTEngine.h @@ -32,6 +32,7 @@ #include #include #include +#include #include #include diff --git a/libexecutive/StateFactoryInterface.h b/libexecutive/StateFactoryInterface.h index 84bac9d5c6..9f6a4d5ac5 100644 --- a/libexecutive/StateFactoryInterface.h +++ b/libexecutive/StateFactoryInterface.h @@ -41,7 +41,7 @@ class StateFactoryInterface StateFactoryInterface() = default; virtual ~StateFactoryInterface(){}; virtual std::shared_ptr getState( - h256 const& _root, std::shared_ptr _factory) = 0; + h256 const& _root, std::shared_ptr _factory) = 0; }; } // namespace executive diff --git a/libledger/DBInitializer.cpp b/libledger/DBInitializer.cpp index 718ff5fce4..d28bc073d8 100644 --- a/libledger/DBInitializer.cpp +++ b/libledger/DBInitializer.cpp @@ -22,7 +22,7 @@ * @date: 2018-10-24 */ #include "DBInitializer.h" -#include "../libstorage/SQLStorage.h" +#include #include "LedgerParam.h" #include #include @@ -30,6 +30,8 @@ #include #include #include +#include +#include using namespace dev; using namespace dev::storage; @@ -111,6 +113,11 @@ void DBInitializer::initLevelDBStorage() std::shared_ptr(pleveldb); leveldb_storage->setDB(leveldb_handler); m_storage = leveldb_storage; + + auto tableFactoryFactory = std::make_shared(); + tableFactoryFactory->setStorage(m_storage); + + m_tableFactoryFactory = tableFactoryFactory; } catch (std::exception& e) { @@ -136,6 +143,11 @@ void DBInitializer::initSQLStorage() sqlStorage->setMaxRetry(m_param->mutableStorageParam().maxRetry); m_storage = sqlStorage; + + auto tableFactoryFactory = std::make_shared(); + tableFactoryFactory->setStorage(m_storage); + + m_tableFactoryFactory = tableFactoryFactory; } /// create ExecutiveContextFactory @@ -148,11 +160,12 @@ void DBInitializer::createExecutiveContext() return; } DBInitializer_LOG(DEBUG) << LOG_DESC("createExecutiveContext..."); - m_executiveContextFac = std::make_shared(); + m_executiveContextFactory = std::make_shared(); /// storage - m_executiveContextFac->setStateStorage(m_storage); + m_executiveContextFactory->setStateStorage(m_storage); // mpt or storage - m_executiveContextFac->setStateFactory(m_stateFactory); + m_executiveContextFactory->setStateFactory(m_stateFactory); + m_executiveContextFactory->setTableFactoryFactory(m_tableFactoryFactory); DBInitializer_LOG(DEBUG) << LOG_DESC("createExecutiveContext SUCC"); } diff --git a/libledger/DBInitializer.h b/libledger/DBInitializer.h index a0e1173b60..ef7deeac3a 100644 --- a/libledger/DBInitializer.h +++ b/libledger/DBInitializer.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -55,11 +56,12 @@ class DBInitializer createExecutiveContext(); } + dev::storage::TableFactoryFactory::Ptr tableFactoryFactory() { return m_tableFactoryFactory; }; dev::storage::Storage::Ptr storage() const { return m_storage; } std::shared_ptr stateFactory() { return m_stateFactory; } std::shared_ptr executiveContextFactory() const { - return m_executiveContextFac; + return m_executiveContextFactory; } virtual void setChannelRPCServer(ChannelRPCServer::Ptr channelRPCServer) @@ -86,8 +88,10 @@ class DBInitializer std::shared_ptr m_param; std::shared_ptr m_stateFactory; dev::storage::Storage::Ptr m_storage = nullptr; - std::shared_ptr m_executiveContextFac; + std::shared_ptr m_executiveContextFactory; std::shared_ptr m_channelRPCServer; + + dev::storage::TableFactoryFactory::Ptr m_tableFactoryFactory; }; } // namespace ledger } // namespace dev diff --git a/libledger/Ledger.cpp b/libledger/Ledger.cpp index 137bb6f59b..4e7d47e859 100644 --- a/libledger/Ledger.cpp +++ b/libledger/Ledger.cpp @@ -368,6 +368,7 @@ bool Ledger::initBlockChain() } std::shared_ptr blockChain = std::make_shared(); blockChain->setStateStorage(m_dbInitializer->storage()); + //blockChain->setTableFactoryFactory(tableFactoryFactory); m_blockChain = blockChain; std::string consensusType = m_param->mutableConsensusParam().consensusType; std::string storageType = m_param->mutableStorageParam().type; diff --git a/libmptstate/MPTStateFactory.cpp b/libmptstate/MPTStateFactory.cpp index cad48a2696..8e1cb0f677 100644 --- a/libmptstate/MPTStateFactory.cpp +++ b/libmptstate/MPTStateFactory.cpp @@ -29,7 +29,7 @@ using namespace dev::eth; using namespace dev::executive; std::shared_ptr MPTStateFactory::getState( - h256 const& _root, std::shared_ptr) + h256 const& _root, std::shared_ptr) { if (_root == dev::h256()) { diff --git a/libmptstate/MPTStateFactory.h b/libmptstate/MPTStateFactory.h index 77cc5358f4..c1f9c16cd7 100644 --- a/libmptstate/MPTStateFactory.h +++ b/libmptstate/MPTStateFactory.h @@ -45,7 +45,7 @@ class MPTStateFactory : public dev::executive::StateFactoryInterface }; virtual ~MPTStateFactory(){}; virtual std::shared_ptr getState( - h256 const& _root, std::shared_ptr _factory) override; + h256 const& _root, std::shared_ptr _factory) override; private: dev::OverlayDB m_db; diff --git a/libstorage/LevelDBStorage.cpp b/libstorage/LevelDBStorage.cpp index 9efef24f63..b724c34dcf 100644 --- a/libstorage/LevelDBStorage.cpp +++ b/libstorage/LevelDBStorage.cpp @@ -23,9 +23,6 @@ #include "Table.h" #include #include -#include -#include -#include #include #include #include @@ -36,8 +33,130 @@ using namespace dev; using namespace dev::storage; -Entries::Ptr LevelDBStorage::select( - h256, int, const std::string& table, const std::string& key, Condition::Ptr condition) +Entries::Ptr LevelDBStorage::select(h256, int, const std::string& table, const std::string& key, Condition::Ptr condition) +{ + (void)condition; + try + { + std::string entryKey = table; + entryKey.append("_").append(key); + + std::string value; + // ReadGuard l(m_remoteDBMutex); + auto s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(entryKey), &value); + // l.unlock(); + if (!s.ok() && !s.IsNotFound()) + { + STORAGE_LEVELDB_LOG(ERROR) + << LOG_DESC("Query leveldb failed") << LOG_KV("status", s.ToString()); + + BOOST_THROW_EXCEPTION(StorageException(-1, "Query leveldb exception:" + s.ToString())); + } + + Entries::Ptr entries = std::make_shared(); + if (!s.IsNotFound()) + { + bytesConstRef frame = bytesConstRef(value); + RLP entriesRlps = RLP(frame); + + for (size_t i = 0; i < entriesRlps.itemCount(); i++) + { + RLP entryRlp = entriesRlps[i]; + Entry::Ptr entry = std::make_shared(); + entry->setField("_hash_", entryRlp[0].toHash().hex()); + entry->setField("_num_", std::to_string(entryRlp[1].toPositiveInt64())); + + RLP fieldsRlp = entryRlp[2]; // fields + for (size_t j = 0; j < fieldsRlp.itemCount(); j++) + { + RLP fieldRlp = fieldsRlp[j]; + entry->setField(fieldRlp[0].toString(), fieldRlp[1].toString()); + } + + if (entry->getStatus() == Entry::Status::NORMAL) + { + entry->setDirty(false); + entries->addEntry(entry); + } + } + } + + return entries; + } + catch (std::exception& e) + { + STORAGE_LEVELDB_LOG(ERROR) << LOG_DESC("Query leveldb exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + + BOOST_THROW_EXCEPTION(e); + } + + return Entries::Ptr(); +} + +size_t LevelDBStorage::commitTableDataRange(std::shared_ptr& batch, + TableData::Ptr tableData, h256 hash, int64_t num, size_t from, size_t to) +{ + // commit table data of given range, thread safe + // range to commit: [from, to) + if (from >= to) + return 0; + size_t total = 0; + auto dataIt = tableData->data.begin(); + std::advance(dataIt, from); + + while (from < to && dataIt != tableData->data.end()) + { + if (dataIt->second->size() == 0u) + { + dataIt++; + from++; + continue; + } + std::string entryKey = tableData->tableName + "_" + dataIt->first; + Json::Value entry; + + size_t entriesSize = dataIt->second->size(); + RLPStream rlps; + rlps.appendList(entriesSize); + for (size_t i = 0; i < entriesSize; ++i) + { + rlps.appendList(3); + rlps.append(hash); + rlps.append(bigint(num)); + + size_t fiedsSize = dataIt->second->get(i)->fields()->size(); + rlps.appendList(fiedsSize); + for (auto& fieldIt : *(dataIt->second->get(i)->fields())) + { + rlps.appendList(2); + rlps.append(fieldIt.first); + rlps.append(fieldIt.second); + } + } + + bytes rlpBytes; + rlps.swapOut(rlpBytes); + + vector_ref refBytes = + vector_ref((const char*)rlpBytes.data(), rlpBytes.size()); + + batch->insertSlice( + leveldb::Slice(entryKey), leveldb::Slice(refBytes.data(), refBytes.size())); + ++total; + // ssOut.seekg(0, std::ios::end); + STORAGE_LEVELDB_LOG(TRACE) + << LOG_KV("commit key", entryKey) << LOG_KV("entries", entriesSize); + //<< LOG_KV("len", ssOut.tellg()); + dataIt++; + from++; + } + + return total; +} + +/* +Entries::Ptr LevelDBStorage::select(h256, int, const std::string& table, const std::string& key) { try { @@ -76,8 +195,7 @@ Entries::Ptr LevelDBStorage::select( entry->setField(valueIt.key().asString(), valueIt->asString()); } - if (entry->getStatus() == Entry::Status::NORMAL && - Table::processCondition(entry, condition)) + if (entry->getStatus() == Entry::Status::NORMAL) { entry->setDirty(false); entries->addEntry(entry); @@ -98,114 +216,123 @@ Entries::Ptr LevelDBStorage::select( return Entries::Ptr(); } -size_t LevelDBStorage::commit( - h256 hash, int64_t num, const std::vector& datas, h256 const&) +size_t LevelDBStorage::commitTableDataRange(std::shared_ptr& batch, + TableData::Ptr tableData, h256 hash, int64_t num, size_t from, size_t to) { - try + // commit table data of given range, thread safe + // range to commit: [from, to) + if (from >= to) + return 0; + size_t total = 0; + auto dataIt = tableData->data.begin(); + std::advance(dataIt, from); + + while (from < to && dataIt != tableData->data.end()) { - size_t counter = 0; - std::string counterValue; - auto s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(COUNTER_KEY), &counterValue); - if (s.ok() && !s.IsNotFound()) + if (dataIt->second->size() == 0u) { - counter = boost::lexical_cast(counterValue); + dataIt++; + from++; + continue; } + std::string entryKey = tableData->tableName + "_" + dataIt->first; + Json::Value entry; - auto hex = hash.hex(); - - std::shared_ptr batch = m_db->createWriteBatch(); - for (size_t i = 0; i < datas.size(); ++i) + for (size_t i = 0; i < dataIt->second->size(); ++i) { - std::map key2value; - - auto tableInfo = datas[i]->info; - auto entries = datas[i]->entries; - - for (size_t j = 0; j < entries->size(); ++j) + Json::Value value; + for (auto& fieldIt : *(dataIt->second->get(i)->fields())) { - auto entry = entries->get(j); - auto key = entry->getField(tableInfo->key); - - auto it = key2value.find(key); - if (it == key2value.end()) - { - std::string entryKey = tableInfo->name; - entryKey.append("_").append(key); + value[fieldIt.first] = fieldIt.second; + } + value["_hash_"] = hash.hex(); + value["_num_"] = num; + entry["values"].append(value); + } - std::string value; - auto s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(entryKey), &value); - // l.unlock(); - if (!s.ok() && !s.IsNotFound()) - { - STORAGE_LEVELDB_LOG(ERROR) - << LOG_DESC("Query leveldb failed") << LOG_KV("status", s.ToString()); + std::stringstream ssOut; + ssOut << entry; + + batch->insertSlice(leveldb::Slice(entryKey), leveldb::Slice(ssOut.str())); + ++total; + // ssOut.seekg(0, std::ios::end); + STORAGE_LEVELDB_LOG(TRACE) + << LOG_KV("commit key", entryKey) << LOG_KV("entries", dataIt->second->size()); + //<< LOG_KV("len", ssOut.tellg()); + dataIt++; + from++; + } - BOOST_THROW_EXCEPTION( - StorageException(-1, "Query leveldb exception:" + s.ToString())); - } + return total; +} +*/ - if (s.IsNotFound()) - { - it = key2value.insert(std::make_pair(key, Json::Value())).first; - } - else - { - std::stringstream ssIn; - ssIn << value; +static const size_t c_commitTableDataRangeEachThread = 128; // 128 is good after testing +size_t LevelDBStorage::commit( + h256 hash, int64_t num, const std::vector& datas, h256 const&) +{ + try + { + auto start_time = utcTime(); + auto record_time = utcTime(); + auto encode_time_cost = 0; + auto writeDB_time_cost = 0; - Json::Value valueJson; - ssIn >> valueJson; + std::atomic total; + total = 0; - it = key2value.emplace(key, valueJson).first; + for (size_t i = 0; i < datas.size(); ++i) + { + TableData::Ptr tableData = datas[i]; + size_t totalSize = tableData->data.size(); + + // Parallel encode and batch + size_t batchesSize = (totalSize + c_commitTableDataRangeEachThread - 1) / + c_commitTableDataRangeEachThread; + std::shared_ptr batch = nullptr; + batch = m_db->createWriteBatch(); + + std::vector> batches(batchesSize, nullptr); + tbb::parallel_for(tbb::blocked_range(0, batchesSize), + [&](const tbb::blocked_range& _r) { + for (size_t j = _r.begin(); j != _r.end(); ++j) + { + size_t from = c_commitTableDataRangeEachThread * j; + size_t to = std::min(c_commitTableDataRangeEachThread * (j + 1), totalSize); + size_t threadTotal = + commitTableDataRange(batch, tableData, hash, num, from, to); + total += threadTotal; } - } + }); + encode_time_cost += utcTime() - record_time; + record_time = utcTime(); - Json::Value value; - for (auto& fieldIt : *(entry->fields())) - { - value[fieldIt.first] = fieldIt.second; - } - value["_hash_"] = hex; - value["_num_"] = boost::lexical_cast(num); - // value["_id_"] = ++counter; - // it->second["values"].append(value); - - auto searchIt = - std::lower_bound(it->second["values"].begin(), it->second["values"].end(), - value, [](const Json::Value& lhs, const Json::Value& rhs) { - // LOG(ERROR) << "lhs: " << lhs.toStyledString() << "rhs: " << - // rhs.toStyledString(); - return boost::lexical_cast(lhs["_id_"].asString()) < - boost::lexical_cast(rhs["_id_"].asString()); - return false; - }); - - if (searchIt != it->second["values"].end() && (*searchIt)["_id_"] == value["_id_"]) - { - *searchIt = value; - } - else - { - value["_id_"] = ++counter; - it->second["values"].append(value); - } - } + // write batch + leveldb::WriteOptions writeOptions; + writeOptions.sync = false; + auto s = m_db->Write(writeOptions, &(batch->writeBatch())); - for (auto it : key2value) + if (!s.ok()) { - std::string entryKey = tableInfo->name + "_" + it.first; - std::stringstream ssOut; - ssOut << it.second; - - batch->insertSlice(leveldb::Slice(entryKey), leveldb::Slice(ssOut.str())); + STORAGE_LEVELDB_LOG(ERROR) << LOG_DESC( + "Commit leveldb crashed! Please remove all " + "data and sync data from other nodes") + << LOG_KV("status", s.ToString()); + + BOOST_THROW_EXCEPTION( + StorageException(-1, "Commit leveldb exception:" + s.ToString())); + return 0; } - } - counterValue = boost::lexical_cast(counterValue); - batch->insertSlice(leveldb::Slice(COUNTER_KEY), leveldb::Slice(counterValue)); + writeDB_time_cost += utcTime() - record_time; + record_time = utcTime(); + } + STORAGE_LEVELDB_LOG(DEBUG) << LOG_BADGE("Commit") << LOG_DESC("Write to db") + << LOG_KV("encodeTimeCost", encode_time_cost) + << LOG_KV("writeDBTimeCost", writeDB_time_cost) + << LOG_KV("totalTimeCost", utcTime() - start_time); - m_db->Write(leveldb::WriteOptions(), &batch->writeBatch()); - return datas.size(); + return total.load(); } catch (std::exception& e) { diff --git a/libstorage/LevelDBStorage.h b/libstorage/LevelDBStorage.h index e2389ca75b..defa4115b9 100644 --- a/libstorage/LevelDBStorage.h +++ b/libstorage/LevelDBStorage.h @@ -40,8 +40,8 @@ class LevelDBStorage : public Storage virtual ~LevelDBStorage(){}; - virtual Entries::Ptr select(h256 hash, int num, const std::string& table, - const std::string& key, Condition::Ptr condition) override; + virtual Entries::Ptr select( + h256 hash, int num, const std::string& table, const std::string& key, Condition::Ptr condition = nullptr) override; virtual size_t commit( h256 hash, int64_t num, const std::vector& datas, h256 const&) override; virtual bool onlyDirty() override; @@ -49,10 +49,10 @@ class LevelDBStorage : public Storage void setDB(std::shared_ptr db); private: + size_t commitTableDataRange(std::shared_ptr& batch, + TableData::Ptr tableData, h256 hash, int64_t num, size_t from, size_t to); std::shared_ptr m_db; dev::SharedMutex m_remoteDBMutex; - - const char* COUNTER_KEY = "_sys_counter_"; }; } // namespace storage diff --git a/libstorage/LevelDBStorage2.cpp b/libstorage/LevelDBStorage2.cpp new file mode 100644 index 0000000000..976688341a --- /dev/null +++ b/libstorage/LevelDBStorage2.cpp @@ -0,0 +1,228 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file SealerPrecompiled.h + * @author ancelmo + * @date 20180921 + */ + +#include "LevelDBStorage2.h" +#include "Table.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace dev::storage; + +Entries::Ptr LevelDBStorage2::select( + h256, int, const std::string& table, const std::string& key, Condition::Ptr condition) +{ + try + { + std::string entryKey = table; + entryKey.append("_").append(key); + + std::string value; + // ReadGuard l(m_remoteDBMutex); + auto s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(entryKey), &value); + // l.unlock(); + if (!s.ok() && !s.IsNotFound()) + { + STORAGE_LEVELDB_LOG(ERROR) + << LOG_DESC("Query leveldb failed") << LOG_KV("status", s.ToString()); + + BOOST_THROW_EXCEPTION(StorageException(-1, "Query leveldb exception:" + s.ToString())); + } + + Entries::Ptr entries = std::make_shared(); + if (!s.IsNotFound()) + { + // parse json + std::stringstream ssIn; + ssIn << value; + + Json::Value valueJson; + ssIn >> valueJson; + + Json::Value values = valueJson["values"]; + for (auto it = values.begin(); it != values.end(); ++it) + { + Entry::Ptr entry = std::make_shared(); + + for (auto valueIt = it->begin(); valueIt != it->end(); ++valueIt) + { + entry->setField(valueIt.key().asString(), valueIt->asString()); + } + + if (entry->getStatus() == Entry::Status::NORMAL && + Table::processCondition(entry, condition)) + { + entry->setDirty(false); + entries->addEntry(entry); + } + } + } + + return entries; + } + catch (std::exception& e) + { + STORAGE_LEVELDB_LOG(ERROR) << LOG_DESC("Query leveldb exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + + BOOST_THROW_EXCEPTION(e); + } + + return Entries::Ptr(); +} + +size_t LevelDBStorage2::commit( + h256 hash, int64_t num, const std::vector& datas, h256 const&) +{ + try + { + size_t counter = 0; + std::string counterValue; + auto s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(COUNTER_KEY), &counterValue); + if (s.ok() && !s.IsNotFound()) + { + counter = boost::lexical_cast(counterValue); + } + + auto hex = hash.hex(); + + std::shared_ptr batch = m_db->createWriteBatch(); + for (size_t i = 0; i < datas.size(); ++i) + { + std::map key2value; + + auto tableInfo = datas[i]->info; + auto entries = datas[i]->entries; + + for (size_t j = 0; j < entries->size(); ++j) + { + auto entry = entries->get(j); + auto key = entry->getField(tableInfo->key); + + auto it = key2value.find(key); + if (it == key2value.end()) + { + std::string entryKey = tableInfo->name; + entryKey.append("_").append(key); + + std::string value; + auto s = m_db->Get(leveldb::ReadOptions(), leveldb::Slice(entryKey), &value); + // l.unlock(); + if (!s.ok() && !s.IsNotFound()) + { + STORAGE_LEVELDB_LOG(ERROR) + << LOG_DESC("Query leveldb failed") << LOG_KV("status", s.ToString()); + + BOOST_THROW_EXCEPTION( + StorageException(-1, "Query leveldb exception:" + s.ToString())); + } + + if (s.IsNotFound()) + { + it = key2value.insert(std::make_pair(key, Json::Value())).first; + } + else + { + std::stringstream ssIn; + ssIn << value; + + Json::Value valueJson; + ssIn >> valueJson; + + it = key2value.emplace(key, valueJson).first; + } + } + + Json::Value value; + for (auto& fieldIt : *(entry->fields())) + { + value[fieldIt.first] = fieldIt.second; + } + value["_hash_"] = hex; + value["_num_"] = boost::lexical_cast(num); + // value["_id_"] = ++counter; + // it->second["values"].append(value); + + auto searchIt = + std::lower_bound(it->second["values"].begin(), it->second["values"].end(), + value, [](const Json::Value& lhs, const Json::Value& rhs) { + // LOG(ERROR) << "lhs: " << lhs.toStyledString() << "rhs: " << + // rhs.toStyledString(); + return boost::lexical_cast(lhs["_id_"].asString()) < + boost::lexical_cast(rhs["_id_"].asString()); + return false; + }); + + if (searchIt != it->second["values"].end() && (*searchIt)["_id_"] == value["_id_"]) + { + *searchIt = value; + } + else + { + value["_id_"] = ++counter; + it->second["values"].append(value); + } + } + + for (auto it : key2value) + { + std::string entryKey = tableInfo->name + "_" + it.first; + std::stringstream ssOut; + ssOut << it.second; + + batch->insertSlice(leveldb::Slice(entryKey), leveldb::Slice(ssOut.str())); + } + } + + counterValue = boost::lexical_cast(counterValue); + batch->insertSlice(leveldb::Slice(COUNTER_KEY), leveldb::Slice(counterValue)); + + m_db->Write(leveldb::WriteOptions(), &batch->writeBatch()); + return datas.size(); + } + catch (std::exception& e) + { + STORAGE_LEVELDB_LOG(ERROR) << LOG_DESC("Commit leveldb exception") + << LOG_KV("msg", boost::diagnostic_information(e)); + BOOST_THROW_EXCEPTION(e); + } + + return 0; +} + +bool LevelDBStorage2::onlyDirty() +{ + return false; +} + +void LevelDBStorage2::setDB(std::shared_ptr db) +{ + m_db = db; +} diff --git a/libstorage/LevelDBStorage2.h b/libstorage/LevelDBStorage2.h new file mode 100644 index 0000000000..6665d93d58 --- /dev/null +++ b/libstorage/LevelDBStorage2.h @@ -0,0 +1,60 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file SealerPrecompiled.h + * @author ancelmo + * @date 20180921 + */ +#pragma once + +#include "Storage.h" +#include "StorageException.h" +#include "Table.h" +#include +#include +#include +#include +#include + +namespace dev +{ +namespace storage +{ +class LevelDBStorage2 : public Storage +{ +public: + typedef std::shared_ptr Ptr; + + virtual ~LevelDBStorage2(){}; + + virtual Entries::Ptr select(h256 hash, int num, const std::string& table, + const std::string& key, Condition::Ptr condition) override; + virtual size_t commit( + h256 hash, int64_t num, const std::vector& datas, h256 const&) override; + virtual bool onlyDirty() override; + + void setDB(std::shared_ptr db); + +private: + std::shared_ptr m_db; + dev::SharedMutex m_remoteDBMutex; + + const char* COUNTER_KEY = "_sys_counter_"; +}; + +} // namespace storage + +} // namespace dev diff --git a/libstorage/MemoryTable.h b/libstorage/MemoryTable.h index 7a76f52492..79bdb51609 100644 --- a/libstorage/MemoryTable.h +++ b/libstorage/MemoryTable.h @@ -37,33 +37,269 @@ namespace dev { namespace storage { +template class MemoryTable : public Table { public: - using CacheType = std::map; + using CacheType = typename std::conditional, + std::map>::type; using CacheItr = typename CacheType::iterator; - using Ptr = std::shared_ptr; - - MemoryTable(); + using Ptr = std::shared_ptr>; virtual ~MemoryTable(){}; - virtual Entries::Ptr select(const std::string& key, Condition::Ptr condition) override; + virtual typename Entries::Ptr select(const std::string& key, Condition::Ptr condition) override + { + try + { + typename Entries::Ptr entries = std::make_shared(); + + CacheItr it; + it = m_cache.find(key); + // beacuse there is no interface to erase element in tbb::concurrent_unordered_map, + // so we set a value to nullptr as a flag to tell table to update it later when a + // key-value is invalid + if (it == m_cache.end() || it->second == nullptr) + { + if (m_remoteDB) + { + entries = m_remoteDB->select(m_blockHash, m_blockNum, m_tableInfo->name, key); + m_cache[key] = entries; + } + } + else + { + entries = it->second; + } - Entries::Ptr selectNoLock(const std::string& key, Condition::Ptr condition); + if (!entries) + { + return std::make_shared(); + } + auto indexes = processEntries(entries, condition); + typename Entries::Ptr resultEntries = std::make_shared(); + for (auto& i : indexes) + { + resultEntries->addEntry(entries->get(i)); + } + return resultEntries; + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable") << LOG_DESC("Table select failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return std::make_shared(); + } virtual int update(const std::string& key, Entry::Ptr entry, Condition::Ptr condition, - AccessOptions::Ptr options = std::make_shared()) override; + AccessOptions::Ptr options = std::make_shared()) override + { + try + { + if (options->check && !checkAuthority(options->origin)) + { + return storage::CODE_NO_AUTHORIZED; + } + + typename Entries::Ptr entries = std::make_shared(); + + CacheItr it; + it = m_cache.find(key); + // beacuse there is no interface to erase element in tbb::concurrent_unordered_map, + // so we set a value to nullptr as a flag to tell table to update it later when a + // key-value is invalid + if (it == m_cache.end() || it->second == nullptr) + { + if (m_remoteDB) + { + entries = m_remoteDB->select(m_blockHash, m_blockNum, m_tableInfo->name, key); + m_cache[key] = entries; + } + } + else + { + entries = it->second; + } + + if (!entries) + { + return 0; + } + + checkField(entry); + auto indexes = processEntries(entries, condition); + std::vector records; + + for (auto& i : indexes) + { + Entry::Ptr updateEntry = entries->get(i); + for (auto& it : *(entry->fields())) + { + records.emplace_back(i, it.first, updateEntry->getField(it.first)); + updateEntry->setField(it.first, it.second); + } + } + this->m_recorder(this->shared_from_this(), Change::Update, key, records); + + entries->setDirty(true); + + return indexes.size(); + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable") + << LOG_DESC("Access MemoryTable failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return 0; + } virtual int insert(const std::string& key, Entry::Ptr entry, AccessOptions::Ptr options = std::make_shared(), - bool needSelect = true) override; + bool needSelect = true) override + { + try + { + if (options->check && !checkAuthority(options->origin)) + { + return storage::CODE_NO_AUTHORIZED; + } + + typename Entries::Ptr entries = std::make_shared(); + Condition::Ptr condition = std::make_shared(); + + CacheItr it; + it = m_cache.find(key); + // beacuse there is no interface to erase element in tbb::concurrent_unordered_map, + // so we set a value to nullptr as a flag to tell table to update it later when a + // key-value is invalid + if (it == m_cache.end() || it->second == nullptr) + { + if (m_remoteDB) + { + if (needSelect) + { + entries = + m_remoteDB->select(m_blockHash, m_blockNum, m_tableInfo->name, key); + m_cache[key] = entries; + } + } + } + else + { + entries = it->second; + } + checkField(entry); + Change::Record record(entries->size()); + std::vector value{record}; + this->m_recorder(this->shared_from_this(), Change::Insert, key, value); + if (entries->size() == 0) + { + entries->addEntry(entry); + m_cache[key] = entries; + return 1; + } + else + { + entries->addEntry(entry); + return 1; + } + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable") + << LOG_DESC("Access MemoryTable failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return 1; + } virtual int remove(const std::string& key, Condition::Ptr condition, - AccessOptions::Ptr options = std::make_shared()) override; + AccessOptions::Ptr options = std::make_shared()) override + { + if (options->check && !checkAuthority(options->origin)) + { + return storage::CODE_NO_AUTHORIZED; + } + + typename Entries::Ptr entries = std::make_shared(); + + CacheItr it; + it = m_cache.find(key); + // beacuse there is no interface to erase element in tbb::concurrent_unordered_map, + // so we set a value to nullptr as a flag to tell table to update it later when a + // key-value is invalid + if (it == m_cache.end() || it->second == nullptr) + { + if (m_remoteDB) + { + entries = m_remoteDB->select(m_blockHash, m_blockNum, m_tableInfo->name, key); + m_cache[key] = entries; + } + } + else + { + entries = it->second; + } + + auto indexes = processEntries(entries, condition); + + std::vector records; + for (auto& i : indexes) + { + Entry::Ptr removeEntry = entries->get(i); + + removeEntry->setStatus(1); + records.emplace_back(i); + } + this->m_recorder(this->shared_from_this(), Change::Remove, key, records); + + entries->setDirty(true); + + return indexes.size(); + } + + virtual h256 hash() override + { + std::map tmpOrderedCache(m_cache.begin(), m_cache.end()); + bytes data; + for (auto& it : tmpOrderedCache) + { + if (it.second->dirty()) + { + data.insert(data.end(), it.first.begin(), it.first.end()); + for (size_t i = 0; i < it.second->size(); ++i) + { + if (it.second->get(i)->dirty()) + { + for (auto& fieldIt : *(it.second->get(i)->fields())) + { + if (isHashField(fieldIt.first)) + { + data.insert(data.end(), fieldIt.first.begin(), fieldIt.first.end()); + data.insert( + data.end(), fieldIt.second.begin(), fieldIt.second.end()); + } + } + } + } + } + } - virtual h256 hash() override; + if (data.empty()) + { + return h256(); + } + bytesConstRef bR(data.data(), data.size()); + h256 hash = dev::sha256(bR); + return hash; + } virtual void clear() override { m_cache.clear(); } virtual bool empty() override { @@ -91,26 +327,72 @@ class MemoryTable : public Table return it != m_tableInfo->authorizedAddress.cend(); } - virtual TableData::Ptr dump() override + virtual void setRecorder(std::function&)> + _recorder) override { - auto data = std::make_shared(); + m_recorder = _recorder; + } - data->info = m_tableInfo; - data->entries = std::make_shared(); + virtual bool dump(TableData::Ptr _data) override + { + bool dirtyTable = false; for (auto it : m_cache) { - data->entries->addEntry(it.second); + _data->data.insert(make_pair(it.first, it.second)); + + if (it.second->dirty()) + { + dirtyTable = true; + } } + return dirtyTable; + } - for (size_t i = 0; i < m_newEntries->size(); ++i) + virtual void rollback(const Change& _change) override + { { - data->entries->addEntry(m_newEntries->get(i)); - } + switch (_change.kind) + { + case Change::Insert: + { + auto entries = m_cache[_change.key]; + entries->removeEntry(_change.value[0].index); + if (entries->size() == 0u) + { + m_cache[_change.key] = nullptr; + } + break; + } + case Change::Update: + { + auto entries = m_cache[_change.key]; + for (auto& record : _change.value) + { + auto entry = entries->get(record.index); + entry->setField(record.key, record.oldValue); + } + break; + } + case Change::Remove: + { + auto entries = m_cache[_change.key]; + for (auto& record : _change.value) + { + auto entry = entries->get(record.index); + entry->setStatus(0); + } + break; + } + case Change::Select: - return data; + default: + break; + } + } } - virtual void rollback(const Change& _change) override; + size_t cacheSize() override { return m_cache.size(); } private: std::vector processEntries(Entries::Ptr entries, Condition::Ptr condition) @@ -245,26 +527,24 @@ class MemoryTable : public Table { for (auto& it : *(entry->fields())) { + if(it.first == "_id_") { + continue; + } if (m_tableInfo->fields.end() == find(m_tableInfo->fields.begin(), m_tableInfo->fields.end(), it.first)) { - STORAGE_LOG(ERROR) - << LOG_BADGE("MemoryTable") << LOG_DESC("field doen not exist") - << LOG_KV("table name", m_tableInfo->name) << LOG_KV("field", it.first); - throw std::invalid_argument("Invalid key."); } } } - dev::SharedMutex m_mutex; Storage::Ptr m_remoteDB; TableInfo::Ptr m_tableInfo; - std::map m_cache; - Entries::Ptr m_newEntries; + CacheType m_cache; h256 m_blockHash; int m_blockNum = 0; -}; // namespace storage - + std::function&)> + m_recorder; +}; } // namespace storage } // namespace dev diff --git a/libstorage/MemoryTable2.cpp b/libstorage/MemoryTable2.cpp new file mode 100644 index 0000000000..f22e4cac66 --- /dev/null +++ b/libstorage/MemoryTable2.cpp @@ -0,0 +1,367 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file MemoryTable2.cpp + * @author ancelmo + * @date 20180921 + */ +#include "MemoryTable2.h" +#include "Common.h" +#include "Table.h" +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace dev::storage; +using namespace dev::precompiled; + +MemoryTable2::MemoryTable2() +{ + m_newEntries = std::make_shared(); +} + +Entries::Ptr MemoryTable2::select(const std::string& key, Condition::Ptr condition) +{ + dev::ReadGuard lock(m_mutex); + + return selectNoLock(key, condition); +} + +Entries::Ptr MemoryTable2::selectNoLock(const std::string& key, Condition::Ptr condition) +{ + try + { + auto entries = std::make_shared(); + condition->EQ(m_tableInfo->key, key); + if (m_remoteDB) + { + // query remoteDB anyway + Entries::Ptr dbEntries = + m_remoteDB->select(m_blockHash, m_blockNum, m_tableInfo->name, key, condition); + + if (!dbEntries) + { + return std::make_shared(); + } + + + for (size_t i = 0; i < dbEntries->size(); ++i) + { + auto entryIt = m_cache.find(dbEntries->get(i)->getID()); + if (entryIt != m_cache.end()) + { + entries->addEntry(entryIt->second); + } + else + { + entries->addEntry(dbEntries->get(i)); + } + } + } + + auto indices = processEntries(m_newEntries, condition); + for (auto it : indices) + { + m_newEntries->get(it)->setTempIndex(it); + entries->addEntry(m_newEntries->get(it)); + } + + return entries; + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable2") << LOG_DESC("Table select failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return std::make_shared(); +} + +int MemoryTable2::update( + const std::string& key, Entry::Ptr entry, Condition::Ptr condition, AccessOptions::Ptr options) +{ + try + { + if (options->check && !checkAuthority(options->origin)) + { + STORAGE_LOG(WARNING) << LOG_BADGE("MemoryTable2") << LOG_DESC("update non-authorized") + << LOG_KV("origin", options->origin.hex()) << LOG_KV("key", key); + + return storage::CODE_NO_AUTHORIZED; + } + + dev::WriteGuard lock(m_mutex); + checkField(entry); + + auto entries = selectNoLock(key, condition); + + std::vector records; + + for (size_t i = 0; i < entries->size(); ++i) + { + auto updateEntry = entries->get(i); + + // if id equals to zero and not in the m_cache, must be new dirty entry + if (updateEntry->getID() != 0 && m_cache.find(updateEntry->getID()) == m_cache.end()) + { + m_cache.insert(std::make_pair(updateEntry->getID(), updateEntry)); + } + + for (auto& it : *(entry->fields())) + { + records.emplace_back(updateEntry->getTempIndex(), it.first, + updateEntry->getField(it.first), updateEntry->getID()); + updateEntry->setField(it.first, it.second); + } + } + + m_recorder(shared_from_this(), Change::Update, key, records); + + return entries->size(); + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable2") << LOG_DESC("Access MemoryTable2 failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return 0; +} + +int MemoryTable2::insert( + const std::string& key, Entry::Ptr entry, AccessOptions::Ptr options, bool needSelect) +{ + try + { + (void)needSelect; + + if (options->check && !checkAuthority(options->origin)) + { + STORAGE_LOG(WARNING) << LOG_BADGE("MemoryTable2") << LOG_DESC("insert non-authorized") + << LOG_KV("origin", options->origin.hex()) << LOG_KV("key", key); + return storage::CODE_NO_AUTHORIZED; + } + + dev::WriteGuard lock(m_mutex); + checkField(entry); + + entry->setField(m_tableInfo->key, key); + Change::Record record(m_newEntries->size()); + m_newEntries->addEntry(entry); + + std::vector value{record}; + m_recorder(shared_from_this(), Change::Insert, key, value); + + return 1; + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable2") << LOG_DESC("Access MemoryTable2 failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return 0; +} + +int MemoryTable2::remove( + const std::string& key, Condition::Ptr condition, AccessOptions::Ptr options) +{ + try + { + if (options->check && !checkAuthority(options->origin)) + { + STORAGE_LOG(WARNING) << LOG_BADGE("MemoryTable2") << LOG_DESC("remove non-authorized") + << LOG_KV("origin", options->origin.hex()) << LOG_KV("key", key); + return storage::CODE_NO_AUTHORIZED; + } + + dev::WriteGuard lock(m_mutex); + auto entries = selectNoLock(key, condition); + + std::vector records; + for (size_t i = 0; i < entries->size(); ++i) + { + auto removeEntry = entries->get(i); + + removeEntry->setStatus(1); + + // if id equals to zero and not in the m_cache, must be new dirty entry + if (removeEntry->getID() != 0 && m_cache.find(removeEntry->getID()) == m_cache.end()) + { + m_cache.insert(std::make_pair(removeEntry->getID(), removeEntry)); + } + + records.emplace_back(removeEntry->getTempIndex(), "", "", removeEntry->getID()); + } + + m_recorder(shared_from_this(), Change::Remove, key, records); + + return entries->size(); + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable2") << LOG_DESC("Access MemoryTable2 failed for") + << LOG_KV("msg", boost::diagnostic_information(e)); + } + + return 0; +} + +dev::h256 MemoryTable2::hash() +{ + dev::ReadGuard lock(m_mutex); + bytes data; + + for (auto it : m_cache) + { + auto id = htonl(it.first); + data.insert(data.end(), (char*)&id, (char*)&id + sizeof(id)); + for (auto fieldIt : *(it.second->fields())) + { + if (isHashField(fieldIt.first)) + { + data.insert(data.end(), fieldIt.first.begin(), fieldIt.first.end()); + data.insert(data.end(), fieldIt.second.begin(), fieldIt.second.end()); + } + } + } + + for (size_t i = 0; i < m_newEntries->size(); ++i) + { + auto entry = m_newEntries->get(i); + for (auto fieldIt : *(entry->fields())) + { + if (isHashField(fieldIt.first)) + { + data.insert(data.end(), fieldIt.first.begin(), fieldIt.first.end()); + data.insert(data.end(), fieldIt.second.begin(), fieldIt.second.end()); + } + } + } + + if (data.empty()) + { + return h256(); + } + + bytesConstRef bR(data.data(), data.size()); + h256 hash = dev::sha256(bR); + + return hash; +} + +void MemoryTable2::rollback(const Change& _change) +{ + dev::WriteGuard lock(m_mutex); + LOG(TRACE) << "Before rollback newEntries size: " << m_newEntries->size(); + for (size_t i = 0; i < m_newEntries->size(); ++i) + { + auto entry = m_newEntries->get(i); + + std::stringstream ss; + ss << i; + ss << "," << entry->getStatus(); + + for (auto it : *(entry->fields())) + { + ss << "," << it.first << ":" << it.second; + } + LOG(TRACE) << ss.str(); + } + + switch (_change.kind) + { + case Change::Insert: + { + LOG(TRACE) << "Rollback insert record newIndex: " << _change.value[0].index; + + auto entry = m_newEntries->get(_change.value[0].index); + entry->setStatus(1); + // m_newEntries->removeEntry(_change.value[0].newIndex); + break; + } + case Change::Update: + { + for (auto& record : _change.value) + { + LOG(TRACE) << "Rollback update record id: " << record.id + << " newIndex: " << record.index; + if (record.id) + { + auto it = m_cache.find(record.id); + if (it != m_cache.end()) + { + it->second->setField(record.key, record.oldValue); + } + } + else + { + auto entry = m_newEntries->get(record.index); + entry->setField(record.key, record.oldValue); + } + } + break; + } + case Change::Remove: + { + for (auto& record : _change.value) + { + LOG(TRACE) << "Rollback remove record id: " << record.id + << " newIndex: " << record.index; + if (record.id) + { + auto it = m_cache.find(record.id); + if (it != m_cache.end()) + { + it->second->setStatus(0); + } + } + else + { + auto entry = m_newEntries->get(record.index); + entry->setStatus(0); + } + } + break; + } + case Change::Select: + + default: + break; + } + + LOG(TRACE) << "After rollback newEntries size: " << m_newEntries->size(); + for (size_t i = 0; i < m_newEntries->size(); ++i) + { + auto entry = m_newEntries->get(i); + + std::stringstream ss; + ss << i; + ss << "," << entry->getStatus(); + + for (auto it : *(entry->fields())) + { + ss << "," << it.first << ":" << it.second; + } + LOG(TRACE) << ss.str(); + } +} diff --git a/libstorage/MemoryTable2.h b/libstorage/MemoryTable2.h new file mode 100644 index 0000000000..ad9144e00f --- /dev/null +++ b/libstorage/MemoryTable2.h @@ -0,0 +1,267 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file MemoryTable.h + * @author ancelmo + * @date 20180921 + */ +#pragma once + +#include "Storage.h" +#include "Table.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dev +{ +namespace storage +{ +class MemoryTable2 : public Table +{ +public: + using CacheType = std::map; + using CacheItr = typename CacheType::iterator; + using Ptr = std::shared_ptr; + + MemoryTable2(); + + virtual ~MemoryTable2(){}; + + virtual Entries::Ptr select(const std::string& key, Condition::Ptr condition) override; + + Entries::Ptr selectNoLock(const std::string& key, Condition::Ptr condition); + + virtual int update(const std::string& key, Entry::Ptr entry, Condition::Ptr condition, + AccessOptions::Ptr options = std::make_shared()) override; + + virtual int insert(const std::string& key, Entry::Ptr entry, + AccessOptions::Ptr options = std::make_shared(), + bool needSelect = true) override; + + virtual int remove(const std::string& key, Condition::Ptr condition, + AccessOptions::Ptr options = std::make_shared()) override; + + virtual h256 hash() override; + + virtual void clear() override { m_cache.clear(); } + virtual bool empty() override + { + for (auto iter : m_cache) + { + if (iter.second != nullptr) + { + return false; + } + } + return true; + } + + void setStateStorage(Storage::Ptr amopDB) override { m_remoteDB = amopDB; } + void setBlockHash(h256 blockHash) override { m_blockHash = blockHash; } + void setBlockNum(int blockNum) override { m_blockNum = blockNum; } + void setTableInfo(TableInfo::Ptr tableInfo) override { m_tableInfo = tableInfo; } + + bool checkAuthority(Address const& _origin) const override + { + if (m_tableInfo->authorizedAddress.empty()) + return true; + auto it = find(m_tableInfo->authorizedAddress.cbegin(), + m_tableInfo->authorizedAddress.cend(), _origin); + return it != m_tableInfo->authorizedAddress.cend(); + } + + virtual bool dump(dev::storage::TableData::Ptr data) override + { + data->info = m_tableInfo; + data->entries = std::make_shared(); + for (auto it : m_cache) + { + data->entries->addEntry(it.second); + } + + for (size_t i = 0; i < m_newEntries->size(); ++i) + { + data->entries->addEntry(m_newEntries->get(i)); + } + + return true; + } + + virtual void rollback(const Change& _change) override; + +private: + std::vector processEntries(Entries::Ptr entries, Condition::Ptr condition) + { + std::vector indexes; + indexes.reserve(entries->size()); + if (condition->getConditions()->empty()) + { + for (size_t i = 0; i < entries->size(); ++i) + indexes.emplace_back(i); + return indexes; + } + + for (size_t i = 0; i < entries->size(); ++i) + { + Entry::Ptr entry = entries->get(i); + if (processCondition(entry, condition)) + { + indexes.push_back(i); + } + } + + return indexes; + } + + bool processCondition(Entry::Ptr entry, Condition::Ptr condition) + { + try + { + for (auto& it : *condition->getConditions()) + { + if (entry->getStatus() == Entry::Status::DELETED) + { + return false; + } + + std::string lhs = entry->getField(it.first); + std::string rhs = it.second.second; + + if (it.second.first == Condition::Op::eq) + { + if (lhs != rhs) + { + return false; + } + } + else if (it.second.first == Condition::Op::ne) + { + if (lhs == rhs) + { + return false; + } + } + else + { + if (lhs.empty()) + { + lhs = "0"; + } + if (rhs.empty()) + { + rhs = "0"; + } + + int lhsNum = boost::lexical_cast(lhs); + int rhsNum = boost::lexical_cast(rhs); + + switch (it.second.first) + { + case Condition::Op::eq: + case Condition::Op::ne: + { + break; + } + case Condition::Op::gt: + { + if (lhsNum <= rhsNum) + { + return false; + } + break; + } + case Condition::Op::ge: + { + if (lhsNum < rhsNum) + { + return false; + } + break; + } + case Condition::Op::lt: + { + if (lhsNum >= rhsNum) + { + return false; + } + break; + } + case Condition::Op::le: + { + if (lhsNum > rhsNum) + { + return false; + } + break; + } + } + } + } + } + catch (std::exception& e) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTable") << LOG_DESC("Compare error") + << LOG_KV("msg", boost::diagnostic_information(e)); + return false; + } + + return true; + } + + bool isHashField(const std::string& _key) + { + if (!_key.empty()) + { + return ((_key.substr(0, 1) != "_" && _key.substr(_key.size() - 1, 1) != "_") || + (_key == STATUS)); + } + return false; + } + + void checkField(Entry::Ptr entry) + { + for (auto& it : *(entry->fields())) + { + if (m_tableInfo->fields.end() == + find(m_tableInfo->fields.begin(), m_tableInfo->fields.end(), it.first)) + { + STORAGE_LOG(ERROR) + << LOG_BADGE("MemoryTable") << LOG_DESC("field doen not exist") + << LOG_KV("table name", m_tableInfo->name) << LOG_KV("field", it.first); + + throw std::invalid_argument("Invalid key."); + } + } + } + + dev::SharedMutex m_mutex; + Storage::Ptr m_remoteDB; + TableInfo::Ptr m_tableInfo; + std::map m_cache; + Entries::Ptr m_newEntries; + h256 m_blockHash; + int m_blockNum = 0; +}; // namespace storage + +} // namespace storage +} // namespace dev diff --git a/libstorage/MemoryTableFactory.cpp b/libstorage/MemoryTableFactory.cpp index 4df7d0153b..40a840ad6d 100644 --- a/libstorage/MemoryTableFactory.cpp +++ b/libstorage/MemoryTableFactory.cpp @@ -21,10 +21,9 @@ #include "MemoryTableFactory.h" #include "Common.h" #include "MemoryTable.h" +#include "StorageException.h" #include "TablePrecompiled.h" #include -#include -#include #include #include #include @@ -84,16 +83,15 @@ Table::Ptr MemoryTableFactory::openTable( tableInfo->fields.emplace_back(tableInfo->key); tableInfo->fields.emplace_back("_hash_"); tableInfo->fields.emplace_back("_num_"); - tableInfo->fields.emplace_back("_id_"); Table::Ptr memoryTable = nullptr; if (isPara) { - memoryTable = std::make_shared(); + memoryTable = std::make_shared>(); } else { - memoryTable = std::make_shared(); + memoryTable = std::make_shared>(); } memoryTable->setStateStorage(m_stateStorage); @@ -146,7 +144,6 @@ Table::Ptr MemoryTableFactory::createTable(const std::string& tableName, STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTableFactory") << LOG_DESC("table already exist in _sys_tables_") << LOG_KV("table name", tableName); - createTableCode = 0; return nullptr; } // Write table entry @@ -154,13 +151,15 @@ Table::Ptr MemoryTableFactory::createTable(const std::string& tableName, tableEntry->setField("table_name", tableName); tableEntry->setField("key_field", keyField); tableEntry->setField("value_field", valueField); - createTableCode = sysTable->insert( + auto result = sysTable->insert( tableName, tableEntry, std::make_shared(_origin, authorityFlag)); - if (createTableCode == storage::CODE_NO_AUTHORIZED) + if (result == storage::CODE_NO_AUTHORIZED) { STORAGE_LOG(WARNING) << LOG_BADGE("MemoryTableFactory") << LOG_DESC("create table non-authorized") << LOG_KV("origin", _origin.hex()) << LOG_KV("table name", tableName); + + BOOST_THROW_EXCEPTION(StorageException(result, "create table non-authorized")); return nullptr; } return openTable(tableName, authorityFlag, isPara); @@ -228,7 +227,7 @@ void MemoryTableFactory::rollback(size_t _savepoint) auto& changeLog = getChangeLog(); while (_savepoint < changeLog.size()) { - auto change = changeLog.back(); + auto& change = changeLog.back(); // Public MemoryTable API cannot be used here because it will add another // change log entry. @@ -238,9 +237,7 @@ void MemoryTableFactory::rollback(size_t _savepoint) } } -void MemoryTableFactory::commit() {} - -void MemoryTableFactory::commitDB(dev::h256 const& _blockHash, int64_t _blockNumber) +void MemoryTableFactory::commitDB(h256 const& _blockHash, int64_t _blockNumber) { auto start_time = utcTime(); auto record_time = utcTime(); @@ -250,9 +247,12 @@ void MemoryTableFactory::commitDB(dev::h256 const& _blockHash, int64_t _blockNum { auto table = std::dynamic_pointer_cast(dbIt.second); - auto tableData = table->dump(); + dev::storage::TableData::Ptr tableData = make_shared(); + tableData->tableName = dbIt.first; + + bool dirtyTable = table->dump(tableData); - if (tableData->entries->size() > 0) + if (!tableData->data.empty() && dirtyTable) { datas.push_back(tableData); } diff --git a/libstorage/MemoryTableFactory.h b/libstorage/MemoryTableFactory.h index 3ffed0e45b..e3629c1973 100644 --- a/libstorage/MemoryTableFactory.h +++ b/libstorage/MemoryTableFactory.h @@ -39,17 +39,17 @@ class ExecutiveContext; } namespace storage { -class MemoryTableFactory : public StateDBFactory +class MemoryTableFactory : public TableFactory { public: typedef std::shared_ptr Ptr; MemoryTableFactory(); virtual ~MemoryTableFactory() {} virtual Table::Ptr openTable( - const std::string& tableName, bool authorityFlag = true, bool isPara = true); + const std::string& tableName, bool authorityFlag = true, bool isPara = true) override; virtual Table::Ptr createTable(const std::string& tableName, const std::string& keyField, const std::string& valueField, bool authorityFlag = true, - Address const& _origin = Address(), bool isPara = true); + Address const& _origin = Address(), bool isPara = true) override; virtual Storage::Ptr stateStorage() { return m_stateStorage; } virtual void setStateStorage(Storage::Ptr stateStorage) { m_stateStorage = stateStorage; } @@ -57,11 +57,10 @@ class MemoryTableFactory : public StateDBFactory void setBlockHash(h256 blockHash); void setBlockNum(int64_t blockNum); - h256 hash(); - size_t savepoint(); - void rollback(size_t _savepoint); - void commit(); - void commitDB(h256 const& _blockHash, int64_t _blockNumber); + virtual h256 hash() override; + virtual size_t savepoint() override; + virtual void rollback(size_t _savepoint) override; + virtual void commitDB(h256 const& _blockHash, int64_t _blockNumber) override; int getCreateTableCode() { return createTableCode; } private: diff --git a/libstorage/MemoryTableFactory2.cpp b/libstorage/MemoryTableFactory2.cpp new file mode 100644 index 0000000000..81cbda9eca --- /dev/null +++ b/libstorage/MemoryTableFactory2.cpp @@ -0,0 +1,352 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file MemoryTableFactory2.h + * @author ancelmo + * @date 20180921 + */ +#include "MemoryTableFactory2.h" +#include "Common.h" +#include "MemoryTable2.h" +#include "TablePrecompiled.h" +#include "StorageException.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace dev::storage; +using namespace std; + +MemoryTableFactory2::MemoryTableFactory2() : m_blockHash(h256(0)), m_blockNum(0) +{ + m_sysTables.push_back(SYS_CONSENSUS); + m_sysTables.push_back(SYS_TABLES); + m_sysTables.push_back(SYS_ACCESS_TABLE); + m_sysTables.push_back(SYS_CURRENT_STATE); + m_sysTables.push_back(SYS_NUMBER_2_HASH); + m_sysTables.push_back(SYS_TX_HASH_2_BLOCK); + m_sysTables.push_back(SYS_HASH_2_BLOCK); + m_sysTables.push_back(SYS_CNS); + m_sysTables.push_back(SYS_CONFIG); + m_sysTables.push_back(SYS_BLOCK_2_NONCES); +} + + +Table::Ptr MemoryTableFactory2::openTable( + const std::string& tableName, bool authorityFlag, bool isPara) +{ + RecursiveGuard l(x_name2Table); + auto it = m_name2Table.find(tableName); + if (it != m_name2Table.end()) + { + return it->second; + } + auto tableInfo = std::make_shared(); + + if (m_sysTables.end() != find(m_sysTables.begin(), m_sysTables.end(), tableName)) + { + tableInfo = getSysTableInfo(tableName); + } + else + { + auto tempSysTable = openTable(SYS_TABLES); + auto tableEntries = tempSysTable->select(tableName, tempSysTable->newCondition()); + if (tableEntries->size() == 0u) + { + return nullptr; + } + auto entry = tableEntries->get(0); + tableInfo->name = tableName; + tableInfo->key = entry->getField("key_field"); + std::string valueFields = entry->getField("value_field"); + boost::split(tableInfo->fields, valueFields, boost::is_any_of(",")); + } + tableInfo->fields.emplace_back(STATUS); + tableInfo->fields.emplace_back(tableInfo->key); + tableInfo->fields.emplace_back("_hash_"); + tableInfo->fields.emplace_back("_num_"); + tableInfo->fields.emplace_back("_id_"); + + Table::Ptr memoryTable = nullptr; + if (isPara) + { + memoryTable = std::make_shared(); + } + else + { + memoryTable = std::make_shared(); + } + + memoryTable->setStateStorage(m_stateStorage); + memoryTable->setBlockHash(m_blockHash); + memoryTable->setBlockNum(m_blockNum); + memoryTable->setTableInfo(tableInfo); + + // authority flag + if (authorityFlag) + { + // set authorized address to memoryTable + if (tableName != std::string(SYS_ACCESS_TABLE)) + { + setAuthorizedAddress(tableInfo); + } + else + { + auto tableEntries = memoryTable->select(SYS_ACCESS_TABLE, memoryTable->newCondition()); + for (size_t i = 0; i < tableEntries->size(); ++i) + { + auto entry = tableEntries->get(i); + if (std::stoi(entry->getField("enable_num")) <= m_blockNum) + { + tableInfo->authorizedAddress.emplace_back(entry->getField("address")); + } + } + } + } + + memoryTable->setTableInfo(tableInfo); + memoryTable->setRecorder([&](Table::Ptr _table, Change::Kind _kind, std::string const& _key, + std::vector& _records) { + auto& changeLog = getChangeLog(); + changeLog.emplace_back(_table, _kind, _key, _records); + }); + + m_name2Table.insert({tableName, memoryTable}); + return memoryTable; +} + +Table::Ptr MemoryTableFactory2::createTable(const std::string& tableName, + const std::string& keyField, const std::string& valueField, bool authorityFlag, + Address const& _origin, bool isPara) +{ + auto sysTable = openTable(SYS_TABLES, authorityFlag); + // To make sure the table exists + auto tableEntries = sysTable->select(tableName, sysTable->newCondition()); + if (tableEntries->size() != 0) + { + STORAGE_LOG(ERROR) << LOG_BADGE("MemoryTableFactory2") + << LOG_DESC("table already exist in _sys_tables_") + << LOG_KV("table name", tableName); + return nullptr; + } + // Write table entry + auto tableEntry = sysTable->newEntry(); + tableEntry->setField("table_name", tableName); + tableEntry->setField("key_field", keyField); + tableEntry->setField("value_field", valueField); + auto result = sysTable->insert( + tableName, tableEntry, std::make_shared(_origin, authorityFlag)); + if (result == storage::CODE_NO_AUTHORIZED) + { + STORAGE_LOG(WARNING) << LOG_BADGE("MemoryTableFactory2") + << LOG_DESC("create table non-authorized") + << LOG_KV("origin", _origin.hex()) << LOG_KV("table name", tableName); + BOOST_THROW_EXCEPTION(StorageException(result, "create table non-authorized")); + return nullptr; + } + return openTable(tableName, authorityFlag, isPara); +} + +size_t MemoryTableFactory2::savepoint() +{ + auto& changeLog = getChangeLog(); + return changeLog.size(); +} + +void MemoryTableFactory2::setBlockHash(h256 blockHash) +{ + m_blockHash = blockHash; +} + +void MemoryTableFactory2::setBlockNum(int64_t blockNum) +{ + m_blockNum = blockNum; +} + +h256 MemoryTableFactory2::hash() +{ + bytes data; + for (auto& it : m_name2Table) + { + auto table = it.second; + h256 hash = table->hash(); + if (hash == h256()) + { + continue; + } + + bytes tableHash = hash.asBytes(); + // LOG(DEBUG) << LOG_BADGE("Report") << LOG_DESC("tableHash") + //<< LOG_KV(it.first, dev::sha256(ref(tableHash))); + + data.insert(data.end(), tableHash.begin(), tableHash.end()); + } + if (data.empty()) + { + return h256(); + } + m_hash = dev::sha256(&data); + return m_hash; +} + +std::vector& MemoryTableFactory2::getChangeLog() +{ + auto changeLog = m_changeLog.get(); + if (m_changeLog.get() != nullptr) + { + return *changeLog; + } + else + { + changeLog = new std::vector(); + m_changeLog.reset(changeLog); + return *changeLog; + } +} + +void MemoryTableFactory2::rollback(size_t _savepoint) +{ + auto& changeLog = getChangeLog(); + while (_savepoint < changeLog.size()) + { + auto change = changeLog.back(); + + // Public MemoryTable API cannot be used here because it will add another + // change log entry. + change.table->rollback(change); + + changeLog.pop_back(); + } +} + +void MemoryTableFactory2::commitDB(dev::h256 const& _blockHash, int64_t _blockNumber) +{ + auto start_time = utcTime(); + auto record_time = utcTime(); + vector datas; + + for (auto& dbIt : m_name2Table) + { + auto table = std::dynamic_pointer_cast
(dbIt.second); + + auto tableData = std::make_shared(); + table->dump(tableData); + + if (tableData->entries->size() > 0) + { + datas.push_back(tableData); + } + } + auto getData_time_cost = utcTime() - record_time; + record_time = utcTime(); + + if (!datas.empty()) + { + stateStorage()->commit(_blockHash, _blockNumber, datas, _blockHash); + } + auto commit_time_cost = utcTime() - record_time; + record_time = utcTime(); + + m_name2Table.clear(); + auto clear_time_cost = utcTime() - record_time; + STORAGE_LOG(DEBUG) << LOG_BADGE("Commit") << LOG_DESC("Commit db time record") + << LOG_KV("getDataTimeCost", getData_time_cost) + << LOG_KV("commitTimeCost", commit_time_cost) + << LOG_KV("clearTimeCost", clear_time_cost) + << LOG_KV("totalTimeCost", utcTime() - start_time); +} + +storage::TableInfo::Ptr MemoryTableFactory2::getSysTableInfo(const std::string& tableName) +{ + auto tableInfo = make_shared(); + tableInfo->name = tableName; + if (tableName == SYS_CONSENSUS) + { + tableInfo->key = "name"; + tableInfo->fields = vector{"type", "node_id", "enable_num"}; + } + else if (tableName == SYS_TABLES) + { + tableInfo->key = "table_name"; + tableInfo->fields = vector{"key_field", "value_field"}; + } + else if (tableName == SYS_ACCESS_TABLE) + { + tableInfo->key = "table_name"; + tableInfo->fields = vector{"address", "enable_num"}; + } + else if (tableName == SYS_CURRENT_STATE) + { + tableInfo->key = SYS_KEY; + tableInfo->fields = std::vector{"value"}; + } + else if (tableName == SYS_NUMBER_2_HASH) + { + tableInfo->key = "number"; + tableInfo->fields = std::vector{"value"}; + } + else if (tableName == SYS_TX_HASH_2_BLOCK) + { + tableInfo->key = "hash"; + tableInfo->fields = std::vector{"value", "index"}; + } + else if (tableName == SYS_HASH_2_BLOCK) + { + tableInfo->key = "hash"; + tableInfo->fields = std::vector{"value"}; + } + else if (tableName == SYS_CNS) + { + tableInfo->key = "name"; + tableInfo->fields = std::vector{"version", "address", "abi"}; + } + else if (tableName == SYS_CONFIG) + { + tableInfo->key = "key"; + tableInfo->fields = std::vector{"value", "enable_num"}; + } + else if (tableName == SYS_BLOCK_2_NONCES) + { + tableInfo->key = "number"; + tableInfo->fields = std::vector{SYS_VALUE}; + } + return tableInfo; +} + + +void MemoryTableFactory2::setAuthorizedAddress(storage::TableInfo::Ptr _tableInfo) +{ + typename Table::Ptr accessTable = openTable(SYS_ACCESS_TABLE); + if (accessTable) + { + auto tableEntries = accessTable->select(_tableInfo->name, accessTable->newCondition()); + for (size_t i = 0; i < tableEntries->size(); ++i) + { + auto entry = tableEntries->get(i); + if (std::stoi(entry->getField("enable_num")) <= m_blockNum) + { + _tableInfo->authorizedAddress.emplace_back(entry->getField("address")); + } + } + } +} diff --git a/libstorage/MemoryTableFactory2.h b/libstorage/MemoryTableFactory2.h new file mode 100644 index 0000000000..c2cd34fb5a --- /dev/null +++ b/libstorage/MemoryTableFactory2.h @@ -0,0 +1,84 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file MemoryTableFactory.h + * @author ancelmo + * @date 20180921 + */ +#pragma once + +#include "Common.h" +#include "MemoryTable.h" +#include "Storage.h" +#include "Table.h" +#include "TablePrecompiled.h" +#include +#include +#include +#include +#include + +namespace dev +{ +namespace blockverifier +{ +class ExecutiveContext; +} +namespace storage +{ +class MemoryTableFactory2 : public TableFactory +{ +public: + typedef std::shared_ptr Ptr; + MemoryTableFactory2(); + virtual ~MemoryTableFactory2() {} + virtual Table::Ptr openTable( + const std::string& tableName, bool authorityFlag = true, bool isPara = true) override; + virtual Table::Ptr createTable(const std::string& tableName, const std::string& keyField, + const std::string& valueField, bool authorityFlag = true, + Address const& _origin = Address(), bool isPara = true) override; + + virtual Storage::Ptr stateStorage() { return m_stateStorage; } + virtual void setStateStorage(Storage::Ptr stateStorage) { m_stateStorage = stateStorage; } + + void setBlockHash(h256 blockHash); + void setBlockNum(int64_t blockNum); + + virtual h256 hash() override; + virtual size_t savepoint() override; + virtual void rollback(size_t _savepoint) override; + virtual void commitDB(h256 const& _blockHash, int64_t _blockNumber) override; + +private: + storage::TableInfo::Ptr getSysTableInfo(const std::string& tableName); + void setAuthorizedAddress(storage::TableInfo::Ptr _tableInfo); + std::vector& getChangeLog(); + Storage::Ptr m_stateStorage; + h256 m_blockHash; + int m_blockNum; + // this map can't be changed, hash() need ordered data + std::map m_name2Table; + boost::thread_specific_ptr> m_changeLog; + h256 m_hash; + std::vector m_sysTables; + + // mutex + mutable RecursiveMutex x_name2Table; +}; + +} // namespace storage + +} // namespace dev diff --git a/libstorage/MemoryTableFactoryFactory.h b/libstorage/MemoryTableFactoryFactory.h new file mode 100644 index 0000000000..5c1f7c2282 --- /dev/null +++ b/libstorage/MemoryTableFactoryFactory.h @@ -0,0 +1,54 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file MemoryTableFactoryFactory.h + * @author ancelmo + * @date 20190328 + */ + +#pragma once + +#include "Table.h" +#include "Storage.h" +#include "MemoryTableFactory.h" +#include + +namespace dev { + +namespace storage { + +class MemoryTableFactoryFactory: public TableFactoryFactory { +public: + virtual TableFactory::Ptr newTableFactory(dev::h256 hash, int64_t number) override { + MemoryTableFactory::Ptr tableFactory = std::make_shared(); + tableFactory->setStateStorage(m_stroage); + tableFactory->setBlockHash(hash); + tableFactory->setBlockNum(number); + + return tableFactory; + } + + void setStorage(dev::storage::Storage::Ptr storage) { + m_stroage = storage; + } + +private: + dev::storage::Storage::Ptr m_stroage; +}; + +} + +} diff --git a/libstorage/MemoryTableFactoryFactory2.h b/libstorage/MemoryTableFactoryFactory2.h new file mode 100644 index 0000000000..f9fd5ba0ad --- /dev/null +++ b/libstorage/MemoryTableFactoryFactory2.h @@ -0,0 +1,54 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ +/** @file MemoryTableFactoryFactory.h + * @author ancelmo + * @date 20190328 + */ + +#pragma once + +#include "Table.h" +#include "Storage.h" +#include "MemoryTableFactory2.h" +#include + +namespace dev { + +namespace storage { + +class MemoryTableFactoryFactory2: public TableFactoryFactory { +public: + virtual TableFactory::Ptr newTableFactory(dev::h256 hash, int64_t number) override { + MemoryTableFactory2::Ptr tableFactory = std::make_shared(); + tableFactory->setStateStorage(m_stroage); + tableFactory->setBlockHash(hash); + tableFactory->setBlockNum(number); + + return tableFactory; + } + + void setStorage(dev::storage::Storage::Ptr storage) { + m_stroage = storage; + } + +private: + dev::storage::Storage::Ptr m_stroage; +}; + +} + +} diff --git a/libstorage/Storage.h b/libstorage/Storage.h index 6376a246e9..e1bfca8039 100644 --- a/libstorage/Storage.h +++ b/libstorage/Storage.h @@ -34,7 +34,7 @@ class Storage : public std::enable_shared_from_this virtual ~Storage(){}; virtual Entries::Ptr select(h256 hash, int num, const std::string& table, - const std::string& key, Condition::Ptr condition) = 0; + const std::string& key, Condition::Ptr condition = nullptr) = 0; virtual size_t commit(h256 hash, int64_t num, const std::vector& datas, h256 const& blockHash) = 0; virtual bool onlyDirty() = 0; diff --git a/libstorage/Table.h b/libstorage/Table.h index b809e391f5..8d50a19111 100644 --- a/libstorage/Table.h +++ b/libstorage/Table.h @@ -163,13 +163,14 @@ struct Change std::string key; struct Record { - size_t id; - size_t newIndex; + + size_t index; std::string key; std::string oldValue; - Record(size_t _id, size_t index = 0, std::string _key = std::string(), - std::string _oldValue = std::string()) - : id(_id), newIndex(index), key(_key), oldValue(_oldValue) + size_t id; + Record(size_t _index, std::string _key = std::string(), + std::string _oldValue = std::string(), size_t _id = 0) + : index(_index), key(_key), oldValue(_oldValue), id(_id) {} }; std::vector value; @@ -190,8 +191,13 @@ class TableData entries = std::make_shared(); } + //for memorytable2 TableInfo::Ptr info; Entries::Ptr entries; + + //for memorytable + std::string tableName; + std::map data; }; // Construction of transaction execution @@ -215,9 +221,9 @@ class Table : public std::enable_shared_from_this
virtual bool checkAuthority(Address const& _origin) const = 0; virtual h256 hash() = 0; virtual void clear() = 0; - virtual TableData::Ptr dump() = 0; - virtual void rollback(const Change& _change) = 0; + virtual bool dump(dev::storage::TableData::Ptr _data) = 0; + virtual void rollback(const Change& _change) = 0; virtual bool empty() = 0; virtual void setRecorder( std::function&)> @@ -230,6 +236,7 @@ class Table : public std::enable_shared_from_this
virtual void setBlockHash(h256 blockHash) = 0; virtual void setBlockNum(int blockNum) = 0; virtual void setTableInfo(TableInfo::Ptr tableInfo) = 0; + virtual size_t cacheSize() { return 0; } static bool processCondition(Entry::Ptr entry, Condition::Ptr condition); @@ -238,19 +245,34 @@ class Table : public std::enable_shared_from_this
m_recorder; }; -// Block execution time construction -class StateDBFactory : public std::enable_shared_from_this +// Block execution time construction by TableFactoryFactory +class TableFactory : public std::enable_shared_from_this { public: - typedef std::shared_ptr Ptr; + typedef std::shared_ptr Ptr; - virtual ~StateDBFactory() {} + virtual ~TableFactory() {} virtual Table::Ptr openTable( const std::string& table, bool authorityFlag = true, bool isPara = false) = 0; virtual Table::Ptr createTable(const std::string& tableName, const std::string& keyField, const std::string& valueField, bool authorityFlag, Address const& _origin = Address(), bool isPara = false) = 0; -}; // namespace storage + + virtual h256 hash() = 0; + virtual size_t savepoint() = 0; + virtual void rollback(size_t _savepoint) = 0; + virtual void commitDB(h256 const& _blockHash, int64_t _blockNumber) = 0; +}; + +class TableFactoryFactory : public std::enable_shared_from_this { +public: + typedef std::shared_ptr Ptr; + + virtual ~TableFactoryFactory() {}; + + virtual TableFactory::Ptr newTableFactory(dev::h256 hash, int64_t number) = 0; +}; + } // namespace storage } // namespace dev diff --git a/libstorage/TableFactoryPrecompiled.cpp b/libstorage/TableFactoryPrecompiled.cpp index 43544a0985..48902a5197 100644 --- a/libstorage/TableFactoryPrecompiled.cpp +++ b/libstorage/TableFactoryPrecompiled.cpp @@ -19,9 +19,9 @@ * @date 20180921 */ #include "TableFactoryPrecompiled.h" -#include "MemoryTable.h" -#include "MemoryTableFactory.h" +#include "Table.h" #include "TablePrecompiled.h" +#include "StorageException.h" #include #include #include @@ -99,11 +99,16 @@ bytes TableFactoryPrecompiled::call( boost::trim(str); valueFiled = boost::join(fieldNameList, ","); tableName = storage::USER_TABLE_PREFIX + tableName; - auto table = - m_memoryTableFactory->createTable(tableName, keyField, valueFiled, true, origin); - // set createTableCode - int createTableCode = m_memoryTableFactory->getCreateTableCode(); - out = abi.abiIn("", createTableCode); + auto result = 0; + try { + auto table = m_memoryTableFactory->createTable(tableName, keyField, valueFiled, true, origin); + // set createTableCode + } + catch(dev::storage::StorageException &e) { + STORAGE_LOG(ERROR) << "Create table failed: " << boost::diagnostic_information(e); + result = e.errorCode(); + } + out = abi.abiIn("", result); } else { diff --git a/libstorage/TableFactoryPrecompiled.h b/libstorage/TableFactoryPrecompiled.h index 2f5c7eb923..c53adff9d2 100644 --- a/libstorage/TableFactoryPrecompiled.h +++ b/libstorage/TableFactoryPrecompiled.h @@ -54,12 +54,12 @@ class TableFactoryPrecompiled : public Precompiled virtual bytes call(std::shared_ptr context, bytesConstRef param, Address const& origin = Address()); - void setMemoryTableFactory(std::shared_ptr memoryTableFactory) + void setMemoryTableFactory(std::shared_ptr memoryTableFactory) { m_memoryTableFactory = memoryTableFactory; } - std::shared_ptr getMemoryTableFactory() + std::shared_ptr getMemoryTableFactory() { return m_memoryTableFactory; } @@ -67,7 +67,7 @@ class TableFactoryPrecompiled : public Precompiled h256 hash(); private: - std::shared_ptr m_memoryTableFactory; + std::shared_ptr m_memoryTableFactory; }; } // namespace blockverifier diff --git a/libstoragestate/StorageState.cpp b/libstoragestate/StorageState.cpp index 3979389ae9..05e012c2c1 100644 --- a/libstoragestate/StorageState.cpp +++ b/libstoragestate/StorageState.cpp @@ -25,7 +25,7 @@ #include "StorageState.h" #include "libdevcrypto/Hash.h" #include "libethcore/Exceptions.h" -#include "libstorage/MemoryTableFactory.h" +#include "libstorage/Table.h" using namespace dev; using namespace dev::eth; @@ -350,7 +350,7 @@ h256 StorageState::rootHash(bool needCalculate) const void StorageState::commit() { - m_memoryTableFactory->commit(); + //m_memoryTableFactory->commit(); } void StorageState::dbCommit(h256 const&, int64_t) diff --git a/libstoragestate/StorageState.h b/libstoragestate/StorageState.h index 530b23cfb3..62e87edffa 100644 --- a/libstoragestate/StorageState.h +++ b/libstoragestate/StorageState.h @@ -167,7 +167,7 @@ class StorageState : public dev::executive::StateFace bool checkAuthority(Address const& _origin, Address const& _contract) const override; void setMemoryTableFactory( - std::shared_ptr _memoryTableFactory) + std::shared_ptr _memoryTableFactory) { m_memoryTableFactory = _memoryTableFactory; } @@ -177,7 +177,7 @@ class StorageState : public dev::executive::StateFace std::shared_ptr getTable(Address const& _address) const; /// check authority by caller u256 m_accountStartNonce; - std::shared_ptr m_memoryTableFactory; + std::shared_ptr m_memoryTableFactory; }; } // namespace storagestate } // namespace dev diff --git a/libstoragestate/StorageStateFactory.cpp b/libstoragestate/StorageStateFactory.cpp index a972cf8730..7f5a65b862 100644 --- a/libstoragestate/StorageStateFactory.cpp +++ b/libstoragestate/StorageStateFactory.cpp @@ -29,7 +29,7 @@ using namespace dev; using namespace dev::storagestate; using namespace dev::executive; std::shared_ptr StorageStateFactory::getState( - h256 const&, std::shared_ptr _factory) + h256 const&, std::shared_ptr _factory) { auto storageState = make_shared(m_accountStartNonce); storageState->setMemoryTableFactory(_factory); diff --git a/libstoragestate/StorageStateFactory.h b/libstoragestate/StorageStateFactory.h index 4b8f52a50d..47d5b95d11 100644 --- a/libstoragestate/StorageStateFactory.h +++ b/libstoragestate/StorageStateFactory.h @@ -36,7 +36,7 @@ class StorageStateFactory : public dev::executive::StateFactoryInterface StorageStateFactory(u256 const& _accountStartNonce) : m_accountStartNonce(_accountStartNonce) {} virtual ~StorageStateFactory() {} std::shared_ptr getState( - h256 const& _root, std::shared_ptr _factory) override; + h256 const& _root, std::shared_ptr _factory) override; private: u256 m_accountStartNonce; diff --git a/test/unittests/libblockchain/BlockChainImp.cpp b/test/unittests/libblockchain/BlockChainImp.cpp index 106616ea56..7b942202df 100644 --- a/test/unittests/libblockchain/BlockChainImp.cpp +++ b/test/unittests/libblockchain/BlockChainImp.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -59,7 +58,7 @@ static std::string const c_commonHash = "067150c07dab4facb7160e075548007e067150c07dab4facb7160e075548007e"; static std::string const c_commonHashPrefix = std::string("0x").append(c_commonHash); -class MockTable : public dev::storage::MemoryTable +class MockTable : public dev::storage::MemoryTable { public: typedef std::shared_ptr Ptr; @@ -152,16 +151,16 @@ class MockMemoryTableFactory : public dev::storage::MemoryTableFactory class MockBlockChainImp : public BlockChainImp { public: - std::shared_ptr getMemoryTableFactory() override + std::shared_ptr getMemoryTableFactory() override { return m_memoryTableFactory; } - void setMemoryTableFactory(std::shared_ptr _m) + void setMemoryTableFactory(std::shared_ptr _m) { m_memoryTableFactory = _m; } - std::shared_ptr m_memoryTableFactory; + std::shared_ptr m_memoryTableFactory; }; class MockState : public StorageState diff --git a/test/unittests/libevm/VMTest.cpp b/test/unittests/libevm/VMTest.cpp index 75ecae4cd2..ceca815a44 100644 --- a/test/unittests/libevm/VMTest.cpp +++ b/test/unittests/libevm/VMTest.cpp @@ -25,6 +25,7 @@ #include #include #include +#include using namespace std; using namespace dev; diff --git a/test/unittests/libexecutive/ExecuteVMTest.cpp b/test/unittests/libexecutive/ExecuteVMTest.cpp index d9c10571b4..ac0004e65d 100644 --- a/test/unittests/libexecutive/ExecuteVMTest.cpp +++ b/test/unittests/libexecutive/ExecuteVMTest.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include @@ -208,4 +209,4 @@ BOOST_AUTO_TEST_CASE(DeployGetSetContractTest) BOOST_AUTO_TEST_SUITE_END() } // namespace test -} // namespace dev \ No newline at end of file +} // namespace dev diff --git a/test/unittests/libstorage/MemoryStorage.h b/test/unittests/libstorage/MemoryStorage.h index 34a5752ead..bb36398be2 100644 --- a/test/unittests/libstorage/MemoryStorage.h +++ b/test/unittests/libstorage/MemoryStorage.h @@ -36,142 +36,27 @@ class MemoryStorage : public Storage virtual ~MemoryStorage(){}; - std::vector processEntries(Entries::Ptr entries, Condition::Ptr condition) + virtual Entries::Ptr select( + h256, int, const std::string& table, const std::string& key, Condition::Ptr) override { - std::vector indexes; - indexes.reserve(entries->size()); - if (condition->getConditions()->empty()) + auto search = data.find(table); + if (search != data.end()) { - for (size_t i = 0; i < entries->size(); ++i) - indexes.emplace_back(i); - return indexes; - } - - for (size_t i = 0; i < entries->size(); ++i) + auto tableData = search->second; + auto it = tableData->data.find(key); + if (it != tableData->data.end()) { - Entry::Ptr entry = entries->get(i); - if (processCondition(entry, condition)) + for (size_t i = 0; i < it->second->size(); ++i) { - indexes.push_back(i); - } - } - - return indexes; - } - - bool processCondition(Entry::Ptr entry, Condition::Ptr condition) + if (it->second->get(i)->getStatus() == Entry::Status::DELETED) { - try - { - for (auto& it : *condition->getConditions()) - { - if (entry->getStatus() == Entry::Status::DELETED) - { - return false; + it->second->removeEntry(i); } - std::string lhs = entry->getField(it.first); - std::string rhs = it.second.second; - - if (it.second.first == Condition::Op::eq) - { - if (lhs != rhs) - { - return false; - } - } - else if (it.second.first == Condition::Op::ne) - { - if (lhs == rhs) - { - return false; } + return it->second; } - else - { - if (lhs.empty()) - { - lhs = "0"; - } - if (rhs.empty()) - { - rhs = "0"; - } - - int lhsNum = boost::lexical_cast(lhs); - int rhsNum = boost::lexical_cast(rhs); - - switch (it.second.first) - { - case Condition::Op::eq: - case Condition::Op::ne: - { - break; - } - case Condition::Op::gt: - { - if (lhsNum <= rhsNum) - { - return false; - } - break; - } - case Condition::Op::ge: - { - if (lhsNum < rhsNum) - { - return false; - } - break; - } - case Condition::Op::lt: - { - if (lhsNum >= rhsNum) - { - return false; - } - break; - } - case Condition::Op::le: - { - if (lhsNum > rhsNum) - { - return false; - } - break; - } } - } - } - } - catch (std::exception& e) - { - return false; - } - - return true; - } - - virtual Entries::Ptr select(h256 hash, int num, const std::string& table, - const std::string& key, Condition::Ptr condition) override - { - (void)hash; - (void)num; - auto it = tableData.find(table); - - if (it != tableData.end()) - { - condition->EQ(it->second->info->key, key); - auto indices = processEntries(it->second->entries, condition); - - auto entries = std::make_shared(); - for (auto indexIt : indices) - { - entries->addEntry(it->second->entries->get(indexIt)); - } - - return entries; - } return std::make_shared(); } @@ -180,50 +65,14 @@ class MemoryStorage : public Storage { for (auto it : datas) { - auto table = it->info->name; - auto tableIt = tableData.find(table); - if (tableIt != tableData.end()) - { - size_t count = 0; - auto countIt = tableCounter.find(table); - if (countIt != tableCounter.end()) - { - count = countIt->second; - } - - for (size_t i = 0; i < it->entries->size(); ++i) - { - auto entry = it->entries->get(i); - if (entry->getID() == 0) - { - entry->setID(++count); - tableIt->second->entries->addEntry(entry); - } - else - { - for (size_t j = 0; j < tableIt->second->entries->size(); ++j) - { - if (tableIt->second->entries->get(j)->getID() == entry->getID()) - { - for (auto fieldIt : *(entry->fields())) - { - tableIt->second->entries->get(j)->setField( - fieldIt.first, fieldIt.second); - } - break; - } - } - } - } - } + data[it->tableName] = it; } return datas.size(); } virtual bool onlyDirty() override { return false; } private: - std::map tableData; - std::map tableCounter; + std::map data; }; } // namespace storage diff --git a/test/unittests/libstorage/MemoryStorage2.h b/test/unittests/libstorage/MemoryStorage2.h new file mode 100644 index 0000000000..34a5752ead --- /dev/null +++ b/test/unittests/libstorage/MemoryStorage2.h @@ -0,0 +1,230 @@ +/** + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + * + * @brief + * + * @file ExecuteVMTest.cpp + * @author: xingqiangbai + * @date 2018-10-25 + */ + +#pragma once + +#include "libstorage/Storage.h" + +namespace dev +{ +namespace storage +{ +class MemoryStorage : public Storage +{ +public: + typedef std::shared_ptr Ptr; + + virtual ~MemoryStorage(){}; + + std::vector processEntries(Entries::Ptr entries, Condition::Ptr condition) + { + std::vector indexes; + indexes.reserve(entries->size()); + if (condition->getConditions()->empty()) + { + for (size_t i = 0; i < entries->size(); ++i) + indexes.emplace_back(i); + return indexes; + } + + for (size_t i = 0; i < entries->size(); ++i) + { + Entry::Ptr entry = entries->get(i); + if (processCondition(entry, condition)) + { + indexes.push_back(i); + } + } + + return indexes; + } + + bool processCondition(Entry::Ptr entry, Condition::Ptr condition) + { + try + { + for (auto& it : *condition->getConditions()) + { + if (entry->getStatus() == Entry::Status::DELETED) + { + return false; + } + + std::string lhs = entry->getField(it.first); + std::string rhs = it.second.second; + + if (it.second.first == Condition::Op::eq) + { + if (lhs != rhs) + { + return false; + } + } + else if (it.second.first == Condition::Op::ne) + { + if (lhs == rhs) + { + return false; + } + } + else + { + if (lhs.empty()) + { + lhs = "0"; + } + if (rhs.empty()) + { + rhs = "0"; + } + + int lhsNum = boost::lexical_cast(lhs); + int rhsNum = boost::lexical_cast(rhs); + + switch (it.second.first) + { + case Condition::Op::eq: + case Condition::Op::ne: + { + break; + } + case Condition::Op::gt: + { + if (lhsNum <= rhsNum) + { + return false; + } + break; + } + case Condition::Op::ge: + { + if (lhsNum < rhsNum) + { + return false; + } + break; + } + case Condition::Op::lt: + { + if (lhsNum >= rhsNum) + { + return false; + } + break; + } + case Condition::Op::le: + { + if (lhsNum > rhsNum) + { + return false; + } + break; + } + } + } + } + } + catch (std::exception& e) + { + return false; + } + + return true; + } + + virtual Entries::Ptr select(h256 hash, int num, const std::string& table, + const std::string& key, Condition::Ptr condition) override + { + (void)hash; + (void)num; + auto it = tableData.find(table); + + if (it != tableData.end()) + { + condition->EQ(it->second->info->key, key); + auto indices = processEntries(it->second->entries, condition); + + auto entries = std::make_shared(); + for (auto indexIt : indices) + { + entries->addEntry(it->second->entries->get(indexIt)); + } + + return entries; + } + + return std::make_shared(); + } + virtual size_t commit( + h256, int64_t, const std::vector& datas, h256 const&) override + { + for (auto it : datas) + { + auto table = it->info->name; + auto tableIt = tableData.find(table); + if (tableIt != tableData.end()) + { + size_t count = 0; + auto countIt = tableCounter.find(table); + if (countIt != tableCounter.end()) + { + count = countIt->second; + } + + for (size_t i = 0; i < it->entries->size(); ++i) + { + auto entry = it->entries->get(i); + if (entry->getID() == 0) + { + entry->setID(++count); + tableIt->second->entries->addEntry(entry); + } + else + { + for (size_t j = 0; j < tableIt->second->entries->size(); ++j) + { + if (tableIt->second->entries->get(j)->getID() == entry->getID()) + { + for (auto fieldIt : *(entry->fields())) + { + tableIt->second->entries->get(j)->setField( + fieldIt.first, fieldIt.second); + } + break; + } + } + } + } + } + } + return datas.size(); + } + virtual bool onlyDirty() override { return false; } + +private: + std::map tableData; + std::map tableCounter; +}; +} // namespace storage + +} // namespace dev diff --git a/test/unittests/libstorage/test_AuthorityPrecompiled.cpp b/test/unittests/libstorage/test_AuthorityPrecompiled.cpp index 5e190cf7b2..7f8e15b14b 100644 --- a/test/unittests/libstorage/test_AuthorityPrecompiled.cpp +++ b/test/unittests/libstorage/test_AuthorityPrecompiled.cpp @@ -31,6 +31,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -50,8 +51,10 @@ struct AuthorityPrecompiledFixture ExecutiveContextFactory factory; auto storage = std::make_shared(); auto storageStateFactory = std::make_shared(h256(0)); + auto tableFactoryFactory = std::make_shared(); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); authorityPrecompiled = context->getPrecompiled(Address(0x1005)); memoryTableFactory = context->getMemoryTableFactory(); @@ -60,7 +63,7 @@ struct AuthorityPrecompiledFixture ~AuthorityPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; Precompiled::Ptr authorityPrecompiled; BlockInfo blockInfo; }; diff --git a/test/unittests/libstorage/test_CNSPrecompiled.cpp b/test/unittests/libstorage/test_CNSPrecompiled.cpp index 7592aef0fc..98ae6434e8 100644 --- a/test/unittests/libstorage/test_CNSPrecompiled.cpp +++ b/test/unittests/libstorage/test_CNSPrecompiled.cpp @@ -31,6 +31,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -50,8 +51,10 @@ struct CNSPrecompiledFixture ExecutiveContextFactory factory; auto storage = std::make_shared(); auto storageStateFactory = std::make_shared(h256(0)); + auto tableFactoryFactory = std::make_shared(); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); cnsPrecompiled = std::make_shared(); memoryTableFactory = context->getMemoryTableFactory(); @@ -60,7 +63,7 @@ struct CNSPrecompiledFixture ~CNSPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; CNSPrecompiled::Ptr cnsPrecompiled; BlockInfo blockInfo; }; @@ -183,4 +186,4 @@ BOOST_AUTO_TEST_CASE(errFunc) BOOST_AUTO_TEST_SUITE_END() -} // namespace test_CNSPrecompiled \ No newline at end of file +} // namespace test_CNSPrecompiled diff --git a/test/unittests/libstorage/test_CRUDPrecompiled.cpp b/test/unittests/libstorage/test_CRUDPrecompiled.cpp index b30718cec6..7c57448295 100644 --- a/test/unittests/libstorage/test_CRUDPrecompiled.cpp +++ b/test/unittests/libstorage/test_CRUDPrecompiled.cpp @@ -24,6 +24,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -45,6 +46,8 @@ struct CRUDPrecompiledFixture auto storageStateFactory = std::make_shared(h256(0)); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + auto tableFactoryFactory = std::make_shared(); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); crudPrecompiled = std::make_shared(); memoryTableFactory = context->getMemoryTableFactory(); @@ -53,7 +56,7 @@ struct CRUDPrecompiledFixture ~CRUDPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; CRUDPrecompiled::Ptr crudPrecompiled; BlockInfo blockInfo; }; diff --git a/test/unittests/libstorage/test_ConsensusPrecompiled.cpp b/test/unittests/libstorage/test_ConsensusPrecompiled.cpp index be844c5365..36ae8ba6bc 100644 --- a/test/unittests/libstorage/test_ConsensusPrecompiled.cpp +++ b/test/unittests/libstorage/test_ConsensusPrecompiled.cpp @@ -8,6 +8,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -27,8 +28,10 @@ struct ConsensusPrecompiledFixture ExecutiveContextFactory factory; auto storage = std::make_shared(); auto storageStateFactory = std::make_shared(h256(0)); + auto tableFactoryFactory = std::make_shared(); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); consensusPrecompiled = std::make_shared(); memoryTableFactory = context->getMemoryTableFactory(); @@ -37,7 +40,7 @@ struct ConsensusPrecompiledFixture ~ConsensusPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; ConsensusPrecompiled::Ptr consensusPrecompiled; BlockInfo blockInfo; }; diff --git a/test/unittests/libstorage/test_DagTransferPrecompiled.cpp b/test/unittests/libstorage/test_DagTransferPrecompiled.cpp index b0a69fd329..4c49084167 100644 --- a/test/unittests/libstorage/test_DagTransferPrecompiled.cpp +++ b/test/unittests/libstorage/test_DagTransferPrecompiled.cpp @@ -29,6 +29,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -50,6 +51,8 @@ struct DagTransferPrecompiledFixture auto storageStateFactory = std::make_shared(h256(0)); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + auto tableFactoryFactory = std::make_shared(); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); dtPrecompiled = std::make_shared(); memoryTableFactory = context->getMemoryTableFactory(); @@ -58,7 +61,7 @@ struct DagTransferPrecompiledFixture ~DagTransferPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; DagTransferPrecompiled::Ptr dtPrecompiled; BlockInfo blockInfo; @@ -483,4 +486,4 @@ BOOST_AUTO_TEST_CASE(userTransfer) BOOST_AUTO_TEST_SUITE_END() -} // namespace test_DagTransferPrecompiled \ No newline at end of file +} // namespace test_DagTransferPrecompiled diff --git a/test/unittests/libstorage/test_HelloWorldPrecompiled.cpp b/test/unittests/libstorage/test_HelloWorldPrecompiled.cpp index f8d53cc6a3..09f80e315a 100644 --- a/test/unittests/libstorage/test_HelloWorldPrecompiled.cpp +++ b/test/unittests/libstorage/test_HelloWorldPrecompiled.cpp @@ -31,6 +31,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -52,6 +53,8 @@ struct HelloWorldPrecompiledFixture auto storageStateFactory = std::make_shared(h256(0)); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + auto tableFactoryFactory = std::make_shared(); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); helloWorldPrecompiled = std::make_shared(); memoryTableFactory = context->getMemoryTableFactory(); @@ -60,7 +63,7 @@ struct HelloWorldPrecompiledFixture ~HelloWorldPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; HelloWorldPrecompiled::Ptr helloWorldPrecompiled; BlockInfo blockInfo; @@ -138,4 +141,4 @@ BOOST_AUTO_TEST_CASE(set) BOOST_AUTO_TEST_SUITE_END() -} // namespace test_HelloWorldPrecompiled \ No newline at end of file +} // namespace test_HelloWorldPrecompiled diff --git a/test/unittests/libstorage/test_LevelDBStateStorage.cpp b/test/unittests/libstorage/test_LevelDBStateStorage.cpp index 4e8dfe3d79..2e027ee3a6 100644 --- a/test/unittests/libstorage/test_LevelDBStateStorage.cpp +++ b/test/unittests/libstorage/test_LevelDBStateStorage.cpp @@ -214,7 +214,7 @@ BOOST_AUTO_TEST_CASE(empty_select) int num = 1; std::string table("t_test"); std::string key("id"); - Entries::Ptr entries = levelDB->select(h, num, table, key, std::make_shared()); + Entries::Ptr entries = levelDB->select(h, num, table, key); BOOST_CHECK_EQUAL(entries->size(), 0u); } @@ -225,17 +225,18 @@ BOOST_AUTO_TEST_CASE(commit) h256 blockHash(0x11231); std::vector datas; dev::storage::TableData::Ptr tableData = std::make_shared(); - tableData->info->name = "t_test"; - tableData->info->key = "Name"; - tableData->info->fields.push_back("id"); + tableData->tableName = "t_test"; Entries::Ptr entries = getEntries(); - tableData->entries = entries; + tableData->data.insert(std::make_pair(std::string("LiSi"), entries)); + if (!tableData->data.empty()) + { datas.push_back(tableData); + } size_t c = levelDB->commit(h, num, datas, blockHash); BOOST_CHECK_EQUAL(c, 1u); std::string table("t_test"); std::string key("LiSi"); - entries = levelDB->select(h, num, table, key, std::make_shared()); + entries = levelDB->select(h, num, table, key); BOOST_CHECK_EQUAL(entries->size(), 1u); } @@ -246,19 +247,15 @@ BOOST_AUTO_TEST_CASE(exception) h256 blockHash(0x11231); std::vector datas; dev::storage::TableData::Ptr tableData = std::make_shared(); - tableData->info->name = "e"; - tableData->info->key = "Name"; - tableData->info->fields.push_back("id"); + tableData->tableName = "e"; Entries::Ptr entries = getEntries(); - entries->get(0)->setField("Name", "Exception"); - tableData->entries = entries; + tableData->data.insert(std::make_pair(std::string("Exception"), entries)); datas.push_back(tableData); BOOST_CHECK_THROW(levelDB->commit(h, num, datas, blockHash), boost::exception); std::string table("e"); std::string key("Exception"); - BOOST_CHECK_THROW( - levelDB->select(h, num, table, key, std::make_shared()), boost::exception); + BOOST_CHECK_THROW(levelDB->select(h, num, table, key), boost::exception); } BOOST_AUTO_TEST_SUITE_END() diff --git a/test/unittests/libstorage/test_LevelDBStateStorage2.cpp b/test/unittests/libstorage/test_LevelDBStateStorage2.cpp new file mode 100644 index 0000000000..b0bb084165 --- /dev/null +++ b/test/unittests/libstorage/test_LevelDBStateStorage2.cpp @@ -0,0 +1,266 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ + +#include "libstorage/LevelDBStorage2.h" +#include +#include +#include +#include +#include + +using namespace dev; +using namespace dev::storage; +using namespace leveldb; + +namespace leveldb +{ +class Value; +} + +namespace test_LevelDBStateStorage2 +{ +inline uint32_t DecodeFixed32(const char* ptr, bool LittleEndian = true) +{ + if (LittleEndian) + { + // Load the raw bytes + uint32_t result; + memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load + return result; + } + return ((static_cast(static_cast(ptr[0]))) | + (static_cast(static_cast(ptr[1])) << 8) | + (static_cast(static_cast(ptr[2])) << 16) | + (static_cast(static_cast(ptr[3])) << 24)); +} + +const char* GetVarint32PtrFallback(const char* p, const char* limit, uint32_t* value) +{ + uint32_t result = 0; + for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) + { + uint32_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) + { + // More bytes are present + result |= ((byte & 127) << shift); + } + else + { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return nullptr; +} + +inline const char* GetVarint32Ptr(const char* p, const char* limit, uint32_t* value) +{ + if (p < limit) + { + uint32_t result = *(reinterpret_cast(p)); + if ((result & 128) == 0) + { + *value = result; + return p + 1; + } + } + return GetVarint32PtrFallback(p, limit, value); +} + +bool GetVarint32(Slice* input, uint32_t* value) +{ + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint32Ptr(p, limit, value); + if (q == nullptr) + { + return false; + } + else + { + *input = Slice(q, limit - q); + return true; + } +} + +bool GetLengthPrefixedSlice(Slice* input, Slice* result) +{ + uint32_t len; + if (GetVarint32(input, &len) && input->size() >= len) + { + *result = Slice(input->data(), len); + input->remove_prefix(len); + return true; + } + else + { + return false; + } +} + +struct MockWriteBatch +{ + std::string rep_; + int Count() { return DecodeFixed32(rep_.data() + 8); } +}; + +class MockLevelDB : public dev::db::BasicLevelDB +{ +public: + MockLevelDB() : BasicLevelDB(dev::db::LevelDB::defaultDBOptions(), "test_LevelDBStateStorage") + {} + virtual ~MockLevelDB() {} + + virtual Status Put(const WriteOptions&, const Slice&, const Slice&) { return Status::OK(); } + + virtual Status Delete(const WriteOptions&, const Slice& key) + { + db.erase(key.ToString()); + return Status::OK(); + } + + virtual Status Write(const WriteOptions&, WriteBatch* updates) + { + if (updates == nullptr) + return Status::InvalidArgument(Slice("InvalidArgument")); + auto batch = reinterpret_cast(updates); + size_t count = batch->Count(); + Slice input(batch->rep_); + input.remove_prefix(12); + for (size_t i = 0; i < count; ++i) + { + Slice key, value; + input.remove_prefix(1); + GetLengthPrefixedSlice(&input, &key); + GetLengthPrefixedSlice(&input, &value); + if (key.ToString() == "e_Exception") + return Status::InvalidArgument(Slice("InvalidArgument")); + db.insert(std::make_pair(key.ToString(), value.ToString())); + } + return Status::OK(); + } + + virtual Status Get(const ReadOptions&, const Slice& key, std::string* value) + { + if (value == nullptr || key.empty() || key == "e_Exception") + return Status::InvalidArgument(Slice("InvalidArgument")); + auto it = db.find(key.ToString()); + if (it == db.end()) + return Status::NotFound(Slice("NotFound")); + *value = it->second; + return Status::OK(); + } + + virtual Status Get(const leveldb::ReadOptions&, const leveldb::Slice&, leveldb::Value*) + { + return Status::OK(); + }; + +private: + // No copying allowed + MockLevelDB(const MockLevelDB&) = delete; + void operator=(const MockLevelDB&) = delete; + std::map db; +}; + +struct LevelDBFixture2 +{ + LevelDBFixture2() + { + levelDB = std::make_shared(); + std::shared_ptr mockLevelDB = std::make_shared(); + levelDB->setDB(mockLevelDB); + } + Entries::Ptr getEntries() + { + Entries::Ptr entries = std::make_shared(); + Entry::Ptr entry = std::make_shared(); + entry->setField("Name", "LiSi"); + entry->setField("id", "1"); + entries->addEntry(entry); + return entries; + } + dev::storage::LevelDBStorage2::Ptr levelDB; +}; + +BOOST_FIXTURE_TEST_SUITE(LevelDB2, LevelDBFixture2) + + +BOOST_AUTO_TEST_CASE(onlyDirty) +{ + BOOST_CHECK_EQUAL(levelDB->onlyDirty(), false); +} + +BOOST_AUTO_TEST_CASE(empty_select) +{ + h256 h(0x01); + int num = 1; + std::string table("t_test"); + std::string key("id"); + Entries::Ptr entries = levelDB->select(h, num, table, key, std::make_shared()); + BOOST_CHECK_EQUAL(entries->size(), 0u); +} + +BOOST_AUTO_TEST_CASE(commit) +{ + h256 h(0x01); + int num = 1; + h256 blockHash(0x11231); + std::vector datas; + dev::storage::TableData::Ptr tableData = std::make_shared(); + tableData->info->name = "t_test"; + tableData->info->key = "Name"; + tableData->info->fields.push_back("id"); + Entries::Ptr entries = getEntries(); + tableData->entries = entries; + datas.push_back(tableData); + size_t c = levelDB->commit(h, num, datas, blockHash); + BOOST_CHECK_EQUAL(c, 1u); + std::string table("t_test"); + std::string key("LiSi"); + entries = levelDB->select(h, num, table, key, std::make_shared()); + BOOST_CHECK_EQUAL(entries->size(), 1u); +} + +BOOST_AUTO_TEST_CASE(exception) +{ + h256 h(0x01); + int num = 1; + h256 blockHash(0x11231); + std::vector datas; + dev::storage::TableData::Ptr tableData = std::make_shared(); + tableData->info->name = "e"; + tableData->info->key = "Name"; + tableData->info->fields.push_back("id"); + Entries::Ptr entries = getEntries(); + entries->get(0)->setField("Name", "Exception"); + tableData->entries = entries; + datas.push_back(tableData); + BOOST_CHECK_THROW(levelDB->commit(h, num, datas, blockHash), boost::exception); + std::string table("e"); + std::string key("Exception"); + + BOOST_CHECK_THROW( + levelDB->select(h, num, table, key, std::make_shared()), boost::exception); +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace test_LevelDBStateStorage diff --git a/test/unittests/libstorage/test_MemoryTableFactory.cpp b/test/unittests/libstorage/test_MemoryTableFactory.cpp index b5a66011fe..cea58b1f0c 100644 --- a/test/unittests/libstorage/test_MemoryTableFactory.cpp +++ b/test/unittests/libstorage/test_MemoryTableFactory.cpp @@ -39,8 +39,7 @@ class MockAMOPDB : public dev::storage::Storage virtual ~MockAMOPDB() {} - virtual Entries::Ptr select( - h256, int, const std::string&, const std::string&, Condition::Ptr) override + virtual Entries::Ptr select(h256, int, const std::string&, const std::string&, Condition::Ptr) override { Entries::Ptr entries = std::make_shared(); return entries; @@ -78,8 +77,7 @@ BOOST_AUTO_TEST_CASE(open_Table) std::string keyField("key"); std::string valueField("value"); memoryDBFactory->createTable(tableName, keyField, valueField, true, Address(), false); - MemoryTable::Ptr table = - std::dynamic_pointer_cast(memoryDBFactory->openTable("t_test", true, false)); + Table::Ptr table = memoryDBFactory->openTable("t_test", true, false); table->remove("name", table->newCondition()); auto entry = table->newEntry(); entry->setField("key", "name"); @@ -124,7 +122,7 @@ BOOST_AUTO_TEST_CASE(parallel_openTable) std::string tableName("t_test"); std::string keyField("key"); std::string valueField("value"); - MemoryTable::Ptr table = std::dynamic_pointer_cast( + MemoryTable::Ptr table = std::dynamic_pointer_cast>( memoryDBFactory->createTable(tableName, keyField, valueField, false, Address(), true)); @@ -134,6 +132,7 @@ BOOST_AUTO_TEST_CASE(parallel_openTable) threads.push_back(std::thread([this, i, table]() { auto entry = table->newEntry(); // entry->setField("key", "balance"); + entry->setField("key", "balance"); auto initBalance = std::to_string(500 + i); entry->setField("value", initBalance); auto key = std::to_string(i); @@ -153,6 +152,7 @@ BOOST_AUTO_TEST_CASE(parallel_openTable) entry = table->newEntry(); // entry->setField("key", "balance"); + entry->setField("key", "balance"); entry->setField("value", std::to_string((i + 1) * 100)); table->update(key, entry, table->newCondition()); entries = table->select(key, table->newCondition()); @@ -170,6 +170,7 @@ BOOST_AUTO_TEST_CASE(parallel_openTable) entry = table->newEntry(); // entry->setField("key", "name"); + entry->setField("key", "name"); entry->setField("value", "Vita"); table->insert(key, entry); @@ -180,11 +181,9 @@ BOOST_AUTO_TEST_CASE(parallel_openTable) table->remove(key, table->newCondition()); entries = table->select(key, table->newCondition()); -#if 0 BOOST_TEST(entries->size() == 1); BOOST_TEST(entries->get(0)->getStatus() == 1); -#endif - BOOST_TEST(entries->size() == 0); + memoryDBFactory->rollback(savepoint2); entries = table->select(key, table->newCondition()); @@ -205,6 +204,7 @@ BOOST_AUTO_TEST_CASE(parallel_openTable) threads.push_back(std::thread([this, i, table]() { auto entry = table->newEntry(); // entry->setField("key", "balance"); + entry->setField("key", "balance"); auto initBalance = std::to_string(500 + i); entry->setField("value", initBalance); auto key = std::to_string(i + 10); diff --git a/test/unittests/libstorage/test_MemoryTableFactory2.cpp b/test/unittests/libstorage/test_MemoryTableFactory2.cpp new file mode 100644 index 0000000000..06d4f7690d --- /dev/null +++ b/test/unittests/libstorage/test_MemoryTableFactory2.cpp @@ -0,0 +1,252 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ + +#include "Common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace dev::storage; + +namespace test_MemoryTableFactory2 +{ +class MockAMOPDB : public dev::storage::Storage +{ +public: + virtual ~MockAMOPDB() {} + + + virtual Entries::Ptr select( + h256, int, const std::string&, const std::string&, Condition::Ptr) override + { + Entries::Ptr entries = std::make_shared(); + return entries; + } + + virtual size_t commit(h256, int64_t, const std::vector&, h256 const&) override + { + return 0; + } + + virtual bool onlyDirty() override { return false; } +}; + +struct MemoryTableFactoryFixture2 +{ + MemoryTableFactoryFixture2() + { + std::shared_ptr mockAMOPDB = std::make_shared(); + + memoryDBFactory = std::make_shared(); + memoryDBFactory->setStateStorage(mockAMOPDB); + + BOOST_TEST_TRUE(memoryDBFactory->stateStorage() == mockAMOPDB); + } + + dev::storage::MemoryTableFactory2::Ptr memoryDBFactory; +}; + +BOOST_FIXTURE_TEST_SUITE(MemoryTableFactory2, MemoryTableFactoryFixture2) + +BOOST_AUTO_TEST_CASE(open_Table) +{ + h256 blockHash(0x0101); + std::string tableName("t_test"); + std::string keyField("key"); + std::string valueField("value"); + memoryDBFactory->createTable(tableName, keyField, valueField, true, Address(), false); + Table::Ptr table = (memoryDBFactory->openTable("t_test", true, false)); + table->remove("name", table->newCondition()); + auto entry = table->newEntry(); + entry->setField("key", "name"); + entry->setField("value", "Lili"); + table->insert("name", entry); + entry = table->newEntry(); + entry->setField("key", "id"); + entry->setField("value", "12345"); + table->update("id", entry, table->newCondition()); + table->insert("id", entry); + entry->setField("key", "balance"); + entry->setField("value", "500"); + table->insert("balance", entry); + auto savePoint = memoryDBFactory->savepoint(); + auto condition = table->newCondition(); + condition->EQ("key", "name"); + condition->NE("value", "name"); + table->remove("name", condition); + memoryDBFactory->rollback(savePoint); + condition = table->newCondition(); + condition->EQ("key", "balance"); + condition->GT("value", "404"); + table->select("balance", condition); + condition = table->newCondition(); + condition->EQ("key", "balance"); + condition->GE("value", "403"); + table->select("balance", condition); + condition = table->newCondition(); + condition->EQ("key", "balance"); + condition->LT("value", "505"); + table->select("balance", condition); + condition = table->newCondition(); + condition->EQ("key", "balance"); + condition->LE("value", "504"); + table->select("balance", condition); + memoryDBFactory->commitDB(h256(0), 2); +} + +BOOST_AUTO_TEST_CASE(parallel_openTable) +{ + h256 blockHash(0x0101); + std::string tableName("t_test"); + std::string keyField("key"); + std::string valueField("value"); + Table::Ptr table = + memoryDBFactory->createTable(tableName, keyField, valueField, false, Address(), true); + + + std::vector threads; + for (auto i = 0; i < 3; ++i) + { + threads.push_back(std::thread([this, i, table]() { + auto entry = table->newEntry(); + // entry->setField("key", "balance"); + auto initBalance = std::to_string(500 + i); + entry->setField("value", initBalance); + auto key = std::to_string(i); + + auto savepoint0 = memoryDBFactory->savepoint(); + BOOST_TEST(savepoint0 == 0); + table->insert(key, entry); + + auto entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 1); + BOOST_TEST(entries->get(0)->getField("value") == initBalance); + + std::this_thread::sleep_for(std::chrono::milliseconds((i + 1) * 100)); + + auto savepoint1 = memoryDBFactory->savepoint(); + BOOST_TEST(savepoint1 == 1); + + entry = table->newEntry(); + // entry->setField("key", "balance"); + entry->setField("value", std::to_string((i + 1) * 100)); + table->update(key, entry, table->newCondition()); + entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 1); + BOOST_TEST(entries->get(0)->getField("value") == std::to_string((i + 1) * 100)); + + memoryDBFactory->rollback(savepoint1); + entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 1); + BOOST_TEST(entries->get(0)->getField("value") == initBalance); + + memoryDBFactory->rollback(savepoint0); + entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 0); + + entry = table->newEntry(); + // entry->setField("key", "name"); + entry->setField("value", "Vita"); + table->insert(key, entry); + + entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 1); + + auto savepoint2 = memoryDBFactory->savepoint(); + + table->remove(key, table->newCondition()); + entries = table->select(key, table->newCondition()); +#if 0 + BOOST_TEST(entries->size() == 1); + BOOST_TEST(entries->get(0)->getStatus() == 1); +#endif + BOOST_TEST(entries->size() == 0); + + memoryDBFactory->rollback(savepoint2); + entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 1); + BOOST_TEST(entries->get(0)->getStatus() == 0); + })); + } + + for (auto& t : threads) + { + t.join(); + } + + threads.clear(); + + for (auto i = 0; i < 3; ++i) + { + threads.push_back(std::thread([this, i, table]() { + auto entry = table->newEntry(); + // entry->setField("key", "balance"); + auto initBalance = std::to_string(500 + i); + entry->setField("value", initBalance); + auto key = std::to_string(i + 10); + + auto savepoint0 = memoryDBFactory->savepoint(); + BOOST_TEST(savepoint0 == 0); + table->insert(key, entry); + + auto entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 1); + BOOST_TEST(entries->get(0)->getField("value") == initBalance); + + memoryDBFactory->rollback(savepoint0); + entries = table->select(key, table->newCondition()); + BOOST_TEST(entries->size() == 0); + })); + } + + for (auto& t : threads) + { + t.join(); + } + memoryDBFactory->commitDB(h256(0), 2); +} + +BOOST_AUTO_TEST_CASE(open_sysTables) +{ + auto table = memoryDBFactory->openTable(SYS_CURRENT_STATE); + table = memoryDBFactory->openTable(SYS_NUMBER_2_HASH); + table = memoryDBFactory->openTable(SYS_TX_HASH_2_BLOCK); + table = memoryDBFactory->openTable(SYS_HASH_2_BLOCK); +} + +BOOST_AUTO_TEST_CASE(setBlockHash) +{ + memoryDBFactory->setBlockHash(h256(0x12345)); +} + +BOOST_AUTO_TEST_CASE(setBlockNum) +{ + memoryDBFactory->setBlockNum(2); +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace test_MemoryTableFactory diff --git a/test/unittests/libstorage/test_ParallelConfigPrecompiled.cpp b/test/unittests/libstorage/test_ParallelConfigPrecompiled.cpp index 2cb99755bc..6075e8f93b 100644 --- a/test/unittests/libstorage/test_ParallelConfigPrecompiled.cpp +++ b/test/unittests/libstorage/test_ParallelConfigPrecompiled.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -70,6 +71,8 @@ class ParallelConfigPrecompiledFixture : TestOutputHelperFixture auto storageStateFactory = std::make_shared(h256(0)); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + auto tableFactoryFactory = std::make_shared(); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); memoryTableFactory = context->getMemoryTableFactory(); parallelConfigPrecompiled = std::dynamic_pointer_cast( @@ -111,7 +114,7 @@ class ParallelConfigPrecompiledFixture : TestOutputHelperFixture public: ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; BlockInfo blockInfo; ParallelConfigPrecompiled::Ptr parallelConfigPrecompiled; }; @@ -165,4 +168,4 @@ BOOST_AUTO_TEST_CASE(registerParallelFunction) BOOST_AUTO_TEST_SUITE_END() } // namespace test -} // namespace dev \ No newline at end of file +} // namespace dev diff --git a/test/unittests/libstorage/test_SystemConfigPrecompiled.cpp b/test/unittests/libstorage/test_SystemConfigPrecompiled.cpp index 25e465d301..afbb570164 100644 --- a/test/unittests/libstorage/test_SystemConfigPrecompiled.cpp +++ b/test/unittests/libstorage/test_SystemConfigPrecompiled.cpp @@ -9,6 +9,7 @@ #include #include #include +#include using namespace dev; using namespace dev::blockverifier; @@ -30,6 +31,8 @@ struct SystemConfigPrecompiledFixture auto storageStateFactory = std::make_shared(h256(0)); factory.setStateStorage(storage); factory.setStateFactory(storageStateFactory); + auto tableFactoryFactory = std::make_shared(); + factory.setTableFactoryFactory(tableFactoryFactory); factory.initExecutiveContext(blockInfo, h256(0), context); systemConfigPrecompiled = std::make_shared(); memoryTableFactory = context->getMemoryTableFactory(); @@ -38,7 +41,7 @@ struct SystemConfigPrecompiledFixture ~SystemConfigPrecompiledFixture() {} ExecutiveContext::Ptr context; - MemoryTableFactory::Ptr memoryTableFactory; + TableFactory::Ptr memoryTableFactory; SystemConfigPrecompiled::Ptr systemConfigPrecompiled; BlockInfo blockInfo; }; diff --git a/test/unittests/libstorage/test_TablePrecompiled.cpp b/test/unittests/libstorage/test_TablePrecompiled.cpp index 71fce42146..0d450ba3fb 100644 --- a/test/unittests/libstorage/test_TablePrecompiled.cpp +++ b/test/unittests/libstorage/test_TablePrecompiled.cpp @@ -39,7 +39,7 @@ class MockPrecompiledEngine : public dev::blockverifier::ExecutiveContext virtual ~MockPrecompiledEngine() {} }; -class MockMemoryDB : public dev::storage::MemoryTable +class MockMemoryDB : public dev::storage::MemoryTable { public: virtual ~MockMemoryDB() {} @@ -53,7 +53,6 @@ struct TablePrecompiledFixture tablePrecompiled = std::make_shared(); auto table = std::make_shared(); TableInfo::Ptr info = std::make_shared(); - info->fields.emplace_back("_id_"); info->fields.emplace_back("name"); info->fields.emplace_back("_status_"); table->setTableInfo(info); diff --git a/test/unittests/libstorage/test_TablePrecompiled2.cpp b/test/unittests/libstorage/test_TablePrecompiled2.cpp new file mode 100644 index 0000000000..49d1557cfe --- /dev/null +++ b/test/unittests/libstorage/test_TablePrecompiled2.cpp @@ -0,0 +1,186 @@ +/* + * @CopyRight: + * FISCO-BCOS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * FISCO-BCOS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with FISCO-BCOS. If not, see + * (c) 2016-2018 fisco-dev contributors. + */ + +#include "Common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace dev; +using namespace std; +using namespace dev::blockverifier; +using namespace dev::storage; + +namespace test_TablePrecompiled2 +{ +class MockPrecompiledEngine : public dev::blockverifier::ExecutiveContext +{ +public: + virtual ~MockPrecompiledEngine() {} +}; + +class MockMemoryDB : public dev::storage::MemoryTable2 +{ +public: + virtual ~MockMemoryDB() {} +}; + +struct TablePrecompiledFixture2 +{ + TablePrecompiledFixture2() + { + context = std::make_shared(); + tablePrecompiled = std::make_shared(); + auto table = std::make_shared(); + TableInfo::Ptr info = std::make_shared(); + info->fields.emplace_back("_id_"); + info->fields.emplace_back("name"); + info->fields.emplace_back("_status_"); + table->setTableInfo(info); + table->setRecorder( + [&](Table::Ptr, Change::Kind, string const&, vector&) {}); + tablePrecompiled->setTable(table); + } + + ~TablePrecompiledFixture2() {} + + dev::blockverifier::TablePrecompiled::Ptr tablePrecompiled; + ExecutiveContext::Ptr context; + BlockInfo blockInfo; + int addressCount = 0x10000; +}; + +BOOST_FIXTURE_TEST_SUITE(TablePrecompiled2, TablePrecompiledFixture2) + +BOOST_AUTO_TEST_CASE(getDB) +{ + tablePrecompiled->getTable(); +} + +BOOST_AUTO_TEST_CASE(hash) +{ + tablePrecompiled->hash(); +} + +BOOST_AUTO_TEST_CASE(clear) +{ + auto table = tablePrecompiled->getTable(); + table->clear(); +} + +BOOST_AUTO_TEST_CASE(toString) +{ + BOOST_CHECK_EQUAL(tablePrecompiled->toString(), "Table"); +} + +BOOST_AUTO_TEST_CASE(call_select) +{ + storage::Condition::Ptr condition = std::make_shared(); + condition->EQ("name", "LiSi"); + auto conditionPrecompiled = std::make_shared(); + conditionPrecompiled->setCondition(condition); + Address conditionAddress = context->registerPrecompiled(conditionPrecompiled); + eth::ContractABI abi; + bytes in = abi.abiIn("select(string,address)", "name", conditionAddress); + bytes out = tablePrecompiled->call(context, bytesConstRef(&in)); + Address entriesAddress; + abi.abiOut(bytesConstRef(&out), entriesAddress); + auto entriesPrecompiled = + std::dynamic_pointer_cast(context->getPrecompiled(entriesAddress)); + auto entries = entriesPrecompiled->getEntries(); + BOOST_TEST(entries->size() == 0u); +} + +BOOST_AUTO_TEST_CASE(call_insert) +{ + auto entry = std::make_shared(); + entry->setField("name", "WangWu"); + auto entryPrecompiled = std::make_shared(); + entryPrecompiled->setEntry(entry); + + auto entryAddress = context->registerPrecompiled(entryPrecompiled); + eth::ContractABI abi; + bytes in = abi.abiIn("insert(string,address)", "name", entryAddress); + bytes out = tablePrecompiled->call(context, bytesConstRef(&in)); + u256 num; + abi.abiOut(bytesConstRef(&out), num); + BOOST_TEST(num == 1u); +} + +BOOST_AUTO_TEST_CASE(call_newCondition) +{ + eth::ContractABI abi; + bytes in = abi.abiIn("newCondition()"); + bytes out1 = tablePrecompiled->call(context, bytesConstRef(&in)); + Address address(++addressCount); + bytes out2 = abi.abiIn("", address); + BOOST_TEST(out1 == out2); +} + +BOOST_AUTO_TEST_CASE(call_newEntry) +{ + eth::ContractABI abi; + bytes in = abi.abiIn("newEntry()"); + bytes out1 = tablePrecompiled->call(context, bytesConstRef(&in)); + Address address(++addressCount); + bytes out2 = abi.abiIn("", address); + BOOST_CHECK(out1 == out2); +} + +BOOST_AUTO_TEST_CASE(call_remove) +{ + storage::Condition::Ptr condition = std::make_shared(); + condition->EQ("name", "LiSi"); + auto conditionPrecompiled = std::make_shared(); + conditionPrecompiled->setCondition(condition); + Address conditionAddress = context->registerPrecompiled(conditionPrecompiled); + eth::ContractABI abi; + bytes in = abi.abiIn("remove(string,address)", "name", conditionAddress); + bytes out = tablePrecompiled->call(context, bytesConstRef(&in)); + u256 num; + abi.abiOut(bytesConstRef(&out), num); + BOOST_TEST(num == 0u); +} + +BOOST_AUTO_TEST_CASE(call_update2) +{ + storage::Condition::Ptr condition = std::make_shared(); + condition->EQ("name", "LiSi"); + auto conditionPrecompiled = std::make_shared(); + conditionPrecompiled->setCondition(condition); + Address conditionAddress = context->registerPrecompiled(conditionPrecompiled); + auto entry = std::make_shared(); + auto entryPrecompiled = std::make_shared(); + entryPrecompiled->setEntry(entry); + auto entryAddress = context->registerPrecompiled(entryPrecompiled); + eth::ContractABI abi; + bytes in = abi.abiIn("update(string,address,address)", "name", entryAddress, conditionAddress); + bytes out = tablePrecompiled->call(context, bytesConstRef(&in)); + u256 num; + abi.abiOut(bytesConstRef(&out), num); + BOOST_TEST(num == 0u); +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace test_TablePrecompiled