diff --git a/constants.xml.native.tests b/constants.xml.native.tests deleted file mode 100644 index 099b921b58..0000000000 --- a/constants.xml.native.tests +++ /dev/null @@ -1,523 +0,0 @@ - - - - - 1 - false - false - 25 - 1 - 3 - - 03B70CF2ABEAE4E86DAEF1A36243E44CD61138B89055099C0D220B58FB86FF588A - - . - 250000 - true - 200 - false - 0 - 15 - 15360 - true - - - 1 - 1 - 2 - 1 - 1 - 1 - 1 - 1 - 1 - 1 - - - false - false - true - 5 - 100000 - 5 - 10 - false - 33135 - 20 - 2 - - - - txn - 27017 - false - - 1000 - 10000 - - - 0.667 - 10 - 10 - 20 - 2 - 1 - 80 - 1000 - - - true - 10 - 10 - 3 - 5 - 5 - 10 - 5 - - - false - 1 - - - - 2000 - 10 - 500 - 3000 - 500 - 180 - 80 - 500 - 60 - 20 - 5 - 3500 - 1000 - 55 - 5 - 3 - 5 - 5 - 10 - 3 - 3 - 12 - 10 - 1 - - - 21000 - 500000 - 400000 - 50 - 10 - 50 - 80 - 10 - 70 - 2000000000 - 2 - 10 - 5 - 15 - 10 - 5 - 127.0.0.1 - - - false - false - true - - 2 - 3 - 6 - - 10 - 10 - 1000 - 0 - 18 - true - true - - - - 0 - - 128 - 8192 - 0 - - - 128 - 8192 - 2 - 4 - - - - false - false - - 300 - 0.334 - 200 - - - 10 - - - - 4201 - 5 - - 4301 - 127.0.0.1 - true - /tmp/zilliqa.sock - /tmp/scilla-server.sock - scilla-server - false - 4401 - - false - 4501 - false - 2500 - 3 - 1000 - true - - 60 - - 0 - true - /home/stephen/dev/Zilliqa/evm-ds/target/release/evm-ds - /tmp/evm-server.sock - /usr/local/etc/log4rs.yml - - 1000000 - - 59 - - true - - true - - - - 200 - 5 - - 0.05 - - 3 - - 50 - 10 - 10 - - - 60 - 600 - 3 - 600 - 200 - 3 - 128 - 1 - 128 - 5000000 - 0 - 10000000 - 5 - 100 - 20 - 5 - 5 - - - true - false - false - http://127.0.0.1:4202/api - 15000 - 5 - - 5 - - false - 4202 - 5 - 3 - 8 - 32 - 2 - 1000 - - 99 - - 5 - 4 - - 1800 - - 1900 - 4 - 5 - - 120 - 10 - 2 - 3 - 360 - 5 - false - - - - 0 - 40 - 5 - true - 30 - 5 - false - 1 - 2 - 5 - 1 - - - true - /home/stephen/dev/scilla - bin/scilla-checker - bin/scilla-runner - scilla_files - _build - src/stdlib - init.json - input_state.json - input_blockchain.json - input_message.json - output.json - input - .scilla - .scillib - scilla_libs - false - false - false - 1500 - 10 - - - false - 100 - false - false - 0 - 0 - - - - 8400000000000000000000 - 176000000000000000 - 12600000000000000000000 - 20 - 40 - 76800 - 20 - 100 - 300 - 3600 - 50 - 0 - 1 - 1572864 - 1024 - 2000000 - true - 0 - 10 - 5 - xxxxxxxxxxx - txnsbackup - false - false - 0.000001 - 3 - - - - localhost - 8090 - - PROMETHEUS - 1000 - 500 - https://opentelemetry.io/schemas/1.2.0 - 1.2.0 - - NONE - - - - - 0.0.0.0 - 4318 - - OTLPHTTP - - NONE - - - - - localhost - 8090 - - NONE - - - - - - 175334 - 0 - - - 175637 - 0 - - - 175637 - 1 - - - 175637 - 2 - - - 175701 - 0 - - - 175701 - 1 - - - false - - - 1664279 - 0 - - - - - 10 - 15 - 10 - 600 - - - - - cc02a3c906612cc5bdb087a30e6093c9f0aa04fc - - - - - - ee06b3c906612cc5bdb087a30e6093c9f0aa04fd - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/libNetwork/P2PServer.cpp b/src/libNetwork/P2PServer.cpp index 7ac23be6f0..e9173e038e 100644 --- a/src/libNetwork/P2PServer.cpp +++ b/src/libNetwork/P2PServer.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include "Blacklist.h" #include "libUtils/Logger.h" @@ -202,9 +203,10 @@ std::shared_ptr P2PServer::CreateAndStart(AsioContext& asio, throw std::runtime_error(error_msg); } -P2PServerConnection::P2PServerConnection( - std::weak_ptr owner, uint64_t this_id, Peer&& remote_peer, - TcpSocket socket, size_t max_message_size) +P2PServerConnection::P2PServerConnection(std::weak_ptr owner, + uint64_t this_id, Peer&& remote_peer, + TcpSocket socket, + size_t max_message_size) : m_owner(std::move(owner)), m_id(this_id), m_remotePeer(std::move(remote_peer)), @@ -229,6 +231,9 @@ void P2PServerConnection::ReadNextMessage() { if (!ec) { assert(n == HDR_LEN); } + if (ec) { + // LOG_GENERAL(WARNING, "Got error code: " << ec.message()); + } if (ec != OPERATION_ABORTED) { self->OnHeaderRead(ec); } @@ -237,10 +242,9 @@ void P2PServerConnection::ReadNextMessage() { void P2PServerConnection::OnHeaderRead(const ErrorCode& ec) { if (ec) { - if (ec != END_OF_FILE) { - LOG_GENERAL(INFO, - "Peer " << m_remotePeer << " read error: " << ec.message()); - } + LOG_GENERAL(INFO, + "Peer " << m_remotePeer << " read error: " << ec.message()); + CloseSocket(); OnConnectionClosed(); return; } @@ -254,7 +258,9 @@ void P2PServerConnection::OnHeaderRead(const ErrorCode& ec) { << " Adding sending node " << m_remotePeer.GetPrintableIPAddress() << " as strictly blacklisted"); - Blacklist::GetInstance().Add({m_remotePeer.GetIpAddress(),m_remotePeer.GetListenPortHost(),m_remotePeer.GetNodeIndentifier()}); + Blacklist::GetInstance().Add({m_remotePeer.GetIpAddress(), + m_remotePeer.GetListenPortHost(), + m_remotePeer.GetNodeIndentifier()}); CloseSocket(); OnConnectionClosed(); @@ -269,6 +275,9 @@ void P2PServerConnection::OnHeaderRead(const ErrorCode& ec) { if (!ec) { assert(n == self->m_readBuffer.size() - HDR_LEN); } + if (ec) { + // LOG_GENERAL(WARNING, "Got error code: " << ec.message()); + } if (ec != OPERATION_ABORTED) { self->OnBodyRead(ec); } @@ -278,17 +287,19 @@ void P2PServerConnection::OnHeaderRead(const ErrorCode& ec) { void P2PServerConnection::OnBodyRead(const ErrorCode& ec) { if (ec) { LOG_GENERAL(INFO, "Read error: " << ec.message()); + CloseSocket(); OnConnectionClosed(); return; } - ReadMessageResult result; auto state = TryReadMessage(m_readBuffer.data(), m_readBuffer.size(), result); if (state != ReadState::SUCCESS) { LOG_GENERAL(WARNING, "Message deserialize error: blacklisting " << m_remotePeer.GetPrintableIPAddress()); - Blacklist::GetInstance().Add({m_remotePeer.GetIpAddress(),m_remotePeer.GetListenPortHost(),m_remotePeer.GetNodeIndentifier()}); + Blacklist::GetInstance().Add({m_remotePeer.GetIpAddress(), + m_remotePeer.GetListenPortHost(), + m_remotePeer.GetNodeIndentifier()}); CloseSocket(); OnConnectionClosed(); @@ -298,6 +309,7 @@ void P2PServerConnection::OnBodyRead(const ErrorCode& ec) { auto owner = m_owner.lock(); if (!owner || !owner->OnMessage(m_id, m_remotePeer, result)) { CloseSocket(); + OnConnectionClosed(); return; } @@ -311,8 +323,44 @@ void P2PServerConnection::Close() { void P2PServerConnection::CloseSocket() { ErrorCode ec; + // both close and shutdown should be none blocking calls certainly on current linux + // shutdown marks the socket as blocked for both read and write + // shutdown tells the OS to begin the graceful closedown of the TCP connection. + // close() is a blocking call that waits for the OS to complete the closedown. + // close also frees the OS resources from the program so should be called even if + // an error condition is encountered m_socket.shutdown(boost::asio::socket_base::shutdown_both, ec); + if (ec) { + m_socket.close(ec); + if (ec) { + LOG_GENERAL(INFO, "Informational, not an issue - Error closing socket: " << ec.message()); + } + return; + } + size_t unread = m_socket.available(ec); + if (ec) { + m_socket.close(ec); + return; + } + // On Linux, the close() function for sockets does not necessarily wait + // for the operating system to complete the operation before returning. + // It typically marks the socket as closed and releases any resources + // associated with it immediately. However, this does not guarantee that all + // pending data has been sent or received. It is important to handle any + // necessary error checking and ensure all data transmission is complete + // before calling close() on a socket. + if (unread > 0) { + do { + boost::container::small_vector buf; + buf.resize(unread); + m_socket.read_some(boost::asio::mutable_buffer(buf.data(), unread), ec); + LOG_GENERAL(INFO, "Draining remaining IO before close" << m_remotePeer.GetPrintableIPAddress()); + } while (!ec && (unread = m_socket.available(ec)) > 0); + } m_socket.close(ec); + if (ec) { + LOG_GENERAL(INFO, "Informational, not an issue - Error closing socket: " << ec.message()); + } } void P2PServerConnection::OnConnectionClosed() { diff --git a/src/libNetwork/SendJobs.cpp b/src/libNetwork/SendJobs.cpp index 3eeff91fa7..b55bfd21f8 100644 --- a/src/libNetwork/SendJobs.cpp +++ b/src/libNetwork/SendJobs.cpp @@ -215,6 +215,8 @@ class GracefulCloseImpl LOG_GENERAL(DEBUG, "Expected EOF, got ec=" << ec.message() << " n=" << n); } + ErrorCode ignored; + self->m_socket.close(ignored); }); } }; @@ -226,10 +228,12 @@ void CloseGracefully(Socket&& socket) { } socket.shutdown(boost::asio::socket_base::shutdown_both, ec); if (ec) { + socket.close(ec); return; } size_t unread = socket.available(ec); if (ec) { + socket.close(ec); return; } if (unread > 0) { @@ -239,6 +243,8 @@ void CloseGracefully(Socket&& socket) { } if (!ec) { std::make_shared(std::move(socket))->Close(); + } else { + socket.close(ec); } } @@ -378,6 +384,9 @@ class PeerSendQueue : public std::enable_shared_from_this { boost::asio::async_write( m_socket, boost::asio::const_buffer(msg.data.get(), msg.size), [self = shared_from_this()](const ErrorCode& ec, size_t) { + if (ec) { + // LOG_GENERAL(WARNING, "Got error code: " << ec.message()); + } if (ec != OPERATION_ABORTED) { self->OnWritten(ec); } @@ -385,6 +394,11 @@ class PeerSendQueue : public std::enable_shared_from_this { } void OnWritten(const ErrorCode& ec) { + if (!ec && !m_closed) { + // LOG_GENERAL(INFO, "Successfully sent message to: " + // << m_peer.GetPrintableIPAddress() + // << " with size: " << m_queue.front().msg.size); + } if (m_closed) { return; } @@ -415,11 +429,12 @@ class PeerSendQueue : public std::enable_shared_from_this { return; } - WaitTimer(m_timer, Milliseconds{RECONNECT_INTERVAL_IN_MS}, [this]() { Reconnect(); }); + WaitTimer(m_timer, Milliseconds{RECONNECT_INTERVAL_IN_MS}, + [this]() { Reconnect(); }); } void Reconnect() { - LOG_GENERAL(DEBUG, "Peer " << m_peer << " reconnects"); + LOG_GENERAL(INFO, "Peer " << m_peer << " reconnects"); CloseGracefully(std::move(m_socket)); m_socket = Socket(m_asioContext); Connect(); @@ -493,7 +508,8 @@ class SendJobsImpl : public SendJobs, return; } - LOG_GENERAL(DEBUG, "Enqueueing message, size=" << message.size); + LOG_GENERAL(DEBUG, "Enqueueing message, size=" << message.size + << " peer = " << peer); // this fn enqueues the lambda to be executed on WorkerThread with // sequential guarantees for messages from every calling thread diff --git a/src/libScilla/UnixDomainSocketServer.cpp b/src/libScilla/UnixDomainSocketServer.cpp index 194e8cc38c..0d51051630 100644 --- a/src/libScilla/UnixDomainSocketServer.cpp +++ b/src/libScilla/UnixDomainSocketServer.cpp @@ -96,10 +96,12 @@ void UnixDomainSocketServer::WorkerThread() { boost::system::error_code ec; m_acceptor->accept(socket, ec); if (ec && m_started) { + socket.close(ec); throw std::runtime_error(ec.message()); } if (!m_started) { + socket.close(ec); break; } @@ -115,6 +117,7 @@ void UnixDomainSocketServer::WorkerThread() { } if (!m_started) { + socket.close(ec); break; } @@ -130,6 +133,7 @@ void UnixDomainSocketServer::WorkerThread() { } if (!m_started) { + socket.close(ec); break; } @@ -147,6 +151,7 @@ void UnixDomainSocketServer::WorkerThread() { if (ec) { LOG_GENERAL(WARNING, "Write to " << m_path << " failed: " << ec.message()); + socket.close(ec); continue; } @@ -154,6 +159,7 @@ void UnixDomainSocketServer::WorkerThread() { boost::asio::local::stream_protocol::socket::shutdown_both, ec); if (ec) { LOG_GENERAL(WARNING, "Shutdown failed: " << ec.message()); + socket.close(ec); continue; } } diff --git a/src/libServer/APIServerImpl.cpp b/src/libServer/APIServerImpl.cpp index 42f51594b6..95817d5b63 100644 --- a/src/libServer/APIServerImpl.cpp +++ b/src/libServer/APIServerImpl.cpp @@ -93,6 +93,8 @@ class APIServerImpl::Connection if (!owner) { // server is closed m_stream.socket().shutdown(tcp::socket::shutdown_both, ec); + m_stream.socket().close(ec); + OnClosed(); return; } @@ -101,6 +103,8 @@ class APIServerImpl::Connection LOG_GENERAL(DEBUG, "Read error: " << ec.message()); m_stream.socket().shutdown(tcp::socket::shutdown_both, ec); } + m_stream.socket().shutdown(tcp::socket::shutdown_both, ec); + m_stream.socket().close(ec); OnClosed(); return; } @@ -205,6 +209,8 @@ class APIServerImpl::Connection if (sz == 0 && m_owner.expired()) { beast::error_code ec; m_stream.socket().shutdown(tcp::socket::shutdown_both, ec); + m_stream.socket().close(ec); + OnClosed(); return; } @@ -346,7 +352,7 @@ bool APIServerImpl::DoListen() { #define CHECK_EC() \ if (ec) { \ - LOG_GENERAL(FATAL, "Cannot start API server: " << ec.message()); \ + LOG_GENERAL(FATAL, "Cannot start API server: " << ec.message() << " port " << m_options.port); \ return false; \ } diff --git a/src/libServer/LocalAPIServer.cpp b/src/libServer/LocalAPIServer.cpp index 23e6244a9b..6b1f5afafb 100644 --- a/src/libServer/LocalAPIServer.cpp +++ b/src/libServer/LocalAPIServer.cpp @@ -46,7 +46,7 @@ bool LocalAPIServer::StartListening() { #define CHECK_EC() \ if (ec) { \ - LOG_GENERAL(WARNING, "Cannot start API server: " << ec.message()); \ + LOG_GENERAL(WARNING, "Cannot start API server: " << ec.message() << " port =" << m_port); \ return false; \ } @@ -117,10 +117,12 @@ void LocalAPIServer::WorkerThread() { boost::system::error_code ec; m_acceptor->accept(socket, ec); if (ec && m_started) { + socket.close(ec); throw std::runtime_error(ec.message()); } if (!m_started) { + socket.close(ec); break; } @@ -131,10 +133,12 @@ void LocalAPIServer::WorkerThread() { DEFAULT_DELIMITER_CHAR, ec); if (ec || n <= 1) { LOG_GENERAL(WARNING, "Read (" << m_ip << ") failed: " << ec.message()); + socket.close(ec); continue; } if (!m_started) { + socket.close(ec); break; } @@ -150,6 +154,7 @@ void LocalAPIServer::WorkerThread() { } if (!m_started) { + socket.close(ec); break; } @@ -166,6 +171,7 @@ void LocalAPIServer::WorkerThread() { boost::asio::write(socket, boost::asio::buffer(response), ec); if (ec) { LOG_GENERAL(WARNING, "Write (" << m_ip << ") failed: " << ec.message()); + socket.close(ec); continue; } @@ -173,8 +179,8 @@ void LocalAPIServer::WorkerThread() { boost::asio::local::stream_protocol::socket::shutdown_both, ec); if (ec) { LOG_GENERAL(WARNING, "Shutdown failed: " << ec.message()); - continue; } + socket.close(ec); } } catch (const std::exception& e) { LOG_GENERAL(WARNING, "Listening to " << m_ip << " failed: " << e.what()); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e9c5cd25d..4973ad389d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,11 +21,15 @@ add_subdirectory (Server) add_subdirectory (EvmFiltersAPI) add_subdirectory (EthRpcMethods) add_subdirectory (Utils) -#add_subdirectory (Zilliqa) +add_subdirectory (Zilliqa) +add_subdirectory (native) #add_subdirectory (RemoteStorageDB) Works only if you have a local mongo server running file(COPY ${CMAKE_SOURCE_DIR}/constants_local.xml DESTINATION ${CMAKE_BINARY_DIR}) -#FIXME: constants.xml is not used for local_run, thus shouldn't be copied # presently, it's a workaround to silence the error thrown by tests_zilliqa_local.py file(COPY ${CMAKE_SOURCE_DIR}/constants.xml DESTINATION ${CMAKE_BINARY_DIR}) file(COPY Node DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) +file(COPY ${CMAKE_SOURCE_DIR}/tests/Zilliqa/test_zilliqa_local.py DESTINATION ${CMAKE_SOURCE_DIR}/build/tests/zilliqa) +file(COPY ${CMAKE_SOURCE_DIR}/tests/Zilliqa/test_zilliqa_lookup.py DESTINATION ${CMAKE_SOURCE_DIR}/build/tests/zilliqa) +file(COPY ${CMAKE_SOURCE_DIR}/tests/Zilliqa/test_zilliqa_seedpub.py DESTINATION ${CMAKE_SOURCE_DIR}/build/tests/zilliqa) +file(COPY ${CMAKE_SOURCE_DIR}/tests/Zilliqa/test_zilliqa_late.py DESTINATION ${CMAKE_SOURCE_DIR}/build/tests/zilliqa) \ No newline at end of file diff --git a/tests/Node/pre_run.sh b/tests/Node/pre_run.sh index b61b4fbeee..19ff696de8 100755 --- a/tests/Node/pre_run.sh +++ b/tests/Node/pre_run.sh @@ -18,23 +18,18 @@ rm -rf local_run/node* -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ulimit -s unlimited; -python tests/Zilliqa/test_zilliqa_local.py stop -python tests/Zilliqa/test_zilliqa_local.py setup 10 -python tests/Zilliqa/test_zilliqa_local.py prestart 5 +python tests/zilliqa/test_zilliqa_local.py stop +python tests/zilliqa/test_zilliqa_local.py setup 10 +python tests/zilliqa/test_zilliqa_local.py prestart 5 # clean up persistence storage rm -rf lookup_local_run/node* -python tests/Zilliqa/test_zilliqa_lookup.py setup 1 +python tests/zilliqa/test_zilliqa_lookup.py setup 1 +#python tests/zilliqa/test_zilliqa_seedpub.py setup 1 diff --git a/tests/Node/pre_run_guard.sh b/tests/Node/pre_run_guard.sh index d51b3d11ca..61168a18ee 100755 --- a/tests/Node/pre_run_guard.sh +++ b/tests/Node/pre_run_guard.sh @@ -18,12 +18,6 @@ rm -rf local_run/node* -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; diff --git a/tests/Node/test_node_NAT.sh b/tests/Node/test_node_NAT.sh index 54a74543be..af817303e8 100755 --- a/tests/Node/test_node_NAT.sh +++ b/tests/Node/test_node_NAT.sh @@ -14,12 +14,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; python tests/Zilliqa/test_zilliqa_NAT.py setup 5 python tests/Zilliqa/test_zilliqa_NAT.py start diff --git a/tests/Node/test_node_late.sh b/tests/Node/test_node_late.sh index a9d73e9504..2766228679 100755 --- a/tests/Node/test_node_late.sh +++ b/tests/Node/test_node_late.sh @@ -14,12 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; + python tests/Zilliqa/test_zilliqa_late.py setup 5 python tests/Zilliqa/test_zilliqa_late.py start diff --git a/tests/Node/test_node_lookup.sh b/tests/Node/test_node_lookup.sh index d6943cbc11..c00a86094a 100755 --- a/tests/Node/test_node_lookup.sh +++ b/tests/Node/test_node_lookup.sh @@ -14,13 +14,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; -python tests/Zilliqa/test_zilliqa_lookup.py start + +python tests/zilliqa/test_zilliqa_lookup.py start echo "end" diff --git a/tests/Node/test_node_rejoindsguardnode2.sh b/tests/Node/test_node_rejoindsguardnode2.sh index eefe780b4e..e6b4f9359b 100755 --- a/tests/Node/test_node_rejoindsguardnode2.sh +++ b/tests/Node/test_node_rejoindsguardnode2.sh @@ -14,12 +14,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; + python tests/Zilliqa/test_zilliqa_local.py setupdsguard2 1 python tests/Zilliqa/test_zilliqa_local.py rejoindsguard2 diff --git a/tests/Node/test_node_seedpub.sh b/tests/Node/test_node_seedpub.sh new file mode 100755 index 0000000000..e41bf08152 --- /dev/null +++ b/tests/Node/test_node_seedpub.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (C) 2019 Zilliqa +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + + +python tests/zilliqa/test_zilliqa_seedpub.py start + +echo "end" diff --git a/tests/Node/test_node_server.sh b/tests/Node/test_node_server.sh index f627f524c5..cc8618396b 100755 --- a/tests/Node/test_node_server.sh +++ b/tests/Node/test_node_server.sh @@ -15,12 +15,7 @@ # along with this program. If not, see . # This script will run 20 DS and 60 nodes. This script should be run on a high performance machine. -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; + python tests/Zilliqa/test_zilliqa_local.py stop python tests/Zilliqa/test_zilliqa_local.py clean diff --git a/tests/Node/test_node_simple.sh b/tests/Node/test_node_simple.sh index c983f6bc45..6c8da5db52 100755 --- a/tests/Node/test_node_simple.sh +++ b/tests/Node/test_node_simple.sh @@ -15,18 +15,13 @@ # along with this program. If not, see . -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; + ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ulimit -s unlimited; -python tests/Zilliqa/test_zilliqa_local.py start 5 +python tests/zilliqa/test_zilliqa_local.py start 5 sleep 40 echo "starting..." diff --git a/tests/Node/test_node_validateBackupDB.sh b/tests/Node/test_node_validateBackupDB.sh index af23366d45..59ede3b33f 100755 --- a/tests/Node/test_node_validateBackupDB.sh +++ b/tests/Node/test_node_validateBackupDB.sh @@ -15,12 +15,7 @@ # along with this program. If not, see . -sudo sysctl net.core.somaxconn=102400; -sudo sysctl net.core.netdev_max_backlog=65536; -sudo sysctl net.ipv4.tcp_tw_reuse=1; -sudo sysctl -w net.ipv4.tcp_rmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_wmem='65536 873800 1534217728'; -sudo sysctl -w net.ipv4.tcp_mem='65536 873800 1534217728'; + ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; diff --git a/tests/Zilliqa/daemon_restart_local.py b/tests/Zilliqa/daemon_restart_local.py index 30ee039b48..9458e0a88f 100755 --- a/tests/Zilliqa/daemon_restart_local.py +++ b/tests/Zilliqa/daemon_restart_local.py @@ -66,7 +66,7 @@ def run_restart(pubKey, privKey, port, typ, path, name): for x in range(0, 1): keypair = keypairs[x].split(" ") - os.system('cd ' + path + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/' + name + ' --privk ' + keypair[0] + ' --pubk ' + keypair[1] + ' --address ' + nodeIP + ' --port ' + port + ' --synctype ' + typ + ' --recovery >> ./error_log_zilliqa 2>&1 &') + os.system('cd ' + path + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/' + name + ' --privk ' + keypair[0] + ' --pubk ' + keypair[1] + ' --address ' + nodeIP + ' --port ' + port + ' --synctype ' + typ + ' --recovery '+ ' --identity normal-' + str(x) + ' >> ./error_log_zilliqa 2>&1 &') if __name__ == "__main__": main() diff --git a/tests/Zilliqa/test_zilliqa_NAT.py b/tests/Zilliqa/test_zilliqa_NAT.py index eb7df339e6..d80ff23567 100755 --- a/tests/Zilliqa/test_zilliqa_NAT.py +++ b/tests/Zilliqa/test_zilliqa_NAT.py @@ -68,7 +68,7 @@ def run_setup(numnodes, printnodes): for x in range(0, numnodes): testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x+1).zfill(4) os.makedirs(testsubdir) - shutil.copyfile('./tests/Zilliqa/zilliqa', testsubdir + '/latezilliqa') + shutil.copyfile('./bin/zilliqa', testsubdir + '/latezilliqa') st = os.stat(testsubdir + '/latezilliqa') os.chmod(testsubdir + '/latezilliqa', st.st_mode | stat.S_IEXEC) @@ -84,9 +84,22 @@ def run_start(): count = len(testfolders_list) keypairs = [] + dev_root = os.getenv("DEV_TREE_ROOT") + if dev_root is None: + print("DEV_TREE_ROOT is not set") + return + + dev_root += "/Zilliqa" + fp = dev_root + "/" + "constants_local.xml.native" + + if not os.path.exists(fp): + print( fp +" not found") + return + + # Generate keypairs (sort by public key) for x in range(0, count): - process = Popen(["./tests/Zilliqa/genkeypair"], stdout=PIPE, universal_newlines=True) + process = Popen(["./bin/genkeypair"], stdout=PIPE, universal_newlines=True) (output, err) = process.communicate() exit_code = process.wait() keypairs.append(output.strip()) @@ -97,7 +110,7 @@ def run_start(): for x in range(0, count): keys_file.write(keypairs[x] + '\n') shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') - shutil.copyfile('constants_local.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + shutil.copyfile(fp, LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') keys_file.close() @@ -105,7 +118,7 @@ def run_start(): for x in range(0, count): keypair = keypairs[x].split(" ") os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ./latezilliqa ' + keypair[1] + ' ' + keypair[0] + ' ' + 'NAT' + ' ' + str(NODE_LISTEN_PORT + x) + ' 0 1 0 > ./error_log_zilliqa 2>&1 &') - os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ./latezilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + 'NAT' + ' --port ' + str(NODE_LISTEN_PORT + x) + '--synctype 1 ' + '> ./error_log_zilliqa 2>&1 &') + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ./latezilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + 'NAT' + ' --port ' + str(NODE_LISTEN_PORT + x) + '--synctype 1 ' + ' --identity normal-' + str(x) + ' > ./error_log_zilliqa 2>&1 &') if __name__ == "__main__": diff --git a/tests/Zilliqa/test_zilliqa_late.py b/tests/Zilliqa/test_zilliqa_late.py index 5fac33ac8c..f7351f19b2 100755 --- a/tests/Zilliqa/test_zilliqa_late.py +++ b/tests/Zilliqa/test_zilliqa_late.py @@ -67,7 +67,7 @@ def run_setup(numnodes, printnodes): for x in range(0, numnodes): testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x+1).zfill(4) os.makedirs(testsubdir) - shutil.copyfile('./tests/Zilliqa/zilliqa', testsubdir + '/latezilliqa') + shutil.copyfile('./bin/zilliqa', testsubdir + '/latezilliqa') st = os.stat(testsubdir + '/latezilliqa') os.chmod(testsubdir + '/latezilliqa', st.st_mode | stat.S_IEXEC) @@ -85,7 +85,7 @@ def run_start(): # Generate keypairs (sort by public key) for x in range(0, count): - process = Popen(["./tests/Zilliqa/genkeypair"], stdout=PIPE, universal_newlines=True) + process = Popen(["./bin/genkeypair"], stdout=PIPE, universal_newlines=True) (output, err) = process.communicate() exit_code = process.wait() keypairs.append(output.strip()) @@ -95,14 +95,14 @@ def run_start(): keys_file = open(LOCAL_RUN_FOLDER + 'keys.txt', "w") for x in range(0, count): keys_file.write(keypairs[x] + '\n') - shutil.copyfile('constants_local.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + shutil.copyfile('constants_local.xml.native', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') keys_file.close() # Launch node zilliqa process for x in range(0, count): keypair = keypairs[x].split(" ") - os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/latezilliqa' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' --synctype 1 ' + '> ./error_log_zilliqa 2>&1 &') + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/latezilliqa' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' --synctype 1 ' + ' --identity normal-' + str(x) + ' > ./error_log_zilliqa 2>&1 &') if __name__ == "__main__": main() diff --git a/tests/Zilliqa/test_zilliqa_local.py b/tests/Zilliqa/test_zilliqa_local.py index 58e5f40c7e..b6bd404814 100755 --- a/tests/Zilliqa/test_zilliqa_local.py +++ b/tests/Zilliqa/test_zilliqa_local.py @@ -19,7 +19,7 @@ import sys import shutil import stat -import time +import time from subprocess import Popen, PIPE @@ -30,382 +30,446 @@ LOCAL_RUN_FOLDER = './local_run/' REJOIN_DS_GUARD_RUN_FOLDER = './dsguard_rejoin_local_run/' + def print_usage(): - print ("Testing multiple Zilliqa nodes in local machine\n" - "===============================================\n" - "Usage:\n\tpython " + sys.argv[0] + " [command] [command parameters]\n" - "Available commands:\n" - "\tTest Execution:\n" - "\t\tsetup [num-nodes] - Set up the nodes\n" - "\t\tsetupdsguard2 [num-nodes] - Start ds guard 2 executable and folder\n" - "\t\tstart [num-nodes] - Start node processes\n" - "\t\trejoindsguard2 - Start ds guard 2 and rejoin the network\n" - "\t\tvalidateBackupDB - Validate DB correctness\n" - "\t\tconnect - Connect everyone\n" - "\t\tconnect [num-nodes] - Connect first num-nodes nodes\n" - "\t\tstop - Stop node processes\n" - "\t\tclean - Remove test output files (e.g., logs)\n" - "\t\tsendcmd [nodenum] [hex msg] - Send hex msg to port\n" - "\t\tsendcmdrandom [nodenum] [msgsize] - Send msg to port\n" - "\t\tstartpow [nodenum] [ds count] [blocknum] [diff] [rand1] [rand2] - Send STARTPOW to node\n" - "\t\tcreatetx [nodenum] [from] [to] [amount] - Send CREATETRANSACTION to node\n" - "\t\tdelete - Delete the set-up nodes\n") + print("Testing multiple Zilliqa nodes in local machine\n" + "===============================================\n" + "Usage:\n\tpython " + sys.argv[0] + " [command] [command parameters]\n" + "Available commands:\n" + "\tTest Execution:\n" + "\t\tsetup [num-nodes] - Set up the nodes\n" + "\t\tsetupdsguard2 [num-nodes] - Start ds guard 2 executable and folder\n" + "\t\tstart [num-nodes] - Start node processes\n" + "\t\trejoindsguard2 - Start ds guard 2 and rejoin the network\n" + "\t\tvalidateBackupDB - Validate DB correctness\n" + "\t\tconnect - Connect everyone\n" + "\t\tconnect [num-nodes] - Connect first num-nodes nodes\n" + "\t\tstop - Stop node processes\n" + "\t\tclean - Remove test output files (e.g., logs)\n" + "\t\tsendcmd [nodenum] [hex msg] - Send hex msg to port\n" + "\t\tsendcmdrandom [nodenum] [msgsize] - Send msg to port\n" + "\t\tstartpow [nodenum] [ds count] [blocknum] [diff] [rand1] [rand2] - Send STARTPOW to node\n" + "\t\tcreatetx [nodenum] [from] [to] [amount] - Send CREATETRANSACTION to node\n" + "\t\tdelete - Delete the set-up nodes\n") + def main(): - numargs = len(sys.argv) - if (numargs < 2): - print_usage() - else: - command = sys.argv[1] - if (command == 'setup'): - print_usage() if (numargs != 3) else run_setup(numnodes=int(sys.argv[2]), printnodes=True) - elif (command == 'setupdsguard2'): - print_usage() if (numargs != 3) else run_setup_dsguard(numnodes=int(sys.argv[2]), printnodes=True) - elif(command == 'prestart'): - print_usage() if (numargs != 3) else run_prestart(numdsnodes=int(sys.argv[2])) - elif(command == 'prestartguard'): - print_usage() if (numargs != 3) else run_prestart(numdsnodes=int(sys.argv[2]), guard_mode=True) - elif (command == 'start'): - print_usage() if (numargs != 3) else run_start(numdsnodes=int(sys.argv[2])) - elif (command == 'rejoindsguard2'): - print_usage() if (numargs != 2) else run_start_dsguard2() - elif (command == 'validateBackupDB'): - print_usage() if (numargs != 2) else run_start_validateBackupDB() - elif (command == 'connect'): - if (numargs == 2): - run_connect(numnodes=0) - elif (numargs == 3): - run_connect(numnodes=int(sys.argv[2])) - else: - print_usage() - elif (command == 'stop'): - print_usage() if (numargs != 2) else run_stop() - elif (command == 'clean'): - print_usage() if (numargs != 2) else run_clean() - elif (command == 'sendcmd'): - print_usage() if (numargs != 4) else run_sendcmd(nodenum=int(sys.argv[2]), msg=sys.argv[3]) - elif (command == 'sendcmdrandom'): - print_usage() if (numargs != 4) else run_sendcmdrandom(nodenum=int(sys.argv[2]), msg_size=sys.argv[3]) - elif (command == 'startpow'): - print_usage() if (numargs != 9) else run_startpow(nodenum=int(sys.argv[2]), dscount=int(sys.argv[3]), blocknum=sys.argv[4], dsdiff=sys.argv[5], diff=sys.argv[6], rand1=sys.argv[7], rand2=sys.argv[8]) - elif (command == 'createtx'): - print_usage() if (numargs != 6) else run_createtx(nodenum=int(sys.argv[2]), fromnode=int(sys.argv[3]), tonode=int(sys.argv[4]), amount=int(sys.argv[5])) - elif (command == 'delete'): - print_usage() if (numargs != 2) else run_delete() - else: - print_usage() + numargs = len(sys.argv) + if (numargs < 2): + print_usage() + else: + command = sys.argv[1] + if (command == 'setup'): + print_usage() if (numargs != 3) else run_setup(numnodes=int(sys.argv[2]), printnodes=True) + elif (command == 'setupdsguard2'): + print_usage() if (numargs != 3) else run_setup_dsguard(numnodes=int(sys.argv[2]), printnodes=True) + elif (command == 'prestart'): + print_usage() if (numargs != 3) else run_prestart(numdsnodes=int(sys.argv[2])) + elif (command == 'prestartguard'): + print_usage() if (numargs != 3) else run_prestart(numdsnodes=int(sys.argv[2]), guard_mode=True) + elif (command == 'start'): + print_usage() if (numargs != 3) else run_start(numdsnodes=int(sys.argv[2])) + elif (command == 'rejoindsguard2'): + print_usage() if (numargs != 2) else run_start_dsguard2() + elif (command == 'validateBackupDB'): + print_usage() if (numargs != 2) else run_start_validateBackupDB() + elif (command == 'connect'): + if (numargs == 2): + run_connect(numnodes=0) + elif (numargs == 3): + run_connect(numnodes=int(sys.argv[2])) + else: + print_usage() + elif (command == 'stop'): + print_usage() if (numargs != 2) else run_stop() + elif (command == 'clean'): + print_usage() if (numargs != 2) else run_clean() + elif (command == 'sendcmd'): + print_usage() if (numargs != 4) else run_sendcmd(nodenum=int(sys.argv[2]), msg=sys.argv[3]) + elif (command == 'sendcmdrandom'): + print_usage() if (numargs != 4) else run_sendcmdrandom(nodenum=int(sys.argv[2]), msg_size=sys.argv[3]) + elif (command == 'startpow'): + print_usage() if (numargs != 9) else run_startpow(nodenum=int(sys.argv[2]), dscount=int(sys.argv[3]), + blocknum=sys.argv[4], dsdiff=sys.argv[5], + diff=sys.argv[6], rand1=sys.argv[7], rand2=sys.argv[8]) + elif (command == 'createtx'): + print_usage() if (numargs != 6) else run_createtx(nodenum=int(sys.argv[2]), fromnode=int(sys.argv[3]), + tonode=int(sys.argv[4]), amount=int(sys.argv[5])) + elif (command == 'delete'): + print_usage() if (numargs != 2) else run_delete() + else: + print_usage() + # ================ # Helper Functions # ================ def get_immediate_subdirectories(a_dir): - subdirs = [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))] - subdirs.sort() - return subdirs + subdirs = [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))] + subdirs.sort() + return subdirs + # ======================== # Test Execution Functions # ======================== def run_setup(numnodes, printnodes): - if os.path.exists(LOCAL_RUN_FOLDER) != True : - # shutil.rmtree(LOCAL_RUN_FOLDER) - os.makedirs(LOCAL_RUN_FOLDER) - for x in range(0, numnodes): - testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x+1).zfill(4) - if os.path.exists(testsubdir) != True : - os.makedirs(testsubdir) - shutil.copyfile('./tests/Zilliqa/zilliqa', testsubdir + '/zilliqa') - shutil.copyfile('./tests/Zilliqa/sendcmd', testsubdir + '/sendcmd') - - st = os.stat(testsubdir + '/zilliqa') - os.chmod(testsubdir + '/zilliqa', st.st_mode | stat.S_IEXEC) - st = os.stat(testsubdir + '/sendcmd') - os.chmod(testsubdir + '/sendcmd', st.st_mode | stat.S_IEXEC) - - if printnodes: - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - for x in range(0, count): - print ('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(NODE_LISTEN_PORT + x) - + '] [Status Server Port '+ str(STATUS_SERVER_LISTEN_PORT + x) - + '] ' + LOCAL_RUN_FOLDER + testfolders_list[x]) + if os.path.exists(LOCAL_RUN_FOLDER) != True: + # shutil.rmtree(LOCAL_RUN_FOLDER) + os.makedirs(LOCAL_RUN_FOLDER) + for x in range(0, numnodes): + testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x + 1).zfill(4) + if os.path.exists(testsubdir) != True: + os.makedirs(testsubdir) + shutil.copyfile('./bin/zilliqa', testsubdir + '/zilliqa') + shutil.copyfile('./bin/sendcmd', testsubdir + '/sendcmd') + ''' + We probably need to configure the S3 settings for download too + ''' + shutil.copyfile('../scripts/download_incr_DB.py', testsubdir + '/download_incr_DB.py') + shutil.copyfile('../scripts/download_static_DB.py', testsubdir + '/download_static_DB.py') + shutil.copyfile('../scripts/upload_incr_DB.py', testsubdir + '/upload_incr_DB.py') + shutil.copyfile('../scripts/auto_backup.py', testsubdir + '/auto_backup.py') + + st = os.stat(testsubdir + '/zilliqa') + os.chmod(testsubdir + '/zilliqa', st.st_mode | stat.S_IEXEC) + st = os.stat(testsubdir + '/sendcmd') + os.chmod(testsubdir + '/sendcmd', st.st_mode | stat.S_IEXEC) + + if printnodes: + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + for x in range(0, count): + print('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(NODE_LISTEN_PORT + x) + + '] [Status Server Port ' + str(STATUS_SERVER_LISTEN_PORT + x) + + '] ' + LOCAL_RUN_FOLDER + testfolders_list[x]) + def run_setup_dsguard(numnodes, printnodes): - if (os.path.exists(REJOIN_DS_GUARD_RUN_FOLDER)): - shutil.rmtree(REJOIN_DS_GUARD_RUN_FOLDER) - os.makedirs(REJOIN_DS_GUARD_RUN_FOLDER) - for x in range(0, numnodes): - testsubdir = REJOIN_DS_GUARD_RUN_FOLDER + 'node_' + str(x+1).zfill(4) - os.makedirs(testsubdir) - shutil.copyfile('./tests/Zilliqa/zilliqa', testsubdir + '/zilliqa_ds_guard_rejoin') - - st = os.stat(testsubdir + '/zilliqa_ds_guard_rejoin') - os.chmod(testsubdir + '/zilliqa_ds_guard_rejoin', st.st_mode | stat.S_IEXEC) - - if printnodes: - testfolders_list = get_immediate_subdirectories(REJOIN_DS_GUARD_RUN_FOLDER) - count = len(testfolders_list) - for x in range(0, count): - print ('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(7001) + '] ' + REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x]) + if (os.path.exists(REJOIN_DS_GUARD_RUN_FOLDER)): + shutil.rmtree(REJOIN_DS_GUARD_RUN_FOLDER) + os.makedirs(REJOIN_DS_GUARD_RUN_FOLDER) + for x in range(0, numnodes): + testsubdir = REJOIN_DS_GUARD_RUN_FOLDER + 'node_' + str(x + 1).zfill(4) + os.makedirs(testsubdir) + shutil.copyfile('./bin/zilliqa', testsubdir + '/zilliqa_ds_guard_rejoin') + + st = os.stat(testsubdir + '/zilliqa_ds_guard_rejoin') + os.chmod(testsubdir + '/zilliqa_ds_guard_rejoin', st.st_mode | stat.S_IEXEC) + + if printnodes: + testfolders_list = get_immediate_subdirectories(REJOIN_DS_GUARD_RUN_FOLDER) + count = len(testfolders_list) + for x in range(0, count): + print('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(7001) + '] ' + REJOIN_DS_GUARD_RUN_FOLDER + + testfolders_list[x]) + def run_prestart(numdsnodes, guard_mode=False): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - keypairs = [] - - # Generate keypairs (sort by public key) - for x in range(0, count): - process = Popen(["./tests/Zilliqa/genkeypair"], stdout=PIPE, universal_newlines=True) - (output, err) = process.communicate() - exit_code = process.wait() - keypairs.append(output.strip()) - keypairs.sort() - - # Use pre generated key for guard mode in local testing - # These keys are non critical and are only used for testing purposes - if guard_mode == True: - keypairs = ["02028CC4DEC0A756B42BD54905237B4E22FCC69D88CFEAA3797AEECF01D6A69E85 55009317F8B1FC7889EDF83742F684FB700EE8F970F7EDB8BDD6286A0F0A4CF1", - "021D99F2E5ACBA39ED5ACC5DCA5EE2ADDE780FFD998E1DBF440FE364C3BE360A7B 50C26000FCC08867FC3B9C03385015179E4B63282CB356014233BB1877FCDBDD", - "025C8ACD69AE4075D3F02CE07641329CEFAF6C1B24BE64187D2ECDBDD55CF934A2 67E57D32E7EF421B704C23B05A7600A56808F2910FC2944CD7C7F87684CF0F49", - "025EA8FFF868B64D5722F16FB07FB93CBFF38C5381F975CA5D0A728AEA19DBC6BC BE0CE7A97F90D433D58C873A9334819FDBF2646E08F61B470ACF996C082F0BB7", - "0268B2A0B5FFE2ADE7A38DF9A878281A8BFFA4F8EE18A20EE53F5ABA3BDDC6BF00 A351988F0776D25CE203EC21BDBAF4402E98A2A9A724C28A8E4FEC81F030AF55", - "027612A13BFA87AB22C0B3166B14873C7BEB77F0A27970BB0D1788EAA5F1BB885A 9C0A58E554511887E39E1E9BC25874B921A13FAFBB7DD4D57C8EBE6D72938C7D", - "028ED45F00C33680BFDBBADD8DDC98627BF18E7B49E83420C00DE9C9752FE8F33D 33DB54623E8AFDFE735E1D73C62F68C924811DD5CD3300FE5203B580E330A373", - "02B006BEAAFABFB738822EB4DFE3C0872A426173FF4C154D083A9C0EC5EB78A0C6 51D0976A2A9E72198D78BF229E8AB34DD9AE9E093CB1B71B4853C6839130CB6B", - "02B5D018B064A26998AD4553BD5D394E898043A9C5A4E414C9EA71F1B26E1CEF3F 13A8B01932B072D1225BDD550C5D4048F664D3AE2476F24FD1EFF123573563F5", - "02C28CDFC2CE6CE00DC38F5DBE565034BE50D382CB63FC260DA91E6828F806AAB6 C106836B85D5498112A13081A60326988012F6B0ED39480E3AB5683C69E04CB7", - "02E1626300DA30EFC798DEB27ED546BB2B4D1D8771E0907B1DAD3A0AD3BE1381CD 3016CB647ABBD35B85939FC716155777DCC41AEB54D5498A0A99BCE74A2BF119", - "02F650040BC0F3158B5D1A870EFC0ABE84FF02A4021A222FC49F9AD070ACFA2DA7 797FD9258980F9669A86E006A679815D15BD899B607454C3EDF284846F13E7E7", - "030DFD4A0CFD68016DF6EFC4A6BF0B67BA42ABEAF8D9AD65C25E97B9DA90CA4DB1 50481C07CA036990945EBE3A0B7D71FFCCF27CFF4B1DA03B1C8FB3660DA89552", - "03129EAEF8A136355FB1485941A593B4BEC4DFC5504D1114138A6D92332005DC59 CC3927D2C18849E6CFA3EE6D9322718961D2F035A1359AD1F37EEF527B5FBB15", - "037D49C420B04B9862BC1F0660544FF27F1D81EA0E5E7C161F1647FBF239F8780E EB4D499149C3582AD84CCB28E697EB0BCAF331CD7CBE43D5672C21BB9C17A477", - "03A6738E9081002097DDC71E3B72F9BBA2C0482034B8E80512D1B2DA5FFFFCBF8E 1430F36192D10C8ABCD9A22B036788C13F17830E807FEC7073B5E14E4B171265", - "03C58CD6B4C6A0E4FEC1D989D3218B67AFC82149039E174074C5FDFE58CB427028 38EB4FB242BCD8D4BBD9114307F9B7F1F90D8DB7DE0AEC356691BEC98642C062", - "03D616566DE986ADAC0E51BAF8147155993D56CCE4834607CFDDF832C3CAFD00F7 B459705C716E12044AB8263F812A9E9153269DD1449FA3B40EAA489F844BF839", - "03E4DA9E02B0830C98E99737FE63B34D068D086184583449C583A68F4DBA79BE64 7541CE8176B260B4A6D28578D2433D17A7948F3810E464C5288E86E7DCDC71AE", - "03FB81D476B3CF161AFD1AE0B861ECC907111AB891DF82028DD3D3085E2460A574 224B31816F0B529F21B14D9F04C42E7F277A024758A57BB6E1B3DEBF39A38E72"] - - nodes = ET.Element("nodes") - dsnodes = ET.Element("dsnodes"); - - # Store sorted keys list in text file - keys_file = open(LOCAL_RUN_FOLDER + 'keys.txt', "w") - for x in range(0, count): - keys_file.write(keypairs[x] + '\n') - keypair = keypairs[x].split(" ") - - if (x < numdsnodes): - ET.SubElement(dsnodes, "pubk").text = keypair[0]; - peer = ET.SubElement(nodes, "peer") - ET.SubElement(peer, "pubk").text = keypair[0] - ET.SubElement(peer, "ip").text = '127.0.0.1' - ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) - keys_file.close() - - #Create dsnodes file - dsTree = ET.ElementTree(dsnodes) - dsTree.write("dsnodes.xml") - dsnodes.clear() - - # Create config_normal.xml with pubkey and IP info of all DS nodes - tree = ET.ElementTree(nodes) - tree.write("config_normal.xml") - - # Clear the element tree - nodes.clear() - - # ds_whitelist.xml generation - keys_file = open(LOCAL_RUN_FOLDER + 'keys.txt', "w") - for x in range(0, count): - keys_file.write(keypairs[x] + '\n') - keypair = keypairs[x].split(" ") - peer = ET.SubElement(nodes, "peer") - ET.SubElement(peer, "pubk").text = keypair[0] - ET.SubElement(peer, "ip").text = '127.0.0.1' - ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) - keys_file.close() - - # Create ds_whitelist.xml with pubkey and IP info of all DS nodes - tree = ET.ElementTree(nodes) - tree.write("ds_whitelist.xml") - - # clear from ds_whitelist - nodes.clear() - - address_nodes = ET.Element("address") - # shard_whitelist.xml generation - keys_file = open(LOCAL_RUN_FOLDER + 'keys.txt', "w") - for x in range(0, count): - keys_file.write(keypairs[x] + '\n') - keypair = keypairs[x].split(" ") - ET.SubElement(address_nodes, "pubk").text = keypair[0] - keys_file.close() - - # Create shard_whitelist.xml with pubkey - tree = ET.ElementTree(address_nodes) - tree.write("shard_whitelist.xml") + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + keypairs = [] + + # Generate keypairs (sort by public key) + for x in range(0, count): + process = Popen(["./bin/genkeypair"], stdout=PIPE, universal_newlines=True) + (output, err) = process.communicate() + exit_code = process.wait() + keypairs.append(output.strip()) + keypairs.sort() + + # Use pre generated key for guard mode in local testing + # These keys are non critical and are only used for testing purposes + if guard_mode == True: + keypairs = [ + "02028CC4DEC0A756B42BD54905237B4E22FCC69D88CFEAA3797AEECF01D6A69E85 55009317F8B1FC7889EDF83742F684FB700EE8F970F7EDB8BDD6286A0F0A4CF1", + "021D99F2E5ACBA39ED5ACC5DCA5EE2ADDE780FFD998E1DBF440FE364C3BE360A7B 50C26000FCC08867FC3B9C03385015179E4B63282CB356014233BB1877FCDBDD", + "025C8ACD69AE4075D3F02CE07641329CEFAF6C1B24BE64187D2ECDBDD55CF934A2 67E57D32E7EF421B704C23B05A7600A56808F2910FC2944CD7C7F87684CF0F49", + "025EA8FFF868B64D5722F16FB07FB93CBFF38C5381F975CA5D0A728AEA19DBC6BC BE0CE7A97F90D433D58C873A9334819FDBF2646E08F61B470ACF996C082F0BB7", + "0268B2A0B5FFE2ADE7A38DF9A878281A8BFFA4F8EE18A20EE53F5ABA3BDDC6BF00 A351988F0776D25CE203EC21BDBAF4402E98A2A9A724C28A8E4FEC81F030AF55", + "027612A13BFA87AB22C0B3166B14873C7BEB77F0A27970BB0D1788EAA5F1BB885A 9C0A58E554511887E39E1E9BC25874B921A13FAFBB7DD4D57C8EBE6D72938C7D", + "028ED45F00C33680BFDBBADD8DDC98627BF18E7B49E83420C00DE9C9752FE8F33D 33DB54623E8AFDFE735E1D73C62F68C924811DD5CD3300FE5203B580E330A373", + "02B006BEAAFABFB738822EB4DFE3C0872A426173FF4C154D083A9C0EC5EB78A0C6 51D0976A2A9E72198D78BF229E8AB34DD9AE9E093CB1B71B4853C6839130CB6B", + "02B5D018B064A26998AD4553BD5D394E898043A9C5A4E414C9EA71F1B26E1CEF3F 13A8B01932B072D1225BDD550C5D4048F664D3AE2476F24FD1EFF123573563F5", + "02C28CDFC2CE6CE00DC38F5DBE565034BE50D382CB63FC260DA91E6828F806AAB6 C106836B85D5498112A13081A60326988012F6B0ED39480E3AB5683C69E04CB7", + "02E1626300DA30EFC798DEB27ED546BB2B4D1D8771E0907B1DAD3A0AD3BE1381CD 3016CB647ABBD35B85939FC716155777DCC41AEB54D5498A0A99BCE74A2BF119", + "02F650040BC0F3158B5D1A870EFC0ABE84FF02A4021A222FC49F9AD070ACFA2DA7 797FD9258980F9669A86E006A679815D15BD899B607454C3EDF284846F13E7E7", + "030DFD4A0CFD68016DF6EFC4A6BF0B67BA42ABEAF8D9AD65C25E97B9DA90CA4DB1 50481C07CA036990945EBE3A0B7D71FFCCF27CFF4B1DA03B1C8FB3660DA89552", + "03129EAEF8A136355FB1485941A593B4BEC4DFC5504D1114138A6D92332005DC59 CC3927D2C18849E6CFA3EE6D9322718961D2F035A1359AD1F37EEF527B5FBB15", + "037D49C420B04B9862BC1F0660544FF27F1D81EA0E5E7C161F1647FBF239F8780E EB4D499149C3582AD84CCB28E697EB0BCAF331CD7CBE43D5672C21BB9C17A477", + "03A6738E9081002097DDC71E3B72F9BBA2C0482034B8E80512D1B2DA5FFFFCBF8E 1430F36192D10C8ABCD9A22B036788C13F17830E807FEC7073B5E14E4B171265", + "03C58CD6B4C6A0E4FEC1D989D3218B67AFC82149039E174074C5FDFE58CB427028 38EB4FB242BCD8D4BBD9114307F9B7F1F90D8DB7DE0AEC356691BEC98642C062", + "03D616566DE986ADAC0E51BAF8147155993D56CCE4834607CFDDF832C3CAFD00F7 B459705C716E12044AB8263F812A9E9153269DD1449FA3B40EAA489F844BF839", + "03E4DA9E02B0830C98E99737FE63B34D068D086184583449C583A68F4DBA79BE64 7541CE8176B260B4A6D28578D2433D17A7948F3810E464C5288E86E7DCDC71AE", + "03FB81D476B3CF161AFD1AE0B861ECC907111AB891DF82028DD3D3085E2460A574 224B31816F0B529F21B14D9F04C42E7F277A024758A57BB6E1B3DEBF39A38E72"] + + nodes = ET.Element("nodes") + dsnodes = ET.Element("dsnodes"); + + # Store sorted keys list in text file + keys_file = open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "w") + for x in range(0, count): + keys_file.write(keypairs[x] + '\n') + keypair = keypairs[x].split(" ") + + if (x < numdsnodes): + ET.SubElement(dsnodes, "pubk").text = keypair[0]; + peer = ET.SubElement(nodes, "peer") + ET.SubElement(peer, "pubk").text = keypair[0] + ET.SubElement(peer, "ip").text = '127.0.0.1' + ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) + keys_file.close() + + # Create dsnodes file + dsTree = ET.ElementTree(dsnodes) + dsTree.write("dsnodes.xml") + dsnodes.clear() + + # Create config_normal.xml with pubkey and IP info of all DS nodes + tree = ET.ElementTree(nodes) + tree.write("config_normal.xml") + + # Clear the element tree + nodes.clear() + + # ds_whitelist.xml generation + keys_file = open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "w") + for x in range(0, count): + keys_file.write(keypairs[x] + '\n') + keypair = keypairs[x].split(" ") + peer = ET.SubElement(nodes, "peer") + ET.SubElement(peer, "pubk").text = keypair[0] + ET.SubElement(peer, "ip").text = '127.0.0.1' + ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) + keys_file.close() + + # Create ds_whitelist.xml with pubkey and IP info of all DS nodes + tree = ET.ElementTree(nodes) + tree.write("ds_whitelist.xml") + + # clear from ds_whitelist + nodes.clear() + + address_nodes = ET.Element("address") + # shard_whitelist.xml generation + keys_file = open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "w") + for x in range(0, count): + keys_file.write(keypairs[x] + '\n') + keypair = keypairs[x].split(" ") + ET.SubElement(address_nodes, "pubk").text = keypair[0] + keys_file.close() + + # Create shard_whitelist.xml with pubkey + tree = ET.ElementTree(address_nodes) + tree.write("shard_whitelist.xml") + def run_start(numdsnodes): + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + + dev_root = os.getenv("DEV_TREE_ROOT") + if dev_root is None: + print("DEV_TREE_ROOT is not set") + return + + dev_root += "/Zilliqa" + fp = dev_root + "/" + "constants_local.xml.native" + + if not os.path.exists(fp): + print( fp +" not found") + return + + # Load the keypairs + keypairs = [] + with open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "r") as f: + keypairs = f.readlines() + keypairs = [x.strip() for x in keypairs] + + # Launch node Zilliqa process + for x in range(0, count): + keypair = keypairs[x].split(" ") + shutil.copyfile('ds_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/ds_whitelist.xml') + shutil.copyfile('shard_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/shard_whitelist.xml') + shutil.copyfile(fp, LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + ipc_path = "/tmp/zilliqa" + str(NODE_LISTEN_PORT + x) + ".sock" + status_server_port = str(STATUS_SERVER_LISTEN_PORT + x) + patch_param_in_xml(LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml', ipc_path, status_server_port) + + shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') + + if (x < numdsnodes): + shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/config.xml') + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[ + 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/zilliqa ' + ' --privk ' + + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str( + NODE_LISTEN_PORT + x) + ' --identity normal-' + str(x) + ' > ./error_log_zilliqa 2>&1 &') + else: + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[ + 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/zilliqa ' + ' --privk ' + + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str( + NODE_LISTEN_PORT + x) + ' --identity normal-' + str(x) + ' > ./error_log_zilliqa 2>&1 &') + + - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - - # Load the keypairs - keypairs = [] - with open(LOCAL_RUN_FOLDER + 'keys.txt', "r") as f: - keypairs = f.readlines() - keypairs = [x.strip() for x in keypairs] - - # Launch node Zilliqa process - for x in range(0, count): - keypair = keypairs[x].split(" ") - shutil.copyfile('ds_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/ds_whitelist.xml') - shutil.copyfile('shard_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/shard_whitelist.xml') - shutil.copyfile('constants_local.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') - ipc_path = "/tmp/zilliqa" + str(NODE_LISTEN_PORT + x) + ".sock" - status_server_port = str(STATUS_SERVER_LISTEN_PORT + x) - patch_param_in_xml(LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml', ipc_path, status_server_port) - - shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') - - if (x < numdsnodes): - shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/config.xml') - os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/zilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' > ./error_log_zilliqa 2>&1 &') - else: - os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/zilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' > ./error_log_zilliqa 2>&1 &') def patch_param_in_xml(filepath, ipc_path, status_server_port): - root = ET.parse(filepath).getroot() - td = root.find('jsonrpc') - td.find('SCILLA_IPC_SOCKET_PATH').text = ipc_path - td.find('STATUS_RPC_PORT').text = status_server_port + root = ET.parse(filepath).getroot() + + td = root.find('jsonrpc') + ''' TODO will need extending for other params''' + td.find('SCILLA_IPC_SOCKET_PATH').text = ipc_path + td.find('STATUS_RPC_PORT').text = status_server_port + + tree = ET.ElementTree(root) + tree.write(filepath) - tree = ET.ElementTree(root) - tree.write(filepath) # To rejoin ds guard index 2 def run_start_dsguard2(): - testfolders_list = get_immediate_subdirectories(REJOIN_DS_GUARD_RUN_FOLDER) - count = len(testfolders_list) + testfolders_list = get_immediate_subdirectories(REJOIN_DS_GUARD_RUN_FOLDER) + count = len(testfolders_list) + + for x in range(0, count): + shutil.copyfile('dsnodes.xml', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') + shutil.copyfile('constants_local.xml.native', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/constants.xml') - for x in range(0, count): - shutil.copyfile('dsnodes.xml', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') - shutil.copyfile('constants_local.xml', REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + # These keys are non critical and are only used for testing purposes + keypairs = "021D99F2E5ACBA39ED5ACC5DCA5EE2ADDE780FFD998E1DBF440FE364C3BE360A7B 50C26000FCC08867FC3B9C03385015179E4B63282CB356014233BB1877FCDBDD" - # These keys are non critical and are only used for testing purposes - keypairs = "021D99F2E5ACBA39ED5ACC5DCA5EE2ADDE780FFD998E1DBF440FE364C3BE360A7B 50C26000FCC08867FC3B9C03385015179E4B63282CB356014233BB1877FCDBDD" + # Launch node zilliqa process + keypair = keypairs.split(" ") + os.system('cd ' + REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[ + 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ./zilliqa_ds_guard_rejoin ' + + keypair[1] + ' ' + keypair[0] + ' ' + '127.0.0.1' + ' ' + str( + 7001) + ' 0 7 0 > ./error_log_zilliqa 2>&1 &') + print("Running and rejoining ds guard at port " + str(7001)) - # Launch node zilliqa process - keypair = keypairs.split(" ") - os.system('cd ' + REJOIN_DS_GUARD_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; ./zilliqa_ds_guard_rejoin ' + keypair[1] + ' ' + keypair[0] + ' ' + '127.0.0.1' + ' ' + str(7001) + ' 0 7 0 > ./error_log_zilliqa 2>&1 &') - print("Running and rejoining ds guard at port "+ str(7001)) # To validate the correctness of backup persistence def run_start_validateBackupDB(): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - - # Load the keypairs - keypairs = [] - with open(LOCAL_RUN_FOLDER + 'keys.txt', "r") as f: - keypairs = f.readlines() - keypairs = [x.strip() for x in keypairs] - - # Launch node Zilliqa process - keypair = keypairs[0].split(" ") - shutil.copyfile('ds_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/ds_whitelist.xml') - shutil.copyfile('shard_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/shard_whitelist.xml') - shutil.copyfile('constants_local.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/constants.xml') - shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/dsnodes.xml') - shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/config.xml') - os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[0] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/zilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + 0) + ' --synctype 5 --recovery 1 > ./error_log_zilliqa 2>&1 &') + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + + # Load the keypairs + keypairs = [] + with open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "r") as f: + keypairs = f.readlines() + keypairs = [x.strip() for x in keypairs] + + # Launch node Zilliqa process + keypair = keypairs[0].split(" ") + shutil.copyfile('ds_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/ds_whitelist.xml') + shutil.copyfile('shard_whitelist.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/shard_whitelist.xml') + shutil.copyfile('constants_local.xml.native', LOCAL_RUN_FOLDER + testfolders_list[0] + '/constants.xml') + shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/dsnodes.xml') + shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[0] + '/config.xml') + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[0] + '; echo \"' + keypair[0] + ' ' + keypair[ + 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/zilliqa ' + ' --privk ' + + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str( + NODE_LISTEN_PORT + 0) + ' --synctype 5 --recovery 1 '+ ' --identity normal-' + str(x) + '> ./error_log_zilliqa 2>&1 &') + def run_connect(numnodes): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - if ((numnodes == 0) or (numnodes > count)): - numnodes = count - - # Load the keypairs - keypairs = [] - with open(LOCAL_RUN_FOLDER + 'keys.txt', "r") as f: - keypairs = f.readlines() - keypairs = [x.strip() for x in keypairs] - - # Connect nodes (exchange hello messages) - edges = set() - for x in range(0, numnodes): - connect_cmd = 'cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; ulimit -Sc unlimited; ulimit -Hc unlimited; ./sendcmd' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' --cmd ' + 'addpeers' - has_peers_to_connect = False - for y in range(x + 1, numnodes): - index = y - if ((x + 1, index + 1) in edges): - continue - elif ((index + 1, x + 1) in edges): - continue - elif (x == index): - continue - else: - has_peers_to_connect = True - keypair = keypairs[index].split(" ") - print ('connecting node ' + str(x + 1) + ' (port ' + str(NODE_LISTEN_PORT + x) + ') to node ' + str(index + 1) + ' (' + str(NODE_LISTEN_PORT + index) + ')') - connect_cmd = connect_cmd + ' --cmdarg ' + keypair[0] + ' --cmdarg ' + '127.0.0.1' + ' --cmdarg ' + str(NODE_LISTEN_PORT + index) - if (x < index): - edges.add((x + 1, index + 1)) - else: - edges.add((index + 1, x + 1)) - if has_peers_to_connect: - os.system(connect_cmd + ' &') - time.sleep(1) - - print ('Total num of edges connected: ' + str(len(edges))) + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + if ((numnodes == 0) or (numnodes > count)): + numnodes = count + + # Load the keypairs + keypairs = [] + with open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "r") as f: + keypairs = f.readlines() + keypairs = [x.strip() for x in keypairs] + + # Connect nodes (exchange hello messages) + edges = set() + for x in range(0, numnodes): + connect_cmd = 'cd ' + LOCAL_RUN_FOLDER + testfolders_list[ + x] + '; ulimit -Sc unlimited; ulimit -Hc unlimited; ./sendcmd' + ' --port ' + str( + NODE_LISTEN_PORT + x) + ' --cmd ' + 'addpeers' + has_peers_to_connect = False + for y in range(x + 1, numnodes): + index = y + if ((x + 1, index + 1) in edges): + continue + elif ((index + 1, x + 1) in edges): + continue + elif (x == index): + continue + else: + has_peers_to_connect = True + keypair = keypairs[index].split(" ") + print('connecting node ' + str(x + 1) + ' (port ' + str(NODE_LISTEN_PORT + x) + ') to node ' + str( + index + 1) + ' (' + str(NODE_LISTEN_PORT + index) + ')') + connect_cmd = connect_cmd + ' --cmdarg ' + keypair[0] + ' --cmdarg ' + '127.0.0.1' + ' --cmdarg ' + str( + NODE_LISTEN_PORT + index) + if (x < index): + edges.add((x + 1, index + 1)) + else: + edges.add((index + 1, x + 1)) + if has_peers_to_connect: + os.system(connect_cmd + ' &') + time.sleep(1) + + print('Total num of edges connected: ' + str(len(edges))) + def run_stop(): - os.system('killall zilliqa') - if os.path.exists(LOCAL_RUN_FOLDER): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - for x in range(0, count): - os.system('fuser -k ' + str(NODE_LISTEN_PORT + x) + '/tcp') + os.system('killall zilliqa') + if os.path.exists(LOCAL_RUN_FOLDER): + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + for x in range(0, count): + os.system('fuser -k ' + str(NODE_LISTEN_PORT + x) + '/tcp') + def run_clean(): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - run_setup(count, False) + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + run_setup(count, False) + def run_sendcmd(nodenum, msg): - os.system('tests/Zilliqa/sendcmd' + ' --port ' + str(NODE_LISTEN_PORT + nodenum - 1) + ' --cmd ' + 'cmd' + ' --cmdarg ' + msg) - + os.system( + './bin/sendcmd' + ' --port ' + str(NODE_LISTEN_PORT + nodenum - 1) + ' --cmd ' + 'cmd' + ' --cmdarg ' + msg) + + def run_sendcmdrandom(nodenum, msg_size): - # msg = "000400" + 'A' * msg_size * 2 - # os.system('tests/Zilliqa/sendcmd ' + str(NODE_LISTEN_PORT + nodenum - 1) + ' cmd ' + msg) - os.system('tests/Zilliqa/sendcmd' + ' --port ' + str(NODE_LISTEN_PORT) + ' --cmd ' + 'broadcast' + ' --cmdarg ' + msg_size) + # msg = "000400" + 'A' * msg_size * 2 + # os.system('./bin/sendcmd ' + str(NODE_LISTEN_PORT + nodenum - 1) + ' cmd ' + msg) + os.system('./bin/sendcmd' + ' --port ' + str(NODE_LISTEN_PORT) + ' --cmd ' + 'broadcast' + ' --cmdarg ' + msg_size) + def run_startpow(nodenum, dscount, blocknum, dsdiff, diff, rand1, rand2): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) - # Load the keypairs - keypairs = [] - with open(LOCAL_RUN_FOLDER + 'keys.txt', "r") as f: - keypairs = f.readlines() - keypairs = [x.strip() for x in keypairs] + # Load the keypairs + keypairs = [] + with open(LOCAL_RUN_FOLDER + 'normal_keys.txt', "r") as f: + keypairs = f.readlines() + keypairs = [x.strip() for x in keypairs] - # Assemble the STARTPOW message - startpow_cmd = 'tests/Zilliqa/sendcmd' + ' --port ' + str(NODE_LISTEN_PORT + nodenum - 1) + ' --cmd ' + 'cmd' + ' --cmdarg 0200' + blocknum + dsdiff + diff + rand1 + rand2 - for x in range(0, dscount): - keypair = keypairs[x].split(" ") - startpow_cmd = startpow_cmd + keypair[0] + '0000000000000000000000000100007F' + "{0:0{1}x}".format(NODE_LISTEN_PORT + x, 8) + # Assemble the STARTPOW message + startpow_cmd = './bin/sendcmd' + ' --port ' + str( + NODE_LISTEN_PORT + nodenum - 1) + ' --cmd ' + 'cmd' + ' --cmdarg 0200' + blocknum + dsdiff + diff + rand1 + rand2 + for x in range(0, dscount): + keypair = keypairs[x].split(" ") + startpow_cmd = startpow_cmd + keypair[0] + '0000000000000000000000000100007F' + "{0:0{1}x}".format( + NODE_LISTEN_PORT + x, 8) + + # Send to node + os.system(startpow_cmd) - # Send to node - os.system(startpow_cmd) def run_delete(): - if (os.path.exists(LOCAL_RUN_FOLDER)): - shutil.rmtree(LOCAL_RUN_FOLDER) + if (os.path.exists(LOCAL_RUN_FOLDER)): + shutil.rmtree(LOCAL_RUN_FOLDER) + if __name__ == "__main__": - main() + main() diff --git a/tests/Zilliqa/test_zilliqa_lookup.py b/tests/Zilliqa/test_zilliqa_lookup.py index 5fac9a1f3f..e5cb78acf1 100755 --- a/tests/Zilliqa/test_zilliqa_lookup.py +++ b/tests/Zilliqa/test_zilliqa_lookup.py @@ -14,17 +14,15 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import subprocess import os import sys import shutil import stat -import time from subprocess import Popen, PIPE import xml.etree.cElementTree as ET -NODE_LISTEN_PORT = 4001 +NODE_LISTEN_PORT = 40092 LOCAL_RUN_FOLDER = './lookup_local_run/' LOCAL_FOLDER = "./" @@ -33,131 +31,141 @@ # need to be an absolute path TXN_PATH = "/tmp/zilliqa_txns" + def print_usage(): - print ("Testing multiple Zilliqa nodes in local machine\n" - "===============================================\n" - "Usage:\n\tpython " + sys.argv[0] + " [command] [command parameters]\n" - "Available commands:\n" - "\tTest Execution:\n" - "\t\tsetup [num-nodes] - Set up the nodes\n" - "\t\tstart - Start node processes\n" - "\t\tgentxn [seconds] - generate transactions\n") + print("Testing multiple Zilliqa nodes in local machine\n" + "===============================================\n" + "Usage:\n\tpython " + sys.argv[0] + " [command] [command parameters]\n" + "Available commands:\n" + "\tTest Execution:\n" + "\t\tsetup [num-nodes] - Set up the nodes\n" + "\t\tstart - Start node processes\n" + "\t\tgentxn [seconds] - generate transactions\n") + def main(): - numargs = len(sys.argv) - if (numargs < 2): - print_usage() - else: - command = sys.argv[1] - if (command == 'setup'): - print_usage() if (numargs != 3) else run_setup(numnodes=int(sys.argv[2]), printnodes=True) - elif (command == 'start'): - print_usage() if (numargs != 2) else run_start() - elif (command == 'gentxn'): - print_usage() if (numargs != 3) else run_gentxn(batch=int(sys.argv[2])) - else: - print_usage() + numargs = len(sys.argv) + if numargs < 2: + print_usage() + else: + command = sys.argv[1] + if command == 'setup': + print_usage() if (numargs != 3) else run_setup(numnodes=int(sys.argv[2]), printnodes=True) + elif command == 'start': + print_usage() if (numargs != 2) else run_start() + elif command == 'gentxn': + print_usage() if (numargs != 3) else run_gentxn(batch=int(sys.argv[2])) + else: + print_usage() + # ================ # Helper Functions # ================ def get_immediate_subdirectories(a_dir): - subdirs = [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name)) and name.startswith('node')] - subdirs.sort() - return subdirs + subdirs = [name for name in os.listdir(a_dir) if + os.path.isdir(os.path.join(a_dir, name)) and name.startswith('node')] + subdirs.sort() + return subdirs + # ======================== # Test Execution Functions # ======================== def run_setup(numnodes, printnodes): - os.system('killall lzilliqa') - if os.path.exists(LOCAL_RUN_FOLDER) != True: - # shutil.rmtree(LOCAL_RUN_FOLDER) - os.makedirs(LOCAL_RUN_FOLDER) - for x in range(0, numnodes): - testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x+1).zfill(4) - if os.path.exists(testsubdir) != True : - os.makedirs(testsubdir) - shutil.copyfile('./tests/Zilliqa/zilliqa', testsubdir + '/lzilliqa') - - st = os.stat(testsubdir + '/lzilliqa') - os.chmod(testsubdir + '/lzilliqa', st.st_mode | stat.S_IEXEC) - - if printnodes: - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - for x in range(0, count): - print ('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(NODE_LISTEN_PORT + x) + '] ' + LOCAL_RUN_FOLDER + testfolders_list[x]) - - - keypairs = [] - # Generate keypairs (sort by public key) - for x in range(0, count): - process = Popen(["./tests/Zilliqa/genkeypair"], stdout=PIPE, universal_newlines=True) - (output, err) = process.communicate() - exit_code = process.wait() - keypairs.append(output.strip()) - keypairs.sort() - - patch_lookup_pubkey(LOCAL_FOLDER + "/constants_local.xml", keypairs, count) - patch_seed_pubkey(LOCAL_FOLDER + "/constants_local.xml", keypairs, count) - nodes = ET.Element("nodes") - - # Store sorted keys list in text file - keys_file = open(LOCAL_RUN_FOLDER + 'keys.txt', "w") - for x in range(0, count): - keys_file.write(keypairs[x] + '\n') - keypair = keypairs[x].split(" ") - if (x < count): - peer = ET.SubElement(nodes, "peer") - ET.SubElement(peer, "pubk").text = keypair[0] - ET.SubElement(peer, "ip").text = '127.0.0.1' - ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) - keys_file.close() + os.system('killall lzilliqa') + if os.path.exists(LOCAL_RUN_FOLDER) is not True: + # shutil.rmtree(LOCAL_RUN_FOLDER) + os.makedirs(LOCAL_RUN_FOLDER) + for x in range(0, numnodes): + testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x + 1).zfill(4) + if os.path.exists(testsubdir) is not True: + os.makedirs(testsubdir) + shutil.copyfile('./bin/zilliqa', testsubdir + '/lzilliqa') + + st = os.stat(testsubdir + '/lzilliqa') + os.chmod(testsubdir + '/lzilliqa', st.st_mode | stat.S_IEXEC) + + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + + if printnodes: + for x in range(0, count): + print('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(NODE_LISTEN_PORT + x) + '] ' + LOCAL_RUN_FOLDER + + testfolders_list[x]) + + keypairs = [] + # Generate keypairs (sort by public key) + for x in range(0, count): + process = Popen(["./bin/genkeypair"], stdout=PIPE, universal_newlines=True) + (output, err) = process.communicate() + exit_code = process.wait() + keypairs.append(output.strip()) + keypairs.sort() + + ''' Only deals with one instance for now ''' + + patch_lookup_pubkey(LOCAL_FOLDER + "/constants_local.xml", keypairs, count) + patch_lookup_port(LOCAL_FOLDER + "/constants_local.xml", NODE_LISTEN_PORT, count) + patch_seed_pubkey(LOCAL_FOLDER + "/constants_local.xml", keypairs, count) + patch_seed_port(LOCAL_FOLDER + "/constants_local.xml", NODE_LISTEN_PORT, count) + + nodes = ET.Element("nodes") + + # Store sorted keys list in text file + keys_file = open(LOCAL_RUN_FOLDER + 'lookup_keys.txt', "w") + for x in range(0, count): + keys_file.write(keypairs[x] + '\n') + keypair = keypairs[x].split(" ") + if x < count: + peer = ET.SubElement(nodes, "peer") + ET.SubElement(peer, "pubk").text = keypair[0] + ET.SubElement(peer, "ip").text = '127.0.0.1' + ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) + keys_file.close() + def patch_constants_xml(filepath, read_txn=False): - root = ET.parse(filepath).getroot() + root = ET.parse(filepath).getroot() - td = root.find('dispatcher') - td.find('TXN_PATH').text = TXN_PATH - if read_txn: - td.find('USE_REMOTE_TXN_CREATOR').text='true' + general = root.find('general') + general.find('LOOKUP_NODE_MODE').text = 'true' - general = root.find('general') - general.find('LOOKUP_NODE_MODE').text = 'true' + tree = ET.ElementTree(root) + tree.write(filepath) - tree = ET.ElementTree(root) - tree.write(filepath) def run_gentxn(batch=100): - if not os.path.exists(TXN_PATH): - os.makedirs(TXN_PATH) + if not os.path.exists(TXN_PATH): + os.makedirs(TXN_PATH) + + if not os.path.exists(GENTXN_WORKING_DIR): + os.makedirs(GENTXN_WORKING_DIR) - if not os.path.exists(GENTXN_WORKING_DIR): - os.makedirs(GENTXN_WORKING_DIR) + print("Created gentxn working folder: " + GENTXN_WORKING_DIR) + shutil.copy('bin/gentxn', os.path.join(GENTXN_WORKING_DIR, 'gentxn')) + gentxn_constants_xml_path = os.path.join(GENTXN_WORKING_DIR, 'constants.xml') + shutil.copyfile('constants_local.xml', gentxn_constants_xml_path) - print("Created gentxn working folder: " + GENTXN_WORKING_DIR) - shutil.copy('bin/gentxn', os.path.join(GENTXN_WORKING_DIR, 'gentxn')) - gentxn_constants_xml_path = os.path.join(GENTXN_WORKING_DIR, 'constants.xml') - shutil.copyfile('constants_local.xml', gentxn_constants_xml_path) + if os.path.exists(gentxn_constants_xml_path): + patch_constants_xml(gentxn_constants_xml_path) - if os.path.exists(gentxn_constants_xml_path): - patch_constants_xml(gentxn_constants_xml_path) + print("Waiting gentxn for creating {} batches".format(batch)) + os.system('cd ' + GENTXN_WORKING_DIR + '; ./gentxn --begin 0 --end {}'.format(batch)) - print("Waiting gentxn for creating {} batches".format(batch)) - os.system('cd ' + GENTXN_WORKING_DIR + '; ./gentxn --begin 0 --end {}'.format(batch)) def patch_lookup_pubkey(filepath, keypairs, count): - root = ET.parse(filepath).getroot() - td = root.find('lookups') - elems = td.findall('peer/pubkey') - for x in range(0, count): - keypair = keypairs[x].split(" ") - elems[x].text = keypair[0] - tree = ET.ElementTree(root) - tree.write(filepath) + root = ET.parse(filepath).getroot() + td = root.find('lookups') + elems = td.findall('peer/pubkey') + for x in range(0, count): + keypair = keypairs[x].split(" ") + elems[x].text = keypair[0] + tree = ET.ElementTree(root) + tree.write(filepath) + def patch_seed_pubkey(filepath, keypairs, count): root = ET.parse(filepath).getroot() @@ -168,34 +176,65 @@ def patch_seed_pubkey(filepath, keypairs, count): elems[x].text = keypair[0] tree = ET.ElementTree(root) tree.write(filepath) - +def patch_lookup_port(filepath, port, count): + root = ET.parse(filepath).getroot() + td = root.find('lookups') + elems = td.findall('peer/port') + for x in range(0, count): + elems[x].text = str(port) + tree = ET.ElementTree(root) + tree.write(filepath) +def patch_seed_port(filepath, port, count): + root = ET.parse(filepath).getroot() + td = root.find('upper_seed') + elems = td.findall('peer/port') + for x in range(0, count): + elems[x].text = str(port) + tree = ET.ElementTree(root) + tree.write(filepath) def run_start(): - testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) - count = len(testfolders_list) - - # Load the keypairs - keypairs = [] - with open(LOCAL_RUN_FOLDER + 'keys.txt', "r") as f: - keypairs = f.readlines() - keypairs = [x.strip() for x in keypairs] - - - for x in range(0, count): - shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/config.xml') - shutil.copyfile('constants_local.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') - shutil.copyfile('dsnodes.xml',LOCAL_RUN_FOLDER+testfolders_list[x]+'/dsnodes.xml') - # FIXME: every lookup node has the option USE_REMOTE_TXN_CREATOR set to true, which seemingly - # enable transaction dispatching on every lookup running locally. However, the truth is only the - # one with the jsonrpc server running will do the transaction dispatching and coincidentally there - # will be only one as there is an unknown issue that multiple lookup nodes are having port collision - # on 4201 and eventually only one will get the port and others won't be able to start jsonrpc server - patch_constants_xml(LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml', True) - - # Launch node zilliqa process - for x in range(0, count): - keypair = keypairs[x].split(" ") - os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/lzilliqa ' + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + str(NODE_LISTEN_PORT + x) + ' > ./error_log_zilliqa 2>&1 &') + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + + dev_root = os.getenv("DEV_TREE_ROOT") + if dev_root is None: + print("DEV_TREE_ROOT is not set") + return + + dev_root += "/Zilliqa" + fp = dev_root + "/" + "constants_local.xml.native" + + if not os.path.exists(fp): + print( fp +" not found") + return + + + # Load the keypairs + keypairs = [] + with open(LOCAL_RUN_FOLDER + 'lookup_keys.txt', "r") as f: + keypairs = f.readlines() + keypairs = [x.strip() for x in keypairs] + + for x in range(0, count): + shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/config.xml') + shutil.copyfile(fp, LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') + # FIXME: every lookup node has the option USE_REMOTE_TXN_CREATOR set to true, which seemingly + # enable transaction dispatching on every lookup running locally. However, the truth is only the + # one with the jsonrpc server running will do the transaction dispatching and coincidentally there + # will be only one as there is an unknown issue that multiple lookup nodes are having port collision + # on 4201 and eventually only one will get the port and others won't be able to start jsonrpc server + patch_constants_xml(LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml', True) + + # Launch node zilliqa process + for x in range(0, count): + keypair = keypairs[x].split(" ") + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[ + 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/lzilliqa ' + + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + + str(NODE_LISTEN_PORT + x) + ' --identity ' + 'lookup-' + str(x) + ' > ./error_log_zilliqa 2>&1 &') + if __name__ == "__main__": - main() + main() diff --git a/tests/Zilliqa/test_zilliqa_seedpub.py b/tests/Zilliqa/test_zilliqa_seedpub.py new file mode 100755 index 0000000000..123a7057e5 --- /dev/null +++ b/tests/Zilliqa/test_zilliqa_seedpub.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +# Copyright (C) 2019 Zilliqa +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import sys +import shutil +import stat + +from subprocess import Popen, PIPE +import xml.etree.cElementTree as ET + + +NODE_LISTEN_PORT = 4201 + +LOCAL_RUN_FOLDER = './seedpub_local_run/' +LOCAL_FOLDER = "./" + +GENTXN_WORKING_DIR = os.path.join(LOCAL_RUN_FOLDER, 'gentxn') + +# need to be an absolute path +TXN_PATH = "/tmp/zilliqa_txns" + + +def print_usage(): + print("Testing multiple Zilliqa nodes in local machine\n" + "===============================================\n" + "Usage:\n\tpython " + sys.argv[0] + " [command] [command parameters]\n" + "Available commands:\n" + "\tTest Execution:\n" + "\t\tsetup [num-nodes] - Set up the nodes\n" + "\t\tstart - Start node processes\n" + "\t\tgentxn [seconds] - generate transactions\n") + + +def main(): + numargs = len(sys.argv) + if numargs < 2: + print_usage() + else: + command = sys.argv[1] + if command == 'setup': + print_usage() if (numargs != 3) else run_setup(numnodes=int(sys.argv[2]), printnodes=True) + elif command == 'start': + print_usage() if (numargs != 2) else run_start() + elif command == 'gentxn': + print_usage() if (numargs != 3) else run_gentxn(batch=int(sys.argv[2])) + else: + print_usage() + + +# ================ +# Helper Functions +# ================ + +def get_immediate_subdirectories(a_dir): + subdirs = [name for name in os.listdir(a_dir) if + os.path.isdir(os.path.join(a_dir, name)) and name.startswith('node')] + subdirs.sort() + return subdirs + + +# ======================== +# Test Execution Functions +# ======================== + +def run_setup(numnodes, printnodes): + os.system('killall seedpub') + if os.path.exists(LOCAL_RUN_FOLDER) is not True: + # shutil.rmtree(LOCAL_RUN_FOLDER) + os.makedirs(LOCAL_RUN_FOLDER) + for x in range(0, numnodes): + testsubdir = LOCAL_RUN_FOLDER + 'node_' + str(x + 1).zfill(4) + if os.path.exists(testsubdir) is not True: + os.makedirs(testsubdir) + shutil.copyfile('./bin/zilliqa', testsubdir + '/seedpub') + + st = os.stat(testsubdir + '/seedpub') + os.chmod(testsubdir + '/seedpub', st.st_mode | stat.S_IEXEC) + + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + + if printnodes: + for x in range(0, count): + print('[Node ' + str(x + 1).ljust(3) + '] [Port ' + str(NODE_LISTEN_PORT + x) + '] ' + LOCAL_RUN_FOLDER + + testfolders_list[x]) + + keypairs = [] + # Generate keypairs (sort by public key) + for x in range(0, count): + process = Popen(["./bin/genkeypair"], stdout=PIPE, universal_newlines=True) + (output, err) = process.communicate() + exit_code = process.wait() + keypairs.append(output.strip()) + keypairs.sort() + + + nodes = ET.Element("nodes") + + # Store sorted keys list in text file + keys_file = open(LOCAL_RUN_FOLDER + 'seedpub_keys.txt', "w") + for x in range(0, count): + keys_file.write(keypairs[x] + '\n') + keypair = keypairs[x].split(" ") + if x < count: + peer = ET.SubElement(nodes, "peer") + ET.SubElement(peer, "pubk").text = keypair[0] + ET.SubElement(peer, "ip").text = '127.0.0.1' + ET.SubElement(peer, "port").text = str(NODE_LISTEN_PORT + x) + keys_file.close() + + +def patch_constants_xml(filepath, read_txn=False): + return + +def run_gentxn(batch=100): + if not os.path.exists(TXN_PATH): + os.makedirs(TXN_PATH) + + if not os.path.exists(GENTXN_WORKING_DIR): + os.makedirs(GENTXN_WORKING_DIR) + + print("Created gentxn working folder: " + GENTXN_WORKING_DIR) + shutil.copy('bin/gentxn', os.path.join(GENTXN_WORKING_DIR, 'gentxn')) + gentxn_constants_xml_path = os.path.join(GENTXN_WORKING_DIR, 'constants.xml') + shutil.copyfile('constants_local.xml', gentxn_constants_xml_path) + + if os.path.exists(gentxn_constants_xml_path): + patch_constants_xml(gentxn_constants_xml_path) + + print("Waiting gentxn for creating {} batches".format(batch)) + os.system('cd ' + GENTXN_WORKING_DIR + '; ./gentxn --begin 0 --end {}'.format(batch)) + + +def patch_lookup_pubkey(filepath, keypairs, count): + root = ET.parse(filepath).getroot() + td = root.find('lookups') + elems = td.findall('peer/pubkey') + for x in range(0, count): + keypair = keypairs[x].split(" ") + elems[x].text = keypair[0] + tree = ET.ElementTree(root) + tree.write(filepath) + + +def patch_seed_pubkey(filepath, keypairs, count): + root = ET.parse(filepath).getroot() + td = root.find('upper_seed') + elems = td.findall('peer/pubkey') + for x in range(0, count): + keypair = keypairs[x].split(" ") + elems[x].text = keypair[0] + tree = ET.ElementTree(root) + tree.write(filepath) + + +def run_start(): + testfolders_list = get_immediate_subdirectories(LOCAL_RUN_FOLDER) + count = len(testfolders_list) + dev_root = os.getenv("DEV_TREE_ROOT") + if dev_root is None: + print("DEV_TREE_ROOT is not set") + return + + dev_root += "/Zilliqa" + fp = dev_root + "/" + "constants_local.xml.native" + + if not os.path.exists(fp): + print( fp +" not found") + return + + + + # Load the keypairs + keypairs = [] + with open(LOCAL_RUN_FOLDER + 'seedpub_keys.txt', "r") as f: + keypairs = f.readlines() + keypairs = [x.strip() for x in keypairs] + + for x in range(0, count): + shutil.copyfile('config_normal.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/config.xml') + shutil.copyfile(fp, LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml') + shutil.copyfile('dsnodes.xml', LOCAL_RUN_FOLDER + testfolders_list[x] + '/dsnodes.xml') + # FIXME: every lookup node has the option USE_REMOTE_TXN_CREATOR set to true, which seemingly + # enable transaction dispatching on every lookup running locally. However, the truth is only the + # one with the jsonrpc server running will do the transaction dispatching and coincidentally there + # will be only one as there is an unknown issue that multiple lookup nodes are having port collision + # on 4201 and eventually only one will get the port and others won't be able to start jsonrpc server + patch_constants_xml(LOCAL_RUN_FOLDER + testfolders_list[x] + '/constants.xml', True) + + # Launch node zilliqa process + for x in range(0, count): + keypair = keypairs[x].split(" ") + os.system('cd ' + LOCAL_RUN_FOLDER + testfolders_list[x] + '; echo \"' + keypair[0] + ' ' + keypair[ + 1] + '\" > mykey.txt' + '; ulimit -n 65535; ulimit -Sc unlimited; ulimit -Hc unlimited; $(pwd)/seedpub ' + + ' --privk ' + keypair[1] + ' --pubk ' + keypair[0] + ' --address ' + '127.0.0.1' + ' --port ' + + str(NODE_LISTEN_PORT + x) + ' --identity ' + 'seedpub-' + str(x) + ' > ./error_log_zilliqa 2>&1 &') + + +if __name__ == "__main__": + main() diff --git a/tests/native/CMakeLists.txt b/tests/native/CMakeLists.txt new file mode 100644 index 0000000000..f4ec8a10cd --- /dev/null +++ b/tests/native/CMakeLists.txt @@ -0,0 +1,2 @@ +file(GLOB TEST_ZILLIQA_FILES *.sh *.py) +file(COPY ${TEST_ZILLIQA_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/native/prepare_constants.py b/tests/native/prepare_constants.py new file mode 100644 index 0000000000..2cc2869f8d --- /dev/null +++ b/tests/native/prepare_constants.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# Copyright (C) 2019 Zilliqa +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +import shutil +import sys +from xml.etree import cElementTree as ET +def update_xml_files(source_file, target_file): + + root = ET.parse(source_file).getroot() + if not root: + print("Failed to parse xml file") + os.abort() + + DEV_TREE_ROOT=os.environ.get('DEV_TREE_ROOT') + + if not DEV_TREE_ROOT: + print("DEV_TREE_ROOT is not set") + os.abort() + + general = root.find('general') + if general: + general.find('LOOKUP_NODE_MODE').text = 'false' + general.find('DEBUG_LEVEL').text = '4' + + scilla_root = root.find('smart_contract') + if scilla_root: + scilla_root.find('SCILLA_ROOT').text = DEV_TREE_ROOT +'/scilla' + scilla_root.find('ENABLE_SCILLA_MULTI_VERSION').text = 'false' + + jsonrpc = root.find('jsonrpc') + if jsonrpc: + jsonrpc.find('EVM_SERVER_BINARY').text = DEV_TREE_ROOT + '/Zilliqa/evm-ds/target/release/evm-ds' + jsonrpc.find('ENABLE_STATUS_RPC').text = 'false' + + metric = root.find('metric/zilliqa') + if metric: + metric.find('METRIC_ZILLIQA_PROVIDER').text = 'NONE' + metric.find('METRIC_ZILLIQA_MASK').text = 'NONE' + + trace = root.find('trace/zilliqa') + if trace: + trace.find('TRACE_ZILLIQA_PROVIDER').text = 'NONE' + trace.find('TRACE_ZILLIQA_MASK').text = 'NONE' + + logging = root.find('logging/zilliqa') + if logging: + logging.find('LOGGING_ZILLIQA_PROVIDER').text = 'NONE' + + tree = ET.ElementTree(root) + tree.write(target_file) + +def main(): + numargs = len(sys.argv) + if (numargs < 2): + os.abort() + else: + in_file = sys.argv[1] + out_file = sys.argv[2] + + update_xml_files(in_file, out_file) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/tests/native/start_native.sh b/tests/native/start_native.sh new file mode 100755 index 0000000000..949e32b98c --- /dev/null +++ b/tests/native/start_native.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright (C) 2019 Zilliqa +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +export DEV_TREE_ROOT=`readlink -f $(pwd)/../..` + +# This script and following scripts use DEV_TREE_ROOT to find the root of the dev tree +# this is also referenced in the python code to allow moving trees and OSes + + +echo "DEV_TREE_ROOT: $DEV_TREE_ROOT" + +# takes a copy of constants.xml and constants_local.xml and prepares new versions with .native extension + +python ./tests/native/prepare_constants.py $DEV_TREE_ROOT/Zilliqa/constants.xml $DEV_TREE_ROOT/Zilliqa/constants.xml.native +python ./tests/native/prepare_constants.py $DEV_TREE_ROOT/Zilliqa/constants_local.xml $DEV_TREE_ROOT/Zilliqa/constants_local.xml.native + +./tests/Node/pre_run.sh && ./tests/Node/test_node_lookup.sh && ./tests/Node/test_node_simple.sh