From a442616edf97dc65f6e768d249eb318fa317c747 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sat, 6 Jul 2019 23:01:10 -0400 Subject: move the demo app to examples/ --- .gitignore | 6 +- CMakeLists.txt | 15 +- examples/.gitignore | 3 + examples/CMakeLists.txt | 25 +++ examples/hotstuff_app.cpp | 425 ++++++++++++++++++++++++++++++++++++ examples/hotstuff_client.cpp | 181 ++++++++++++++++ scripts/run.sh | 501 ------------------------------------------- scripts/run_client.sh | 357 ------------------------------ scripts/run_demo.sh | 6 +- scripts/run_demo_client.sh | 2 +- src/hotstuff_app.cpp | 425 ------------------------------------ src/hotstuff_client.cpp | 181 ---------------- 12 files changed, 646 insertions(+), 1481 deletions(-) create mode 100644 examples/.gitignore create mode 100644 examples/CMakeLists.txt create mode 100644 examples/hotstuff_app.cpp create mode 100644 examples/hotstuff_client.cpp delete mode 100755 scripts/run.sh delete mode 100755 scripts/run_client.sh delete mode 100644 src/hotstuff_app.cpp delete mode 100644 src/hotstuff_client.cpp diff --git a/.gitignore b/.gitignore index b9dda4a..4de281a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,10 +5,8 @@ CMakeDoxyfile.in CMakeCache.txt cmake-build-debug/ libsecp256k1-prefix/ -hotstuff-app -hotstuff-client -hotstuff-keygen -hotstuff-tls-keygen +/hotstuff-keygen +/hotstuff-tls-keygen libhotstuff.a src/*.swo src/*.swp diff --git a/CMakeLists.txt b/CMakeLists.txt index f93b944..c6af4bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,19 +79,16 @@ option(HOTSTUFF_PROTO_LOG "enable protocol log" OFF) option(HOTSTUFF_MSG_STAT "eanble message statistics" ON) option(HOTSTUFF_BLK_PROFILE "enable block profiling" OFF) option(HOTSTUFF_TWO_STEP "use two-step HotStuff (instead of three-step HS)" OFF) +option(BUILD_EXAMPLES "build examples" ON) configure_file(src/config.h.in include/hotstuff/config.h @ONLY) -# add executables - -add_executable(hotstuff-app - src/hotstuff_app.cpp) -target_link_libraries(hotstuff-app hotstuff_static) - -add_executable(hotstuff-client - src/hotstuff_client.cpp) -target_link_libraries(hotstuff-client hotstuff_static) +# build examples +if(BUILD_EXAMPLES) + add_subdirectory(examples) +endif() +# build tools add_executable(hotstuff-keygen src/hotstuff_keygen.cpp) target_link_libraries(hotstuff-keygen hotstuff_static) diff --git a/examples/.gitignore b/examples/.gitignore new file mode 100644 index 0000000..e3bc5e0 --- /dev/null +++ b/examples/.gitignore @@ -0,0 +1,3 @@ +/Makefile +/hotstuff-app +/hotstuff-client diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt new file mode 100644 index 0000000..e0e3e2a --- /dev/null +++ b/examples/CMakeLists.txt @@ -0,0 +1,25 @@ +# Copyright (c) 2019 Ted Yin +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +add_executable(hotstuff-app hotstuff_app.cpp) +target_link_libraries(hotstuff-app hotstuff_static) + +add_executable(hotstuff-client hotstuff_client.cpp) +target_link_libraries(hotstuff-client hotstuff_static) diff --git a/examples/hotstuff_app.cpp b/examples/hotstuff_app.cpp new file mode 100644 index 0000000..a0fe80b --- /dev/null +++ b/examples/hotstuff_app.cpp @@ -0,0 +1,425 @@ +/** + * Copyright 2018 VMware + * Copyright 2018 Ted Yin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "salticidae/stream.h" +#include "salticidae/util.h" +#include "salticidae/network.h" +#include "salticidae/msg.h" + +#include "hotstuff/promise.hpp" +#include "hotstuff/type.h" +#include "hotstuff/entity.h" +#include "hotstuff/util.h" +#include "hotstuff/client.h" +#include "hotstuff/hotstuff.h" +#include "hotstuff/liveness.h" + +using salticidae::MsgNetwork; +using salticidae::ClientNetwork; +using salticidae::ElapsedTime; +using salticidae::Config; +using salticidae::_1; +using salticidae::_2; +using salticidae::static_pointer_cast; +using salticidae::trim_all; +using salticidae::split; + +using hotstuff::TimerEvent; +using hotstuff::EventContext; +using hotstuff::NetAddr; +using hotstuff::HotStuffError; +using hotstuff::CommandDummy; +using hotstuff::Finality; +using hotstuff::command_t; +using hotstuff::uint256_t; +using hotstuff::opcode_t; +using hotstuff::bytearray_t; +using hotstuff::DataStream; +using hotstuff::ReplicaID; +using hotstuff::MsgReqCmd; +using hotstuff::MsgRespCmd; +using hotstuff::get_hash; +using hotstuff::promise_t; + +using HotStuff = hotstuff::HotStuffSecp256k1; + +class HotStuffApp: public HotStuff { + double stat_period; + double impeach_timeout; + EventContext ec; + EventContext req_ec; + EventContext resp_ec; + /** Network messaging between a replica and its client. */ + ClientNetwork cn; + /** Timer object to schedule a periodic printing of system statistics */ + TimerEvent ev_stat_timer; + /** Timer object to monitor the progress for simple impeachment */ + TimerEvent impeach_timer; + /** The listen address for client RPC */ + NetAddr clisten_addr; + + std::unordered_map unconfirmed; + + using conn_t = ClientNetwork::conn_t; + using resp_queue_t = salticidae::MPSCQueueEventDriven; + + /* for the dedicated thread sending responses to the clients */ + std::thread req_thread; + std::thread resp_thread; + resp_queue_t resp_queue; + salticidae::BoxObj resp_tcall; + salticidae::BoxObj req_tcall; + + void client_request_cmd_handler(MsgReqCmd &&, const conn_t &); + + static command_t parse_cmd(DataStream &s) { + auto cmd = new CommandDummy(); + s >> *cmd; + return cmd; + } + + void reset_imp_timer() { + impeach_timer.del(); + impeach_timer.add(impeach_timeout); + } + + void state_machine_execute(const Finality &fin) override { + reset_imp_timer(); +#ifndef HOTSTUFF_ENABLE_BENCHMARK + HOTSTUFF_LOG_INFO("replicated %s", std::string(fin).c_str()); +#endif + resp_queue.enqueue(fin); + } + +#ifdef HOTSTUFF_MSG_STAT + std::unordered_set client_conns; + void print_stat() const; +#endif + + public: + HotStuffApp(uint32_t blk_size, + double stat_period, + double impeach_timeout, + ReplicaID idx, + const bytearray_t &raw_privkey, + NetAddr plisten_addr, + NetAddr clisten_addr, + hotstuff::pacemaker_bt pmaker, + const EventContext &ec, + size_t nworker, + const Net::Config &repnet_config, + const ClientNetwork::Config &clinet_config); + + void start(const std::vector> &reps); + void stop(); +}; + +std::pair split_ip_port_cport(const std::string &s) { + auto ret = trim_all(split(s, ";")); + if (ret.size() != 2) + throw std::invalid_argument("invalid cport format"); + return std::make_pair(ret[0], ret[1]); +} + +salticidae::BoxObj papp = nullptr; + +int main(int argc, char **argv) { + Config config("hotstuff.conf"); + + ElapsedTime elapsed; + elapsed.start(); + + auto opt_blk_size = Config::OptValInt::create(1); + auto opt_parent_limit = Config::OptValInt::create(-1); + auto opt_stat_period = Config::OptValDouble::create(10); + auto opt_replicas = Config::OptValStrVec::create(); + auto opt_idx = Config::OptValInt::create(0); + auto opt_client_port = Config::OptValInt::create(-1); + auto opt_privkey = Config::OptValStr::create(); + auto opt_tls_privkey = Config::OptValStr::create(); + auto opt_tls_cert = Config::OptValStr::create(); + auto opt_help = Config::OptValFlag::create(false); + auto opt_pace_maker = Config::OptValStr::create("dummy"); + auto opt_fixed_proposer = Config::OptValInt::create(1); + auto opt_base_timeout = Config::OptValDouble::create(1); + auto opt_prop_delay = Config::OptValDouble::create(1); + auto opt_imp_timeout = Config::OptValDouble::create(11); + auto opt_nworker = Config::OptValInt::create(1); + auto opt_repnworker = Config::OptValInt::create(1); + auto opt_repburst = Config::OptValInt::create(100); + auto opt_clinworker = Config::OptValInt::create(8); + auto opt_cliburst = Config::OptValInt::create(1000); + auto opt_notls = Config::OptValFlag::create(false); + + config.add_opt("block-size", opt_blk_size, Config::SET_VAL); + config.add_opt("parent-limit", opt_parent_limit, Config::SET_VAL); + config.add_opt("stat-period", opt_stat_period, Config::SET_VAL); + config.add_opt("replica", opt_replicas, Config::APPEND, 'a', "add an replica to the list"); + config.add_opt("idx", opt_idx, Config::SET_VAL, 'i', "specify the index in the replica list"); + config.add_opt("cport", opt_client_port, Config::SET_VAL, 'c', "specify the port listening for clients"); + config.add_opt("privkey", opt_privkey, Config::SET_VAL); + config.add_opt("tls-privkey", opt_tls_privkey, Config::SET_VAL); + config.add_opt("tls-cert", opt_tls_cert, Config::SET_VAL); + config.add_opt("pace-maker", opt_pace_maker, Config::SET_VAL, 'p', "specify pace maker (dummy, rr)"); + config.add_opt("proposer", opt_fixed_proposer, Config::SET_VAL, 'l', "set the fixed proposer (for dummy)"); + config.add_opt("base-timeout", opt_base_timeout, Config::SET_VAL, 't', "set the initial timeout for the Round-Robin Pacemaker"); + config.add_opt("prop-delay", opt_prop_delay, Config::SET_VAL, 't', "set the delay that follows the timeout for the Round-Robin Pacemaker"); + config.add_opt("imp-timeout", opt_imp_timeout, Config::SET_VAL, 'u', "set impeachment timeout (for sticky)"); + config.add_opt("nworker", opt_nworker, Config::SET_VAL, 'n', "the number of threads for verification"); + config.add_opt("repnworker", opt_repnworker, Config::SET_VAL, 'm', "the number of threads for replica network"); + config.add_opt("repburst", opt_repburst, Config::SET_VAL, 'b', ""); + config.add_opt("clinworker", opt_clinworker, Config::SET_VAL, 'M', "the number of threads for client network"); + config.add_opt("cliburst", opt_cliburst, Config::SET_VAL, 'B', ""); + config.add_opt("notls", opt_notls, Config::SWITCH_ON, 's', "disable TLS"); + config.add_opt("help", opt_help, Config::SWITCH_ON, 'h', "show this help info"); + + EventContext ec; + config.parse(argc, argv); + if (opt_help->get()) + { + config.print_help(); + exit(0); + } + auto idx = opt_idx->get(); + auto client_port = opt_client_port->get(); + std::vector> replicas; + for (const auto &s: opt_replicas->get()) + { + auto res = trim_all(split(s, ",")); + if (res.size() != 3) + throw HotStuffError("invalid replica info"); + replicas.push_back(std::make_tuple(res[0], res[1], res[2])); + } + + if (!(0 <= idx && (size_t)idx < replicas.size())) + throw HotStuffError("replica idx out of range"); + std::string binding_addr = std::get<0>(replicas[idx]); + if (client_port == -1) + { + auto p = split_ip_port_cport(binding_addr); + size_t idx; + try { + client_port = stoi(p.second, &idx); + } catch (std::invalid_argument &) { + throw HotStuffError("client port not specified"); + } + } + + NetAddr plisten_addr{split_ip_port_cport(binding_addr).first}; + + auto parent_limit = opt_parent_limit->get(); + hotstuff::pacemaker_bt pmaker; + if (opt_pace_maker->get() == "dummy") + pmaker = new hotstuff::PaceMakerDummyFixed(opt_fixed_proposer->get(), parent_limit); + else + pmaker = new hotstuff::PaceMakerRR(ec, parent_limit, opt_base_timeout->get(), opt_prop_delay->get()); + + HotStuffApp::Net::Config repnet_config; + ClientNetwork::Config clinet_config; + if (!opt_tls_privkey->get().empty() && !opt_notls->get()) + { + auto tls_priv_key = new salticidae::PKey( + salticidae::PKey::create_privkey_from_der( + hotstuff::from_hex(opt_tls_privkey->get()))); + auto tls_cert = new salticidae::X509( + salticidae::X509::create_from_der( + hotstuff::from_hex(opt_tls_cert->get()))); + repnet_config + .enable_tls(true) + .tls_key(tls_priv_key) + .tls_cert(tls_cert); + } + repnet_config + .burst_size(opt_repburst->get()) + .nworker(opt_repnworker->get()); + clinet_config + .burst_size(opt_cliburst->get()) + .nworker(opt_clinworker->get()); + papp = new HotStuffApp(opt_blk_size->get(), + opt_stat_period->get(), + opt_imp_timeout->get(), + idx, + hotstuff::from_hex(opt_privkey->get()), + plisten_addr, + NetAddr("0.0.0.0", client_port), + std::move(pmaker), + ec, + opt_nworker->get(), + repnet_config, + clinet_config); + std::vector> reps; + for (auto &r: replicas) + { + auto p = split_ip_port_cport(std::get<0>(r)); + reps.push_back(std::make_tuple( + NetAddr(p.first), + hotstuff::from_hex(std::get<1>(r)), + hotstuff::from_hex(std::get<2>(r)))); + } + auto shutdown = [&](int) { papp->stop(); }; + salticidae::SigEvent ev_sigint(ec, shutdown); + salticidae::SigEvent ev_sigterm(ec, shutdown); + ev_sigint.add(SIGINT); + ev_sigterm.add(SIGTERM); + + papp->start(reps); + elapsed.stop(true); + return 0; +} + +HotStuffApp::HotStuffApp(uint32_t blk_size, + double stat_period, + double impeach_timeout, + ReplicaID idx, + const bytearray_t &raw_privkey, + NetAddr plisten_addr, + NetAddr clisten_addr, + hotstuff::pacemaker_bt pmaker, + const EventContext &ec, + size_t nworker, + const Net::Config &repnet_config, + const ClientNetwork::Config &clinet_config): + HotStuff(blk_size, idx, raw_privkey, + plisten_addr, std::move(pmaker), ec, nworker, repnet_config), + stat_period(stat_period), + impeach_timeout(impeach_timeout), + ec(ec), + cn(req_ec, clinet_config), + clisten_addr(clisten_addr) { + /* prepare the thread used for sending back confirmations */ + resp_tcall = new salticidae::ThreadCall(resp_ec); + req_tcall = new salticidae::ThreadCall(req_ec); + resp_queue.reg_handler(resp_ec, [this](resp_queue_t &q) { + Finality fin; + while (q.try_dequeue(fin)) + { + auto it = unconfirmed.find(fin.cmd_hash); + if (it != unconfirmed.end()) + { + it->second.resolve(fin); + unconfirmed.erase(it); + } + } + return false; + }); + + /* register the handlers for msg from clients */ + cn.reg_handler(salticidae::generic_bind(&HotStuffApp::client_request_cmd_handler, this, _1, _2)); + cn.start(); + cn.listen(clisten_addr); +} + +void HotStuffApp::client_request_cmd_handler(MsgReqCmd &&msg, const conn_t &conn) { + const NetAddr addr = conn->get_addr(); + auto cmd = parse_cmd(msg.serialized); + const auto &cmd_hash = cmd->get_hash(); + HOTSTUFF_LOG_DEBUG("processing %s", std::string(*cmd).c_str()); + exec_command(cmd_hash, [this, addr](Finality fin) { + resp_queue.enqueue(fin); + }); + /* the following function is executed on the dedicated thread for confirming commands */ + resp_tcall->async_call([this, addr, cmd_hash](salticidae::ThreadCall::Handle &) { + auto it = unconfirmed.find(cmd_hash); + if (it == unconfirmed.end()) + it = unconfirmed.insert( + std::make_pair(cmd_hash, promise_t([](promise_t &){}))).first; + it->second.then([this, addr](const Finality &fin) { + try { + cn.send_msg(MsgRespCmd(std::move(fin)), addr); + } catch (std::exception &err) { + HOTSTUFF_LOG_WARN("unable to send to the client: %s", err.what()); + } + }); + }); +} + +void HotStuffApp::start(const std::vector> &reps) { + ev_stat_timer = TimerEvent(ec, [this](TimerEvent &) { + HotStuff::print_stat(); + HotStuffApp::print_stat(); + //HotStuffCore::prune(100); + ev_stat_timer.add(stat_period); + }); + ev_stat_timer.add(stat_period); + impeach_timer = TimerEvent(ec, [this](TimerEvent &) { + if (get_decision_waiting().size()) + get_pace_maker()->impeach(); + reset_imp_timer(); + }); + impeach_timer.add(impeach_timeout); + HOTSTUFF_LOG_INFO("** starting the system with parameters **"); + HOTSTUFF_LOG_INFO("blk_size = %lu", blk_size); + HOTSTUFF_LOG_INFO("conns = %lu", HotStuff::size()); + HOTSTUFF_LOG_INFO("** starting the event loop..."); + HotStuff::start(reps); + cn.reg_conn_handler([this](const salticidae::ConnPool::conn_t &_conn, bool connected) { + auto conn = salticidae::static_pointer_cast(_conn); + if (connected) + client_conns.insert(conn); + else + client_conns.erase(conn); + return true; + }); + req_thread = std::thread([this]() { req_ec.dispatch(); }); + resp_thread = std::thread([this]() { resp_ec.dispatch(); }); + /* enter the event main loop */ + ec.dispatch(); +} + +void HotStuffApp::stop() { + papp->req_tcall->async_call([this](salticidae::ThreadCall::Handle &) { + req_ec.stop(); + }); + papp->resp_tcall->async_call([this](salticidae::ThreadCall::Handle &) { + resp_ec.stop(); + }); + + req_thread.join(); + resp_thread.join(); + ec.stop(); +} + +void HotStuffApp::print_stat() const { +#ifdef HOTSTUFF_MSG_STAT + HOTSTUFF_LOG_INFO("--- client msg. (10s) ---"); + size_t _nsent = 0; + size_t _nrecv = 0; + for (const auto &conn: client_conns) + { + if (conn == nullptr) continue; + size_t ns = conn->get_nsent(); + size_t nr = conn->get_nrecv(); + size_t nsb = conn->get_nsentb(); + size_t nrb = conn->get_nrecvb(); + conn->clear_msgstat(); + HOTSTUFF_LOG_INFO("%s: %u(%u), %u(%u)", + std::string(conn->get_addr()).c_str(), ns, nsb, nr, nrb); + _nsent += ns; + _nrecv += nr; + } + HOTSTUFF_LOG_INFO("--- end client msg. ---"); +#endif +} diff --git a/examples/hotstuff_client.cpp b/examples/hotstuff_client.cpp new file mode 100644 index 0000000..831b7ff --- /dev/null +++ b/examples/hotstuff_client.cpp @@ -0,0 +1,181 @@ +/** + * Copyright 2018 VMware + * Copyright 2018 Ted Yin + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "salticidae/type.h" +#include "salticidae/netaddr.h" +#include "salticidae/network.h" +#include "salticidae/util.h" + +#include "hotstuff/util.h" +#include "hotstuff/type.h" +#include "hotstuff/client.h" + +using salticidae::Config; + +using hotstuff::ReplicaID; +using hotstuff::NetAddr; +using hotstuff::EventContext; +using hotstuff::MsgReqCmd; +using hotstuff::MsgRespCmd; +using hotstuff::CommandDummy; +using hotstuff::HotStuffError; +using hotstuff::uint256_t; +using hotstuff::opcode_t; +using hotstuff::command_t; + +EventContext ec; +ReplicaID proposer; +size_t max_async_num; +int max_iter_num; +uint32_t cid; +uint32_t cnt = 0; +uint32_t nfaulty; + +struct Request { + command_t cmd; + size_t confirmed; + salticidae::ElapsedTime et; + Request(const command_t &cmd): cmd(cmd), confirmed(0) { et.start(); } +}; + +using Net = salticidae::MsgNetwork; + +std::unordered_map conns; +std::unordered_map waiting; +std::vector replicas; +std::vector> elapsed; +Net mn(ec, Net::Config()); + +void connect_all() { + for (size_t i = 0; i < replicas.size(); i++) + conns.insert(std::make_pair(i, mn.connect_sync(replicas[i]))); +} + +bool try_send(bool check = true) { + if ((!check || waiting.size() < max_async_num) && max_iter_num) + { + auto cmd = new CommandDummy(cid, cnt++); + MsgReqCmd msg(*cmd); + for (auto &p: conns) mn.send_msg(msg, p.second); +#ifndef HOTSTUFF_ENABLE_BENCHMARK + HOTSTUFF_LOG_INFO("send new cmd %.10s", + get_hex(cmd->get_hash()).c_str()); +#endif + waiting.insert(std::make_pair( + cmd->get_hash(), Request(cmd))); + if (max_iter_num > 0) + max_iter_num--; + return true; + } + return false; +} + +void client_resp_cmd_handler(MsgRespCmd &&msg, const Net::conn_t &) { + auto &fin = msg.fin; + HOTSTUFF_LOG_DEBUG("got %s", std::string(msg.fin).c_str()); + const uint256_t &cmd_hash = fin.cmd_hash; + auto it = waiting.find(cmd_hash); + auto &et = it->second.et; + if (it == waiting.end()) return; + et.stop(); + if (++it->second.confirmed <= nfaulty) return; // wait for f + 1 ack +#ifndef HOTSTUFF_ENABLE_BENCHMARK + HOTSTUFF_LOG_INFO("got %s, wall: %.3f, cpu: %.3f", + std::string(fin).c_str(), + et.elapsed_sec, et.cpu_elapsed_sec); +#else + struct timeval tv; + gettimeofday(&tv, nullptr); + elapsed.push_back(std::make_pair(tv, et.elapsed_sec)); +#endif + waiting.erase(it); + while (try_send()); +} + +std::pair split_ip_port_cport(const std::string &s) { + auto ret = salticidae::trim_all(salticidae::split(s, ";")); + return std::make_pair(ret[0], ret[1]); +} + +int main(int argc, char **argv) { + Config config("hotstuff.conf"); + + auto opt_idx = Config::OptValInt::create(0); + auto opt_replicas = Config::OptValStrVec::create(); + auto opt_max_iter_num = Config::OptValInt::create(100); + auto opt_max_async_num = Config::OptValInt::create(10); + auto opt_cid = Config::OptValInt::create(-1); + + auto shutdown = [&](int) { ec.stop(); }; + salticidae::SigEvent ev_sigint(ec, shutdown); + salticidae::SigEvent ev_sigterm(ec, shutdown); + ev_sigint.add(SIGINT); + ev_sigterm.add(SIGTERM); + + mn.reg_handler(client_resp_cmd_handler); + mn.start(); + + config.add_opt("idx", opt_idx, Config::SET_VAL); + config.add_opt("cid", opt_cid, Config::SET_VAL); + config.add_opt("replica", opt_replicas, Config::APPEND); + config.add_opt("iter", opt_max_iter_num, Config::SET_VAL); + config.add_opt("max-async", opt_max_async_num, Config::SET_VAL); + config.parse(argc, argv); + auto idx = opt_idx->get(); + max_iter_num = opt_max_iter_num->get(); + max_async_num = opt_max_async_num->get(); + std::vector raw; + for (const auto &s: opt_replicas->get()) + { + auto res = salticidae::trim_all(salticidae::split(s, ",")); + if (res.size() < 1) + throw HotStuffError("format error"); + raw.push_back(res[0]); + } + + if (!(0 <= idx && (size_t)idx < raw.size() && raw.size() > 0)) + throw std::invalid_argument("out of range"); + cid = opt_cid->get() != -1 ? opt_cid->get() : idx; + for (const auto &p: raw) + { + auto _p = split_ip_port_cport(p); + size_t _; + replicas.push_back(NetAddr(NetAddr(_p.first).ip, htons(stoi(_p.second, &_)))); + } + + nfaulty = (replicas.size() - 1) / 3; + HOTSTUFF_LOG_INFO("nfaulty = %zu", nfaulty); + connect_all(); + while (try_send()); + ec.dispatch(); + +#ifdef HOTSTUFF_ENABLE_BENCHMARK + for (const auto &e: elapsed) + { + char fmt[64]; + struct tm *tmp = localtime(&e.first.tv_sec); + strftime(fmt, sizeof fmt, "%Y-%m-%d %H:%M:%S.%%06u [hotstuff info] %%.6f\n", tmp); + fprintf(stderr, fmt, e.first.tv_usec, e.second); + } +#endif + return 0; +} diff --git a/scripts/run.sh b/scripts/run.sh deleted file mode 100755 index aa8fca1..0000000 --- a/scripts/run.sh +++ /dev/null @@ -1,501 +0,0 @@ -#!/bin/bash - -proj_server_bin="hotstuff-app" -proj_server_path="/home/ted/hot-stuff/$proj_server_bin" -proj_conf_name="hotstuff.conf" - -peer_list="./nodes.txt" # the list of nodes -conf_src="./hotstuff.gen.conf" -server_map="./server_map.txt" # optional mapping from node ip to server ip -template_dir="template" # the dir that keeps the content shared among all nodes -remote_base="/home/ted/testbed" # remote dir used to keep files for the experiment -#remote_base="/tmp/" # remote dir used to keep files for the experiment -remote_log="log" # log filename -remote_user="ted" -copy_to_remote_pat="rsync -avz @:" -copy_from_remote_pat="rsync -avz @: " -exe_remote_pat="ssh @ bash" -run_remote_pat="cd \"\"; gdb -ex r -ex bt -ex generate-core-file -ex q --args '$proj_server_path' --conf \"hotstuff.gen-sec.conf\"" -reset_remote_pat="pgrep -f '$proj_server_bin' | xargs kill -9" - -fin_keyword="error:" # the keyword indicating completion of execution -fin_chk_period=1 -fin_chk_skip_pat='^([A-O][0-9]*)|(_ctl)$' -force_peer_list=0 -async_num=128 - -function join { local IFS="$1"; shift; echo "$*"; } -function split { - local IFS="$1" - local arr=($2) - echo "${arr[@]}" -} - -function die { echo "$1"; exit 1; } - -declare -A nodes -declare -A node_confs -nodes_cnt=0 -function get_node_info { - pl="$1" - if [[ "$force_peer_list" == 1 ]]; then - pl="$peer_list" - fi - OIFS="$IFS" - IFS=$'\n' - node_list=($(cat "$pl")) - IFS="$OIFS" - for tuple in "${node_list[@]}"; do - tup0=($(split $'\t' "$tuple")) - tup=($(split : "${tup0[0]}")) - nodes[${tup[0]}]="${tup[1]}:${tup[2]}" - node_confs[${tup[0]}]="${tup0[@]:1}" - echo "${tup[0]} => ${nodes[${tup[0]}]} & ${node_confs[${tup[0]}]}" - let nodes_cnt++ - done -} - -declare -A server_map -function get_server_map { - { - IFS=$'\n' - map_list=($(cat "$1")) - } - IFS=$'\n \t' - for pair in "${map_list[@]}"; do - p=($pair) - server_map[${p[0]}]="${p[1]}" - echo "mapping ${p[0]} => ${p[1]}" - done -} - - -function get_addr { - tup=($(split ';' $1)) - echo "${tup[0]}" -} - -function get_ip { - tup=($(split : $1)) - echo "${tup[0]}" -} - -function get_peer_port { - tup=($(split : $1)) - tup2=($(split ';' ${tup[1]})) - echo "${tup2[0]}" -} - - -function get_client_port { - tup=($(split : $1)) - tup2=($(split ';' ${tup[1]})) - echo "${tup2[1]}" -} - - -function get_ip_by_id { - get_ip "${nodes[$1]}" -} - -function get_peer_port_by_id { - get_peer_port "${nodes[$1]}" -} - - -function get_client_port_by_id { - get_client_port "${nodes[$1]}" -} - -function copy_file { - local pat="$1" - local cmd="${pat///$2}" - cmd="${cmd///$3}" - cmd="${cmd///$remote_user}" - cmd="${cmd///$4}" - echo $cmd - eval "$cmd" -} >> log 2>&1 - -function execute_remote_cmd_pid { - local node_ip="$1" - local c="$2" - local l="$3" - local cmd="${exe_remote_pat///$node_ip}" - cmd="${cmd///$remote_user}" - eval $cmd << EOF -$c > $l 2>&1 & echo \$! -EOF -} - - - -function execute_remote_cmd_stat { - local node_ip="$1" - local c="$2" - local l="$3" - local cmd="${exe_remote_pat///$node_ip}" - cmd="${cmd///$remote_user}" - eval $cmd << EOF -$c > $l 2>&1 ; echo \$? -EOF -} - - -function _remote_load { - local workdir="$1" - local rworkdir="$2" - local node_ip="$3" - local rid="$4" - local extra_conf=($5) - local tmpldir="$workdir/$template_dir/" - local node_tmpldir="$workdir/$rid" - [[ $(execute_remote_cmd_stat "$node_ip" \ - "mkdir -p \"$rworkdir\"" \ - /dev/null) == 0 ]] || die "failed to create directory $rworkdir" - copy_file "$copy_to_remote_pat" "$tmpldir" "$node_ip" "$rworkdir" - for conf in "${extra_conf[@]}"; do - copy_file "$copy_to_remote_pat" "$node_tmpldir/$conf" "$node_ip" "$rworkdir" - done -} - -function _remote_start { - local workdir="$1" - local rworkdir="$2" - local node_id="$3" - local node_ip="$4" - local client_port="$5" - local cmd="${run_remote_pat///$rworkdir}" - cmd="${cmd///$node_id}" - cmd="${cmd///$client_port}" - execute_remote_cmd_pid "$node_ip" "$cmd" \ - "\"$rworkdir/$remote_log\"" > "$workdir/${node_id}.pid" -} - -function _remote_exec { - local workdir="$1" - local rworkdir="$2" - local node_ip="$3" - local cmd="$4" - [[ $(execute_remote_cmd_stat "$node_ip" "$cmd" /dev/null) == 0 ]] -} - -function _remote_stop { - local node_pid="$4" - _remote_exec "$1" "$2" "$3" "kill $node_pid" -} - -function _remote_status { - local node_pid="$4" - _remote_exec "$1" "$2" "$3" "kill -0 $node_pid" -} - -function _remote_finished { - _remote_exec "$1" "$2" "$3" "grep \"$fin_keyword\" \"$rworkdir/$remote_log\"" -} - -function _remote_fetch { - local workdir="$1" - local rworkdir="$2" - local node_id="$3" - local node_ip="$4" - copy_file "$copy_from_remote_pat" "$workdir/${node_id}.log" "$node_ip" "$rworkdir/$remote_log" -} - -function start_all { - local workdir="$1" - local tmpldir="$workdir/$template_dir/" - mkdir "$workdir" > /dev/null 2>&1 || die "workdir already exists" - rm -rf "$tmpldir" - mkdir "$tmpldir" - cp "$peer_list" "$workdir/peer_list.txt" - cp "$server_map" "$workdir/server_map.txt" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - echo "copying configuration file" - rsync -avP "$conf_src" "$tmpldir/$proj_conf_name" - echo "${node_list[@]}" - cnt="${#nodes[@]}" - #for rid in "${!nodes[@]}"; do - rid=0 - c=0 - while [[ "$rid" -lt "$cnt" ]]; do - local node_tmpldir="$workdir/$rid" - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local pport="$(get_peer_port_by_id $rid)" - local cport="$(get_client_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - local extra_conf_=(${node_confs[$rid]}) - rm -rf "$node_tmpldir" - mkdir "$node_tmpldir" - ( - local extra_conf=() - for conf in "${extra_conf_[@]}"; do - cp "$conf" "$node_tmpldir/" - extra_conf+=($(basename "$conf")) - copy_file "$copy_to_remote_pat" "$tmpldir/$conf" "$node_ip" "$rworkdir" - done - echo "Starting $rid @ $ip, $pport and $cport" - _remote_load "$workdir" "$rworkdir" "$ip" "$rid" "${extra_conf[@]}" - echo "$rid loaded" - ) & - let rid++ - let c++ - if [[ "$c" -eq "$async_num" ]]; then - c=0 - wait - fi - done - wait - rid=0 - c=0 - #for rid in "${!nodes[@]}"; do - while [[ "$rid" -lt "$cnt" ]]; do - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local pport="$(get_peer_port_by_id $rid)" - local cport="$(get_client_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - ( - echo "Starting $rid @ $ip, $pport and $cport" - _remote_start "$workdir" "$rworkdir" "$rid" "$ip" "$cport" - echo "$rid started" - ) & - let rid++ - let c++ - if [[ "$c" -eq "$async_num" ]]; then - c=0 - wait - fi - done - wait -} - -function fetch_all { - local workdir="$1" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - for rid in "${!nodes[@]}"; do - #if [[ "$rid" != 0 ]]; then - # continue - #fi - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local port="$(get_peer_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - local pid="$(cat $workdir/${rid}.pid)" - local msg="Fetching $rid @ $ip, $port " - _remote_fetch "$workdir" "$rworkdir" "$rid" "$ip" && echo "$msg: copied" || echo "$msg: failed" & - done - wait -} - -function exec_all { - local workdir="$1" - local cmd="$2" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - cnt="${#nodes[@]}" - rid=0 - c=0 - #for rid in "${!nodes[@]}"; do - while [[ "$rid" -lt "$cnt" ]]; do - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local port="$(get_peer_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - local msg="Executing $rid @ $ip, $port " - _remote_exec "$workdir" "$rworkdir" "$ip" "$cmd" && echo "$msg: succeeded" || echo "$msg: failed" & - let rid++ - let c++ - if [[ "$c" -eq "$async_num" ]]; then - c=0 - wait - fi - done - wait -} - -function reset_all { - exec_all "$1" "$reset_remote_pat" -} - -function stop_all { - local workdir="$1" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - for rid in "${!nodes[@]}"; do - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local port="$(get_peer_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - local pid="$(cat $workdir/${rid}.pid)" - local msg="Killing $rid @ $ip, $port " - _remote_stop "$workdir" "$rworkdir" "$ip" "$pid" && echo "$msg: stopped" || echo "$msg: failed" & - done - wait -} - -function status_all { - local workdir="$1" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - cnt="${#nodes[@]}" - rid=0 - c=0 - #for rid in "${!nodes[@]}"; do - while [[ "$rid" -lt "$cnt" ]]; do - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local port="$(get_peer_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - local pid="$(cat $workdir/${rid}.pid)" - local msg="$rid @ $ip, $port " - _remote_status "$workdir" "$rworkdir" "$ip" "$pid" && echo "$msg: running" || echo "$msg: dead" & - let rid++ - let c++ - if [[ "$c" -eq "$async_num" ]]; then - c=0 - wait - fi - done - wait -} - -function finished_all { - local workdir="$1" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - for rid in "${!nodes[@]}"; do - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local port="$(get_peer_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - if [[ "$rid" =~ $fin_chk_skip_pat ]]; then - continue - fi - printf "$rid @ $ip, $port " - _remote_finished "$workdir" "$rworkdir" "$ip" && echo "finished" || echo "in-progress" - done -} - -function wait_all { - local workdir="$1" - get_node_info "$workdir/peer_list.txt" - get_server_map "$workdir/server_map.txt" - while true; do - finished=1 - printf "checking the nodes..." - for rid in "${!nodes[@]}"; do - local ip="$(get_ip_by_id $rid)" - ip="${server_map[$ip]:-$ip}" - local port="$(get_peer_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${rid}" - if [[ "$rid" =~ $fin_chk_skip_pat ]]; then - continue - fi - if ! _remote_finished "$workdir" "$rworkdir" "$ip"; then - finished=0 - break - fi - done - if [[ $finished == 1 ]]; then - break - fi - echo "not finished yet, wait for $fin_chk_period secs" - sleep "$fin_chk_period" - done - echo "finished" -} - -function check_all { - status_all "$1" | grep dead -q - [[ "$?" -eq 0 ]] && die "some nodes are dead" - echo "ok" -} - -function print_help { -echo "Usage: $0 [--bin] [--path] [--conf] [--conf-src] [--peer-list] [--server-map] [--user] [--force-peer-list] [--help] COMMAND WORKDIR - - --help show this help and exit - --bin name of binary executable - --path path to the binary - --conf shared configuration filename - --conf-src shared configuration source file - --peer-list FILE read peer list from FILE (default: $peer_list) - --server-map FILE read server map from FILE (default: $server_map) - --user USER the username to login the remote machines - --force-peer-list force the use of FILE specified by --peer-list - instead of the peer list in WORKDIR" - exit 0 -} - -function check_argnum { - argnum=$(($# - 1)) - [[ "$1" -eq "$argnum" ]] || die "incorrect argnum: got $argnum, $1 expected" -} - -getopt --test > /dev/null -[[ $? -ne 4 ]] && die "getopt unsupported" - -SHORT= -LONG='\ -bin:,path:,conf:,conf-src:,\ -peer-list:,\ -server-map:,\ -remote-base:,\ -remote-user:,\ -copy-to-remote-pat:,\ -copy-from-remote-pat:,\ -exe-remote-pat:,\ -run-remote-pat:,\ -reset-remote-pat:,\ -fin-keyword:,\ -fin-chk-period:,\ -fin-chk-skip-pat:,\ -force-peer-list,\ -help' - -PARSED=$(getopt --options "$SHORT" --longoptions "$LONG" --name "$0" -- "$@") -[[ $? -ne 0 ]] && exit 1 -eval set -- "$PARSED" - -while true; do - case "$1" in - --bin) proj_server_bin="$2"; shift 2;; - --path) proj_server_path="$2"; shift 2;; - --conf) proj_conf_name="$2"; shift 2;; - --conf-src) conf_src="$2"; shift 2;; - --peer-list) peer_list="$2"; shift 2;; - --server-map) server_map="$2"; shift 2;; - --remote-base) remote_base="$2"; shift 2;; - --remote-user) remote_user="$2"; shift 2;; - --copy-to-remote-pat) copy_to_remote_pat="$2"; shift 2;; - --copy-from-remote-pat) copy_from_remote_pat="$2"; shift 2;; - --exe-remote-pat) exe_remote_pat="$2"; shift 2;; - --run-remote-pat) run_remote_pat="$2"; shift 2;; - --reset-remote-pat) reset_remote_pat="$2"; shift 2;; - --fin-keyword) fin_keyword="$2"; shift 2;; - --fin-chk-period) fin_chk_period="$2"; shift 2;; - --fin-chk-skip-pat) fin_chk_skip_pat="$2"; shift 2;; - --force-peer-list) force_peer_list=1; shift 1;; - --help) print_help; shift 1;; - --) shift; break;; - *) die "internal error";; - esac -done -cmd="$1" -shift 1 -case "$cmd" in - start) check_argnum 1 "$@" && start_all "$1" ;; - stop) check_argnum 1 "$@" && stop_all "$1" ;; - status) check_argnum 1 "$@" && status_all "$1" ;; - check) check_argnum 1 "$@" && check_all "$1" ;; - finished) check_argnum 1 "$@" && finished_all "$1" ;; - fetch) check_argnum 1 "$@" && fetch_all "$1" ;; - wait) check_argnum 1 "$@" && wait_all "$1" ;; - reset) check_argnum 1 "$@" && reset_all "$1" ;; - exec) check_argnum 2 "$@" && exec_all "$1" "$2" ;; - *) print_help ;; -esac diff --git a/scripts/run_client.sh b/scripts/run_client.sh deleted file mode 100755 index 7825c02..0000000 --- a/scripts/run_client.sh +++ /dev/null @@ -1,357 +0,0 @@ -#!/bin/bash - -proj_client_bin="hotstuff-client" -proj_client_path="/home/ted/hot-stuff/$proj_client_bin" -proj_conf_name="hotstuff.conf" - -peer_list="./nodes.txt" # the list of nodes -client_list="./clients.txt" # the list of clients -conf_src="./hotstuff.gen.conf" -template_dir="template" # the dir that keeps the content shared among all nodes -remote_base="/home/ted/testbed" # remote dir used to keep files for the experiment -#remote_base="/tmp/" # remote dir used to keep files for the experiment -remote_log="log" # log filename -remote_user="ted" -copy_to_remote_pat="rsync -avz @:" -copy_from_remote_pat="rsync -avz @: " -exe_remote_pat="ssh @ bash" -run_remote_pat="cd \"\"; '$proj_client_path' --idx \"\" --iter -1 --max-async 3" -reset_remote_pat="pgrep -f '$proj_client_bin' | xargs kill -9" -node_id_step=1 - -function join { local IFS="$1"; shift; echo "$*"; } -function split { - local IFS="$1" - local arr=($2) - echo "${arr[@]}" -} - -function die { echo "$1"; exit 1; } - -declare -A nodes -nodes_cnt=0 -function get_node_info { - pl="$1" - if [[ "$force_peer_list" == 1 ]]; then - pl="$peer_list" - fi - OIFS="$IFS" - IFS=$'\n' - node_list=($(cat "$pl")) - IFS="$OIFS" - for tuple in "${node_list[@]}"; do - tup0=($(split $'\t' "$tuple")) - tup=($(split : "${tup0[0]}")) - nodes[${tup[0]}]="${tup[1]}:${tup[2]}" - echo "${tup[0]} => ${nodes[${tup[0]}]}" - let nodes_cnt++ - done -} - -function get_client_info { - cip_list=($(cat "$1")) -} - - -function get_addr { - tup=($(split ';' $1)) - echo "${tup[0]}" -} - -function get_ip { - tup=($(split : $1)) - echo "${tup[0]}" -} - -function get_peer_port { - tup=($(split : $1)) - tup2=($(split ';' ${tup[1]})) - echo "${tup2[0]}" -} - - -function get_client_port { - tup=($(split : $1)) - tup2=($(split ';' ${tup[1]})) - echo "${tup2[1]}" -} - - -function get_ip_by_id { - get_ip "${nodes[$1]}" -} - -function get_peer_port_by_id { - get_peer_port "${nodes[$1]}" -} - - -function get_client_port_by_id { - get_client_port "${nodes[$1]}" -} - -function copy_file { - local pat="$1" - local cmd="${pat///$2}" - cmd="${cmd///$3}" - cmd="${cmd///$remote_user}" - cmd="${cmd///$4}" - echo $cmd - eval "$cmd" -} >> log 2>&1 - -function execute_remote_cmd_pid { - local node_ip="$1" - local c="$2" - local l="$3" - local cmd="${exe_remote_pat///$node_ip}" - cmd="${cmd///$remote_user}" - eval $cmd << EOF -$c > $l 2>&1 & echo \$! -EOF -} - - - -function execute_remote_cmd_stat { - local node_ip="$1" - local c="$2" - local l="$3" - local cmd="${exe_remote_pat///$node_ip}" - cmd="${cmd///$remote_user}" - eval $cmd << EOF -$c > $l 2>&1 ; echo \$? -EOF -} - - -function _remote_load { - local workdir="$1" - local rworkdir="$2" - local node_ip="$3" - local tmpldir="$workdir/$template_dir/" - [[ $(execute_remote_cmd_stat "$node_ip" \ - "mkdir -p \"$rworkdir\"" \ - /dev/null) == 0 ]] || die "failed to create directory $rworkdir" - copy_file "$copy_to_remote_pat" "$tmpldir" "$node_ip" "$rworkdir" -} - -function _remote_start { - local workdir="$1" - local rworkdir="$2" - local node_id="$3" - local node_ip="$4" - local client_port="$5" - local client_ip="$6" - local cmd="${run_remote_pat///$rworkdir}" - cmd="${cmd///$node_id_step}" - cmd="${cmd///$((node_id * node_id_step))}" - cmd="${cmd///$node_ip:$client_port}" - execute_remote_cmd_pid "$client_ip" "$cmd" \ - "\"$rworkdir/$remote_log\"" > "$workdir/${node_id}.pid" -} - -function _remote_exec { - local workdir="$1" - local rworkdir="$2" - local node_ip="$3" - local cmd="$4" - [[ $(execute_remote_cmd_stat "$node_ip" "$cmd" /dev/null) == 0 ]] -} - -function _remote_stop { - local node_pid="$4" - _remote_exec "$1" "$2" "$3" "kill $node_pid" -} - -function _remote_status { - local node_pid="$4" - _remote_exec "$1" "$2" "$3" "kill -0 $node_pid" -} - -function _remote_fetch { - local workdir="$1" - local rworkdir="$2" - local node_id="$3" - local node_ip="$4" - copy_file "$copy_from_remote_pat" "$workdir/${node_id}.log" "$node_ip" "$rworkdir/$remote_log" -} - -function start_all { - local workdir="$1" - local tmpldir="$workdir/$template_dir/" - mkdir "$workdir" > /dev/null 2>&1 || die "workdir already exists" - rm -rf "$tmpldir" - mkdir "$tmpldir" - cp "$peer_list" "$workdir/peer_list.txt" - cp "$client_list" "$workdir/client_list.txt" - get_node_info "$workdir/peer_list.txt" - get_client_info "$workdir/client_list.txt" - echo "coyping configuration file" - rsync -avP "$conf_src" "$tmpldir/$proj_conf_name" - local i=0 - local j=0 - for cip in "${cip_list[@]}"; do - local rid="${nodes[$i]}" - local ip="$(get_ip_by_id $rid)" - local pport="$(get_peer_port_by_id $rid)" - local cport="$(get_client_port_by_id $rid)" - local rworkdir="$remote_base/$workdir/${j}" - ( - echo "Starting a client @ $cip, connecting to server #$rid @ $ip:$cport" - _remote_load "$workdir" "$rworkdir" "$cip" - _remote_start "$workdir" "$rworkdir" "$j" "$ip" "$cport" "$cip" - echo "client #$j started" - ) & - let i++ - let j++ - if [[ "$i" -eq "${#nodes[@]}" ]]; then - i=0 - fi - done - wait -} - -function fetch_all { - local workdir="$1" - get_client_info "$workdir/client_list.txt" - local i=0 - for cip in "${cip_list[@]}"; do - local rworkdir="$remote_base/$workdir/${i}" - local pid="$(cat $workdir/${i}.pid)" - local msg="Fetching $i @ $cip" - _remote_fetch "$workdir" "$rworkdir" "$i" "$cip" && echo "$msg: copied" || echo "$msg: failed" & - let i++ - done - wait -} - -function exec_all { - local workdir="$1" - local cmd="$2" - get_client_info "$workdir/client_list.txt" - local i=0 - for cip in "${cip_list[@]}"; do - local rworkdir="$remote_base/$workdir/${i}" - local msg="Executing $i @ $cip" - _remote_exec "$workdir" "$rworkdir" "$cip" "$cmd" && echo "$msg: succeeded" || echo "$msg: failed" & - let i++ - done - wait -} - -function reset_all { - exec_all "$1" "$reset_remote_pat" -} - -function stop_all { - local workdir="$1" - get_client_info "$workdir/client_list.txt" - local i=0 - for cip in "${cip_list[@]}"; do - local rworkdir="$remote_base/$workdir/${i}" - local pid="$(cat $workdir/${i}.pid)" - local msg="Killing $i @ $cip" - _remote_stop "$workdir" "$rworkdir" "$cip" "$pid" && echo "$msg: stopped" || echo "$msg: failed" & - let i++ - done - wait -} - -function status_all { - local workdir="$1" - get_client_info "$workdir/client_list.txt" - local i=0 - for cip in "${cip_list[@]}"; do - local rworkdir="$remote_base/$workdir/${i}" - local pid="$(cat $workdir/${i}.pid)" - local msg="$i @ $cip" - _remote_status "$workdir" "$rworkdir" "$cip" "$pid" && echo "$msg: running" || echo "$msg: dead" & - let i++ - done - wait -} - -function check_all { - status_all "$1" | grep dead -q - [[ "$?" -eq 0 ]] && die "some nodes are dead" - echo "ok" -} - -function print_help { -echo "Usage: $0 [--bin] [--path] [--conf] [--conf-src] [--peer-list] [--client-list] [--user] [--force-peer-list] [--help] COMMAND WORKDIR - - --help show this help and exit - --bin name of binary executable - --path path to the binary - --conf shared configuration filename - --conf-src shared configuration source file - --peer-list FILE read peer list from FILE (default: $peer_list) - --client-list FILE read client list from FILE (default: $client_list) - --user USER the username to login the remote machines - --force-peer-list force the use of FILE specified by --peer-list - instead of the peer list in WORKDIR" - exit 0 -} - -function check_argnum { - argnum=$(($# - 1)) - [[ "$1" -eq "$argnum" ]] || die "incorrect argnum: got $argnum, $1 expected" -} - -getopt --test > /dev/null -[[ $? -ne 4 ]] && die "getopt unsupported" - -SHORT= -LONG='\ -bin:,path:,conf:,conf-src:,\ -peer-list:,\ -client-list:,\ -remote-base:,\ -remote-user:,\ -copy-to-remote-pat:,\ -copy-from-remote-pat:,\ -exe-remote-pat:,\ -run-remote-pat:,\ -reset-remote-pat:,\ -force-peer-list,\ -node-id-step:,\ -help' - -PARSED=$(getopt --options "$SHORT" --longoptions "$LONG" --name "$0" -- "$@") -[[ $? -ne 0 ]] && exit 1 -eval set -- "$PARSED" - -while true; do - case "$1" in - --bin) proj_client_bin="$2"; shift 2;; - --path) proj_client_path="$2"; shift 2;; - --conf) proj_conf_name="$2"; shift 2;; - --conf-src) conf_src="$2"; shift 2;; - --peer-list) peer_list="$2"; shift 2;; - --client-list) client_list="$2"; shift 2;; - --remote-base) remote_base="$2"; shift 2;; - --remote-user) remote_user="$2"; shift 2;; - --copy-to-remote-pat) copy_to_remote_pat="$2"; shift 2;; - --copy-from-remote-pat) copy_from_remote_pat="$2"; shift 2;; - --exe-remote-pat) exe_remote_pat="$2"; shift 2;; - --run-remote-pat) run_remote_pat="$2"; shift 2;; - --reset-remote-pat) reset_remote_pat="$2"; shift 2;; - --node-id-step) node_id_step="$2"; shift 2;; - --help) print_help; shift 1;; - --) shift; break;; - *) die "internal error";; - esac -done -cmd="$1" -shift 1 -case "$cmd" in - start) check_argnum 1 "$@" && start_all "$1" ;; - stop) check_argnum 1 "$@" && stop_all "$1" ;; - status) check_argnum 1 "$@" && status_all "$1" ;; - check) check_argnum 1 "$@" && check_all "$1" ;; - fetch) check_argnum 1 "$@" && fetch_all "$1" ;; - reset) check_argnum 1 "$@" && reset_all "$1" ;; - exec) check_argnum 2 "$@" && exec_all "$1" "$2" ;; - *) print_help ;; -esac diff --git a/scripts/run_demo.sh b/scripts/run_demo.sh index 5f54787..0ad462e 100755 --- a/scripts/run_demo.sh +++ b/scripts/run_demo.sh @@ -5,8 +5,8 @@ if [[ $# -gt 0 ]]; then fi for i in "${rep[@]}"; do echo "starting replica $i" - #valgrind --leak-check=full ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & - #gdb -ex r -ex bt -ex q --args ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & - ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & + #valgrind --leak-check=full ./examples/hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & + #gdb -ex r -ex bt -ex q --args ./examples/hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & + ./examples/hotstuff-app --conf ./hotstuff-sec${i}.conf > log${i} 2>&1 & done wait diff --git a/scripts/run_demo_client.sh b/scripts/run_demo_client.sh index 5ed274d..f27b44d 100755 --- a/scripts/run_demo_client.sh +++ b/scripts/run_demo_client.sh @@ -1,2 +1,2 @@ #!/bin/bash -./hotstuff-client --idx 0 --iter -1 --max-async 4 +./examples/hotstuff-client --idx 0 --iter -1 --max-async 4 diff --git a/src/hotstuff_app.cpp b/src/hotstuff_app.cpp deleted file mode 100644 index a0fe80b..0000000 --- a/src/hotstuff_app.cpp +++ /dev/null @@ -1,425 +0,0 @@ -/** - * Copyright 2018 VMware - * Copyright 2018 Ted Yin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "salticidae/stream.h" -#include "salticidae/util.h" -#include "salticidae/network.h" -#include "salticidae/msg.h" - -#include "hotstuff/promise.hpp" -#include "hotstuff/type.h" -#include "hotstuff/entity.h" -#include "hotstuff/util.h" -#include "hotstuff/client.h" -#include "hotstuff/hotstuff.h" -#include "hotstuff/liveness.h" - -using salticidae::MsgNetwork; -using salticidae::ClientNetwork; -using salticidae::ElapsedTime; -using salticidae::Config; -using salticidae::_1; -using salticidae::_2; -using salticidae::static_pointer_cast; -using salticidae::trim_all; -using salticidae::split; - -using hotstuff::TimerEvent; -using hotstuff::EventContext; -using hotstuff::NetAddr; -using hotstuff::HotStuffError; -using hotstuff::CommandDummy; -using hotstuff::Finality; -using hotstuff::command_t; -using hotstuff::uint256_t; -using hotstuff::opcode_t; -using hotstuff::bytearray_t; -using hotstuff::DataStream; -using hotstuff::ReplicaID; -using hotstuff::MsgReqCmd; -using hotstuff::MsgRespCmd; -using hotstuff::get_hash; -using hotstuff::promise_t; - -using HotStuff = hotstuff::HotStuffSecp256k1; - -class HotStuffApp: public HotStuff { - double stat_period; - double impeach_timeout; - EventContext ec; - EventContext req_ec; - EventContext resp_ec; - /** Network messaging between a replica and its client. */ - ClientNetwork cn; - /** Timer object to schedule a periodic printing of system statistics */ - TimerEvent ev_stat_timer; - /** Timer object to monitor the progress for simple impeachment */ - TimerEvent impeach_timer; - /** The listen address for client RPC */ - NetAddr clisten_addr; - - std::unordered_map unconfirmed; - - using conn_t = ClientNetwork::conn_t; - using resp_queue_t = salticidae::MPSCQueueEventDriven; - - /* for the dedicated thread sending responses to the clients */ - std::thread req_thread; - std::thread resp_thread; - resp_queue_t resp_queue; - salticidae::BoxObj resp_tcall; - salticidae::BoxObj req_tcall; - - void client_request_cmd_handler(MsgReqCmd &&, const conn_t &); - - static command_t parse_cmd(DataStream &s) { - auto cmd = new CommandDummy(); - s >> *cmd; - return cmd; - } - - void reset_imp_timer() { - impeach_timer.del(); - impeach_timer.add(impeach_timeout); - } - - void state_machine_execute(const Finality &fin) override { - reset_imp_timer(); -#ifndef HOTSTUFF_ENABLE_BENCHMARK - HOTSTUFF_LOG_INFO("replicated %s", std::string(fin).c_str()); -#endif - resp_queue.enqueue(fin); - } - -#ifdef HOTSTUFF_MSG_STAT - std::unordered_set client_conns; - void print_stat() const; -#endif - - public: - HotStuffApp(uint32_t blk_size, - double stat_period, - double impeach_timeout, - ReplicaID idx, - const bytearray_t &raw_privkey, - NetAddr plisten_addr, - NetAddr clisten_addr, - hotstuff::pacemaker_bt pmaker, - const EventContext &ec, - size_t nworker, - const Net::Config &repnet_config, - const ClientNetwork::Config &clinet_config); - - void start(const std::vector> &reps); - void stop(); -}; - -std::pair split_ip_port_cport(const std::string &s) { - auto ret = trim_all(split(s, ";")); - if (ret.size() != 2) - throw std::invalid_argument("invalid cport format"); - return std::make_pair(ret[0], ret[1]); -} - -salticidae::BoxObj papp = nullptr; - -int main(int argc, char **argv) { - Config config("hotstuff.conf"); - - ElapsedTime elapsed; - elapsed.start(); - - auto opt_blk_size = Config::OptValInt::create(1); - auto opt_parent_limit = Config::OptValInt::create(-1); - auto opt_stat_period = Config::OptValDouble::create(10); - auto opt_replicas = Config::OptValStrVec::create(); - auto opt_idx = Config::OptValInt::create(0); - auto opt_client_port = Config::OptValInt::create(-1); - auto opt_privkey = Config::OptValStr::create(); - auto opt_tls_privkey = Config::OptValStr::create(); - auto opt_tls_cert = Config::OptValStr::create(); - auto opt_help = Config::OptValFlag::create(false); - auto opt_pace_maker = Config::OptValStr::create("dummy"); - auto opt_fixed_proposer = Config::OptValInt::create(1); - auto opt_base_timeout = Config::OptValDouble::create(1); - auto opt_prop_delay = Config::OptValDouble::create(1); - auto opt_imp_timeout = Config::OptValDouble::create(11); - auto opt_nworker = Config::OptValInt::create(1); - auto opt_repnworker = Config::OptValInt::create(1); - auto opt_repburst = Config::OptValInt::create(100); - auto opt_clinworker = Config::OptValInt::create(8); - auto opt_cliburst = Config::OptValInt::create(1000); - auto opt_notls = Config::OptValFlag::create(false); - - config.add_opt("block-size", opt_blk_size, Config::SET_VAL); - config.add_opt("parent-limit", opt_parent_limit, Config::SET_VAL); - config.add_opt("stat-period", opt_stat_period, Config::SET_VAL); - config.add_opt("replica", opt_replicas, Config::APPEND, 'a', "add an replica to the list"); - config.add_opt("idx", opt_idx, Config::SET_VAL, 'i', "specify the index in the replica list"); - config.add_opt("cport", opt_client_port, Config::SET_VAL, 'c', "specify the port listening for clients"); - config.add_opt("privkey", opt_privkey, Config::SET_VAL); - config.add_opt("tls-privkey", opt_tls_privkey, Config::SET_VAL); - config.add_opt("tls-cert", opt_tls_cert, Config::SET_VAL); - config.add_opt("pace-maker", opt_pace_maker, Config::SET_VAL, 'p', "specify pace maker (dummy, rr)"); - config.add_opt("proposer", opt_fixed_proposer, Config::SET_VAL, 'l', "set the fixed proposer (for dummy)"); - config.add_opt("base-timeout", opt_base_timeout, Config::SET_VAL, 't', "set the initial timeout for the Round-Robin Pacemaker"); - config.add_opt("prop-delay", opt_prop_delay, Config::SET_VAL, 't', "set the delay that follows the timeout for the Round-Robin Pacemaker"); - config.add_opt("imp-timeout", opt_imp_timeout, Config::SET_VAL, 'u', "set impeachment timeout (for sticky)"); - config.add_opt("nworker", opt_nworker, Config::SET_VAL, 'n', "the number of threads for verification"); - config.add_opt("repnworker", opt_repnworker, Config::SET_VAL, 'm', "the number of threads for replica network"); - config.add_opt("repburst", opt_repburst, Config::SET_VAL, 'b', ""); - config.add_opt("clinworker", opt_clinworker, Config::SET_VAL, 'M', "the number of threads for client network"); - config.add_opt("cliburst", opt_cliburst, Config::SET_VAL, 'B', ""); - config.add_opt("notls", opt_notls, Config::SWITCH_ON, 's', "disable TLS"); - config.add_opt("help", opt_help, Config::SWITCH_ON, 'h', "show this help info"); - - EventContext ec; - config.parse(argc, argv); - if (opt_help->get()) - { - config.print_help(); - exit(0); - } - auto idx = opt_idx->get(); - auto client_port = opt_client_port->get(); - std::vector> replicas; - for (const auto &s: opt_replicas->get()) - { - auto res = trim_all(split(s, ",")); - if (res.size() != 3) - throw HotStuffError("invalid replica info"); - replicas.push_back(std::make_tuple(res[0], res[1], res[2])); - } - - if (!(0 <= idx && (size_t)idx < replicas.size())) - throw HotStuffError("replica idx out of range"); - std::string binding_addr = std::get<0>(replicas[idx]); - if (client_port == -1) - { - auto p = split_ip_port_cport(binding_addr); - size_t idx; - try { - client_port = stoi(p.second, &idx); - } catch (std::invalid_argument &) { - throw HotStuffError("client port not specified"); - } - } - - NetAddr plisten_addr{split_ip_port_cport(binding_addr).first}; - - auto parent_limit = opt_parent_limit->get(); - hotstuff::pacemaker_bt pmaker; - if (opt_pace_maker->get() == "dummy") - pmaker = new hotstuff::PaceMakerDummyFixed(opt_fixed_proposer->get(), parent_limit); - else - pmaker = new hotstuff::PaceMakerRR(ec, parent_limit, opt_base_timeout->get(), opt_prop_delay->get()); - - HotStuffApp::Net::Config repnet_config; - ClientNetwork::Config clinet_config; - if (!opt_tls_privkey->get().empty() && !opt_notls->get()) - { - auto tls_priv_key = new salticidae::PKey( - salticidae::PKey::create_privkey_from_der( - hotstuff::from_hex(opt_tls_privkey->get()))); - auto tls_cert = new salticidae::X509( - salticidae::X509::create_from_der( - hotstuff::from_hex(opt_tls_cert->get()))); - repnet_config - .enable_tls(true) - .tls_key(tls_priv_key) - .tls_cert(tls_cert); - } - repnet_config - .burst_size(opt_repburst->get()) - .nworker(opt_repnworker->get()); - clinet_config - .burst_size(opt_cliburst->get()) - .nworker(opt_clinworker->get()); - papp = new HotStuffApp(opt_blk_size->get(), - opt_stat_period->get(), - opt_imp_timeout->get(), - idx, - hotstuff::from_hex(opt_privkey->get()), - plisten_addr, - NetAddr("0.0.0.0", client_port), - std::move(pmaker), - ec, - opt_nworker->get(), - repnet_config, - clinet_config); - std::vector> reps; - for (auto &r: replicas) - { - auto p = split_ip_port_cport(std::get<0>(r)); - reps.push_back(std::make_tuple( - NetAddr(p.first), - hotstuff::from_hex(std::get<1>(r)), - hotstuff::from_hex(std::get<2>(r)))); - } - auto shutdown = [&](int) { papp->stop(); }; - salticidae::SigEvent ev_sigint(ec, shutdown); - salticidae::SigEvent ev_sigterm(ec, shutdown); - ev_sigint.add(SIGINT); - ev_sigterm.add(SIGTERM); - - papp->start(reps); - elapsed.stop(true); - return 0; -} - -HotStuffApp::HotStuffApp(uint32_t blk_size, - double stat_period, - double impeach_timeout, - ReplicaID idx, - const bytearray_t &raw_privkey, - NetAddr plisten_addr, - NetAddr clisten_addr, - hotstuff::pacemaker_bt pmaker, - const EventContext &ec, - size_t nworker, - const Net::Config &repnet_config, - const ClientNetwork::Config &clinet_config): - HotStuff(blk_size, idx, raw_privkey, - plisten_addr, std::move(pmaker), ec, nworker, repnet_config), - stat_period(stat_period), - impeach_timeout(impeach_timeout), - ec(ec), - cn(req_ec, clinet_config), - clisten_addr(clisten_addr) { - /* prepare the thread used for sending back confirmations */ - resp_tcall = new salticidae::ThreadCall(resp_ec); - req_tcall = new salticidae::ThreadCall(req_ec); - resp_queue.reg_handler(resp_ec, [this](resp_queue_t &q) { - Finality fin; - while (q.try_dequeue(fin)) - { - auto it = unconfirmed.find(fin.cmd_hash); - if (it != unconfirmed.end()) - { - it->second.resolve(fin); - unconfirmed.erase(it); - } - } - return false; - }); - - /* register the handlers for msg from clients */ - cn.reg_handler(salticidae::generic_bind(&HotStuffApp::client_request_cmd_handler, this, _1, _2)); - cn.start(); - cn.listen(clisten_addr); -} - -void HotStuffApp::client_request_cmd_handler(MsgReqCmd &&msg, const conn_t &conn) { - const NetAddr addr = conn->get_addr(); - auto cmd = parse_cmd(msg.serialized); - const auto &cmd_hash = cmd->get_hash(); - HOTSTUFF_LOG_DEBUG("processing %s", std::string(*cmd).c_str()); - exec_command(cmd_hash, [this, addr](Finality fin) { - resp_queue.enqueue(fin); - }); - /* the following function is executed on the dedicated thread for confirming commands */ - resp_tcall->async_call([this, addr, cmd_hash](salticidae::ThreadCall::Handle &) { - auto it = unconfirmed.find(cmd_hash); - if (it == unconfirmed.end()) - it = unconfirmed.insert( - std::make_pair(cmd_hash, promise_t([](promise_t &){}))).first; - it->second.then([this, addr](const Finality &fin) { - try { - cn.send_msg(MsgRespCmd(std::move(fin)), addr); - } catch (std::exception &err) { - HOTSTUFF_LOG_WARN("unable to send to the client: %s", err.what()); - } - }); - }); -} - -void HotStuffApp::start(const std::vector> &reps) { - ev_stat_timer = TimerEvent(ec, [this](TimerEvent &) { - HotStuff::print_stat(); - HotStuffApp::print_stat(); - //HotStuffCore::prune(100); - ev_stat_timer.add(stat_period); - }); - ev_stat_timer.add(stat_period); - impeach_timer = TimerEvent(ec, [this](TimerEvent &) { - if (get_decision_waiting().size()) - get_pace_maker()->impeach(); - reset_imp_timer(); - }); - impeach_timer.add(impeach_timeout); - HOTSTUFF_LOG_INFO("** starting the system with parameters **"); - HOTSTUFF_LOG_INFO("blk_size = %lu", blk_size); - HOTSTUFF_LOG_INFO("conns = %lu", HotStuff::size()); - HOTSTUFF_LOG_INFO("** starting the event loop..."); - HotStuff::start(reps); - cn.reg_conn_handler([this](const salticidae::ConnPool::conn_t &_conn, bool connected) { - auto conn = salticidae::static_pointer_cast(_conn); - if (connected) - client_conns.insert(conn); - else - client_conns.erase(conn); - return true; - }); - req_thread = std::thread([this]() { req_ec.dispatch(); }); - resp_thread = std::thread([this]() { resp_ec.dispatch(); }); - /* enter the event main loop */ - ec.dispatch(); -} - -void HotStuffApp::stop() { - papp->req_tcall->async_call([this](salticidae::ThreadCall::Handle &) { - req_ec.stop(); - }); - papp->resp_tcall->async_call([this](salticidae::ThreadCall::Handle &) { - resp_ec.stop(); - }); - - req_thread.join(); - resp_thread.join(); - ec.stop(); -} - -void HotStuffApp::print_stat() const { -#ifdef HOTSTUFF_MSG_STAT - HOTSTUFF_LOG_INFO("--- client msg. (10s) ---"); - size_t _nsent = 0; - size_t _nrecv = 0; - for (const auto &conn: client_conns) - { - if (conn == nullptr) continue; - size_t ns = conn->get_nsent(); - size_t nr = conn->get_nrecv(); - size_t nsb = conn->get_nsentb(); - size_t nrb = conn->get_nrecvb(); - conn->clear_msgstat(); - HOTSTUFF_LOG_INFO("%s: %u(%u), %u(%u)", - std::string(conn->get_addr()).c_str(), ns, nsb, nr, nrb); - _nsent += ns; - _nrecv += nr; - } - HOTSTUFF_LOG_INFO("--- end client msg. ---"); -#endif -} diff --git a/src/hotstuff_client.cpp b/src/hotstuff_client.cpp deleted file mode 100644 index 831b7ff..0000000 --- a/src/hotstuff_client.cpp +++ /dev/null @@ -1,181 +0,0 @@ -/** - * Copyright 2018 VMware - * Copyright 2018 Ted Yin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "salticidae/type.h" -#include "salticidae/netaddr.h" -#include "salticidae/network.h" -#include "salticidae/util.h" - -#include "hotstuff/util.h" -#include "hotstuff/type.h" -#include "hotstuff/client.h" - -using salticidae::Config; - -using hotstuff::ReplicaID; -using hotstuff::NetAddr; -using hotstuff::EventContext; -using hotstuff::MsgReqCmd; -using hotstuff::MsgRespCmd; -using hotstuff::CommandDummy; -using hotstuff::HotStuffError; -using hotstuff::uint256_t; -using hotstuff::opcode_t; -using hotstuff::command_t; - -EventContext ec; -ReplicaID proposer; -size_t max_async_num; -int max_iter_num; -uint32_t cid; -uint32_t cnt = 0; -uint32_t nfaulty; - -struct Request { - command_t cmd; - size_t confirmed; - salticidae::ElapsedTime et; - Request(const command_t &cmd): cmd(cmd), confirmed(0) { et.start(); } -}; - -using Net = salticidae::MsgNetwork; - -std::unordered_map conns; -std::unordered_map waiting; -std::vector replicas; -std::vector> elapsed; -Net mn(ec, Net::Config()); - -void connect_all() { - for (size_t i = 0; i < replicas.size(); i++) - conns.insert(std::make_pair(i, mn.connect_sync(replicas[i]))); -} - -bool try_send(bool check = true) { - if ((!check || waiting.size() < max_async_num) && max_iter_num) - { - auto cmd = new CommandDummy(cid, cnt++); - MsgReqCmd msg(*cmd); - for (auto &p: conns) mn.send_msg(msg, p.second); -#ifndef HOTSTUFF_ENABLE_BENCHMARK - HOTSTUFF_LOG_INFO("send new cmd %.10s", - get_hex(cmd->get_hash()).c_str()); -#endif - waiting.insert(std::make_pair( - cmd->get_hash(), Request(cmd))); - if (max_iter_num > 0) - max_iter_num--; - return true; - } - return false; -} - -void client_resp_cmd_handler(MsgRespCmd &&msg, const Net::conn_t &) { - auto &fin = msg.fin; - HOTSTUFF_LOG_DEBUG("got %s", std::string(msg.fin).c_str()); - const uint256_t &cmd_hash = fin.cmd_hash; - auto it = waiting.find(cmd_hash); - auto &et = it->second.et; - if (it == waiting.end()) return; - et.stop(); - if (++it->second.confirmed <= nfaulty) return; // wait for f + 1 ack -#ifndef HOTSTUFF_ENABLE_BENCHMARK - HOTSTUFF_LOG_INFO("got %s, wall: %.3f, cpu: %.3f", - std::string(fin).c_str(), - et.elapsed_sec, et.cpu_elapsed_sec); -#else - struct timeval tv; - gettimeofday(&tv, nullptr); - elapsed.push_back(std::make_pair(tv, et.elapsed_sec)); -#endif - waiting.erase(it); - while (try_send()); -} - -std::pair split_ip_port_cport(const std::string &s) { - auto ret = salticidae::trim_all(salticidae::split(s, ";")); - return std::make_pair(ret[0], ret[1]); -} - -int main(int argc, char **argv) { - Config config("hotstuff.conf"); - - auto opt_idx = Config::OptValInt::create(0); - auto opt_replicas = Config::OptValStrVec::create(); - auto opt_max_iter_num = Config::OptValInt::create(100); - auto opt_max_async_num = Config::OptValInt::create(10); - auto opt_cid = Config::OptValInt::create(-1); - - auto shutdown = [&](int) { ec.stop(); }; - salticidae::SigEvent ev_sigint(ec, shutdown); - salticidae::SigEvent ev_sigterm(ec, shutdown); - ev_sigint.add(SIGINT); - ev_sigterm.add(SIGTERM); - - mn.reg_handler(client_resp_cmd_handler); - mn.start(); - - config.add_opt("idx", opt_idx, Config::SET_VAL); - config.add_opt("cid", opt_cid, Config::SET_VAL); - config.add_opt("replica", opt_replicas, Config::APPEND); - config.add_opt("iter", opt_max_iter_num, Config::SET_VAL); - config.add_opt("max-async", opt_max_async_num, Config::SET_VAL); - config.parse(argc, argv); - auto idx = opt_idx->get(); - max_iter_num = opt_max_iter_num->get(); - max_async_num = opt_max_async_num->get(); - std::vector raw; - for (const auto &s: opt_replicas->get()) - { - auto res = salticidae::trim_all(salticidae::split(s, ",")); - if (res.size() < 1) - throw HotStuffError("format error"); - raw.push_back(res[0]); - } - - if (!(0 <= idx && (size_t)idx < raw.size() && raw.size() > 0)) - throw std::invalid_argument("out of range"); - cid = opt_cid->get() != -1 ? opt_cid->get() : idx; - for (const auto &p: raw) - { - auto _p = split_ip_port_cport(p); - size_t _; - replicas.push_back(NetAddr(NetAddr(_p.first).ip, htons(stoi(_p.second, &_)))); - } - - nfaulty = (replicas.size() - 1) / 3; - HOTSTUFF_LOG_INFO("nfaulty = %zu", nfaulty); - connect_all(); - while (try_send()); - ec.dispatch(); - -#ifdef HOTSTUFF_ENABLE_BENCHMARK - for (const auto &e: elapsed) - { - char fmt[64]; - struct tm *tmp = localtime(&e.first.tv_sec); - strftime(fmt, sizeof fmt, "%Y-%m-%d %H:%M:%S.%%06u [hotstuff info] %%.6f\n", tmp); - fprintf(stderr, fmt, e.first.tv_usec, e.second); - } -#endif - return 0; -} -- cgit v1.2.3-70-g09d2