From d15ec0b93def57e5f3832f429a3b948e86a62887 Mon Sep 17 00:00:00 2001 From: Determinant Date: Thu, 27 Jun 2019 20:33:06 -0400 Subject: finish p2p & TLS integration and testing --- include/salticidae/conn.h | 8 +- include/salticidae/crypto.h | 55 ++++++++ include/salticidae/event.h | 4 +- include/salticidae/network.h | 277 ++++++++++++++++++++++--------------- include/salticidae/stream.h | 70 +++++----- src/conn.cpp | 10 +- src/network.cpp | 4 +- test/.gitignore | 1 + test/CMakeLists.txt | 3 + test/test_p2p.cpp | 14 +- test/test_p2p_stress.cpp | 26 ++-- test/test_p2p_tls.cpp | 317 +++++++++++++++++++++++++++++++++++++++++++ 12 files changed, 620 insertions(+), 169 deletions(-) create mode 100644 test/test_p2p_tls.cpp diff --git a/include/salticidae/conn.h b/include/salticidae/conn.h index 87966ac..e39d31d 100644 --- a/include/salticidae/conn.h +++ b/include/salticidae/conn.h @@ -195,7 +195,12 @@ class ConnPool { if (enable_tls && connected) { conn->worker->get_tcall()->async_call([this, conn, ret](ThreadCall::Handle &) { - if (ret) conn->recv_data_func = Conn::_recv_data_tls; + if (ret) + { + conn->recv_data_func = Conn::_recv_data_tls; + conn->ev_socket.del(); + conn->ev_socket.add(FdEvent::READ | FdEvent::WRITE); + } else worker_terminate(conn); }); } @@ -262,6 +267,7 @@ class ConnPool { conn->send_data_func = Conn::_send_data; conn->recv_data_func = Conn::_recv_data; enable_send_buffer(conn, client_fd); + cpool->on_setup(conn); cpool->update_conn(conn, true); } assert(conn->fd != -1); diff --git a/include/salticidae/crypto.h b/include/salticidae/crypto.h index d7231a4..fe4de4f 100644 --- a/include/salticidae/crypto.h +++ b/include/salticidae/crypto.h @@ -31,6 +31,7 @@ #ifdef __cplusplus #include #include +#include namespace salticidae { @@ -137,10 +138,30 @@ static int _skip_CA_check(int, X509_STORE_CTX *) { class PKey { EVP_PKEY *key; friend class TLSContext; + friend class X509; public: PKey(EVP_PKEY *key): key(key) {} PKey(const PKey &) = delete; PKey(PKey &&other): key(other.key) { other.key = nullptr; } + + static PKey create_privkey_rsa(size_t nbits = 2048) { + auto key = EVP_PKEY_new(); + if (key == nullptr) + throw SalticidaeError(SALTI_ERROR_TLS_KEY); + auto e = BN_new(); + BN_set_word(e, 17); + auto rsa = RSA_new(); + auto ret = RSA_generate_key_ex(rsa, nbits, e, nullptr); + BN_free(e); + if (!ret) + { + RSA_free(rsa); + throw SalticidaeError(SALTI_ERROR_TLS_KEY); + } + EVP_PKEY_set1_RSA(key, rsa); + RSA_free(rsa); + return PKey(key); + } static PKey create_privkey_from_pem_file(std::string pem_fname, std::string *passwd = nullptr) { FILE *fp = fopen(pem_fname.c_str(), "r"); @@ -193,6 +214,14 @@ class PKey { return std::move(res); } + void save_privkey_to_file(const std::string &fname) { + FILE *fp = fopen(fname.c_str(), "wb"); + auto ret = PEM_write_PrivateKey(fp, key, nullptr, nullptr, 0, nullptr, nullptr); + fclose(fp); + if (!ret) + throw SalticidaeError(SALTI_ERROR_TLS_X509); + } + ~PKey() { if (key) EVP_PKEY_free(key); } }; @@ -203,6 +232,24 @@ class X509 { X509(::X509 *x509): x509(x509) {} X509(const X509 &) = delete; X509(X509 &&other): x509(other.x509) { other.x509 = nullptr; } + + static X509 create_self_signed_from_pubkey(const PKey &pkey, const char *country = "US", const char *common_name = "localhost") { + auto x509 = X509_new(); + ASN1_INTEGER_set(X509_get_serialNumber(x509), 1); + X509_set_pubkey(x509, pkey.key); + X509_gmtime_adj(X509_get_notBefore(x509), 0); + X509_gmtime_adj(X509_get_notAfter(x509), 0); + auto name = X509_get_subject_name(x509); + X509_NAME_add_entry_by_txt(name, "C", MBSTRING_ASC, + (unsigned char *)country, -1, -1, 0); + X509_NAME_add_entry_by_txt(name, "O", MBSTRING_ASC, + (unsigned char *)"libsalticidae", -1, -1, 0); + X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC, + (unsigned char *)common_name, -1, -1, 0); + X509_set_issuer_name(x509, name); + X509_sign(x509, pkey.key, EVP_sha1()); + return X509(x509); + } static X509 create_from_pem_file(std::string pem_fname, std::string *passwd = nullptr) { FILE *fp = fopen(pem_fname.c_str(), "r"); @@ -251,6 +298,14 @@ class X509 { return std::move(res); } + void save_to_file(const std::string &fname) { + FILE *fp = fopen(fname.c_str(), "wb"); + auto ret = PEM_write_X509(fp, x509); + fclose(fp); + if (!ret) + throw SalticidaeError(SALTI_ERROR_TLS_X509); + } + ~X509() { if (x509) X509_free(x509); } }; diff --git a/include/salticidae/event.h b/include/salticidae/event.h index ad78a6e..a7ea209 100644 --- a/include/salticidae/event.h +++ b/include/salticidae/event.h @@ -565,7 +565,7 @@ class MPSCQueueEventDriven: public MPSCQueue { // memory barrier here, so any load/store in enqueue must be finialized if (wait_sig.exchange(false, std::memory_order_acq_rel)) { - SALTICIDAE_LOG_DEBUG("mpsc notify"); + //SALTICIDAE_LOG_DEBUG("mpsc notify"); write(fd, &dummy, 8); } return true; @@ -616,7 +616,7 @@ class MPMCQueueEventDriven: public MPMCQueue { // memory barrier here, so any load/store in enqueue must be finialized if (wait_sig.exchange(false, std::memory_order_acq_rel)) { - SALTICIDAE_LOG_DEBUG("mpsc notify"); + //SALTICIDAE_LOG_DEBUG("mpmc notify"); write(fd, &dummy, 8); } return true; diff --git a/include/salticidae/network.h b/include/salticidae/network.h index 975084f..d3f3bae 100644 --- a/include/salticidae/network.h +++ b/include/salticidae/network.h @@ -261,7 +261,6 @@ class PeerNetwork: public MsgNetwork { public: using MsgNet = MsgNetwork; using Msg = typename MsgNet::Msg; - using unknown_callback_t = std::function; enum IdentityMode { ADDR_BASED, @@ -282,6 +281,9 @@ class PeerNetwork: public MsgNetwork { public: Conn(): MsgNet::Conn(), peer(nullptr) {} + NetAddr get_peer_addr() { + return peer ? peer->peer_addr : NetAddr(); + } PeerNetwork *get_net() { return static_cast(ConnPool::Conn::get_pool()); @@ -295,11 +297,13 @@ class PeerNetwork: public MsgNetwork { }; using conn_t = ArcObj; + using peer_callback_t = std::function; + using unknown_peer_callback_t = std::function; private: - struct Peer { + class Peer { + friend PeerNetwork; /** connection addr, may be different due to passive mode */ - uint256_t nonce; uint256_t peer_id; NetAddr peer_addr; /** the underlying connection, may be invalid when connected = false */ @@ -316,17 +320,17 @@ class PeerNetwork: public MsgNetwork { double ping_period; Peer() = delete; - Peer(conn_t conn, conn_t inbound_conn, conn_t outbound_conn, const PeerNetwork *pn): - conn(conn), - inbound_conn(inbound_conn), - outbound_conn(outbound_conn), - ev_ping_timer( - TimerEvent(pn->disp_ec, std::bind(&Peer::ping_timer, this, _1))), - connected(false), - outbound_handshake(false), - inbound_handshake(false), - ping_period(pn->ping_period) {} - ~Peer() {} + Peer(const uint256_t &peer_id, conn_t conn, conn_t inbound_conn, conn_t outbound_conn, const PeerNetwork *pn): + peer_id(peer_id), + conn(conn), + inbound_conn(inbound_conn), + outbound_conn(outbound_conn), + ev_ping_timer( + TimerEvent(pn->disp_ec, std::bind(&Peer::ping_timer, this, _1))), + connected(false), + outbound_handshake(false), + inbound_handshake(false), + ping_period(pn->ping_period) {} Peer &operator=(const Peer &) = delete; Peer(const Peer &) = delete; @@ -337,12 +341,15 @@ class PeerNetwork: public MsgNetwork { if (ev_ping_timer) ev_ping_timer.del(); } + public: + ~Peer() {} }; std::unordered_map pending_peers; std::unordered_map known_peers; std::unordered_map> pid2peer; - unknown_callback_t unknown_peer_cb; + peer_callback_t peer_cb; + unknown_peer_callback_t unknown_peer_cb; const IdentityMode id_mode; double retry_conn_delay; @@ -350,41 +357,35 @@ class PeerNetwork: public MsgNetwork { double conn_timeout; NetAddr listen_addr; bool allow_unknown_peer; - uint256_t my_pname; uint256_t my_nonce; struct MsgPing { static const OpcodeType opcode; DataStream serialized; - uint256_t pname; + NetAddr claimed_addr; uint256_t nonce; - uint256_t peer_id; - MsgPing() { serialized << false; } - MsgPing(const uint256_t &_pname, const uint256_t &_nonce) { - serialized << true << _pname << _nonce; + MsgPing() { serialized << (uint8_t)false; } + MsgPing(const NetAddr &_claimed_addr, const uint256_t &_nonce) { + serialized << (uint8_t)true << _claimed_addr << _nonce; } MsgPing(DataStream &&s) { uint8_t flag; s >> flag; if (flag) - { - s >> pname >> nonce; - DataStream tmp; - tmp << pname << nonce; - peer_id = tmp.get_hash(); - } + s >> claimed_addr >> nonce; } }; struct MsgPong: public MsgPing { static const OpcodeType opcode; MsgPong(): MsgPing() {} - MsgPong(const uint256_t &_pname, const uint256_t _nonce): MsgPing(_pname, _nonce) {} + MsgPong(const NetAddr &_claimed_addr, const uint256_t _nonce): + MsgPing(_claimed_addr, _nonce) {} MsgPong(DataStream &&s): MsgPing(std::move(s)) {} }; - void msg_ping(MsgPing &&msg, const conn_t &conn); - void msg_pong(MsgPong &&msg, const conn_t &conn); + void ping_handler(MsgPing &&msg, const conn_t &conn); + void pong_handler(MsgPong &&msg, const conn_t &conn); void _ping_msg_cb(const conn_t &conn, uint16_t port); void _pong_msg_cb(const conn_t &conn, uint16_t port); bool check_handshake(Peer *peer); @@ -400,6 +401,14 @@ class PeerNetwork: public MsgNetwork { } void on_setup(const ConnPool::conn_t &) override; void on_teardown(const ConnPool::conn_t &) override; + uint256_t gen_peer_id(const conn_t &conn, const NetAddr &claimed_addr, const uint256_t &nonce) { + DataStream tmp; + if (!this->enable_tls || id_mode == ADDR_BASED) + tmp << nonce << claimed_addr; + else + tmp << conn->get_peer_cert()->get_der(); + return tmp.get_hash(); + } public: @@ -420,7 +429,7 @@ class PeerNetwork: public MsgNetwork { _ping_period(30), _conn_timeout(180), _allow_unknown_peer(false), - _id_mode(ADDR_BASED) {} + _id_mode(CERT_BASED) {} Config &retry_conn_delay(double x) { @@ -450,14 +459,14 @@ class PeerNetwork: public MsgNetwork { }; PeerNetwork(const EventContext &ec, const Config &config): - MsgNet(ec, config), - id_mode(config._id_mode), - retry_conn_delay(config._retry_conn_delay), - ping_period(config._ping_period), - conn_timeout(config._conn_timeout), - allow_unknown_peer(config._allow_unknown_peer) { - this->reg_handler(generic_bind(&PeerNetwork::msg_ping, this, _1, _2)); - this->reg_handler(generic_bind(&PeerNetwork::msg_pong, this, _1, _2)); + MsgNet(ec, config), + id_mode(config._id_mode), + retry_conn_delay(config._retry_conn_delay), + ping_period(config._ping_period), + conn_timeout(config._conn_timeout), + allow_unknown_peer(config._allow_unknown_peer) { + this->reg_handler(generic_bind(&PeerNetwork::ping_handler, this, _1, _2)); + this->reg_handler(generic_bind(&PeerNetwork::pong_handler, this, _1, _2)); } virtual ~PeerNetwork() { this->stop(); } @@ -481,6 +490,8 @@ class PeerNetwork: public MsgNetwork { conn_t connect(const NetAddr &addr) = delete; template void reg_unknown_peer_handler(Func cb) { unknown_peer_cb = cb; } + template + void reg_peer_handler(Func cb) { peer_cb = cb; } }; /* this callback is run by a worker */ @@ -577,43 +588,63 @@ void PeerNetwork::on_setup(const ConnPool::conn_t &_conn) { auto &ev_timeout = conn->ev_timeout; conn->ev_retry_timer.del(); assert(!ev_timeout); - ev_timeout = TimerEvent(worker->get_ec(), [worker, conn](TimerEvent &) { + ev_timeout = TimerEvent(worker->get_ec(), [listen_addr=this->listen_addr, worker, conn](TimerEvent &) { try { - SALTICIDAE_LOG_INFO("peer ping-pong timeout"); - conn->get_net()->worker_terminate(conn); + SALTICIDAE_LOG_INFO("peer ping-pong timeout %s <-> %s", + std::string(listen_addr).c_str(), + std::string(conn->get_peer_addr()).c_str()); + //conn->get_net()->worker_terminate(conn); } catch (...) { worker->error_callback(std::current_exception()); } }); /* the initial ping-pong to set up the connection */ tcall_reset_timeout(worker, conn, conn_timeout); pending_peers[conn->get_addr()] = conn; if (conn->get_mode() == Conn::ConnMode::ACTIVE) - send_msg(MsgPing(my_pname, my_nonce), conn); + send_msg(MsgPing(listen_addr, my_nonce), conn); } template void PeerNetwork::on_teardown(const ConnPool::conn_t &_conn) { MsgNet::on_teardown(_conn); auto conn = static_pointer_cast(_conn); - const auto &addr = conn->get_addr(); + auto addr = conn->get_addr(); conn->ev_retry_timer.clear(); conn->ev_timeout.clear(); pending_peers.erase(addr); + SALTICIDAE_LOG_INFO("connection lost: %s", std::string(*conn).c_str()); auto p = conn->peer; + if (p) addr = p->peer_addr; + TimerEvent retry_timer(this->disp_ec, [this, addr](TimerEvent &) { + try { + start_active_conn(addr); + } catch (...) { this->disp_error_cb(std::current_exception()); } + }); + auto it = known_peers.find(addr); + if (it == known_peers.end()) return; + pending_peers[addr] = conn; if (p) { if (conn != p->conn) return; + p->inbound_conn = nullptr; + p->outbound_conn = nullptr; p->ev_ping_timer.del(); p->connected = false; + p->outbound_handshake = false; + p->inbound_handshake = false; known_peers[p->peer_addr] = uint256_t(); - // try to reconnect - conn->ev_retry_timer = TimerEvent(this->disp_ec, [this, addr](TimerEvent &) { - try { - start_active_conn(addr); - } catch (...) { this->disp_error_cb(std::current_exception()); } + pid2peer.erase(p->peer_id); + this->user_tcall->async_call([this, conn](ThreadCall::Handle &) { + if (peer_cb) peer_cb(conn, false); }); + conn->ev_retry_timer = std::move(retry_timer); + conn->ev_retry_timer.add(gen_conn_timeout()); + } + else + { + if (!it->second.is_null()) return; + conn->ev_retry_timer = std::move(retry_timer); conn->ev_retry_timer.add(gen_conn_timeout()); } - SALTICIDAE_LOG_INFO("connection lost: %s", std::string(*conn).c_str()); } template @@ -666,13 +697,16 @@ bool PeerNetwork::check_handshake(Peer *p) { color_begin = TTY_COLOR_BLUE; color_end = TTY_COLOR_RESET; } - SALTICIDAE_LOG_INFO("%sPeerNetwork: established connection with %s <-> %s via %s", + SALTICIDAE_LOG_INFO("%sPeerNetwork: established connection %s <-> %s via %s", color_begin, std::string(listen_addr).c_str(), std::string(p->peer_addr).c_str(), std::string(*(p->conn)).c_str(), color_end); } + this->user_tcall->async_call([this, conn=p->conn](ThreadCall::Handle &) { + if (peer_cb) peer_cb(conn, true); + }); return true; } @@ -684,52 +718,67 @@ void PeerNetwork::start_active_conn(const NetAddr &addr) { template inline typename PeerNetwork::conn_t PeerNetwork::_get_peer_conn(const NetAddr &addr) const { - auto it = pending_peers.find(addr); - if (it == pending_peers.end()) + auto it = known_peers.find(addr); + if (it == known_peers.end()) throw PeerNetworkError(SALTI_ERROR_PEER_NOT_EXIST); - return it->second; + auto it2 = pid2peer.find(it->second); + assert(it2 != pid2peer.end()); + return it2->second->conn; } /* end: functions invoked by the dispatcher */ /* begin: functions invoked by the user loop */ template -void PeerNetwork::msg_ping(MsgPing &&msg, const conn_t &conn) { +void PeerNetwork::ping_handler(MsgPing &&msg, const conn_t &conn) { this->disp_tcall->async_call([this, conn, msg=std::move(msg)](ThreadCall::Handle &) { try { auto conn_mode = conn->get_mode(); if (conn_mode == ConnPool::Conn::DEAD) return; - if (!msg.peer_id.is_null()) + if (!msg.claimed_addr.is_null()) { + auto peer_id = gen_peer_id(conn, msg.claimed_addr, msg.nonce); if (conn_mode == Conn::ConnMode::PASSIVE) { - send_msg(MsgPong(my_pname, my_nonce), conn); + if (!known_peers.count(msg.claimed_addr)) + { + this->user_tcall->async_call([this, addr=msg.claimed_addr](ThreadCall::Handle &) { + if (unknown_peer_cb) unknown_peer_cb(addr); + }); + this->disp_terminate(conn); + return; + } SALTICIDAE_LOG_INFO("%s inbound handshake from %s", std::string(listen_addr).c_str(), std::string(*conn).c_str()); - auto it = pid2peer.find(msg.peer_id); + send_msg(MsgPong(listen_addr, my_nonce), conn); + auto it = pid2peer.find(peer_id); if (it != pid2peer.end()) { + auto p = it->second.get(); + if (p->connected) + { + //conn->get_net()->disp_terminate(conn); + return; + } + auto &old_conn = p->inbound_conn; + if (old_conn && old_conn->get_mode() != Conn::ConnMode::DEAD) + { + SALTICIDAE_LOG_DEBUG("%s terminating old connection %s", + std::string(listen_addr).c_str(), + std::string(*old_conn).c_str()); + old_conn->peer = nullptr; + old_conn->get_net()->disp_terminate(old_conn); + } + old_conn = conn; if (msg.nonce < my_nonce) { - auto p = it->second.get(); - auto &old_conn = p->inbound_conn; - if (old_conn && old_conn->get_mode() != Conn::ConnMode::DEAD) - { - SALTICIDAE_LOG_INFO("%s terminating old connection %s", - std::string(listen_addr).c_str(), - std::string(*old_conn).c_str()); - old_conn->peer = nullptr; - old_conn->get_net()->disp_terminate(old_conn); - } - old_conn = conn; p->conn = conn; } } else { - it = pid2peer.insert(std::make_pair( - msg.peer_id, - new Peer(conn, conn, nullptr, this))).first; + it = pid2peer.insert(std::make_pair(peer_id, + new Peer(peer_id, conn, conn, nullptr, this))).first; } auto p = it->second.get(); p->inbound_handshake = true; @@ -749,39 +798,45 @@ void PeerNetwork::msg_ping(MsgPing &&msg, const conn_t &conn) { } template -void PeerNetwork::msg_pong(MsgPong &&msg, const conn_t &conn) { +void PeerNetwork::pong_handler(MsgPong &&msg, const conn_t &conn) { this->disp_tcall->async_call([this, conn, msg=std::move(msg)](ThreadCall::Handle &) { try { auto conn_mode = conn->get_mode(); if (conn_mode == ConnPool::Conn::DEAD) return; - if (!msg.peer_id.is_null()) + if (!msg.claimed_addr.is_null()) { + auto peer_id = gen_peer_id(conn, msg.claimed_addr, msg.nonce); if (conn_mode == Conn::ConnMode::ACTIVE) { SALTICIDAE_LOG_INFO("%s outbound handshake to %s", std::string(listen_addr).c_str(), std::string(*conn).c_str()); - auto it = pid2peer.find(msg.peer_id); + auto it = pid2peer.find(peer_id); if (it != pid2peer.end()) { + auto p = it->second.get(); + if (p->connected) + { + conn->get_net()->disp_terminate(conn); + return; + } + auto &old_conn = p->outbound_conn; + if (old_conn && old_conn->get_mode() != Conn::ConnMode::DEAD) + { + SALTICIDAE_LOG_DEBUG("%s terminating old connection %s", + std::string(listen_addr).c_str(), + std::string(*old_conn).c_str()); + old_conn->peer = nullptr; + old_conn->get_net()->disp_terminate(old_conn); + } + old_conn = conn; if (my_nonce < msg.nonce) { - auto p = it->second.get(); - auto &old_conn = p->outbound_conn; - if (old_conn && old_conn->get_mode() != Conn::ConnMode::DEAD) - { - SALTICIDAE_LOG_INFO("%s terminating old connection %s", - std::string(listen_addr).c_str(), - std::string(*old_conn).c_str()); - old_conn->peer = nullptr; - old_conn->get_net()->disp_terminate(old_conn); - } - old_conn = conn; p->conn = conn; } else { - SALTICIDAE_LOG_INFO("%s terminating low connection %s", + SALTICIDAE_LOG_DEBUG("%s terminating low connection %s", std::string(listen_addr).c_str(), std::string(*conn).c_str()); conn->get_net()->disp_terminate(conn); @@ -789,9 +844,8 @@ void PeerNetwork::msg_pong(MsgPong &&msg, const conn_t &conn) { } else { - it = pid2peer.insert(std::make_pair( - msg.peer_id, - new Peer(conn, nullptr, conn, this))).first; + it = pid2peer.insert(std::make_pair(peer_id, + new Peer(peer_id, conn, nullptr, conn, this))).first; } auto p = it->second.get(); p->outbound_handshake = true; @@ -806,7 +860,11 @@ void PeerNetwork::msg_pong(MsgPong &&msg, const conn_t &conn) { else { auto p = conn->peer; - if (!p) return; + if (!p) + { + SALTICIDAE_LOG_WARN("unexpected poing mesage"); + return; + } p->pong_msg_ok = true; if (p->ping_timer_ok) { @@ -826,18 +884,6 @@ void PeerNetwork::listen(NetAddr _listen_addr) { try { MsgNet::_listen(_listen_addr); listen_addr = _listen_addr; - DataStream pid; - if (id_mode == CERT_BASED) - { - if (!this->enable_tls) - throw PeerNetworkError(SALTI_ERROR_TLS_LOAD_CERT); - pid << this->tls_cert->get_der(); - } - else - { - pid << listen_addr; - } - my_pname = pid.get_hash(); uint8_t rand_bytes[32]; if (!RAND_bytes(rand_bytes, 32)) throw PeerNetworkError(SALTI_ERROR_RAND_SOURCE); @@ -868,14 +914,25 @@ template void PeerNetwork::del_peer(const NetAddr &addr) { this->disp_tcall->async_call([this, addr](ThreadCall::Handle &) { try { - if (!known_peers.erase(addr)) + auto it = known_peers.find(addr); + if (it == known_peers.end()) throw PeerNetworkError(SALTI_ERROR_PEER_NOT_EXIST); - auto it = pending_peers.find(addr); - assert(it != pending_peers.end()); - auto conn = it->second; - auto p = conn->peer; - if (p) pid2peer.erase(p->peer_id); - this->disp_terminate(conn); + auto peer_id = it->second; + known_peers.erase(it); + auto it2 = pending_peers.find(addr); + if (it2 != pending_peers.end()) + { + if (!it2->second->peer) + this->disp_terminate(it2->second); + pending_peers.erase(it2); + } + auto it3 = pid2peer.find(peer_id); + if (it3 != pid2peer.end()) + { + auto p = it3->second.get(); + this->disp_terminate(p->conn); + pid2peer.erase(it3); + } } catch (const PeerNetworkError &) { this->recoverable_error(std::current_exception()); } catch (...) { this->disp_error_cb(std::current_exception()); } diff --git a/include/salticidae/stream.h b/include/salticidae/stream.h index dc47792..2efc532 100644 --- a/include/salticidae/stream.h +++ b/include/salticidae/stream.h @@ -198,8 +198,40 @@ class DataStream { inline uint256_t get_hash() const; }; +class Serializable { + public: + virtual ~Serializable() = default; + virtual void serialize(DataStream &s) const = 0; + virtual void unserialize(DataStream &s) = 0; + + virtual void from_bytes(const bytearray_t &raw_bytes) { + DataStream s(raw_bytes); + s >> *this; + } + + virtual void from_bytes(bytearray_t &&raw_bytes) { + DataStream s(std::move(raw_bytes)); + s >> *this; + } + + + virtual void from_hex(const std::string &hex_str) { + DataStream s; + s.load_hex(hex_str); + s >> *this; + } + + bytearray_t to_bytes() const { + DataStream s; + s << *this; + return std::move(s); + } + + inline std::string to_hex() const; +}; + template -class Blob { +class Blob: public Serializable { using _impl_type = T; static const size_t bit_per_datum = sizeof(_impl_type) * 8; static_assert(!(N % bit_per_datum), "N must be divisible by bit_per_datum"); @@ -252,7 +284,7 @@ class Blob { size_t cheap_hash() const { return *data; } - void serialize(DataStream &s) const { + void serialize(DataStream &s) const override { if (loaded) { for (const _impl_type *ptr = data; ptr < data + _len; ptr++) @@ -265,7 +297,7 @@ class Blob { } } - void unserialize(DataStream &s) { + void unserialize(DataStream &s) override { for (_impl_type *ptr = data; ptr < data + _len; ptr++) { _impl_type x; @@ -424,37 +456,7 @@ inline bytearray_t from_hex(const std::string &hex_str) { return std::move(s); } -class Serializable { - public: - virtual ~Serializable() = default; - virtual void serialize(DataStream &s) const = 0; - virtual void unserialize(DataStream &s) = 0; - - virtual void from_bytes(const bytearray_t &raw_bytes) { - DataStream s(raw_bytes); - s >> *this; - } - - virtual void from_bytes(bytearray_t &&raw_bytes) { - DataStream s(std::move(raw_bytes)); - s >> *this; - } - - - virtual void from_hex(const std::string &hex_str) { - DataStream s; - s.load_hex(hex_str); - s >> *this; - } - - bytearray_t to_bytes() const { - DataStream s; - s << *this; - return std::move(s); - } - - std::string to_hex() const { return get_hex(*this); } -}; +inline std::string Serializable::to_hex() const { return get_hex(*this); } } diff --git a/src/conn.cpp b/src/conn.cpp index ab08399..84f08a4 100644 --- a/src/conn.cpp +++ b/src/conn.cpp @@ -223,9 +223,13 @@ void ConnPool::Conn::_recv_data_tls_handshake(const conn_t &conn, int, int) { /* finishing TLS handshake */ conn->send_data_func = _send_data_tls; conn->recv_data_func = _recv_data_dummy; + conn->ev_socket.del(); + conn->ev_socket.add(FdEvent::WRITE); conn->peer_cert = new X509(conn->tls->get_peer_cert()); conn->worker->enable_send_buffer(conn, conn->fd); - conn->cpool->update_conn(conn, true); + auto cpool = conn->cpool; + cpool->on_setup(conn); + cpool->update_conn(conn, true); } else { @@ -301,7 +305,6 @@ void ConnPool::accept_client(int fd, int) { SALTICIDAE_LOG_INFO("accepted %s", std::string(*conn).c_str()); auto &worker = select_worker(); conn->worker = &worker; - on_setup(conn); worker.feed(conn, client_fd); } } catch (...) { recoverable_error(std::current_exception()); } @@ -315,14 +318,13 @@ void ConnPool::conn_server(const conn_t &conn, int fd, int events) { SALTICIDAE_LOG_INFO("connected to remote %s", std::string(*conn).c_str()); auto &worker = select_worker(); conn->worker = &worker; - on_setup(conn); worker.feed(conn, fd); } else { if (events & TimedFdEvent::TIMEOUT) SALTICIDAE_LOG_INFO("%s connect timeout", std::string(*conn).c_str()); - throw SalticidaeError(SALTI_ERROR_CONNECT); + throw SalticidaeError(SALTI_ERROR_CONNECT, errno); } } catch (...) { disp_terminate(conn); diff --git a/src/network.cpp b/src/network.cpp index 74f4df9..a02ead1 100644 --- a/src/network.cpp +++ b/src/network.cpp @@ -250,8 +250,8 @@ void peernetwork_listen(peernetwork_t *self, const netaddr_t *listen_addr, Salti void peernetwork_reg_unknown_peer_handler(peernetwork_t *self, msgnetwork_unknown_peer_callback_t cb, void *userdata) { - self->reg_unknown_peer_handler([=](const NetAddr &addr) { - cb(&addr, userdata); + self->reg_unknown_peer_handler([=](const NetAddr &claimed_addr) { + cb(&claimed_addr, userdata); }); } diff --git a/test/.gitignore b/test/.gitignore index 7cebf5d..f50f029 100644 --- a/test/.gitignore +++ b/test/.gitignore @@ -2,6 +2,7 @@ test_msg test_bits test_msgnet test_p2p +test_p2p_tls test_p2p_stress test_queue bench_network diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d112b7a..0a1d3f1 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,6 +35,9 @@ target_link_libraries(test_msgnet_tls salticidae_static) add_executable(test_p2p test_p2p.cpp) target_link_libraries(test_p2p salticidae_static) +add_executable(test_p2p_tls test_p2p_tls.cpp) +target_link_libraries(test_p2p_tls salticidae_static) + add_executable(test_p2p_stress test_p2p_stress.cpp) target_link_libraries(test_p2p_stress salticidae_static) diff --git a/test/test_p2p.cpp b/test/test_p2p.cpp index 14304eb..7f80f85 100644 --- a/test/test_p2p.cpp +++ b/test/test_p2p.cpp @@ -87,8 +87,14 @@ struct Net { this->id, fatal ? "fatal" : "recoverable", err.what()); } }); - net->reg_unknown_peer_handler([this](const NetAddr &addr) { - fprintf(stdout, "net %lu: unknown peer attempts to connnect %s\n", this->id, std::string(addr).c_str()); + net->reg_peer_handler([this](const PeerNetwork::conn_t &conn, bool connected) { + fprintf(stdout, "net %lu: %s peer %s\n", this->id, + connected ? "connected to" : "disconnected from", + std::string(conn->get_peer_addr()).c_str()); + }); + net->reg_unknown_peer_handler([this](const NetAddr &claimed_addr) { + fprintf(stdout, "net %lu: unknown peer %s attempts to connnect\n", + this->id, std::string(claimed_addr).c_str()); }); th = std::thread([=](){ try { @@ -258,8 +264,8 @@ int main(int argc, char **argv) { fprintf(stdout, "add -- start a node (create a PeerNetwork instance)\n" "addpeer -- add a peer to a given node\n" - "rmpeer -- add a peer to a given node\n" - "rm -- remove a node (destroy a PeerNetwork instance)\n" + "delpeer -- add a peer to a given node\n" + "del -- remove a node (destroy a PeerNetwork instance)\n" "msg -- send a text message to a node\n" "ls -- list all node ids\n" "exit -- quit the program\n" diff --git a/test/test_p2p_stress.cpp b/test/test_p2p_stress.cpp index 1cb2ca3..1eb4a0d 100644 --- a/test/test_p2p_stress.cpp +++ b/test/test_p2p_stress.cpp @@ -97,29 +97,29 @@ void install_proto(AppContext &app, const size_t &seg_buff_size) { auto &ec = app.ec; auto &net = *app.net; auto send_rand = [&](int size, const MyNet::conn_t &conn) { - auto &tc = app.tc[conn->get_addr()]; + auto addr = conn->get_peer_addr(); + assert(!addr.is_null()); + auto &tc = app.tc[addr]; MsgRand msg(size); tc.hash = msg.serialized.get_hash(); net.send_msg(std::move(msg), conn); }; - net.reg_conn_handler([&, send_rand](const ConnPool::conn_t &conn, bool connected) { + net.reg_peer_handler([&, send_rand](const MyNet::conn_t &conn, bool connected) { if (connected) { - if (conn->get_mode() == ConnPool::Conn::ACTIVE) - { - auto &tc = app.tc[conn->get_addr()]; - tc.state = 1; - SALTICIDAE_LOG_INFO("increasing phase"); - send_rand(tc.state, static_pointer_cast(conn)); - } + auto addr = conn->get_peer_addr(); + assert(!addr.is_null()); + auto &tc = app.tc[addr]; + tc.state = 1; + SALTICIDAE_LOG_INFO("increasing phase"); + send_rand(tc.state, conn); } - return true; }); net.reg_error_handler([ec](const std::exception_ptr _err, bool fatal) { try { std::rethrow_exception(_err); } catch (const std::exception & err) { - SALTICIDAE_LOG_WARN("main thread captured %s error: %s", + SALTICIDAE_LOG_WARN("captured %s error: %s", fatal ? "fatal" : "recoverable", err.what()); } }); @@ -128,7 +128,9 @@ void install_proto(AppContext &app, const size_t &seg_buff_size) { net.send_msg(MsgAck(hash), conn); }); net.reg_handler([&, send_rand](MsgAck &&msg, const MyNet::conn_t &conn) { - auto &tc = app.tc[conn->get_addr()]; + auto addr = conn->get_peer_addr(); + assert(!addr.is_null()); + auto &tc = app.tc[addr]; if (msg.hash != tc.hash) { SALTICIDAE_LOG_ERROR("corrupted I/O!"); diff --git a/test/test_p2p_tls.cpp b/test/test_p2p_tls.cpp new file mode 100644 index 0000000..9fe0aec --- /dev/null +++ b/test/test_p2p_tls.cpp @@ -0,0 +1,317 @@ +/** + * Copyright (c) 2019 Ava Labs, Inc. + * + * Author: Ted Yin + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include +#include +#include +#include + +#include "salticidae/msg.h" +#include "salticidae/event.h" +#include "salticidae/network.h" +#include "salticidae/stream.h" + +using salticidae::NetAddr; +using salticidae::DataStream; +using salticidae::htole; +using salticidae::letoh; +using salticidae::EventContext; +using salticidae::ThreadCall; +using salticidae::PKey; +using std::placeholders::_1; +using std::placeholders::_2; + +using PeerNetwork = salticidae::PeerNetwork; + +struct MsgText { + static const uint8_t opcode = 0x0; + DataStream serialized; + uint64_t id; + std::string text; + + MsgText(uint64_t id, const std::string &text) { + serialized << salticidae::htole(id) << salticidae::htole((uint32_t)text.length()) << text; + } + + MsgText(DataStream &&s) { + uint32_t len; + s >> id; + id = salticidae::letoh(id); + s >> len; + len = salticidae::letoh(len); + text = std::string((const char *)s.get_data_inplace(len), len); + } +}; + +const uint8_t MsgText::opcode; + +struct Net { + uint64_t id; + EventContext ec; + ThreadCall tc; + std::thread th; + PeerNetwork *net; + const std::string listen_addr; + + Net(uint64_t id, uint16_t port): id(id), tc(ec), listen_addr("127.0.0.1:"+ std::to_string(port)) { + auto tls_key = new PKey(PKey::create_privkey_rsa(2048)); + auto tls_cert = new salticidae::X509(salticidae::X509::create_self_signed_from_pubkey(*tls_key)); + tls_key->save_privkey_to_file(std::to_string(port) + "_pkey.pem"); + tls_cert->save_to_file(std::to_string(port) + ".pem"); + net = new PeerNetwork(ec, PeerNetwork::Config(salticidae::ConnPool::Config() + .enable_tls(true) + .tls_key(tls_key) + .tls_cert(tls_cert) + ).conn_timeout(5) + .ping_period(2) + .id_mode(PeerNetwork::IdentityMode::ADDR_BASED)); + net->reg_handler([this](const MsgText &msg, const PeerNetwork::conn_t &) { + fprintf(stdout, "net %lu: peer %lu says %s\n", this->id, msg.id, msg.text.c_str()); + }); + net->reg_conn_handler([this](const salticidae::ConnPool::conn_t &conn, bool connected) { + if (connected) + { + fprintf(stdout, "net %lu: peer's cert is %s\n", this->id, + salticidae::get_hash(conn->get_peer_cert()->get_der()).to_hex().c_str()); + } + return true; + }); + net->reg_error_handler([this](const std::exception_ptr _err, bool fatal) { + try { + std::rethrow_exception(_err); + } catch (const std::exception &err) { + fprintf(stdout, "net %lu: captured %s error during an async call: %s\n", + this->id, fatal ? "fatal" : "recoverable", err.what()); + } + }); + net->reg_peer_handler([this](const PeerNetwork::conn_t &conn, bool connected) { + fprintf(stdout, "net %lu: %s peer %s\n", this->id, + connected ? "connected to" : "disconnected from", + std::string(conn->get_peer_addr()).c_str()); + }); + net->reg_unknown_peer_handler([this](const NetAddr &claimed_addr) { + fprintf(stdout, "net %lu: unknown peer %s attempts to connnect\n", + this->id, std::string(claimed_addr).c_str()); + }); + th = std::thread([=](){ + try { + net->start(); + net->listen(NetAddr(listen_addr)); + fprintf(stdout, "net %lu: listen to %s\n", id, listen_addr.c_str()); + ec.dispatch(); + } catch (std::exception &err) { + fprintf(stdout, "net %lu: got error during a sync call: %s\n", id, err.what()); + } + fprintf(stdout, "net %lu: main loop ended\n", id); + }); + } + + void add_peer(const std::string &listen_addr) { + try { + net->add_peer(NetAddr(listen_addr)); + } catch (std::exception &err) { + fprintf(stdout, "net %lu: got error during a sync call: %s\n", id, err.what()); + } + } + + void del_peer(const std::string &listen_addr) { + try { + net->del_peer(NetAddr(listen_addr)); + } catch (std::exception &err) { + fprintf(stdout, "net %lu: got error during a sync call: %s\n", id, err.what()); + } + } + + void stop_join() { + tc.async_call([ec=this->ec](ThreadCall::Handle &) { ec.stop(); }); + th.join(); + } + + ~Net() { delete net; } +}; + +std::unordered_map nets; +std::unordered_map > cmd_map; + +int read_int(char *buff) { + scanf("%64s", buff); + try { + int t = std::stoi(buff); + if (t < 0) throw std::invalid_argument("negative"); + return t; + } catch (std::invalid_argument) { + fprintf(stdout, "expect a non-negative integer\n"); + return -1; + } +} + +int main(int argc, char **argv) { + int i; + fprintf(stdout, "p2p network library playground (type help for more info)\n"); + fprintf(stdout, "========================================================\n"); + + auto cmd_exit = [](char *) { + for (auto &p: nets) + { + p.second->stop_join(); + delete p.second; + } + exit(0); + }; + + auto cmd_add = [](char *buff) { + int id = read_int(buff); + if (id < 0) return; + if (nets.count(id)) + { + fprintf(stdout, "net id already exists\n"); + return; + } + int port = read_int(buff); + if (port < 0) return; + if (port >= 65536) + { + fprintf(stdout, "port should be < 65536\n"); + return; + } + nets.insert(std::make_pair(id, new Net(id, port))); + }; + + auto cmd_ls = [](char *) { + for (auto &p: nets) + fprintf(stdout, "%d -> %s\n", p.first, p.second->listen_addr.c_str()); + }; + + auto cmd_del = [](char *buff) { + int id = read_int(buff); + if (id < 0) return; + auto it = nets.find(id); + if (it == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + it->second->stop_join(); + delete it->second; + nets.erase(it); + }; + + auto cmd_addpeer = [](char *buff) { + int id = read_int(buff); + if (id < 0) return; + auto it = nets.find(id); + if (it == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + int id2 = read_int(buff); + if (id2 < 0) return; + auto it2 = nets.find(id2); + if (it2 == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + it->second->add_peer(it2->second->listen_addr); + }; + + auto cmd_delpeer = [](char *buff) { + int id = read_int(buff); + if (id < 0) return; + auto it = nets.find(id); + if (it == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + int id2 = read_int(buff); + if (id2 < 0) return; + auto it2 = nets.find(id2); + if (it2 == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + it->second->del_peer(it2->second->listen_addr); + }; + + auto cmd_msg = [](char *buff) { + int id = read_int(buff); + if (id < 0) return; + auto it = nets.find(id); + if (it == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + int id2 = read_int(buff); + if (id2 < 0) return; + auto it2 = nets.find(id2); + if (it2 == nets.end()) + { + fprintf(stdout, "net id does not exist\n"); + return; + } + scanf("%64s", buff); + it->second->net->send_msg(MsgText(id, buff), it2->second->listen_addr); + }; + + auto cmd_help = [](char *) { + fprintf(stdout, + "add -- start a node (create a PeerNetwork instance)\n" + "addpeer -- add a peer to a given node\n" + "delpeer -- add a peer to a given node\n" + "del -- remove a node (destroy a PeerNetwork instance)\n" + "msg -- send a text message to a node\n" + "ls -- list all node ids\n" + "exit -- quit the program\n" + "help -- show this info\n" + ); + }; + + cmd_map.insert(std::make_pair("add", cmd_add)); + cmd_map.insert(std::make_pair("addpeer", cmd_addpeer)); + cmd_map.insert(std::make_pair("del", cmd_del)); + cmd_map.insert(std::make_pair("delpeer", cmd_delpeer)); + cmd_map.insert(std::make_pair("msg", cmd_msg)); + cmd_map.insert(std::make_pair("ls", cmd_ls)); + cmd_map.insert(std::make_pair("exit", cmd_exit)); + cmd_map.insert(std::make_pair("help", cmd_help)); + + for (;;) + { + fprintf(stdout, "> "); + char buff[128]; + if (scanf("%64s", buff) == EOF) break; + auto it = cmd_map.find(buff); + if (it == cmd_map.end()) + fprintf(stdout, "invalid comand \"%s\"\n", buff); + else + (it->second)(buff); + } + + return 0; +} -- cgit v1.2.3-70-g09d2