debug p2p impl

4 files changed, 217 insertions, 54 deletions

diff --git a/include/salticidae/network.h b/include/salticidae/network.h
index 8f8940b..290eaa9 100644
--- a/include/salticidae/network.h
+++ b/include/salticidae/network.h
@@ -200,9 +200,13 @@ class MsgNetwork: public ConnPool {
 /** Simple network that handles client-server requests. */
 template<typename OpcodeType>
 class ClientNetwork: public MsgNetwork<OpcodeType> {
+    public:
     using MsgNet = MsgNetwork<OpcodeType>;
     using Msg = typename MsgNet::Msg;
+
+    private:
     std::unordered_map<NetAddr, typename MsgNet::conn_t> addr2conn;
+    std::mutex cn_mlock;
 
     public:
     class Conn: public MsgNet::Conn {
@@ -248,9 +252,9 @@ template<typename OpcodeType = uint8_t,
         OpcodeType OPCODE_PING = 0xf0,
         OpcodeType OPCODE_PONG = 0xf1>
 class PeerNetwork: public MsgNetwork<OpcodeType> {
+    public:
     using MsgNet = MsgNetwork<OpcodeType>;
     using Msg = typename MsgNet::Msg;
-    public:
     enum IdentityMode {
         IP_BASED,
         IP_PORT_BASED
@@ -319,7 +323,7 @@ class PeerNetwork: public MsgNetwork<OpcodeType> {
     std::unordered_map <NetAddr, BoxObj<Peer>> id2peer;
     std::mutex pn_mlock;
 
-    IdentityMode id_mode;
+    const IdentityMode id_mode;
     double retry_conn_delay;
     double ping_period;
     double conn_timeout;
@@ -351,8 +355,29 @@ class PeerNetwork: public MsgNetwork<OpcodeType> {
         }
     };
 
+    struct PingCmd: public ConnPool::DispatchCmd {
+        conn_t conn;
+        uint16_t port;
+        PingCmd(const conn_t &conn, uint16_t port):
+            conn(conn), port(port) {}
+        void exec(ConnPool *cpool) override {
+            auto pn = static_cast<PeerNetwork *>(cpool);
+            pn->_ping_msg_cb(conn, port);
+        }
+    };
+    
+    struct PongCmd: public PingCmd {
+        using PingCmd::PingCmd;
+        void exec(ConnPool *cpool) override {
+            auto pn = static_cast<PeerNetwork *>(cpool);
+            pn->_pong_msg_cb(this->conn, this->port);
+        }
+    };
+
     void msg_ping(MsgPing &&msg, Conn &conn);
     void msg_pong(MsgPong &&msg, Conn &conn);
+    void _ping_msg_cb(const conn_t &conn, uint16_t port);
+    void _pong_msg_cb(const conn_t &conn, uint16_t port);
     bool check_new_conn(Conn &conn, uint16_t port);
     void start_active_conn(const NetAddr &paddr);
 
@@ -387,9 +412,7 @@ class PeerNetwork: public MsgNetwork<OpcodeType> {
     void send_msg(const MsgType &msg, const NetAddr &paddr);
     void listen(NetAddr listen_addr);
     bool has_peer(const NetAddr &paddr) const;
-    conn_t connect(const NetAddr &addr) {
-        return static_pointer_cast<Conn>(ConnPool::connect(addr));
-    }
+    conn_t connect(const NetAddr &addr) = delete;
 };
 
 /* this callback is run by a worker */
@@ -455,11 +478,11 @@ void PeerNetwork<O, _, __>::Conn::on_setup() {
     if (this->get_mode() == Conn::ConnMode::ACTIVE)
     {
         peer_id = this->get_addr();
-        if (id_mode == IP_BASED) peer_id.port = 0;
+        if (pn->id_mode == IP_BASED) peer_id.port = 0;
     }
     /* the initial ping-pong to set up the connection */
     auto &conn = static_cast<Conn &>(*this);
-    conn->reset_timeout(pn->conn_timeout);
+    reset_timeout(pn->conn_timeout);
     pn->MsgNet::send_msg(MsgPing(pn->listen_port), conn);
 }
 
@@ -572,51 +595,34 @@ bool PeerNetwork<O, _, __>::check_new_conn(Conn &conn, uint16_t port) {
 }
 
 template<typename O, O _, O __>
-class PeerNetworkPing: public ConnPool::DispatchCmd {
-    using conn_t = typename PeerNetwork<O, _, __>::conn_t;
-    conn_t conn;
-    uint16_t port;
-    public:
-    PeerNetworkPing(const conn_t &conn, uint16_t port):
-        conn(conn), port(port) {}
-    void exec(ConnPool *cpool) override {
-        auto pn = static_cast<PeerNetwork<O, _, __> *>(cpool);
-        mutex_lg_t _pn_lg(pn->pn_mlock);
-        if (pn->check_new_conn(conn, port)) return;
-        auto p = pn->id2peer.find(conn.peer_id)->second.get();
-        mutex_lg_t _p_lg(p->mlock);
-        pn->_send_msg(MsgPong(this->listen_port), p);
-    }
-};
+void PeerNetwork<O, _, __>::_ping_msg_cb(const conn_t &conn, uint16_t port) {
+    mutex_lg_t _pn_lg(pn_mlock);
+    if (check_new_conn(*conn, port)) return;
+    auto p = id2peer.find(conn->peer_id)->second.get();
+    mutex_lg_t _p_lg(p->mlock);
+    _send_msg(MsgPong(this->listen_port), p);
+}
 
 template<typename O, O _, O __>
-class PeerNetworkPong: public ConnPool::DispatchCmd {
-    using conn_t = typename PeerNetwork<O, _, __>::conn_t;
-    conn_t conn;
-    uint16_t port;
-    public:
-    PeerNetworkPong(const conn_t &conn, uint16_t port):
-        conn(conn), port(port) {}
-    void exec(ConnPool *cpool) override {
-        auto pn = static_cast<PeerNetwork<O, _, __> *>(cpool);
-        mutex_lg_t _pn_lg(pn->pn_mlock);
-        auto it = pn->id2peer.find(conn->peer_id);
-        if (it == pn->id2peer.end())
-        {
-            SALTICIDAE_LOG_WARN("pong message discarded");
-            return;
-        }
-        if (pn->check_new_conn(conn, port)) return;
-        auto p = it->second.get();
-        mutex_lg_t _p_lg(p->mlock);
-        p->pong_msg_ok = true;
-        if (p->ping_timer_ok)
-        {
-            p->reset_ping_timer();
-            p->send_ping();
-        }
+void PeerNetwork<O, _, __>::_pong_msg_cb(const conn_t &conn, uint16_t port) {
+    mutex_lg_t _pn_lg(pn_mlock);
+    auto it = id2peer.find(conn->peer_id);
+    if (it == id2peer.end())
+    {
+        SALTICIDAE_LOG_WARN("pong message discarded");
+        return;
     }
-};
+    if (check_new_conn(*conn, port)) return;
+    auto p = it->second.get();
+    mutex_lg_t _p_lg(p->mlock);
+    p->pong_msg_ok = true;
+    if (p->ping_timer_ok)
+    {
+        p->reset_ping_timer();
+        p->send_ping();
+    }
+}
+
 /* end: functions invoked by the dispatcher */
 
 /* this function could be both invoked by the dispatcher and the user loop */
@@ -625,7 +631,7 @@ void PeerNetwork<O, _, __>::start_active_conn(const NetAddr &addr) {
     auto p = id2peer.find(addr)->second.get();
     mutex_lg_t _p_lg(p->mlock);
     if (p->connected) return;
-    auto conn = static_pointer_cast<Conn>(connect(addr));
+    auto conn = static_pointer_cast<Conn>(MsgNet::connect(addr));
     assert(p->conn == nullptr);
     p->conn = conn;
 }
@@ -635,13 +641,13 @@ template<typename O, O _, O __>
 void PeerNetwork<O, _, __>::msg_ping(MsgPing &&msg, Conn &conn) {
     uint16_t port = msg.port;
     SALTICIDAE_LOG_INFO("ping from %s, port %u", std::string(conn).c_str(), ntohs(port));
-    auto dcmd = new PeerNetworkPing<O, _, __>(conn.self(), port);
+    auto dcmd = new PingCmd(static_pointer_cast<Conn>(conn.self()), port);
     write(this->dlisten_fd[1], &dcmd, sizeof(dcmd));
 }
 
 template<typename O, O _, O __>
 void PeerNetwork<O, _, __>::msg_pong(MsgPong &&msg, Conn &conn) {
-    auto dcmd = new PeerNetworkPong<O, _, __>(conn.self(), msg.port);
+    auto dcmd = new PongCmd(static_pointer_cast<Conn>(conn.self()), msg.port);
     write(this->dlisten_fd[1], &dcmd, sizeof(dcmd));
 }
 
@@ -710,6 +716,7 @@ void ClientNetwork<OpcodeType>::Conn::on_setup() {
     MsgNet::Conn::on_setup();
     assert(this->get_mode() == Conn::PASSIVE);
     const auto &addr = this->get_addr();
+    mutex_lg_t _cn_lg(cn_mlock);
     auto cn = get_net();
     cn->addr2conn.erase(addr);
     cn->addr2conn.insert(
@@ -721,12 +728,14 @@ template<typename OpcodeType>
 void ClientNetwork<OpcodeType>::Conn::on_teardown() {
     MsgNet::Conn::on_teardown();
     assert(this->get_mode() == Conn::PASSIVE);
+    mutex_lg_t _cn_lg(cn_mlock);
     get_net()->addr2conn.erase(this->get_addr());
 }
 
 template<typename OpcodeType>
 template<typename MsgType>
 void ClientNetwork<OpcodeType>::send_msg(const MsgType &msg, const NetAddr &addr) {
+    mutex_lg_t _cn_lg(cn_mlock);
     auto it = addr2conn.find(addr);
     if (it == addr2conn.end()) return;
     MsgNet::send_msg(msg, *(it->second));
diff --git a/src/conn.cpp b/src/conn.cpp
index edbe9f3..7b7c699 100644
--- a/src/conn.cpp
+++ b/src/conn.cpp
@@ -66,7 +66,7 @@ void ConnPool::Conn::send_data(evutil_socket_t fd, short events) {
                 send_buffer.rewind(std::move(buff_seg));
                 if (ret < 0 && errno != EWOULDBLOCK)
                 {
-                    SALTICIDAE_LOG_INFO("reason: %s", strerror(errno));
+                    SALTICIDAE_LOG_INFO("send(%d) failure: %s", fd, strerror(errno));
                     terminate();
                     return;
                 }
@@ -98,7 +98,7 @@ void ConnPool::Conn::recv_data(evutil_socket_t fd, short events) {
         if (ret < 0)
         {
             if (errno == EWOULDBLOCK) break;
-            SALTICIDAE_LOG_INFO("reason: %s", strerror(errno));
+            SALTICIDAE_LOG_INFO("recv(%d) failure: %s", fd, strerror(errno));
             /* connection err or half-opened connection */
             terminate();
             return;
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 44c984d..3d4dc23 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -29,6 +29,9 @@ target_link_libraries(test_bits salticidae_static)
 add_executable(test_network test_network.cpp)
 target_link_libraries(test_network salticidae_static)
 
+add_executable(test_p2p test_p2p.cpp)
+target_link_libraries(test_p2p salticidae_static)
+
 add_executable(test_queue test_queue.cpp)
 target_link_libraries(test_queue salticidae_static pthread)
 
diff --git a/test/test_p2p.cpp b/test/test_p2p.cpp
new file mode 100644
index 0000000..4146fd9
--- /dev/null
+++ b/test/test_p2p.cpp
@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2018 Cornell University.
+ *
+ * Author: Ted Yin <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <cstdio>
+#include <string>
+#include <functional>
+
+#include "salticidae/msg.h"
+#include "salticidae/event.h"
+#include "salticidae/network.h"
+#include "salticidae/stream.h"
+
+using salticidae::NetAddr;
+using salticidae::DataStream;
+using salticidae::MsgNetwork;
+using salticidae::htole;
+using salticidae::letoh;
+using std::placeholders::_1;
+using std::placeholders::_2;
+
+/** Hello Message. */
+struct MsgHello {
+    static const uint8_t opcode = 0x0;
+    DataStream serialized;
+    std::string name;
+    std::string text;
+    /** Defines how to serialize the msg. */
+    MsgHello(const std::string &name,
+            const std::string &text) {
+        serialized << htole((uint32_t)name.length());
+        serialized << name << text;
+    }
+    /** Defines how to parse the msg. */
+    MsgHello(DataStream &&s) {
+        uint32_t len;
+        s >> len;
+        len = letoh(len);
+        name = std::string((const char *)s.get_data_inplace(len), len);
+        len = s.size();
+        text = std::string((const char *)s.get_data_inplace(len), len);
+    }
+};
+
+/** Acknowledgement Message. */
+struct MsgAck {
+    static const uint8_t opcode = 0x1;
+    DataStream serialized;
+    MsgAck() {}
+    MsgAck(DataStream &&s) {}
+};
+
+const uint8_t MsgHello::opcode;
+const uint8_t MsgAck::opcode;
+
+using PeerNetworkByteOp = salticidae::PeerNetwork<uint8_t>;
+
+struct MyNet: public PeerNetworkByteOp {
+    const std::string name;
+    const NetAddr peer;
+
+    MyNet(const salticidae::EventContext &ec,
+            const std::string name,
+            const NetAddr &peer):
+            PeerNetwork<uint8_t>(ec, 10, 2, 2, 4096, 2, 10),
+            name(name),
+            peer(peer) {
+        /* message handler could be a bound method */
+        reg_handler(salticidae::generic_bind(
+            &MyNet::on_receive_hello, this, _1, _2));
+
+        reg_conn_handler([this](ConnPool::Conn &conn, bool connected) {
+            if (connected)
+            {
+                if (conn.get_mode() == ConnPool::Conn::ACTIVE)
+                {
+                    printf("[%s] Connected, sending hello.\n",
+                            this->name.c_str());
+                    /* send the first message through this connection */
+                    MsgNet::send_msg(MsgHello(this->name, "Hello there!"),
+                            static_cast<MsgNet::Conn &>(conn));
+                }
+                else
+                    printf("[%s] Accepted, waiting for greetings.\n",
+                            this->name.c_str());
+            }
+            else
+            {
+                printf("[%s] Disconnected, retrying.\n", this->name.c_str());
+            }
+        });
+    }
+
+    void on_receive_hello(MsgHello &&msg, MyNet::Conn &conn) {
+        printf("[%s] %s says %s\n",
+                name.c_str(),
+                msg.name.c_str(), msg.text.c_str());
+        /* send acknowledgement */
+        MsgNet::send_msg(MsgAck(), conn);
+    }
+};
+
+
+void on_receive_ack(MsgAck &&msg, MyNet::Conn &conn) {
+    auto net = static_cast<MyNet *>(conn.get_net());
+    printf("[%s] the peer knows\n", net->name.c_str());
+}
+
+salticidae::EventContext ec;
+NetAddr alice_addr("127.0.0.1:12345");
+NetAddr bob_addr("127.0.0.1:12346");
+
+int main() {
+    /* test two nodes */
+    MyNet alice(ec, "Alice", bob_addr);
+    MyNet bob(ec, "Bob", alice_addr);
+
+    /* message handler could be a normal function */
+    alice.reg_handler(on_receive_ack);
+    bob.reg_handler(on_receive_ack);
+
+    alice.listen(alice_addr);
+    bob.listen(bob_addr);
+
+    /* first attempt */
+    alice.add_peer(bob_addr);
+    bob.add_peer(alice_addr);
+
+    ec.dispatch();
+    return 0;
+}