WIP: bounded recv buffer

9 files changed, 249 insertions, 0 deletions

diff --git a/test/.gitignore b/test/.gitignore
index 50e25ba..a83336f 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -11,3 +11,4 @@ Makefile
 test_msgnet_c
 test_msgnet_tls
 bench_network_tls
+test_bounded_recv_buffer
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 15cd414..8f95e14 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -55,3 +55,6 @@ target_link_libraries(bench_network_tls salticidae_static pthread)
 
 add_executable(test_msgnet_c test_msgnet_c.c)
 target_link_libraries(test_msgnet_c salticidae_static pthread)
+
+add_executable(test_bounded_recv_buffer test_bounded_recv_buffer.cpp)
+target_link_libraries(test_bounded_recv_buffer salticidae_static pthread)
diff --git a/test/bench_network.cpp b/test/bench_network.cpp
index 1021ec4..1d52c39 100644
--- a/test/bench_network.cpp
+++ b/test/bench_network.cpp
@@ -130,6 +130,13 @@ salticidae::EventContext ec;
 NetAddr alice_addr("127.0.0.1:1234");
 NetAddr bob_addr("127.0.0.1:1235");
 
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 int main() {
     salticidae::BoxObj<MyNet> alice = new MyNet(ec, "Alice", 10);
     alice->start();
@@ -137,6 +144,7 @@ int main() {
     salticidae::EventContext tec;
     MyNet bob(tec, "Bob");
     std::thread bob_thread([&]() {
+        masksigs();
         bob.start();
         bob.connect(alice_addr);
         tec.dispatch();
diff --git a/test/bench_network_tls.cpp b/test/bench_network_tls.cpp
index 7c682ba..47b9347 100644
--- a/test/bench_network_tls.cpp
+++ b/test/bench_network_tls.cpp
@@ -132,6 +132,13 @@ salticidae::EventContext ec;
 NetAddr alice_addr("127.0.0.1:1234");
 NetAddr bob_addr("127.0.0.1:1235");
 
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 int main() {
     salticidae::BoxObj<MyNet> alice = new MyNet(ec, "Alice", 10);
     alice->start();
@@ -139,6 +146,7 @@ int main() {
     salticidae::EventContext tec;
     MyNet bob(tec, "Bob");
     std::thread bob_thread([&]() {
+        masksigs();
         bob.start();
         bob.connect(alice_addr);
         tec.dispatch();
diff --git a/test/test_bounded_recv_buffer.cpp b/test/test_bounded_recv_buffer.cpp
new file mode 100644
index 0000000..a2ec4e1
--- /dev/null
+++ b/test/test_bounded_recv_buffer.cpp
@@ -0,0 +1,192 @@
+/**
+ * Copyright (c) 2018 Cornell University.
+ *
+ * Author: Ted Yin <[email protected]>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of
+ * this software and associated documentation files (the "Software"), to deal in
+ * the Software without restriction, including without limitation the rights to
+ * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is furnished to do
+ * so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <cstdio>
+#include <string>
+#include <functional>
+#include <thread>
+#include <signal.h>
+
+/* disable SHA256 checksum */
+#define SALTICIDAE_NOCHECKSUM
+
+#include "salticidae/msg.h"
+#include "salticidae/event.h"
+#include "salticidae/network.h"
+#include "salticidae/stream.h"
+
+using salticidae::NetAddr;
+using salticidae::DataStream;
+using salticidae::MsgNetwork;
+using salticidae::htole;
+using salticidae::letoh;
+using salticidae::bytearray_t;
+using salticidae::TimerEvent;
+using salticidae::ThreadCall;
+using std::placeholders::_1;
+using std::placeholders::_2;
+using opcode_t = uint8_t;
+
+struct MsgBytes {
+    static const opcode_t opcode = 0xa;
+    DataStream serialized;
+    bytearray_t bytes;
+    MsgBytes(size_t size) {
+        bytes.resize(size);
+        serialized << htole((uint32_t)size) << bytes;
+    }
+    MsgBytes(DataStream &&s) {
+        uint32_t len;
+        s >> len;
+        len = letoh(len);
+        auto base = s.get_data_inplace(len);
+        bytes = bytearray_t(base, base + len);
+    }
+};
+
+const opcode_t MsgBytes::opcode;
+
+using MsgNetworkByteOp = MsgNetwork<opcode_t>;
+
+struct MyNet: public MsgNetworkByteOp {
+    const std::string name;
+    TimerEvent ev_period_stat;
+    ThreadCall tcall;
+    size_t nrecv;
+    std::function<void(ThreadCall::Handle &)> trigger;
+    std::atomic<bool> stopped;
+
+    MyNet(const salticidae::EventContext &ec,
+            const std::string name,
+            double stat_timeout = -1):
+            MsgNetworkByteOp(ec, MsgNetworkByteOp::Config(
+                ConnPool::Config()
+                    .max_recv_buff_size(10)
+                    .queue_capacity(10)
+                ).burst_size(10)),
+            name(name),
+            ev_period_stat(ec, [this, stat_timeout](TimerEvent &) {
+                SALTICIDAE_LOG_INFO("%.2f mps", nrecv / (double)stat_timeout);
+                fflush(stderr);
+                nrecv = 0;
+                ev_period_stat.add(stat_timeout);
+            }),
+            tcall(ec),
+            nrecv(0), stopped(false) {
+        /* message handler could be a bound method */
+        reg_handler(salticidae::generic_bind(&MyNet::on_receive_bytes, this, _1, _2));
+        if (stat_timeout > 0)
+            ev_period_stat.add(0);
+        reg_conn_handler([this, ec](const ConnPool::conn_t &conn, bool connected) {
+            if (connected)
+            {
+                if (conn->get_mode() == MyNet::Conn::ACTIVE)
+                {
+                    printf("[%s] connected, sending bytes.\n", this->name.c_str());
+                    /* send the first message through this connection */
+                    trigger = [this, conn](ThreadCall::Handle &) {
+                        while (!send_msg(MsgBytes(256), salticidae::static_pointer_cast<Conn>(conn)))
+                        {
+                            if (stopped)
+                            {
+                                stop();
+                                return;
+                            }
+                            fprintf(stderr, "cannot send message, retrying\n");
+                            sleep(1);
+                        }
+                        if (!conn->is_terminated())
+                            tcall.async_call(trigger);
+                    };
+                    tcall.async_call(trigger);
+                }
+                else
+                    printf("[%s] passively connected, waiting for bytes.\n", this->name.c_str());
+            }
+            else
+            {
+                printf("[%s] disconnected, retrying.\n", this->name.c_str());
+                /* try to reconnect to the same address */
+                connect(conn->get_addr());
+            }
+            return true;
+        });
+    }
+
+    void on_receive_bytes(MsgBytes &&msg, const conn_t &conn) {
+        if (stopped)
+        {
+            conn->get_pool()->stop();
+            return;
+        }
+        sleep(1);
+        nrecv++;
+    }
+};
+
+salticidae::EventContext aec, bec, ec;
+NetAddr alice_addr("127.0.0.1:1234");
+NetAddr bob_addr("127.0.0.1:1235");
+
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
+int main() {
+    MyNet alice(aec, "Alice", 10), bob(bec, "Bob");
+    std::thread alice_thread([&]() {
+        masksigs();
+        alice.start();
+        alice.listen(alice_addr);
+        aec.dispatch();
+    });
+    std::thread bob_thread([&]() {
+        masksigs();
+        bob.start();
+        bob.connect(alice_addr);
+        bec.dispatch();
+    });
+    auto shutdown = [&](int) {
+        bob.stopped = true;
+        bob.tcall.async_call([&](salticidae::ThreadCall::Handle &) {
+            bec.stop();
+        });
+        bob_thread.join();
+        alice.stopped = true;
+        alice.tcall.async_call([&](salticidae::ThreadCall::Handle &) {
+            aec.stop();
+        });
+        alice_thread.join();
+        ec.stop();
+    };
+    salticidae::SigEvent ev_sigint(ec, shutdown);
+    salticidae::SigEvent ev_sigterm(ec, shutdown);
+    ev_sigint.add(SIGINT);
+    ev_sigterm.add(SIGTERM);
+    ec.dispatch();
+    return 0;
+}
diff --git a/test/test_p2p.cpp b/test/test_p2p.cpp
index d097562..aff712a 100644
--- a/test/test_p2p.cpp
+++ b/test/test_p2p.cpp
@@ -28,6 +28,7 @@
 #include <functional>
 #include <unordered_map>
 #include <unistd.h>
+#include <signal.h>
 
 #include "salticidae/msg.h"
 #include "salticidae/event.h"
@@ -67,6 +68,13 @@ struct MsgText {
 
 const uint8_t MsgText::opcode;
 
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 struct Net {
     uint64_t id;
     EventContext ec;
@@ -98,6 +106,7 @@ struct Net {
                     this->id, std::string(claimed_addr).c_str());
         });
         th = std::thread([=](){
+            masksigs();
             try {
                 net->start();
                 net->listen(NetAddr(listen_addr));
diff --git a/test/test_p2p_stress.cpp b/test/test_p2p_stress.cpp
index 9fe1b83..49f1b2e 100644
--- a/test/test_p2p_stress.cpp
+++ b/test/test_p2p_stress.cpp
@@ -179,6 +179,13 @@ void install_proto(AppContext &app, const size_t &seg_buff_size) {
     });
 }
 
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 int main(int argc, char **argv) {
     Config config;
     auto opt_no_msg = Config::OptValFlag::create(false);
@@ -225,6 +232,7 @@ int main(int argc, char **argv) {
 
     for (auto &a: apps)
         threads.push_back(std::thread([&]() {
+            masksigs();
             a.net->listen(a.addr);
             for (auto &paddr: addrs)
                 if (paddr != a.addr) a.net->add_peer(paddr);
diff --git a/test/test_p2p_tls.cpp b/test/test_p2p_tls.cpp
index 698bbac..93cefac 100644
--- a/test/test_p2p_tls.cpp
+++ b/test/test_p2p_tls.cpp
@@ -29,6 +29,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <unistd.h>
+#include <signal.h>
 
 #include "salticidae/msg.h"
 #include "salticidae/event.h"
@@ -69,6 +70,13 @@ struct MsgText {
 
 const uint8_t MsgText::opcode;
 
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 std::unordered_set<uint256_t> valid_certs;
 
 struct Net {
@@ -124,6 +132,7 @@ struct Net {
                     this->id, std::string(claimed_addr).c_str());
         });
         th = std::thread([=](){
+            masksigs();
             try {
                 net->start();
                 net->listen(NetAddr(listen_addr));
diff --git a/test/test_queue.cpp b/test/test_queue.cpp
index 5c32dac..9082135 100644
--- a/test/test_queue.cpp
+++ b/test/test_queue.cpp
@@ -8,6 +8,13 @@
 using salticidae::TimerEvent;
 using salticidae::Config;
 
+void masksigs() {
+	sigset_t mask;
+	sigemptyset(&mask);
+    sigfillset(&mask);
+    pthread_sigmask(SIG_BLOCK, &mask, NULL);
+}
+
 void test_mpsc(int nproducers, int nops, size_t burst_size, bool test_rewind) {
     size_t total = nproducers * nops;
     salticidae::EventContext ec;
@@ -33,6 +40,7 @@ void test_mpsc(int nproducers, int nops, size_t burst_size, bool test_rewind) {
     });
     std::vector<std::thread> producers;
     std::thread consumer([&collected, total, &ec]() {
+        masksigs();
         TimerEvent timer(ec, [&ec, &collected, total](TimerEvent &timer) {
             if (collected.load() == total) ec.stop();
             timer.add(1);
@@ -44,6 +52,7 @@ void test_mpsc(int nproducers, int nops, size_t burst_size, bool test_rewind) {
     for (int i = 0; i < nproducers; i++)
     {
         producers.emplace(producers.end(), std::thread([&q, nops, i, nproducers]() {
+            masksigs();
             int x = i;
             for (int j = 0; j < nops; j++)
             {
@@ -89,6 +98,7 @@ void test_mpmc(int nproducers, int nconsumers, int nops, size_t burst_size) {
     {
         consumers.emplace(consumers.end(), std::thread(
                 [&collected, total, &ec = ecs[i]]() {
+            masksigs();
             TimerEvent timer(ec, [&ec, &collected, total](TimerEvent &timer) {
                 if (collected.load() == total) ec.stop();
                 timer.add(1);
@@ -100,6 +110,7 @@ void test_mpmc(int nproducers, int nconsumers, int nops, size_t burst_size) {
     for (int i = 0; i < nproducers; i++)
     {
         producers.emplace(producers.end(), std::thread([&q, nops, i, nproducers]() {
+            masksigs();
             int x = i;
             for (int j = 0; j < nops; j++)
             {