From 8d242e0112c7f9072600a48366ebbd6ce7b5c727 Mon Sep 17 00:00:00 2001
From: Determinant <ted.sybil@gmail.com>
Date: Fri, 28 Jun 2019 22:49:03 -0400
Subject: fix data race in timers

---
 include/salticidae/conn.h    |  2 +-
 include/salticidae/network.h | 24 ++++++++++++------------
 src/conn.cpp                 |  7 +++----
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/include/salticidae/conn.h b/include/salticidae/conn.h
index b693b4f..c34521a 100644
--- a/include/salticidae/conn.h
+++ b/include/salticidae/conn.h
@@ -115,6 +115,7 @@ class ConnPool {
         Conn(Conn &&other) = delete;
     
         virtual ~Conn() {
+            std::atomic_thread_fence(std::memory_order_acquire);
             SALTICIDAE_LOG_INFO("destroyed %s", std::string(*this).c_str());
         }
 
@@ -251,7 +252,6 @@ class ConnPool {
             /* the caller should finalize all the preparation */
             tcall.async_call([this, conn, client_fd](ThreadCall::Handle &) {
                 try {
-                    conn->ev_connect.clear();
                     assert(conn->mode != Conn::ConnMode::DEAD);
                     auto cpool = conn->cpool;
                     if (cpool->enable_tls)
diff --git a/include/salticidae/network.h b/include/salticidae/network.h
index 12392b2..f5fba90 100644
--- a/include/salticidae/network.h
+++ b/include/salticidae/network.h
@@ -291,7 +291,7 @@ class PeerNetwork: public MsgNetwork<OpcodeType> {
 
         protected:
         void stop() override {
-            ev_timeout.del();
+            ev_timeout.clear();
             MsgNet::Conn::stop();
         }
     };
@@ -596,16 +596,17 @@ void PeerNetwork<O, _, __>::on_setup(const ConnPool::conn_t &_conn) {
     MsgNet::on_setup(_conn);
     auto conn = static_pointer_cast<Conn>(_conn);
     auto worker = conn->worker;
-    auto &ev_timeout = conn->ev_timeout;
-    assert(!ev_timeout);
-    ev_timeout = TimerEvent(worker->get_ec(),
-            [listen_addr=this->listen_addr, worker, conn](TimerEvent &) {
-        try {
-            SALTICIDAE_LOG_INFO("peer ping-pong timeout %s <-> %s",
-                std::string(listen_addr).c_str(),
-                std::string(conn->get_peer_addr()).c_str());
-            conn->get_net()->worker_terminate(conn);
-        } catch (...) { worker->error_callback(std::current_exception()); }
+    worker->get_tcall()->async_call([this, conn, worker](ThreadCall::Handle &) {
+        auto &ev_timeout = conn->ev_timeout;
+        assert(!ev_timeout);
+        ev_timeout = TimerEvent(worker->get_ec(), [=](TimerEvent &) {
+            try {
+                SALTICIDAE_LOG_INFO("peer ping-pong timeout %s <-> %s",
+                    std::string(listen_addr).c_str(),
+                    std::string(conn->get_peer_addr()).c_str());
+                this->worker_terminate(conn);
+            } catch (...) { worker->error_callback(std::current_exception()); }
+        });
     });
     /* the initial ping-pong to set up the connection */
     tcall_reset_timeout(worker, conn, conn_timeout);
@@ -619,7 +620,6 @@ void PeerNetwork<O, _, __>::on_teardown(const ConnPool::conn_t &_conn) {
     MsgNet::on_teardown(_conn);
     auto conn = static_pointer_cast<Conn>(_conn);
     auto addr = conn->get_addr();
-    conn->ev_timeout.clear();
     pending_peers.erase(addr);
     SALTICIDAE_LOG_INFO("connection lost: %s", std::string(*conn).c_str());
     auto p = conn->peer;
diff --git a/src/conn.cpp b/src/conn.cpp
index e01690d..b6c6c71 100644
--- a/src/conn.cpp
+++ b/src/conn.cpp
@@ -248,8 +248,8 @@ void ConnPool::Conn::stop() {
     {
         if (worker) worker->unfeed();
         if (tls) tls->shutdown();
-        ev_connect.del();
-        ev_socket.del();
+        ev_socket.clear();
+        ev_connect.clear();
         send_buffer.get_queue().unreg_handler();
         mode = ConnMode::DEAD;
     }
@@ -419,9 +419,8 @@ void ConnPool::del_conn(const conn_t &conn) {
 void ConnPool::release_conn(const conn_t &conn) {
     /* inform the upper layer the connection will be destroyed */
     on_teardown(conn);
-    conn->ev_connect.clear();
-    conn->ev_socket.clear();
     ::close(conn->fd);
+    std::atomic_thread_fence(std::memory_order_release);
 }
 
 ConnPool::conn_t ConnPool::add_conn(const conn_t &conn) {
-- 
cgit v1.2.3-70-g09d2