fix the connection issue with --notls flag on; WIP: deployment example

17 files changed, 470 insertions, 5 deletions

diff --git a/.gitmodules b/.gitmodules
index 834ffa4..132f666 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "salticidae"]
 	path = salticidae
 	url = https://github.com/Determinant/salticidae.git
+[submodule "scripts/deploy/minirun"]
+	path = scripts/deploy/minirun
+	url = https://github.com/Determinant/minirun.git
diff --git a/salticidae b/salticidae
-Subproject 4eff97f09b0c5df6f62f009616ed4d38e3eba75
+Subproject 13c40bb073bbf1bccc3c88787d490b6d401eb8a
diff --git a/scripts/deploy/app/_setup.yml b/scripts/deploy/app/_setup.yml
new file mode 100644
index 0000000..7a9ba28
--- /dev/null
+++ b/scripts/deploy/app/_setup.yml
@@ -0,0 +1,34 @@
+---
+# available vars:
+# workdir -- the local directory of run_id
+
+- name: setup libhotstuff example binaries
+  block:
+    - apt:
+        pkg:
+          - g++
+          - libssl-dev
+          - libuv1-dev
+          - cmake
+          - make
+          - dh-autoreconf
+        state:
+          latest
+      become: true
+      become_user: root
+    - file:
+        path: "{{ hs_repo_dir }}"
+        state: directory
+    - synchronize:
+        archive: true
+        delete: true
+        src: "{{ (workdir, hs_local_repo_dir, '') | path_join }}"
+        dest: "{{ hs_repo_dir }}"
+        mode: push
+        rsync_opts:
+          - "--exclude=.git"
+          - "--exclude=scripts/"
+          - "--exclude=CMakeCache.txt"
+          - "--exclude=CMakeFiles"
+          - "--exclude=libsecp256k1-prefix"
+          - "--delete-excluded"
diff --git a/scripts/deploy/app/build.yml b/scripts/deploy/app/build.yml
new file mode 100644
index 0000000..2a0c24c
--- /dev/null
+++ b/scripts/deploy/app/build.yml
@@ -0,0 +1,27 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- name: build hotstuff example binaries
+  block:
+    - file:
+        path: "{{ (hs_repo_dir, 'build') | path_join }}"
+        state: absent
+    - command: cmake -DCMAKE_BUID_TYPE=Release -DHOTSTUFF_PROTO_LOG=OFF -DCMAKE_CXX_FLAGS={{ hs_flags | default('') }}
+      args:
+        chdir: "{{ hs_repo_dir }}"
+      environment:
+        PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+    - command: make clean
+      args:
+        chdir: "{{ hs_repo_dir }}"
+      environment:
+        PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+    - command: make -j4
+      args:
+        chdir: "{{ hs_repo_dir }}"
+      environment:
+        PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
diff --git a/scripts/deploy/app/hotstuff_app.py b/scripts/deploy/app/hotstuff_app.py
new file mode 100644
index 0000000..2a21e65
--- /dev/null
+++ b/scripts/deploy/app/hotstuff_app.py
@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+
+ANSIBLE_METADATA = {
+    'metadata_version': '0.1',
+    'status': ['preview'],
+    'supported_by': 'Ted Yin'
+}
+
+DOCUMENTATION = '''
+---
+module: hotstuff_app
+
+short_description: Ansible module for hotstuff-app
+
+author:
+    - Ted Yin (@Tederminant)
+'''
+
+EXAMPLES = '''
+'''
+
+RETURN = '''
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        bin=dict(type='str', required=True),
+        cwd=dict(type='str', required=True),
+        conf=dict(type='str', required=True),
+        log_dir=dict(type='str', required=True),
+        tls=dict(type='bool', default=True, required=False),
+    )
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=False
+    )
+
+    expanduser = [
+        'bin',
+        'cwd',
+        'conf',
+        'log_dir',
+    ]
+
+    for arg in expanduser:
+        module.params[arg] = os.path.expanduser(module.params[arg])
+
+    try:
+        cmd = [*(module.params['bin'].split())]
+        cmd += ['--conf', module.params['conf']]
+        if not module.params['tls']:
+            cmd += ['--notls']
+
+        logdir = module.params['log_dir']
+        if not (logdir is None):
+            stdout = open(os.path.expanduser(
+                os.path.join(logdir, 'stdout')), "w")
+            stderr = open(os.path.expanduser(
+                os.path.join(logdir, 'stderr')), "w")
+            nullsrc = open("/dev/null", "r")
+        else:
+            (stdout, stderr) = None, None
+
+        cwd = module.params['cwd']
+
+        pid = subprocess.Popen(
+                cmd,
+                cwd=cwd,
+                stdin=nullsrc,
+                stdout=stdout, stderr=stderr,
+                env=os.environ,
+                shell=False,
+                start_new_session=True).pid
+        module.exit_json(
+                changed=False,
+                status=0, pid=pid, cmd=" ".join(cmd), cwd=cwd)
+    except (OSError, subprocess.SubprocessError) as e:
+        module.fail_json(msg=str(e), changed=False, status=1)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/deploy/app/hotstuff_cli.py b/scripts/deploy/app/hotstuff_cli.py
new file mode 100644
index 0000000..0d4b57a
--- /dev/null
+++ b/scripts/deploy/app/hotstuff_cli.py
@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+
+ANSIBLE_METADATA = {
+    'metadata_version': '0.1',
+    'status': ['preview'],
+    'supported_by': 'Ted Yin'
+}
+
+DOCUMENTATION = '''
+---
+module: hotstuff_cli
+
+short_description: Ansible module for hotstuff-client
+
+author:
+    - Ted Yin (@Tederminant)
+'''
+
+EXAMPLES = '''
+'''
+
+RETURN = '''
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+def run_module():
+    # define available arguments/parameters a user can pass to the module
+    module_args = dict(
+        bin=dict(type='str', required=True),
+        cwd=dict(type='str', required=True),
+        idx=dict(type='int', required=True),
+        cid=dict(type='int', required=True),
+        iter=dict(type='int', required=True),
+        max_async=dict(type='int', required=True),
+        log_dir=dict(type='str', required=True),
+    )
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=False
+    )
+
+    expanduser = [
+        'bin',
+        'cwd',
+        'log_dir',
+    ]
+
+    for arg in expanduser:
+        module.params[arg] = os.path.expanduser(module.params[arg])
+
+    try:
+        cmd = [*(module.params['bin'].split())]
+        cmd += [
+            '--idx', str(module.params['idx']),
+            '--cid', str(module.params['cid']),
+            '--iter', str(module.params['iter']),
+            '--max-async', str(module.params['max_async']),
+        ]
+
+        logdir = module.params['log_dir']
+        if not (logdir is None):
+            stdout = open(os.path.expanduser(
+                os.path.join(logdir, 'stdout')), "w")
+            stderr = open(os.path.expanduser(
+                os.path.join(logdir, 'stderr')), "w")
+            nullsrc = open("/dev/null", "r")
+        else:
+            (stdout, stderr) = None, None
+
+        cwd = module.params['cwd']
+
+        pid = subprocess.Popen(
+                cmd,
+                cwd=cwd,
+                stdin=nullsrc,
+                stdout=stdout, stderr=stderr,
+                env=os.environ,
+                shell=False,
+                start_new_session=True).pid
+        module.exit_json(
+                changed=False,
+                status=0, pid=pid, cmd=" ".join(cmd), cwd=cwd)
+    except (OSError, subprocess.SubprocessError) as e:
+        module.fail_json(msg=str(e), changed=False, status=1)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/deploy/app/reset.yml b/scripts/deploy/app/reset.yml
new file mode 100644
index 0000000..1b80b27
--- /dev/null
+++ b/scripts/deploy/app/reset.yml
@@ -0,0 +1,21 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- name: reset hotstuff
+  vars:
+    log_dir: "{{ (testbed, hs_log_dir) | path_join }}"
+  block:
+    - name: remove confs
+      file:
+        path: "{{ conf_dir }}"
+        state: absent
+      when: "hs_no_reset_conf is not defined or (hs_no_reset_conf == False)"
+    - name: remove old logs
+      file:
+        path: "{{ log_dir }}"
+        state: absent
+      when: "hs_no_reset_log is not defined or (hs_no_reset_log == False)"
diff --git a/scripts/deploy/app/run.yml b/scripts/deploy/app/run.yml
new file mode 100644
index 0000000..260d611
--- /dev/null
+++ b/scripts/deploy/app/run.yml
@@ -0,0 +1,40 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- vars:
+    conf_dir: "{{ (testbed, hs_conf_dir) | path_join }}"
+    log_dir: "{{ (testbed, hs_log_dir) | path_join }}"
+  block:
+    - name: create testbed dirs
+      block:
+        - file:
+            path: "{{ conf_dir }}"
+            state: directory
+        - file:
+            path: "{{ log_dir }}"
+            state: directory
+    - name: copy the base conf file
+      copy:
+        src: "{{ (lookup('env','run_path'), hs_base_conf) | path_join }}"
+        dest: "{{ (conf_dir, 'hotstuff.conf') | path_join }}"
+        mode: '0644'
+    - name: copy per node conf files
+      copy:
+        src: "{{ (lookup('env','run_path'), extra_conf) | path_join }}"
+        dest: "{{ (conf_dir, (extra_conf | basename)) | path_join }}"
+        mode: '0644'
+      when: extra_conf | default('') | length > 0
+    - name: start the hotstuff replica
+      hotstuff_app:
+        bin: "{{ replica_bin | default('~/libhotstuff/examples/hotstuff_app') }}"
+        log_dir: "{{ log_dir }}"
+        cwd: "{{ conf_dir }}"
+        conf: "{{ extra_conf | basename }}"
+        tls: false
+      environment:
+        PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+      register: spawn_results
diff --git a/scripts/deploy/app/run_cli.yml b/scripts/deploy/app/run_cli.yml
new file mode 100644
index 0000000..c882a13
--- /dev/null
+++ b/scripts/deploy/app/run_cli.yml
@@ -0,0 +1,36 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- vars:
+    conf_dir: "{{ (testbed,  hs_conf_dir) | path_join }}"
+    log_dir: "{{ (testbed, hs_log_dir) | path_join }}"
+  block:
+    - name: create testbed dirs
+      block:
+        - file:
+            path: "{{ conf_dir }}"
+            state: directory
+        - file:
+            path: "{{ log_dir }}"
+            state: directory
+    - name: copy the base conf file
+      copy:
+        src: "{{ (lookup('env','run_path'), hs_base_conf) | path_join }}"
+        dest: "{{ (conf_dir, 'hotstuff.conf') | path_join }}"
+        mode: '0644'
+    - name: start the hotstuff client
+      hotstuff_cli:
+        bin: "{{ client_bin | default('~/libhotstuff/examples/hotstuff_client') }}"
+        log_dir: "{{ log_dir }}"
+        cwd: "{{ conf_dir }}"
+        idx: "{{ client_idx | default(1) }}"
+        cid: "{{ cid }}"
+        iter: "{{ max_iter | default(100000) }}"
+        max_async: "{{ max_async }}"
+      environment:
+        PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+      register: spawn_results
diff --git a/scripts/deploy/gen_all.sh b/scripts/deploy/gen_all.sh
new file mode 100755
index 0000000..295bf99
--- /dev/null
+++ b/scripts/deploy/gen_all.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+# NOTE: make sure hotstuff-keygen and hotstuff-tls-keygen exist by building the main repo
+# NOTE: make sure you have replicas.txt and clients.txt written properly:
+# 1) in replicas.txt, each row is "<external_ip_for_ssh> <inter_replica_net_ip>" (without quotes)
+# 2) in clients.txt each row is an IP adress for the machine
+# One IP can appear one or more times in the text files. If the same IP appears
+# several times, it runs multiple replica/client processes at the same time.
+
+python3 ./gen_inventory.py --prefix 'hotstuff.gen' > nodes.ini
+awk '{print $2}' replicas.txt > replicas_inter.txt
+python ../gen_conf.py --ips replicas_inter.txt --iter 1 --prefix 'hotstuff.gen' --keygen ../../hotstuff-keygen --tls-keygen ../../hotstuff-tls-keygen
diff --git a/scripts/deploy/gen_inventory.py b/scripts/deploy/gen_inventory.py
new file mode 100644
index 0000000..23af205
--- /dev/null
+++ b/scripts/deploy/gen_inventory.py
@@ -0,0 +1,34 @@
+import argparse
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Generate configuration files for a run.')
+    parser.add_argument('--replicas', type=str, default='replicas.txt',
+                        help='text file with all replicas IPs (each row is "<external_ip> <inter_replica_net_ip>", repeat a line if you would like to share the same machine)')
+    parser.add_argument('--clients', type=str, default='clients.txt',
+                        help='text file with all clients IPs (each row is "<external_ip>")')
+    parser.add_argument('--prefix', type=str, default='hotstuff.gen')
+    args = parser.parse_args()
+
+    print("[nodes_setup]")
+    host_idx_count = {}
+    replicas = []
+    clients = []
+    with open(args.replicas, "r") as rfile:
+        for line in rfile:
+            (pub_ip, priv_ip) = line.split()
+            replicas.append((pub_ip.strip(), priv_ip.strip()))
+    with open(args.clients, "r") as cfile:
+        for line in cfile:
+            clients.append(line.strip())
+    machines = sorted(set([pub_ip for (pub_ip, _) in replicas] + clients))
+    print("\n".join(machines))
+    print("\n[nodes]")
+    for (i, (pub_ip, priv_ip)) in enumerate(replicas):
+        host_idx = host_idx_count.setdefault(pub_ip, 0)
+        host_idx_count[pub_ip] += 1
+        print("replica{} ansible_host={} host_idx={} extra_conf={}-sec{}.conf".format(
+                i, pub_ip, host_idx, args.prefix, i))
+
+    print("\n[clients]")
+    for (i, ip) in enumerate(clients):
+        print("client{} ansible_host={} cid={}".format(i, ip, i))
diff --git a/scripts/deploy/group_vars/all.yml b/scripts/deploy/group_vars/all.yml
new file mode 100644
index 0000000..d9e9a44
--- /dev/null
+++ b/scripts/deploy/group_vars/all.yml
@@ -0,0 +1,24 @@
+---
+## basic config
+
+ansible_connection: ssh
+ansible_user: ubuntu
+# change to your aws ec2 key file here
+ansible_ssh_private_key_file: ~/.ssh/ted-aws-key2.pem
+# remote directory that keeps the work directory for the running app
+testbed_prefix: "/home/ubuntu/testbed"
+
+## app specific config
+
+# process name that is used by killall in reset
+bin_name: hotstuff-app
+# binary path (remote)
+replica_bin: "/home/ubuntu/libhotstuff/examples/hotstuff-app"
+client_bin: "/home/ubuntu/libhotstuff/examples/hotstuff-client"
+# remote repo path
+hs_repo_dir: "/home/ubuntu/libhotstuff"
+# local source code path (that will be copied to the remote)
+hs_local_repo_dir: "../../"
+hs_conf_dir: "conf"
+hs_log_dir: "log"
+hs_base_conf: "./hotstuff.gen.conf"
diff --git a/scripts/deploy/group_vars/clients.yml b/scripts/deploy/group_vars/clients.yml
new file mode 100644
index 0000000..5c5afd8
--- /dev/null
+++ b/scripts/deploy/group_vars/clients.yml
@@ -0,0 +1,4 @@
+---
+bin_name: "hotstuff-client"
+max_iter: 100000
+max_async: 175
diff --git a/scripts/deploy/run.sh b/scripts/deploy/run.sh
new file mode 100755
index 0000000..46bcb33
--- /dev/null
+++ b/scripts/deploy/run.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+basedir="$(dirname $(realpath "${BASH_SOURCE[0]}"))"
+export node_group=nodes
+export node_setup_group=nodes_setup
+export node_file="$basedir/nodes.ini"
+export group_vars="$basedir/group_vars"
+export build_tasks="$basedir/app/build.yml"
+export reset_tasks="$basedir/app/reset.yml"
+export run_tasks="$basedir/app/run.yml"
+export setup_tasks="$basedir/app/_setup.yml"
+export app_module="$basedir/app/"
+export run_path="$(realpath "$PWD")"
+
+minirun/run.sh "$@"
diff --git a/scripts/deploy/run_cli.sh b/scripts/deploy/run_cli.sh
new file mode 100755
index 0000000..a4eafb8
--- /dev/null
+++ b/scripts/deploy/run_cli.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+basedir="$(dirname $(realpath "${BASH_SOURCE[0]}"))"
+export node_group=clients
+export node_setup_group=nodes_setup
+export node_file="$basedir/nodes.ini"
+export group_vars="$basedir/group_vars"
+export build_tasks="$basedir/app/build.yml"
+export reset_tasks="$basedir/app/reset.yml"
+export run_tasks="$basedir/app/run_cli.yml"
+export setup_tasks="$basedir/app/_setup.yml"
+export app_module="$basedir/app/"
+export run_path="$(realpath "$PWD")"
+
+minirun/run.sh "$@"
diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py
index ca61ea1..428b5b5 100644
--- a/scripts/gen_conf.py
+++ b/scripts/gen_conf.py
@@ -31,9 +31,12 @@ if __name__ == "__main__":
 
     main_conf = open("{}.conf".format(prefix), 'w')
     nodes = open(args.nodes, 'w')
-    replicas = ["{}:{};{}".format(ip, base_pport + i, base_cport + i)
-                for ip in ips
-                for i in range(iter)]
+    port_count = {}
+    replicas = []
+    for ip in ips:
+        i = port_count.setdefault(ip, 0)
+        port_count[ip] += 1
+        replicas.append("{}:{};{}".format(ip, base_pport + i, base_cport + i))
     p = subprocess.Popen([keygen_bin, '--num', str(len(replicas))],
                         stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'))
     keys = [[t[4:] for t in l.decode('ascii').split()] for l in p.stdout]
diff --git a/src/hotstuff.cpp b/src/hotstuff.cpp
index ad2ed55..af1b2b4 100644
--- a/src/hotstuff.cpp
+++ b/src/hotstuff.cpp
@@ -257,9 +257,10 @@ void HotStuffBase::resp_blk_handler(MsgRespBlock &&msg, const Net::conn_t &) {
 bool HotStuffBase::conn_handler(const salticidae::ConnPool::conn_t &conn, bool connected) {
     if (connected)
     {
+        if (!pn.enable_tls) return true;
         auto cert = conn->get_peer_cert();
         //SALTICIDAE_LOG_INFO("%s", salticidae::get_hash(cert->get_der()).to_hex().c_str());
-        return (!cert) || valid_tls_certs.count(salticidae::get_hash(cert->get_der()));
+        return valid_tls_certs.count(salticidae::get_hash(cert->get_der()));
     }
     return true;
 }
@@ -359,6 +360,13 @@ HotStuffBase::HotStuffBase(uint32_t blk_size,
     pn.reg_handler(salticidae::generic_bind(&HotStuffBase::req_blk_handler, this, _1, _2));
     pn.reg_handler(salticidae::generic_bind(&HotStuffBase::resp_blk_handler, this, _1, _2));
     pn.reg_conn_handler(salticidae::generic_bind(&HotStuffBase::conn_handler, this, _1, _2));
+    pn.reg_error_handler([](const std::exception_ptr _err, bool fatal, int32_t async_id) {
+        try {
+            std::rethrow_exception(_err);
+        } catch (const std::exception &err) {
+            HOTSTUFF_LOG_WARN("network async error: %s\n", err.what());
+        }
+    });
     pn.start();
     pn.listen(listen_addr);
 }
@@ -408,7 +416,7 @@ void HotStuffBase::start(
         auto &addr = std::get<0>(replicas[i]);
         auto cert_hash = std::move(std::get<2>(replicas[i]));
         valid_tls_certs.insert(cert_hash);
-        salticidae::PeerId peer{cert_hash};
+        auto peer = pn.enable_tls ? salticidae::PeerId(cert_hash) : salticidae::PeerId(addr);
         HotStuffCore::add_replica(i, peer, std::move(std::get<1>(replicas[i])));
         if (addr != listen_addr)
         {