From d5bf357c8010bb1219d04ede14ce699b409e93c5 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sun, 30 Aug 2020 00:15:08 -0400 Subject: fix the connection issue with --notls flag on; WIP: deployment example --- .gitmodules | 3 ++ salticidae | 2 +- scripts/deploy/app/_setup.yml | 34 ++++++++++++ scripts/deploy/app/build.yml | 27 ++++++++++ scripts/deploy/app/hotstuff_app.py | 93 +++++++++++++++++++++++++++++++++ scripts/deploy/app/hotstuff_cli.py | 97 +++++++++++++++++++++++++++++++++++ scripts/deploy/app/reset.yml | 21 ++++++++ scripts/deploy/app/run.yml | 40 +++++++++++++++ scripts/deploy/app/run_cli.yml | 36 +++++++++++++ scripts/deploy/gen_all.sh | 11 ++++ scripts/deploy/gen_inventory.py | 34 ++++++++++++ scripts/deploy/group_vars/all.yml | 24 +++++++++ scripts/deploy/group_vars/clients.yml | 4 ++ scripts/deploy/run.sh | 15 ++++++ scripts/deploy/run_cli.sh | 15 ++++++ scripts/gen_conf.py | 9 ++-- src/hotstuff.cpp | 12 ++++- 17 files changed, 471 insertions(+), 6 deletions(-) create mode 100644 scripts/deploy/app/_setup.yml create mode 100644 scripts/deploy/app/build.yml create mode 100644 scripts/deploy/app/hotstuff_app.py create mode 100644 scripts/deploy/app/hotstuff_cli.py create mode 100644 scripts/deploy/app/reset.yml create mode 100644 scripts/deploy/app/run.yml create mode 100644 scripts/deploy/app/run_cli.yml create mode 100755 scripts/deploy/gen_all.sh create mode 100644 scripts/deploy/gen_inventory.py create mode 100644 scripts/deploy/group_vars/all.yml create mode 100644 scripts/deploy/group_vars/clients.yml create mode 100755 scripts/deploy/run.sh create mode 100755 scripts/deploy/run_cli.sh diff --git a/.gitmodules b/.gitmodules index 834ffa4..132f666 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "salticidae"] path = salticidae url = https://github.com/Determinant/salticidae.git +[submodule "scripts/deploy/minirun"] + path = scripts/deploy/minirun + url = https://github.com/Determinant/minirun.git diff --git a/salticidae b/salticidae index 4eff97f..13c40bb 160000 --- a/salticidae +++ b/salticidae @@ -1 +1 @@ -Subproject commit 4eff97f09b0c5df6f62f009616ed4d38e3eba753 +Subproject commit 13c40bb073bbf1bccc3c88787d490b6d401eb8aa diff --git a/scripts/deploy/app/_setup.yml b/scripts/deploy/app/_setup.yml new file mode 100644 index 0000000..7a9ba28 --- /dev/null +++ b/scripts/deploy/app/_setup.yml @@ -0,0 +1,34 @@ +--- +# available vars: +# workdir -- the local directory of run_id + +- name: setup libhotstuff example binaries + block: + - apt: + pkg: + - g++ + - libssl-dev + - libuv1-dev + - cmake + - make + - dh-autoreconf + state: + latest + become: true + become_user: root + - file: + path: "{{ hs_repo_dir }}" + state: directory + - synchronize: + archive: true + delete: true + src: "{{ (workdir, hs_local_repo_dir, '') | path_join }}" + dest: "{{ hs_repo_dir }}" + mode: push + rsync_opts: + - "--exclude=.git" + - "--exclude=scripts/" + - "--exclude=CMakeCache.txt" + - "--exclude=CMakeFiles" + - "--exclude=libsecp256k1-prefix" + - "--delete-excluded" diff --git a/scripts/deploy/app/build.yml b/scripts/deploy/app/build.yml new file mode 100644 index 0000000..2a0c24c --- /dev/null +++ b/scripts/deploy/app/build.yml @@ -0,0 +1,27 @@ +--- +# available vars: +# last_state -- the content of state.json +# nid -- host_idx (with 0 as default) +# ngroup -- the group of nodes involved in the build +# testbed -- the remote path of run_id + +- name: build hotstuff example binaries + block: + - file: + path: "{{ (hs_repo_dir, 'build') | path_join }}" + state: absent + - command: cmake -DCMAKE_BUID_TYPE=Release -DHOTSTUFF_PROTO_LOG=OFF -DCMAKE_CXX_FLAGS={{ hs_flags | default('') }} + args: + chdir: "{{ hs_repo_dir }}" + environment: + PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin + - command: make clean + args: + chdir: "{{ hs_repo_dir }}" + environment: + PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin + - command: make -j4 + args: + chdir: "{{ hs_repo_dir }}" + environment: + PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin diff --git a/scripts/deploy/app/hotstuff_app.py b/scripts/deploy/app/hotstuff_app.py new file mode 100644 index 0000000..2a21e65 --- /dev/null +++ b/scripts/deploy/app/hotstuff_app.py @@ -0,0 +1,93 @@ +#!/usr/bin/python3 + +import os +import subprocess + +ANSIBLE_METADATA = { + 'metadata_version': '0.1', + 'status': ['preview'], + 'supported_by': 'Ted Yin' +} + +DOCUMENTATION = ''' +--- +module: hotstuff_app + +short_description: Ansible module for hotstuff-app + +author: + - Ted Yin (@Tederminant) +''' + +EXAMPLES = ''' +''' + +RETURN = ''' +''' + +from ansible.module_utils.basic import AnsibleModule + +def run_module(): + # define available arguments/parameters a user can pass to the module + module_args = dict( + bin=dict(type='str', required=True), + cwd=dict(type='str', required=True), + conf=dict(type='str', required=True), + log_dir=dict(type='str', required=True), + tls=dict(type='bool', default=True, required=False), + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=False + ) + + expanduser = [ + 'bin', + 'cwd', + 'conf', + 'log_dir', + ] + + for arg in expanduser: + module.params[arg] = os.path.expanduser(module.params[arg]) + + try: + cmd = [*(module.params['bin'].split())] + cmd += ['--conf', module.params['conf']] + if not module.params['tls']: + cmd += ['--notls'] + + logdir = module.params['log_dir'] + if not (logdir is None): + stdout = open(os.path.expanduser( + os.path.join(logdir, 'stdout')), "w") + stderr = open(os.path.expanduser( + os.path.join(logdir, 'stderr')), "w") + nullsrc = open("/dev/null", "r") + else: + (stdout, stderr) = None, None + + cwd = module.params['cwd'] + + pid = subprocess.Popen( + cmd, + cwd=cwd, + stdin=nullsrc, + stdout=stdout, stderr=stderr, + env=os.environ, + shell=False, + start_new_session=True).pid + module.exit_json( + changed=False, + status=0, pid=pid, cmd=" ".join(cmd), cwd=cwd) + except (OSError, subprocess.SubprocessError) as e: + module.fail_json(msg=str(e), changed=False, status=1) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/scripts/deploy/app/hotstuff_cli.py b/scripts/deploy/app/hotstuff_cli.py new file mode 100644 index 0000000..0d4b57a --- /dev/null +++ b/scripts/deploy/app/hotstuff_cli.py @@ -0,0 +1,97 @@ +#!/usr/bin/python3 + +import os +import subprocess + +ANSIBLE_METADATA = { + 'metadata_version': '0.1', + 'status': ['preview'], + 'supported_by': 'Ted Yin' +} + +DOCUMENTATION = ''' +--- +module: hotstuff_cli + +short_description: Ansible module for hotstuff-client + +author: + - Ted Yin (@Tederminant) +''' + +EXAMPLES = ''' +''' + +RETURN = ''' +''' + +from ansible.module_utils.basic import AnsibleModule + +def run_module(): + # define available arguments/parameters a user can pass to the module + module_args = dict( + bin=dict(type='str', required=True), + cwd=dict(type='str', required=True), + idx=dict(type='int', required=True), + cid=dict(type='int', required=True), + iter=dict(type='int', required=True), + max_async=dict(type='int', required=True), + log_dir=dict(type='str', required=True), + ) + + module = AnsibleModule( + argument_spec=module_args, + supports_check_mode=False + ) + + expanduser = [ + 'bin', + 'cwd', + 'log_dir', + ] + + for arg in expanduser: + module.params[arg] = os.path.expanduser(module.params[arg]) + + try: + cmd = [*(module.params['bin'].split())] + cmd += [ + '--idx', str(module.params['idx']), + '--cid', str(module.params['cid']), + '--iter', str(module.params['iter']), + '--max-async', str(module.params['max_async']), + ] + + logdir = module.params['log_dir'] + if not (logdir is None): + stdout = open(os.path.expanduser( + os.path.join(logdir, 'stdout')), "w") + stderr = open(os.path.expanduser( + os.path.join(logdir, 'stderr')), "w") + nullsrc = open("/dev/null", "r") + else: + (stdout, stderr) = None, None + + cwd = module.params['cwd'] + + pid = subprocess.Popen( + cmd, + cwd=cwd, + stdin=nullsrc, + stdout=stdout, stderr=stderr, + env=os.environ, + shell=False, + start_new_session=True).pid + module.exit_json( + changed=False, + status=0, pid=pid, cmd=" ".join(cmd), cwd=cwd) + except (OSError, subprocess.SubprocessError) as e: + module.fail_json(msg=str(e), changed=False, status=1) + + +def main(): + run_module() + + +if __name__ == '__main__': + main() diff --git a/scripts/deploy/app/reset.yml b/scripts/deploy/app/reset.yml new file mode 100644 index 0000000..1b80b27 --- /dev/null +++ b/scripts/deploy/app/reset.yml @@ -0,0 +1,21 @@ +--- +# available vars: +# last_state -- the content of state.json +# nid -- host_idx (with 0 as default) +# ngroup -- the group of nodes involved in the build +# testbed -- the remote path of run_id + +- name: reset hotstuff + vars: + log_dir: "{{ (testbed, hs_log_dir) | path_join }}" + block: + - name: remove confs + file: + path: "{{ conf_dir }}" + state: absent + when: "hs_no_reset_conf is not defined or (hs_no_reset_conf == False)" + - name: remove old logs + file: + path: "{{ log_dir }}" + state: absent + when: "hs_no_reset_log is not defined or (hs_no_reset_log == False)" diff --git a/scripts/deploy/app/run.yml b/scripts/deploy/app/run.yml new file mode 100644 index 0000000..260d611 --- /dev/null +++ b/scripts/deploy/app/run.yml @@ -0,0 +1,40 @@ +--- +# available vars: +# last_state -- the content of state.json +# nid -- host_idx (with 0 as default) +# ngroup -- the group of nodes involved in the build +# testbed -- the remote path of run_id + +- vars: + conf_dir: "{{ (testbed, hs_conf_dir) | path_join }}" + log_dir: "{{ (testbed, hs_log_dir) | path_join }}" + block: + - name: create testbed dirs + block: + - file: + path: "{{ conf_dir }}" + state: directory + - file: + path: "{{ log_dir }}" + state: directory + - name: copy the base conf file + copy: + src: "{{ (lookup('env','run_path'), hs_base_conf) | path_join }}" + dest: "{{ (conf_dir, 'hotstuff.conf') | path_join }}" + mode: '0644' + - name: copy per node conf files + copy: + src: "{{ (lookup('env','run_path'), extra_conf) | path_join }}" + dest: "{{ (conf_dir, (extra_conf | basename)) | path_join }}" + mode: '0644' + when: extra_conf | default('') | length > 0 + - name: start the hotstuff replica + hotstuff_app: + bin: "{{ replica_bin | default('~/libhotstuff/examples/hotstuff_app') }}" + log_dir: "{{ log_dir }}" + cwd: "{{ conf_dir }}" + conf: "{{ extra_conf | basename }}" + tls: false + environment: + PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin + register: spawn_results diff --git a/scripts/deploy/app/run_cli.yml b/scripts/deploy/app/run_cli.yml new file mode 100644 index 0000000..c882a13 --- /dev/null +++ b/scripts/deploy/app/run_cli.yml @@ -0,0 +1,36 @@ +--- +# available vars: +# last_state -- the content of state.json +# nid -- host_idx (with 0 as default) +# ngroup -- the group of nodes involved in the build +# testbed -- the remote path of run_id + +- vars: + conf_dir: "{{ (testbed, hs_conf_dir) | path_join }}" + log_dir: "{{ (testbed, hs_log_dir) | path_join }}" + block: + - name: create testbed dirs + block: + - file: + path: "{{ conf_dir }}" + state: directory + - file: + path: "{{ log_dir }}" + state: directory + - name: copy the base conf file + copy: + src: "{{ (lookup('env','run_path'), hs_base_conf) | path_join }}" + dest: "{{ (conf_dir, 'hotstuff.conf') | path_join }}" + mode: '0644' + - name: start the hotstuff client + hotstuff_cli: + bin: "{{ client_bin | default('~/libhotstuff/examples/hotstuff_client') }}" + log_dir: "{{ log_dir }}" + cwd: "{{ conf_dir }}" + idx: "{{ client_idx | default(1) }}" + cid: "{{ cid }}" + iter: "{{ max_iter | default(100000) }}" + max_async: "{{ max_async }}" + environment: + PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin + register: spawn_results diff --git a/scripts/deploy/gen_all.sh b/scripts/deploy/gen_all.sh new file mode 100755 index 0000000..295bf99 --- /dev/null +++ b/scripts/deploy/gen_all.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# NOTE: make sure hotstuff-keygen and hotstuff-tls-keygen exist by building the main repo +# NOTE: make sure you have replicas.txt and clients.txt written properly: +# 1) in replicas.txt, each row is " " (without quotes) +# 2) in clients.txt each row is an IP adress for the machine +# One IP can appear one or more times in the text files. If the same IP appears +# several times, it runs multiple replica/client processes at the same time. + +python3 ./gen_inventory.py --prefix 'hotstuff.gen' > nodes.ini +awk '{print $2}' replicas.txt > replicas_inter.txt +python ../gen_conf.py --ips replicas_inter.txt --iter 1 --prefix 'hotstuff.gen' --keygen ../../hotstuff-keygen --tls-keygen ../../hotstuff-tls-keygen diff --git a/scripts/deploy/gen_inventory.py b/scripts/deploy/gen_inventory.py new file mode 100644 index 0000000..23af205 --- /dev/null +++ b/scripts/deploy/gen_inventory.py @@ -0,0 +1,34 @@ +import argparse + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate configuration files for a run.') + parser.add_argument('--replicas', type=str, default='replicas.txt', + help='text file with all replicas IPs (each row is " ", repeat a line if you would like to share the same machine)') + parser.add_argument('--clients', type=str, default='clients.txt', + help='text file with all clients IPs (each row is "")') + parser.add_argument('--prefix', type=str, default='hotstuff.gen') + args = parser.parse_args() + + print("[nodes_setup]") + host_idx_count = {} + replicas = [] + clients = [] + with open(args.replicas, "r") as rfile: + for line in rfile: + (pub_ip, priv_ip) = line.split() + replicas.append((pub_ip.strip(), priv_ip.strip())) + with open(args.clients, "r") as cfile: + for line in cfile: + clients.append(line.strip()) + machines = sorted(set([pub_ip for (pub_ip, _) in replicas] + clients)) + print("\n".join(machines)) + print("\n[nodes]") + for (i, (pub_ip, priv_ip)) in enumerate(replicas): + host_idx = host_idx_count.setdefault(pub_ip, 0) + host_idx_count[pub_ip] += 1 + print("replica{} ansible_host={} host_idx={} extra_conf={}-sec{}.conf".format( + i, pub_ip, host_idx, args.prefix, i)) + + print("\n[clients]") + for (i, ip) in enumerate(clients): + print("client{} ansible_host={} cid={}".format(i, ip, i)) diff --git a/scripts/deploy/group_vars/all.yml b/scripts/deploy/group_vars/all.yml new file mode 100644 index 0000000..d9e9a44 --- /dev/null +++ b/scripts/deploy/group_vars/all.yml @@ -0,0 +1,24 @@ +--- +## basic config + +ansible_connection: ssh +ansible_user: ubuntu +# change to your aws ec2 key file here +ansible_ssh_private_key_file: ~/.ssh/ted-aws-key2.pem +# remote directory that keeps the work directory for the running app +testbed_prefix: "/home/ubuntu/testbed" + +## app specific config + +# process name that is used by killall in reset +bin_name: hotstuff-app +# binary path (remote) +replica_bin: "/home/ubuntu/libhotstuff/examples/hotstuff-app" +client_bin: "/home/ubuntu/libhotstuff/examples/hotstuff-client" +# remote repo path +hs_repo_dir: "/home/ubuntu/libhotstuff" +# local source code path (that will be copied to the remote) +hs_local_repo_dir: "../../" +hs_conf_dir: "conf" +hs_log_dir: "log" +hs_base_conf: "./hotstuff.gen.conf" diff --git a/scripts/deploy/group_vars/clients.yml b/scripts/deploy/group_vars/clients.yml new file mode 100644 index 0000000..5c5afd8 --- /dev/null +++ b/scripts/deploy/group_vars/clients.yml @@ -0,0 +1,4 @@ +--- +bin_name: "hotstuff-client" +max_iter: 100000 +max_async: 175 diff --git a/scripts/deploy/run.sh b/scripts/deploy/run.sh new file mode 100755 index 0000000..46bcb33 --- /dev/null +++ b/scripts/deploy/run.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +basedir="$(dirname $(realpath "${BASH_SOURCE[0]}"))" +export node_group=nodes +export node_setup_group=nodes_setup +export node_file="$basedir/nodes.ini" +export group_vars="$basedir/group_vars" +export build_tasks="$basedir/app/build.yml" +export reset_tasks="$basedir/app/reset.yml" +export run_tasks="$basedir/app/run.yml" +export setup_tasks="$basedir/app/_setup.yml" +export app_module="$basedir/app/" +export run_path="$(realpath "$PWD")" + +minirun/run.sh "$@" diff --git a/scripts/deploy/run_cli.sh b/scripts/deploy/run_cli.sh new file mode 100755 index 0000000..a4eafb8 --- /dev/null +++ b/scripts/deploy/run_cli.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +basedir="$(dirname $(realpath "${BASH_SOURCE[0]}"))" +export node_group=clients +export node_setup_group=nodes_setup +export node_file="$basedir/nodes.ini" +export group_vars="$basedir/group_vars" +export build_tasks="$basedir/app/build.yml" +export reset_tasks="$basedir/app/reset.yml" +export run_tasks="$basedir/app/run_cli.yml" +export setup_tasks="$basedir/app/_setup.yml" +export app_module="$basedir/app/" +export run_path="$(realpath "$PWD")" + +minirun/run.sh "$@" diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py index ca61ea1..428b5b5 100644 --- a/scripts/gen_conf.py +++ b/scripts/gen_conf.py @@ -31,9 +31,12 @@ if __name__ == "__main__": main_conf = open("{}.conf".format(prefix), 'w') nodes = open(args.nodes, 'w') - replicas = ["{}:{};{}".format(ip, base_pport + i, base_cport + i) - for ip in ips - for i in range(iter)] + port_count = {} + replicas = [] + for ip in ips: + i = port_count.setdefault(ip, 0) + port_count[ip] += 1 + replicas.append("{}:{};{}".format(ip, base_pport + i, base_cport + i)) p = subprocess.Popen([keygen_bin, '--num', str(len(replicas))], stdout=subprocess.PIPE, stderr=open(os.devnull, 'w')) keys = [[t[4:] for t in l.decode('ascii').split()] for l in p.stdout] diff --git a/src/hotstuff.cpp b/src/hotstuff.cpp index ad2ed55..af1b2b4 100644 --- a/src/hotstuff.cpp +++ b/src/hotstuff.cpp @@ -257,9 +257,10 @@ void HotStuffBase::resp_blk_handler(MsgRespBlock &&msg, const Net::conn_t &) { bool HotStuffBase::conn_handler(const salticidae::ConnPool::conn_t &conn, bool connected) { if (connected) { + if (!pn.enable_tls) return true; auto cert = conn->get_peer_cert(); //SALTICIDAE_LOG_INFO("%s", salticidae::get_hash(cert->get_der()).to_hex().c_str()); - return (!cert) || valid_tls_certs.count(salticidae::get_hash(cert->get_der())); + return valid_tls_certs.count(salticidae::get_hash(cert->get_der())); } return true; } @@ -359,6 +360,13 @@ HotStuffBase::HotStuffBase(uint32_t blk_size, pn.reg_handler(salticidae::generic_bind(&HotStuffBase::req_blk_handler, this, _1, _2)); pn.reg_handler(salticidae::generic_bind(&HotStuffBase::resp_blk_handler, this, _1, _2)); pn.reg_conn_handler(salticidae::generic_bind(&HotStuffBase::conn_handler, this, _1, _2)); + pn.reg_error_handler([](const std::exception_ptr _err, bool fatal, int32_t async_id) { + try { + std::rethrow_exception(_err); + } catch (const std::exception &err) { + HOTSTUFF_LOG_WARN("network async error: %s\n", err.what()); + } + }); pn.start(); pn.listen(listen_addr); } @@ -408,7 +416,7 @@ void HotStuffBase::start( auto &addr = std::get<0>(replicas[i]); auto cert_hash = std::move(std::get<2>(replicas[i])); valid_tls_certs.insert(cert_hash); - salticidae::PeerId peer{cert_hash}; + auto peer = pn.enable_tls ? salticidae::PeerId(cert_hash) : salticidae::PeerId(addr); HotStuffCore::add_replica(i, peer, std::move(std::get<1>(replicas[i]))); if (addr != listen_addr) { -- cgit v1.2.3