aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitmodules3
m---------salticidae0
-rw-r--r--scripts/deploy/app/_setup.yml34
-rw-r--r--scripts/deploy/app/build.yml27
-rw-r--r--scripts/deploy/app/hotstuff_app.py93
-rw-r--r--scripts/deploy/app/hotstuff_cli.py97
-rw-r--r--scripts/deploy/app/reset.yml21
-rw-r--r--scripts/deploy/app/run.yml40
-rw-r--r--scripts/deploy/app/run_cli.yml36
-rwxr-xr-xscripts/deploy/gen_all.sh11
-rw-r--r--scripts/deploy/gen_inventory.py34
-rw-r--r--scripts/deploy/group_vars/all.yml24
-rw-r--r--scripts/deploy/group_vars/clients.yml4
-rwxr-xr-xscripts/deploy/run.sh15
-rwxr-xr-xscripts/deploy/run_cli.sh15
-rw-r--r--scripts/gen_conf.py9
-rw-r--r--src/hotstuff.cpp12
17 files changed, 470 insertions, 5 deletions
diff --git a/.gitmodules b/.gitmodules
index 834ffa4..132f666 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,6 @@
[submodule "salticidae"]
path = salticidae
url = https://github.com/Determinant/salticidae.git
+[submodule "scripts/deploy/minirun"]
+ path = scripts/deploy/minirun
+ url = https://github.com/Determinant/minirun.git
diff --git a/salticidae b/salticidae
-Subproject 4eff97f09b0c5df6f62f009616ed4d38e3eba75
+Subproject 13c40bb073bbf1bccc3c88787d490b6d401eb8a
diff --git a/scripts/deploy/app/_setup.yml b/scripts/deploy/app/_setup.yml
new file mode 100644
index 0000000..7a9ba28
--- /dev/null
+++ b/scripts/deploy/app/_setup.yml
@@ -0,0 +1,34 @@
+---
+# available vars:
+# workdir -- the local directory of run_id
+
+- name: setup libhotstuff example binaries
+ block:
+ - apt:
+ pkg:
+ - g++
+ - libssl-dev
+ - libuv1-dev
+ - cmake
+ - make
+ - dh-autoreconf
+ state:
+ latest
+ become: true
+ become_user: root
+ - file:
+ path: "{{ hs_repo_dir }}"
+ state: directory
+ - synchronize:
+ archive: true
+ delete: true
+ src: "{{ (workdir, hs_local_repo_dir, '') | path_join }}"
+ dest: "{{ hs_repo_dir }}"
+ mode: push
+ rsync_opts:
+ - "--exclude=.git"
+ - "--exclude=scripts/"
+ - "--exclude=CMakeCache.txt"
+ - "--exclude=CMakeFiles"
+ - "--exclude=libsecp256k1-prefix"
+ - "--delete-excluded"
diff --git a/scripts/deploy/app/build.yml b/scripts/deploy/app/build.yml
new file mode 100644
index 0000000..2a0c24c
--- /dev/null
+++ b/scripts/deploy/app/build.yml
@@ -0,0 +1,27 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- name: build hotstuff example binaries
+ block:
+ - file:
+ path: "{{ (hs_repo_dir, 'build') | path_join }}"
+ state: absent
+ - command: cmake -DCMAKE_BUID_TYPE=Release -DHOTSTUFF_PROTO_LOG=OFF -DCMAKE_CXX_FLAGS={{ hs_flags | default('') }}
+ args:
+ chdir: "{{ hs_repo_dir }}"
+ environment:
+ PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+ - command: make clean
+ args:
+ chdir: "{{ hs_repo_dir }}"
+ environment:
+ PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+ - command: make -j4
+ args:
+ chdir: "{{ hs_repo_dir }}"
+ environment:
+ PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
diff --git a/scripts/deploy/app/hotstuff_app.py b/scripts/deploy/app/hotstuff_app.py
new file mode 100644
index 0000000..2a21e65
--- /dev/null
+++ b/scripts/deploy/app/hotstuff_app.py
@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+
+ANSIBLE_METADATA = {
+ 'metadata_version': '0.1',
+ 'status': ['preview'],
+ 'supported_by': 'Ted Yin'
+}
+
+DOCUMENTATION = '''
+---
+module: hotstuff_app
+
+short_description: Ansible module for hotstuff-app
+
+author:
+ - Ted Yin (@Tederminant)
+'''
+
+EXAMPLES = '''
+'''
+
+RETURN = '''
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+def run_module():
+ # define available arguments/parameters a user can pass to the module
+ module_args = dict(
+ bin=dict(type='str', required=True),
+ cwd=dict(type='str', required=True),
+ conf=dict(type='str', required=True),
+ log_dir=dict(type='str', required=True),
+ tls=dict(type='bool', default=True, required=False),
+ )
+
+ module = AnsibleModule(
+ argument_spec=module_args,
+ supports_check_mode=False
+ )
+
+ expanduser = [
+ 'bin',
+ 'cwd',
+ 'conf',
+ 'log_dir',
+ ]
+
+ for arg in expanduser:
+ module.params[arg] = os.path.expanduser(module.params[arg])
+
+ try:
+ cmd = [*(module.params['bin'].split())]
+ cmd += ['--conf', module.params['conf']]
+ if not module.params['tls']:
+ cmd += ['--notls']
+
+ logdir = module.params['log_dir']
+ if not (logdir is None):
+ stdout = open(os.path.expanduser(
+ os.path.join(logdir, 'stdout')), "w")
+ stderr = open(os.path.expanduser(
+ os.path.join(logdir, 'stderr')), "w")
+ nullsrc = open("/dev/null", "r")
+ else:
+ (stdout, stderr) = None, None
+
+ cwd = module.params['cwd']
+
+ pid = subprocess.Popen(
+ cmd,
+ cwd=cwd,
+ stdin=nullsrc,
+ stdout=stdout, stderr=stderr,
+ env=os.environ,
+ shell=False,
+ start_new_session=True).pid
+ module.exit_json(
+ changed=False,
+ status=0, pid=pid, cmd=" ".join(cmd), cwd=cwd)
+ except (OSError, subprocess.SubprocessError) as e:
+ module.fail_json(msg=str(e), changed=False, status=1)
+
+
+def main():
+ run_module()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/deploy/app/hotstuff_cli.py b/scripts/deploy/app/hotstuff_cli.py
new file mode 100644
index 0000000..0d4b57a
--- /dev/null
+++ b/scripts/deploy/app/hotstuff_cli.py
@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+
+import os
+import subprocess
+
+ANSIBLE_METADATA = {
+ 'metadata_version': '0.1',
+ 'status': ['preview'],
+ 'supported_by': 'Ted Yin'
+}
+
+DOCUMENTATION = '''
+---
+module: hotstuff_cli
+
+short_description: Ansible module for hotstuff-client
+
+author:
+ - Ted Yin (@Tederminant)
+'''
+
+EXAMPLES = '''
+'''
+
+RETURN = '''
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+
+def run_module():
+ # define available arguments/parameters a user can pass to the module
+ module_args = dict(
+ bin=dict(type='str', required=True),
+ cwd=dict(type='str', required=True),
+ idx=dict(type='int', required=True),
+ cid=dict(type='int', required=True),
+ iter=dict(type='int', required=True),
+ max_async=dict(type='int', required=True),
+ log_dir=dict(type='str', required=True),
+ )
+
+ module = AnsibleModule(
+ argument_spec=module_args,
+ supports_check_mode=False
+ )
+
+ expanduser = [
+ 'bin',
+ 'cwd',
+ 'log_dir',
+ ]
+
+ for arg in expanduser:
+ module.params[arg] = os.path.expanduser(module.params[arg])
+
+ try:
+ cmd = [*(module.params['bin'].split())]
+ cmd += [
+ '--idx', str(module.params['idx']),
+ '--cid', str(module.params['cid']),
+ '--iter', str(module.params['iter']),
+ '--max-async', str(module.params['max_async']),
+ ]
+
+ logdir = module.params['log_dir']
+ if not (logdir is None):
+ stdout = open(os.path.expanduser(
+ os.path.join(logdir, 'stdout')), "w")
+ stderr = open(os.path.expanduser(
+ os.path.join(logdir, 'stderr')), "w")
+ nullsrc = open("/dev/null", "r")
+ else:
+ (stdout, stderr) = None, None
+
+ cwd = module.params['cwd']
+
+ pid = subprocess.Popen(
+ cmd,
+ cwd=cwd,
+ stdin=nullsrc,
+ stdout=stdout, stderr=stderr,
+ env=os.environ,
+ shell=False,
+ start_new_session=True).pid
+ module.exit_json(
+ changed=False,
+ status=0, pid=pid, cmd=" ".join(cmd), cwd=cwd)
+ except (OSError, subprocess.SubprocessError) as e:
+ module.fail_json(msg=str(e), changed=False, status=1)
+
+
+def main():
+ run_module()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/scripts/deploy/app/reset.yml b/scripts/deploy/app/reset.yml
new file mode 100644
index 0000000..1b80b27
--- /dev/null
+++ b/scripts/deploy/app/reset.yml
@@ -0,0 +1,21 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- name: reset hotstuff
+ vars:
+ log_dir: "{{ (testbed, hs_log_dir) | path_join }}"
+ block:
+ - name: remove confs
+ file:
+ path: "{{ conf_dir }}"
+ state: absent
+ when: "hs_no_reset_conf is not defined or (hs_no_reset_conf == False)"
+ - name: remove old logs
+ file:
+ path: "{{ log_dir }}"
+ state: absent
+ when: "hs_no_reset_log is not defined or (hs_no_reset_log == False)"
diff --git a/scripts/deploy/app/run.yml b/scripts/deploy/app/run.yml
new file mode 100644
index 0000000..260d611
--- /dev/null
+++ b/scripts/deploy/app/run.yml
@@ -0,0 +1,40 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- vars:
+ conf_dir: "{{ (testbed, hs_conf_dir) | path_join }}"
+ log_dir: "{{ (testbed, hs_log_dir) | path_join }}"
+ block:
+ - name: create testbed dirs
+ block:
+ - file:
+ path: "{{ conf_dir }}"
+ state: directory
+ - file:
+ path: "{{ log_dir }}"
+ state: directory
+ - name: copy the base conf file
+ copy:
+ src: "{{ (lookup('env','run_path'), hs_base_conf) | path_join }}"
+ dest: "{{ (conf_dir, 'hotstuff.conf') | path_join }}"
+ mode: '0644'
+ - name: copy per node conf files
+ copy:
+ src: "{{ (lookup('env','run_path'), extra_conf) | path_join }}"
+ dest: "{{ (conf_dir, (extra_conf | basename)) | path_join }}"
+ mode: '0644'
+ when: extra_conf | default('') | length > 0
+ - name: start the hotstuff replica
+ hotstuff_app:
+ bin: "{{ replica_bin | default('~/libhotstuff/examples/hotstuff_app') }}"
+ log_dir: "{{ log_dir }}"
+ cwd: "{{ conf_dir }}"
+ conf: "{{ extra_conf | basename }}"
+ tls: false
+ environment:
+ PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+ register: spawn_results
diff --git a/scripts/deploy/app/run_cli.yml b/scripts/deploy/app/run_cli.yml
new file mode 100644
index 0000000..c882a13
--- /dev/null
+++ b/scripts/deploy/app/run_cli.yml
@@ -0,0 +1,36 @@
+---
+# available vars:
+# last_state -- the content of state.json
+# nid -- host_idx (with 0 as default)
+# ngroup -- the group of nodes involved in the build
+# testbed -- the remote path of run_id
+
+- vars:
+ conf_dir: "{{ (testbed, hs_conf_dir) | path_join }}"
+ log_dir: "{{ (testbed, hs_log_dir) | path_join }}"
+ block:
+ - name: create testbed dirs
+ block:
+ - file:
+ path: "{{ conf_dir }}"
+ state: directory
+ - file:
+ path: "{{ log_dir }}"
+ state: directory
+ - name: copy the base conf file
+ copy:
+ src: "{{ (lookup('env','run_path'), hs_base_conf) | path_join }}"
+ dest: "{{ (conf_dir, 'hotstuff.conf') | path_join }}"
+ mode: '0644'
+ - name: start the hotstuff client
+ hotstuff_cli:
+ bin: "{{ client_bin | default('~/libhotstuff/examples/hotstuff_client') }}"
+ log_dir: "{{ log_dir }}"
+ cwd: "{{ conf_dir }}"
+ idx: "{{ client_idx | default(1) }}"
+ cid: "{{ cid }}"
+ iter: "{{ max_iter | default(100000) }}"
+ max_async: "{{ max_async }}"
+ environment:
+ PATH: /sbin:/usr/sbin:/bin:/usr/bin:/usr/local/bin:/snap/bin
+ register: spawn_results
diff --git a/scripts/deploy/gen_all.sh b/scripts/deploy/gen_all.sh
new file mode 100755
index 0000000..295bf99
--- /dev/null
+++ b/scripts/deploy/gen_all.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+# NOTE: make sure hotstuff-keygen and hotstuff-tls-keygen exist by building the main repo
+# NOTE: make sure you have replicas.txt and clients.txt written properly:
+# 1) in replicas.txt, each row is "<external_ip_for_ssh> <inter_replica_net_ip>" (without quotes)
+# 2) in clients.txt each row is an IP adress for the machine
+# One IP can appear one or more times in the text files. If the same IP appears
+# several times, it runs multiple replica/client processes at the same time.
+
+python3 ./gen_inventory.py --prefix 'hotstuff.gen' > nodes.ini
+awk '{print $2}' replicas.txt > replicas_inter.txt
+python ../gen_conf.py --ips replicas_inter.txt --iter 1 --prefix 'hotstuff.gen' --keygen ../../hotstuff-keygen --tls-keygen ../../hotstuff-tls-keygen
diff --git a/scripts/deploy/gen_inventory.py b/scripts/deploy/gen_inventory.py
new file mode 100644
index 0000000..23af205
--- /dev/null
+++ b/scripts/deploy/gen_inventory.py
@@ -0,0 +1,34 @@
+import argparse
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description='Generate configuration files for a run.')
+ parser.add_argument('--replicas', type=str, default='replicas.txt',
+ help='text file with all replicas IPs (each row is "<external_ip> <inter_replica_net_ip>", repeat a line if you would like to share the same machine)')
+ parser.add_argument('--clients', type=str, default='clients.txt',
+ help='text file with all clients IPs (each row is "<external_ip>")')
+ parser.add_argument('--prefix', type=str, default='hotstuff.gen')
+ args = parser.parse_args()
+
+ print("[nodes_setup]")
+ host_idx_count = {}
+ replicas = []
+ clients = []
+ with open(args.replicas, "r") as rfile:
+ for line in rfile:
+ (pub_ip, priv_ip) = line.split()
+ replicas.append((pub_ip.strip(), priv_ip.strip()))
+ with open(args.clients, "r") as cfile:
+ for line in cfile:
+ clients.append(line.strip())
+ machines = sorted(set([pub_ip for (pub_ip, _) in replicas] + clients))
+ print("\n".join(machines))
+ print("\n[nodes]")
+ for (i, (pub_ip, priv_ip)) in enumerate(replicas):
+ host_idx = host_idx_count.setdefault(pub_ip, 0)
+ host_idx_count[pub_ip] += 1
+ print("replica{} ansible_host={} host_idx={} extra_conf={}-sec{}.conf".format(
+ i, pub_ip, host_idx, args.prefix, i))
+
+ print("\n[clients]")
+ for (i, ip) in enumerate(clients):
+ print("client{} ansible_host={} cid={}".format(i, ip, i))
diff --git a/scripts/deploy/group_vars/all.yml b/scripts/deploy/group_vars/all.yml
new file mode 100644
index 0000000..d9e9a44
--- /dev/null
+++ b/scripts/deploy/group_vars/all.yml
@@ -0,0 +1,24 @@
+---
+## basic config
+
+ansible_connection: ssh
+ansible_user: ubuntu
+# change to your aws ec2 key file here
+ansible_ssh_private_key_file: ~/.ssh/ted-aws-key2.pem
+# remote directory that keeps the work directory for the running app
+testbed_prefix: "/home/ubuntu/testbed"
+
+## app specific config
+
+# process name that is used by killall in reset
+bin_name: hotstuff-app
+# binary path (remote)
+replica_bin: "/home/ubuntu/libhotstuff/examples/hotstuff-app"
+client_bin: "/home/ubuntu/libhotstuff/examples/hotstuff-client"
+# remote repo path
+hs_repo_dir: "/home/ubuntu/libhotstuff"
+# local source code path (that will be copied to the remote)
+hs_local_repo_dir: "../../"
+hs_conf_dir: "conf"
+hs_log_dir: "log"
+hs_base_conf: "./hotstuff.gen.conf"
diff --git a/scripts/deploy/group_vars/clients.yml b/scripts/deploy/group_vars/clients.yml
new file mode 100644
index 0000000..5c5afd8
--- /dev/null
+++ b/scripts/deploy/group_vars/clients.yml
@@ -0,0 +1,4 @@
+---
+bin_name: "hotstuff-client"
+max_iter: 100000
+max_async: 175
diff --git a/scripts/deploy/run.sh b/scripts/deploy/run.sh
new file mode 100755
index 0000000..46bcb33
--- /dev/null
+++ b/scripts/deploy/run.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+basedir="$(dirname $(realpath "${BASH_SOURCE[0]}"))"
+export node_group=nodes
+export node_setup_group=nodes_setup
+export node_file="$basedir/nodes.ini"
+export group_vars="$basedir/group_vars"
+export build_tasks="$basedir/app/build.yml"
+export reset_tasks="$basedir/app/reset.yml"
+export run_tasks="$basedir/app/run.yml"
+export setup_tasks="$basedir/app/_setup.yml"
+export app_module="$basedir/app/"
+export run_path="$(realpath "$PWD")"
+
+minirun/run.sh "$@"
diff --git a/scripts/deploy/run_cli.sh b/scripts/deploy/run_cli.sh
new file mode 100755
index 0000000..a4eafb8
--- /dev/null
+++ b/scripts/deploy/run_cli.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+basedir="$(dirname $(realpath "${BASH_SOURCE[0]}"))"
+export node_group=clients
+export node_setup_group=nodes_setup
+export node_file="$basedir/nodes.ini"
+export group_vars="$basedir/group_vars"
+export build_tasks="$basedir/app/build.yml"
+export reset_tasks="$basedir/app/reset.yml"
+export run_tasks="$basedir/app/run_cli.yml"
+export setup_tasks="$basedir/app/_setup.yml"
+export app_module="$basedir/app/"
+export run_path="$(realpath "$PWD")"
+
+minirun/run.sh "$@"
diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py
index ca61ea1..428b5b5 100644
--- a/scripts/gen_conf.py
+++ b/scripts/gen_conf.py
@@ -31,9 +31,12 @@ if __name__ == "__main__":
main_conf = open("{}.conf".format(prefix), 'w')
nodes = open(args.nodes, 'w')
- replicas = ["{}:{};{}".format(ip, base_pport + i, base_cport + i)
- for ip in ips
- for i in range(iter)]
+ port_count = {}
+ replicas = []
+ for ip in ips:
+ i = port_count.setdefault(ip, 0)
+ port_count[ip] += 1
+ replicas.append("{}:{};{}".format(ip, base_pport + i, base_cport + i))
p = subprocess.Popen([keygen_bin, '--num', str(len(replicas))],
stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'))
keys = [[t[4:] for t in l.decode('ascii').split()] for l in p.stdout]
diff --git a/src/hotstuff.cpp b/src/hotstuff.cpp
index ad2ed55..af1b2b4 100644
--- a/src/hotstuff.cpp
+++ b/src/hotstuff.cpp
@@ -257,9 +257,10 @@ void HotStuffBase::resp_blk_handler(MsgRespBlock &&msg, const Net::conn_t &) {
bool HotStuffBase::conn_handler(const salticidae::ConnPool::conn_t &conn, bool connected) {
if (connected)
{
+ if (!pn.enable_tls) return true;
auto cert = conn->get_peer_cert();
//SALTICIDAE_LOG_INFO("%s", salticidae::get_hash(cert->get_der()).to_hex().c_str());
- return (!cert) || valid_tls_certs.count(salticidae::get_hash(cert->get_der()));
+ return valid_tls_certs.count(salticidae::get_hash(cert->get_der()));
}
return true;
}
@@ -359,6 +360,13 @@ HotStuffBase::HotStuffBase(uint32_t blk_size,
pn.reg_handler(salticidae::generic_bind(&HotStuffBase::req_blk_handler, this, _1, _2));
pn.reg_handler(salticidae::generic_bind(&HotStuffBase::resp_blk_handler, this, _1, _2));
pn.reg_conn_handler(salticidae::generic_bind(&HotStuffBase::conn_handler, this, _1, _2));
+ pn.reg_error_handler([](const std::exception_ptr _err, bool fatal, int32_t async_id) {
+ try {
+ std::rethrow_exception(_err);
+ } catch (const std::exception &err) {
+ HOTSTUFF_LOG_WARN("network async error: %s\n", err.what());
+ }
+ });
pn.start();
pn.listen(listen_addr);
}
@@ -408,7 +416,7 @@ void HotStuffBase::start(
auto &addr = std::get<0>(replicas[i]);
auto cert_hash = std::move(std::get<2>(replicas[i]));
valid_tls_certs.insert(cert_hash);
- salticidae::PeerId peer{cert_hash};
+ auto peer = pn.enable_tls ? salticidae::PeerId(cert_hash) : salticidae::PeerId(addr);
HotStuffCore::add_replica(i, peer, std::move(std::get<1>(replicas[i])));
if (addr != listen_addr)
{
Powered by cgit, Themed by Ted Yin.