From 1df3530f1f887ca8280536fb40aeabf7025a67aa Mon Sep 17 00:00:00 2001 From: Determinant Date: Sat, 18 Aug 2018 01:36:58 -0400 Subject: ... --- scripts/gen_conf.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py index bc45540..2cfc54b 100644 --- a/scripts/gen_conf.py +++ b/scripts/gen_conf.py @@ -6,15 +6,16 @@ import argparse if __name__ == "__main__": parser = argparse.ArgumentParser(description='Generate configuration file for a batch of replicas') parser.add_argument('--prefix', type=str, default='hotstuff.gen') - parser.add_argument('--iplist', type=str, default=None) + parser.add_argument('--ips', type=str, default=None) parser.add_argument('--iter', type=int, default=10) parser.add_argument('--pport', type=int, default=10000) parser.add_argument('--cport', type=int, default=20000) parser.add_argument('--keygen', type=str, default='./hotstuff-keygen') + parser.add_argument('--nodes', type=str, default='nodes.txt') args = parser.parse_args() - if args.iplist is None: + if args.ips is None: ips = ['127.0.0.1'] else: ips = [l.strip() for l in open(args.iplist, 'r').readlines()] @@ -22,9 +23,10 @@ if __name__ == "__main__": iter = args.iter base_pport = args.pport base_cport = args.cport - keygen_bin= args.keygen + keygen_bin = args.keygen main_conf = open("{}.conf".format(prefix), 'w') + nodes = open(args.nodes, 'w') replicas = ["{}:{};{}".format(ip, base_pport + i, base_cport + i) for ip in ips for i in range(iter)] @@ -33,6 +35,8 @@ if __name__ == "__main__": keys = [[t[4:] for t in l.decode('ascii').split()] for l in p.stdout] for r in zip(replicas, keys, itertools.count(0)): main_conf.write("replica = {}, {}\n".format(r[0], r[1][0])) - r_conf = open("{}-sec{}.conf".format(prefix, r[2]), 'w') + r_conf_name = "{}-sec{}.conf".format(prefix, r[2]) + nodes.write("{}:{}\t{}\n".format(r[2], r[0], r_conf_name)) + r_conf = open(r_conf_name, 'w') r_conf.write("privkey = {}\n".format(r[1][1])) r_conf.write("idx = {}\n".format(r[2])) -- cgit v1.2.3-70-g09d2 From 7c5f0eb6222639ecb9e2e8422481e4f5f87ba721 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sat, 18 Aug 2018 14:27:55 -0400 Subject: update scripts --- scripts/gen_conf.py | 6 + scripts/run.sh | 457 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 463 insertions(+) create mode 100755 scripts/run.sh diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py index 2cfc54b..7b06826 100644 --- a/scripts/gen_conf.py +++ b/scripts/gen_conf.py @@ -12,6 +12,8 @@ if __name__ == "__main__": parser.add_argument('--cport', type=int, default=20000) parser.add_argument('--keygen', type=str, default='./hotstuff-keygen') parser.add_argument('--nodes', type=str, default='nodes.txt') + parser.add_argument('--block-size', type=int, default=1) + parser.add_argument('--pace-maker', type=str, default='dummy') args = parser.parse_args() @@ -33,6 +35,10 @@ if __name__ == "__main__": p = subprocess.Popen([keygen_bin, '--num', str(len(replicas))], stdout=subprocess.PIPE, stderr=open(os.devnull, 'w')) keys = [[t[4:] for t in l.decode('ascii').split()] for l in p.stdout] + if not (args.block_size is None): + main_conf.write("block-size = {}\n".format(args.block_size)) + if not (args.pace_maker is None): + main_conf.write("pace-maker = {}\n".format(args.pace_maker)) for r in zip(replicas, keys, itertools.count(0)): main_conf.write("replica = {}, {}\n".format(r[0], r[1][0])) r_conf_name = "{}-sec{}.conf".format(prefix, r[2]) diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..48e4ef7 --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,457 @@ +#!/bin/bash + +proj_server_bin="hotstuff-app" +proj_server_path="/home/ted/hot-stuff/$proj_server_bin" +proj_conf_name="hotstuff.conf" + +peer_list="./nodes.txt" # the list of peers (without the controller) +conf_src="./hotstuff.gen.conf" +server_map="./server_map.txt" # optional mapping from node ip to server ip +template_dir="template" # the dir that keeps the content shared among all nodes +remote_base="/home/ted/testbed" # remote dir used to keep files for the experiment +#remote_base="/tmp/" # remote dir used to keep files for the experiment +remote_log="log" # log filename +remote_user="ted" +copy_to_remote_pat="rsync -avz @:" +copy_from_remote_pat="rsync -avz @: " +exe_remote_pat="ssh @ bash" +run_remote_pat="cd \"\"; gdb -ex r -ex bt -ex generate-core-file -ex q --args '$proj_server_path' --conf \"hotstuff.gen-sec.conf\"" +reset_remote_pat="pgrep -f '$proj_server_bin' | xargs kill -9" + +fin_keyword="error:" # the keyword indicating completion of execution +fin_chk_period=1 +fin_chk_skip_pat='^([A-O][0-9]*)|(_ctl)$' +force_peer_list=0 + +function join { local IFS="$1"; shift; echo "$*"; } +function split { + local IFS="$1" + local arr=($2) + echo "${arr[@]}" +} + +function die { echo "$1"; exit 1; } + +declare -A nodes +declare -A node_confs +nodes_cnt=0 +function get_node_info { + pl="$1" + if [[ "$force_peer_list" == 1 ]]; then + pl="$peer_list" + fi + OIFS="$IFS" + IFS=$'\n' + node_list=($(cat "$pl")) + IFS="$OIFS" + for tuple in "${node_list[@]}"; do + tup0=($(split $'\t' "$tuple")) + tup=($(split : "${tup0[0]}")) + nodes[${tup[0]}]="${tup[1]}:${tup[2]}" + node_confs[${tup[0]}]="${tup0[@]:1}" + echo "${tup[0]} => ${nodes[${tup[0]}]} & ${node_confs[${tup[0]}]}" + let nodes_cnt++ + done +} + +declare -A server_map +function get_server_map { + { + IFS=$'\n' + map_list=($(cat "$1")) + } + IFS=$'\n \t' + for pair in "${map_list[@]}"; do + p=($pair) + server_map[${p[0]}]="${p[1]}" + echo "mapping ${p[0]} => ${p[1]}" + done +} + + +function get_addr { + tup=($(split ';' $1)) + echo "${tup[0]}" +} + +function get_ip { + tup=($(split : $1)) + echo "${tup[0]}" +} + +function get_peer_port { + tup=($(split : $1)) + tup2=($(split ';' ${tup[1]})) + echo "${tup2[0]}" +} + + +function get_client_port { + tup=($(split : $1)) + tup2=($(split ';' ${tup[1]})) + echo "${tup2[1]}" +} + + +function get_ip_by_id { + get_ip "${nodes[$1]}" +} + +function get_peer_port_by_id { + get_peer_port "${nodes[$1]}" +} + + +function get_client_port_by_id { + get_client_port "${nodes[$1]}" +} + +function copy_file { + local pat="$1" + local cmd="${pat///$2}" + cmd="${cmd///$3}" + cmd="${cmd///$remote_user}" + cmd="${cmd///$4}" + echo $cmd + eval "$cmd" +} >> log 2>&1 + +function execute_remote_cmd_pid { + local node_ip="$1" + local c="$2" + local l="$3" + local cmd="${exe_remote_pat///$node_ip}" + cmd="${cmd///$remote_user}" + eval $cmd << EOF +$c > $l 2>&1 & echo \$! +EOF +} + + + +function execute_remote_cmd_stat { + local node_ip="$1" + local c="$2" + local l="$3" + local cmd="${exe_remote_pat///$node_ip}" + cmd="${cmd///$remote_user}" + eval $cmd << EOF +$c > $l 2>&1 ; echo \$? +EOF +} + + +function _remote_load { + local workdir="$1" + local rworkdir="$2" + local node_ip="$3" + local rid="$4" + local extra_conf=($5) + local tmpldir="$workdir/$template_dir/" + local node_tmpldir="$workdir/$rid" + [[ $(execute_remote_cmd_stat "$node_ip" \ + "mkdir -p \"$rworkdir\"" \ + /dev/null) == 0 ]] || die "failed to create directory $rworkdir" + copy_file "$copy_to_remote_pat" "$tmpldir" "$node_ip" "$rworkdir" + for conf in "${extra_conf[@]}"; do + copy_file "$copy_to_remote_pat" "$node_tmpldir/$conf" "$node_ip" "$rworkdir" + done +} + +function _remote_start { + local workdir="$1" + local rworkdir="$2" + local node_id="$3" + local node_ip="$4" + local client_port="$5" + local cmd="${run_remote_pat///$rworkdir}" + cmd="${cmd///$node_id}" + cmd="${cmd///$client_port}" + execute_remote_cmd_pid "$node_ip" "$cmd" \ + "\"$rworkdir/$remote_log\"" > "$workdir/${node_id}.pid" +} + +function _remote_exec { + local workdir="$1" + local rworkdir="$2" + local node_ip="$3" + local cmd="$4" + [[ $(execute_remote_cmd_stat "$node_ip" "$cmd" /dev/null) == 0 ]] +} + +function _remote_stop { + local node_pid="$4" + _remote_exec "$1" "$2" "$3" "kill $node_pid" +} + +function _remote_status { + local node_pid="$4" + _remote_exec "$1" "$2" "$3" "kill -0 $node_pid" +} + +function _remote_finished { + _remote_exec "$1" "$2" "$3" "grep \"$fin_keyword\" \"$rworkdir/$remote_log\"" +} + +function _remote_fetch { + local workdir="$1" + local rworkdir="$2" + local node_id="$3" + local node_ip="$4" + copy_file "$copy_from_remote_pat" "$workdir/${node_id}.log" "$node_ip" "$rworkdir/$remote_log" +} + +function start_all { + local workdir="$1" + local tmpldir="$workdir/$template_dir/" + mkdir "$workdir" > /dev/null 2>&1 || die "workdir already exists" + rm -rf "$tmpldir" + mkdir "$tmpldir" + cp "$peer_list" "$workdir/peer_list.txt" + cp "$server_map" "$workdir/server_map.txt" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + echo "generating configuration file using $peer_list" + cp "$conf_src" "$tmpldir/$proj_conf_name" + for rid in "${!nodes[@]}"; do + local node_tmpldir="$workdir/$rid" + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local pport="$(get_peer_port_by_id $rid)" + local cport="$(get_client_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + local extra_conf_=(${node_confs[$rid]}) + rm -rf "$node_tmpldir" + mkdir "$node_tmpldir" + ( + local extra_conf=() + for conf in "${extra_conf_[@]}"; do + cp "$conf" "$node_tmpldir/" + extra_conf+=($(basename "$conf")) + copy_file "$copy_to_remote_pat" "$tmpldir/$conf" "$node_ip" "$rworkdir" + done + echo "Starting $rid @ $ip, $pport and $cport" + _remote_load "$workdir" "$rworkdir" "$ip" "$rid" "${extra_conf[@]}" + echo "$rid loaded" + ) & + done + wait + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local pport="$(get_peer_port_by_id $rid)" + local cport="$(get_client_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + ( + echo "Starting $rid @ $ip, $pport and $cport" + _remote_start "$workdir" "$rworkdir" "$rid" "$ip" "$cport" + echo "$rid started" + ) & + done + wait +} + +function fetch_all { + local workdir="$1" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local port="$(get_peer_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + local pid="$(cat $workdir/${rid}.pid)" + local msg="Fetching $rid @ $ip, $port " + _remote_fetch "$workdir" "$rworkdir" "$rid" "$ip" && echo "$msg: copied" || echo "$msg: failed" & + done + wait +} + +function exec_all { + local workdir="$1" + local cmd="$2" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local port="$(get_peer_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + local msg="Executing $rid @ $ip, $port " + _remote_exec "$workdir" "$rworkdir" "$ip" "$cmd" && echo "$msg: succeeded" || echo "$msg: failed" & + done + wait +} + +function reset_all { + exec_all "$1" "$reset_remote_pat" +} + +function stop_all { + local workdir="$1" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local port="$(get_peer_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + local pid="$(cat $workdir/${rid}.pid)" + local msg="Killing $rid @ $ip, $port " + _remote_stop "$workdir" "$rworkdir" "$ip" "$pid" && echo "$msg: stopped" || echo "$msg: failed" & + done + wait +} + +function status_all { + local workdir="$1" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local port="$(get_peer_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + local pid="$(cat $workdir/${rid}.pid)" + local msg="$rid @ $ip, $port " + _remote_status "$workdir" "$rworkdir" "$ip" "$pid" && echo "$msg: running" || echo "$msg: dead" & + done + wait +} + +function finished_all { + local workdir="$1" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local port="$(get_peer_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + if [[ "$rid" =~ $fin_chk_skip_pat ]]; then + continue + fi + printf "$rid @ $ip, $port " + _remote_finished "$workdir" "$rworkdir" "$ip" && echo "finished" || echo "in-progress" + done +} + +function wait_all { + local workdir="$1" + get_node_info "$workdir/peer_list.txt" + get_server_map "$workdir/server_map.txt" + while true; do + finished=1 + printf "checking the nodes..." + for rid in "${!nodes[@]}"; do + local ip="$(get_ip_by_id $rid)" + ip="${server_map[$ip]:-$ip}" + local port="$(get_peer_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${rid}" + if [[ "$rid" =~ $fin_chk_skip_pat ]]; then + continue + fi + if ! _remote_finished "$workdir" "$rworkdir" "$ip"; then + finished=0 + break + fi + done + if [[ $finished == 1 ]]; then + break + fi + echo "not finished yet, wait for $fin_chk_period secs" + sleep "$fin_chk_period" + done + echo "finished" +} + +function check_all { + status_all "$1" | grep dead -q + [[ "$?" -eq 0 ]] && die "some nodes are dead" + echo "ok" +} + +function print_help { +echo "Usage: $0 [--bin] [--path] [--conf] [--conf-src] [--peer-list] [--server-map] [--user] [--force-peer-list] [--help] COMMAND WORKDIR + + --help show this help and exit + --bin name of binary executable + --path path to the binary + --conf shared configuration filename + --conf-src shared configuration source file + --peer-list FILE read peer list from FILE (default: $peer_list) + --server-map FILE read server map from FILE (default: $server_map) + --user USER the username to login the remote machines + --force-peer-list force the use of FILE specified by --peer-list + instead of the peer list in WORKDIR" + exit 0 +} + +function check_argnum { + argnum=$(($# - 1)) + [[ "$1" -eq "$argnum" ]] || die "incorrect argnum: got $argnum, $1 expected" +} + +getopt --test > /dev/null +[[ $? -ne 4 ]] && die "getopt unsupported" + +SHORT= +LONG='\ +bin:,path:,conf:,conf-src:,\ +peer-list:,\ +server-map:,\ +remote-base:,\ +remote-user:,\ +copy-to-remote-pat:,\ +copy-from-remote-pat:,\ +exe-remote-pat:,\ +run-remote-pat:,\ +reset-remote-pat:,\ +fin-keyword:,\ +fin-chk-period:,\ +fin-chk-skip-pat:,\ +force-peer-list,\ +help' + +PARSED=$(getopt --options "$SHORT" --longoptions "$LONG" --name "$0" -- "$@") +[[ $? -ne 0 ]] && exit 1 +eval set -- "$PARSED" + +while true; do + case "$1" in + --bin) proj_server_bin="$2"; shift 2;; + --path) proj_server_path="$2"; shift 2;; + --conf) proj_conf_name="$2"; shift 2;; + --conf-src) conf_src="$2"; shift 2;; + --peer-list) peer_list="$2"; shift 2;; + --server-map) server_map="$2"; shift 2;; + --remote-base) remote_base="$2"; shift 2;; + --remote-user) remote_user="$2"; shift 2;; + --copy-to-remote-pat) copy_to_remote_pat="$2"; shift 2;; + --copy-from-remote-pat) copy_from_remote_pat="$2"; shift 2;; + --exe-remote-pat) exe_remote_pat="$2"; shift 2;; + --run-remote-pat) run_remote_pat="$2"; shift 2;; + --reset-remote-pat) reset_remote_pat="$2"; shift 2;; + --fin-keyword) fin_keyword="$2"; shift 2;; + --fin-chk-period) fin_chk_period="$2"; shift 2;; + --fin-chk-skip-pat) fin_chk_skip_pat="$2"; shift 2;; + --force-peer-list) force_peer_list=1; shift 1;; + --help) print_help; shift 1;; + --) shift; break;; + *) die "internal error";; + esac +done +cmd="$1" +shift 1 +case "$cmd" in + start) check_argnum 1 "$@" && start_all "$1" ;; + stop) check_argnum 1 "$@" && stop_all "$1" ;; + status) check_argnum 1 "$@" && status_all "$1" ;; + check) check_argnum 1 "$@" && check_all "$1" ;; + finished) check_argnum 1 "$@" && finished_all "$1" ;; + fetch) check_argnum 1 "$@" && fetch_all "$1" ;; + wait) check_argnum 1 "$@" && wait_all "$1" ;; + reset) check_argnum 1 "$@" && reset_all "$1" ;; + exec) check_argnum 2 "$@" && exec_all "$1" "$2" ;; + *) print_help ;; +esac -- cgit v1.2.3-70-g09d2 From 559465ecfa891da2b547c0e144e981a01ff4af33 Mon Sep 17 00:00:00 2001 From: Determinant Date: Sat, 18 Aug 2018 16:28:32 -0400 Subject: add more scripts --- README.rst | 7 +- scripts/run.sh | 4 +- scripts/run_client.sh | 354 ++++++++++++++++++++++++++++++++++++++++++++- scripts/run_demo.sh | 12 ++ scripts/run_demo_client.sh | 2 + scripts/run_replicas.sh | 12 -- 6 files changed, 373 insertions(+), 18 deletions(-) create mode 100755 scripts/run_demo.sh create mode 100755 scripts/run_demo_client.sh delete mode 100755 scripts/run_replicas.sh diff --git a/README.rst b/README.rst index 7c526d5..3eeb61d 100644 --- a/README.rst +++ b/README.rst @@ -38,14 +38,15 @@ section may be incomplete and subject to changes. :: - # clone from the repo + # install from the repo git clone https://github.com/Determinant/hot-stuff.git cd hot-stuff/ + git submodule init git submodule update --recursive # ensure openssl and libevent are installed on your machine cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED=ON -DHOTSTUFF_PROTO_LOG=ON make - # start 4 demo replicas with scripts/run_replicas.sh - # start the demo client with scripts/run_client.sh + # start 4 demo replicas with scripts/run_demo.sh + # start the demo client with scripts/run_demo_client.sh diff --git a/scripts/run.sh b/scripts/run.sh index 48e4ef7..53d9923 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -4,7 +4,7 @@ proj_server_bin="hotstuff-app" proj_server_path="/home/ted/hot-stuff/$proj_server_bin" proj_conf_name="hotstuff.conf" -peer_list="./nodes.txt" # the list of peers (without the controller) +peer_list="./nodes.txt" # the list of nodes conf_src="./hotstuff.gen.conf" server_map="./server_map.txt" # optional mapping from node ip to server ip template_dir="template" # the dir that keeps the content shared among all nodes @@ -211,7 +211,7 @@ function start_all { cp "$server_map" "$workdir/server_map.txt" get_node_info "$workdir/peer_list.txt" get_server_map "$workdir/server_map.txt" - echo "generating configuration file using $peer_list" + echo "copying configuration file" cp "$conf_src" "$tmpldir/$proj_conf_name" for rid in "${!nodes[@]}"; do local node_tmpldir="$workdir/$rid" diff --git a/scripts/run_client.sh b/scripts/run_client.sh index 93a9148..090dce1 100755 --- a/scripts/run_client.sh +++ b/scripts/run_client.sh @@ -1,2 +1,354 @@ #!/bin/bash -./hotstuff-client --idx 0 --iter -1 --max-async 3 + +proj_client_bin="hotstuff-client" +proj_client_path="/home/ted/hot-stuff/$proj_client_bin" +proj_conf_name="hotstuff.conf" + +peer_list="./nodes.txt" # the list of nodes +client_list="./clients.txt" # the list of clients +conf_src="./hotstuff.gen.conf" +template_dir="template" # the dir that keeps the content shared among all nodes +remote_base="/home/ted/testbed" # remote dir used to keep files for the experiment +#remote_base="/tmp/" # remote dir used to keep files for the experiment +remote_log="log" # log filename +remote_user="ted" +copy_to_remote_pat="rsync -avz @:" +copy_from_remote_pat="rsync -avz @: " +exe_remote_pat="ssh @ bash" +run_remote_pat="cd \"\"; '$proj_client_path' --idx \"\" --iter -1 --max-async 3" +reset_remote_pat="pgrep -f '$proj_client_bin' | xargs kill -9" + +function join { local IFS="$1"; shift; echo "$*"; } +function split { + local IFS="$1" + local arr=($2) + echo "${arr[@]}" +} + +function die { echo "$1"; exit 1; } + +declare -A nodes +nodes_cnt=0 +function get_node_info { + pl="$1" + if [[ "$force_peer_list" == 1 ]]; then + pl="$peer_list" + fi + OIFS="$IFS" + IFS=$'\n' + node_list=($(cat "$pl")) + IFS="$OIFS" + for tuple in "${node_list[@]}"; do + tup0=($(split $'\t' "$tuple")) + tup=($(split : "${tup0[0]}")) + nodes[${tup[0]}]="${tup[1]}:${tup[2]}" + echo "${tup[0]} => ${nodes[${tup[0]}]}" + let nodes_cnt++ + done +} + +function get_client_info { + cip_list=($(cat "$1")) +} + + +function get_addr { + tup=($(split ';' $1)) + echo "${tup[0]}" +} + +function get_ip { + tup=($(split : $1)) + echo "${tup[0]}" +} + +function get_peer_port { + tup=($(split : $1)) + tup2=($(split ';' ${tup[1]})) + echo "${tup2[0]}" +} + + +function get_client_port { + tup=($(split : $1)) + tup2=($(split ';' ${tup[1]})) + echo "${tup2[1]}" +} + + +function get_ip_by_id { + get_ip "${nodes[$1]}" +} + +function get_peer_port_by_id { + get_peer_port "${nodes[$1]}" +} + + +function get_client_port_by_id { + get_client_port "${nodes[$1]}" +} + +function copy_file { + local pat="$1" + local cmd="${pat///$2}" + cmd="${cmd///$3}" + cmd="${cmd///$remote_user}" + cmd="${cmd///$4}" + echo $cmd + eval "$cmd" +} >> log 2>&1 + +function execute_remote_cmd_pid { + local node_ip="$1" + local c="$2" + local l="$3" + local cmd="${exe_remote_pat///$node_ip}" + cmd="${cmd///$remote_user}" + eval $cmd << EOF +$c > $l 2>&1 & echo \$! +EOF +} + + + +function execute_remote_cmd_stat { + local node_ip="$1" + local c="$2" + local l="$3" + local cmd="${exe_remote_pat///$node_ip}" + cmd="${cmd///$remote_user}" + eval $cmd << EOF +$c > $l 2>&1 ; echo \$? +EOF +} + + +function _remote_load { + local workdir="$1" + local rworkdir="$2" + local node_ip="$3" + local tmpldir="$workdir/$template_dir/" + [[ $(execute_remote_cmd_stat "$node_ip" \ + "mkdir -p \"$rworkdir\"" \ + /dev/null) == 0 ]] || die "failed to create directory $rworkdir" + copy_file "$copy_to_remote_pat" "$tmpldir" "$node_ip" "$rworkdir" +} + +function _remote_start { + local workdir="$1" + local rworkdir="$2" + local node_id="$3" + local node_ip="$4" + local client_port="$5" + local client_ip="$6" + local cmd="${run_remote_pat///$rworkdir}" + cmd="${cmd///$node_id}" + cmd="${cmd///$node_ip:$client_port}" + execute_remote_cmd_pid "$client_ip" "$cmd" \ + "\"$rworkdir/$remote_log\"" > "$workdir/${node_id}.pid" +} + +function _remote_exec { + local workdir="$1" + local rworkdir="$2" + local node_ip="$3" + local cmd="$4" + [[ $(execute_remote_cmd_stat "$node_ip" "$cmd" /dev/null) == 0 ]] +} + +function _remote_stop { + local node_pid="$4" + _remote_exec "$1" "$2" "$3" "kill $node_pid" +} + +function _remote_status { + local node_pid="$4" + _remote_exec "$1" "$2" "$3" "kill -0 $node_pid" +} + +function _remote_fetch { + local workdir="$1" + local rworkdir="$2" + local node_id="$3" + local node_ip="$4" + copy_file "$copy_from_remote_pat" "$workdir/${node_id}.log" "$node_ip" "$rworkdir/$remote_log" +} + +function start_all { + local workdir="$1" + local tmpldir="$workdir/$template_dir/" + mkdir "$workdir" > /dev/null 2>&1 || die "workdir already exists" + rm -rf "$tmpldir" + mkdir "$tmpldir" + cp "$peer_list" "$workdir/peer_list.txt" + cp "$client_list" "$workdir/client_list.txt" + get_node_info "$workdir/peer_list.txt" + get_client_info "$workdir/client_list.txt" + echo "coyping configuration file" + cp "$conf_src" "$tmpldir/$proj_conf_name" + local nclient="${#cip_list[@]}" + local i=0 + for tuple in "${node_list[@]}"; do + local cip="${cip_list[$i]}" + local tup=($(split : "$tuple")) + local rid="${tup[0]}" + local ip="$(get_ip_by_id $rid)" + local pport="$(get_peer_port_by_id $rid)" + local cport="$(get_client_port_by_id $rid)" + local rworkdir="$remote_base/$workdir/${i}" + ( + echo "Starting a client @ $cip, connecting to server #$rid @ $ip:$cport" + _remote_load "$workdir" "$rworkdir" "$cip" + _remote_start "$workdir" "$rworkdir" "$i" "$ip" "$cport" "$cip" + echo "client #$i started" + ) & + let i++ + if [[ "$i" -eq "$nclient" ]]; then + break + fi + done + wait +} + +function fetch_all { + local workdir="$1" + get_client_info "$workdir/client_list.txt" + local i=0 + for cip in "${cip_list[@]}"; do + local rworkdir="$remote_base/$workdir/${i}" + local pid="$(cat $workdir/${i}.pid)" + local msg="Fetching $i @ $cip" + _remote_fetch "$workdir" "$rworkdir" "$i" "$cip" && echo "$msg: copied" || echo "$msg: failed" & + let i++ + done + wait +} + +function exec_all { + local workdir="$1" + local cmd="$2" + get_client_info "$workdir/client_list.txt" + local i=0 + for cip in "${cip_list[@]}"; do + local rworkdir="$remote_base/$workdir/${i}" + local msg="Executing $i @ $cip" + _remote_exec "$workdir" "$rworkdir" "$cip" "$cmd" && echo "$msg: succeeded" || echo "$msg: failed" & + let i++ + done + wait +} + +function reset_all { + exec_all "$1" "$reset_remote_pat" +} + +function stop_all { + local workdir="$1" + get_client_info "$workdir/client_list.txt" + local i=0 + for cip in "${cip_list[@]}"; do + local rworkdir="$remote_base/$workdir/${i}" + local pid="$(cat $workdir/${i}.pid)" + local msg="Killing $i @ $cip" + _remote_stop "$workdir" "$rworkdir" "$cip" "$pid" && echo "$msg: stopped" || echo "$msg: failed" & + let i++ + done + wait +} + +function status_all { + local workdir="$1" + get_client_info "$workdir/client_list.txt" + local i=0 + for cip in "${cip_list[@]}"; do + local rworkdir="$remote_base/$workdir/${i}" + local pid="$(cat $workdir/${i}.pid)" + local msg="$i @ $cip" + _remote_status "$workdir" "$rworkdir" "$cip" "$pid" && echo "$msg: running" || echo "$msg: dead" & + let i++ + done + wait +} + +function check_all { + status_all "$1" | grep dead -q + [[ "$?" -eq 0 ]] && die "some nodes are dead" + echo "ok" +} + +function print_help { +echo "Usage: $0 [--bin] [--path] [--conf] [--conf-src] [--peer-list] [--client-list] [--user] [--force-peer-list] [--help] COMMAND WORKDIR + + --help show this help and exit + --bin name of binary executable + --path path to the binary + --conf shared configuration filename + --conf-src shared configuration source file + --peer-list FILE read peer list from FILE (default: $peer_list) + --client-list FILE read client list from FILE (default: $client_list) + --user USER the username to login the remote machines + --force-peer-list force the use of FILE specified by --peer-list + instead of the peer list in WORKDIR" + exit 0 +} + +function check_argnum { + argnum=$(($# - 1)) + [[ "$1" -eq "$argnum" ]] || die "incorrect argnum: got $argnum, $1 expected" +} + +getopt --test > /dev/null +[[ $? -ne 4 ]] && die "getopt unsupported" + +SHORT= +LONG='\ +bin:,path:,conf:,conf-src:,\ +peer-list:,\ +client-list:,\ +remote-base:,\ +remote-user:,\ +copy-to-remote-pat:,\ +copy-from-remote-pat:,\ +exe-remote-pat:,\ +run-remote-pat:,\ +reset-remote-pat:,\ +force-peer-list,\ +help' + +PARSED=$(getopt --options "$SHORT" --longoptions "$LONG" --name "$0" -- "$@") +[[ $? -ne 0 ]] && exit 1 +eval set -- "$PARSED" + +while true; do + case "$1" in + --bin) proj_client_bin="$2"; shift 2;; + --path) proj_client_path="$2"; shift 2;; + --conf) proj_conf_name="$2"; shift 2;; + --conf-src) conf_src="$2"; shift 2;; + --peer-list) peer_list="$2"; shift 2;; + --client-list) client_list="$2"; shift 2;; + --remote-base) remote_base="$2"; shift 2;; + --remote-user) remote_user="$2"; shift 2;; + --copy-to-remote-pat) copy_to_remote_pat="$2"; shift 2;; + --copy-from-remote-pat) copy_from_remote_pat="$2"; shift 2;; + --exe-remote-pat) exe_remote_pat="$2"; shift 2;; + --run-remote-pat) run_remote_pat="$2"; shift 2;; + --reset-remote-pat) reset_remote_pat="$2"; shift 2;; + --help) print_help; shift 1;; + --) shift; break;; + *) die "internal error";; + esac +done +cmd="$1" +shift 1 +case "$cmd" in + start) check_argnum 1 "$@" && start_all "$1" ;; + stop) check_argnum 1 "$@" && stop_all "$1" ;; + status) check_argnum 1 "$@" && status_all "$1" ;; + check) check_argnum 1 "$@" && check_all "$1" ;; + fetch) check_argnum 1 "$@" && fetch_all "$1" ;; + reset) check_argnum 1 "$@" && reset_all "$1" ;; + exec) check_argnum 2 "$@" && exec_all "$1" "$2" ;; + *) print_help ;; +esac diff --git a/scripts/run_demo.sh b/scripts/run_demo.sh new file mode 100755 index 0000000..5f54787 --- /dev/null +++ b/scripts/run_demo.sh @@ -0,0 +1,12 @@ +#!/bin/bash +rep=({0..3}) +if [[ $# -gt 0 ]]; then + rep=($@) +fi +for i in "${rep[@]}"; do + echo "starting replica $i" + #valgrind --leak-check=full ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & + #gdb -ex r -ex bt -ex q --args ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & + ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & +done +wait diff --git a/scripts/run_demo_client.sh b/scripts/run_demo_client.sh new file mode 100755 index 0000000..93a9148 --- /dev/null +++ b/scripts/run_demo_client.sh @@ -0,0 +1,2 @@ +#!/bin/bash +./hotstuff-client --idx 0 --iter -1 --max-async 3 diff --git a/scripts/run_replicas.sh b/scripts/run_replicas.sh deleted file mode 100755 index 5f54787..0000000 --- a/scripts/run_replicas.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -rep=({0..3}) -if [[ $# -gt 0 ]]; then - rep=($@) -fi -for i in "${rep[@]}"; do - echo "starting replica $i" - #valgrind --leak-check=full ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & - #gdb -ex r -ex bt -ex q --args ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & - ./hotstuff-app --conf hotstuff-sec${i}.conf > log${i} 2>&1 & -done -wait -- cgit v1.2.3-70-g09d2 From f4eb65192d1ff39dd40c15827c012d5a55045c43 Mon Sep 17 00:00:00 2001 From: Determinant Date: Mon, 20 Aug 2018 12:20:16 -0400 Subject: ... --- scripts/gen_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py index 7b06826..391e0d6 100644 --- a/scripts/gen_conf.py +++ b/scripts/gen_conf.py @@ -20,7 +20,7 @@ if __name__ == "__main__": if args.ips is None: ips = ['127.0.0.1'] else: - ips = [l.strip() for l in open(args.iplist, 'r').readlines()] + ips = [l.strip() for l in open(args.ips, 'r').readlines()] prefix = args.prefix iter = args.iter base_pport = args.pport -- cgit v1.2.3-70-g09d2 From cf3f1c3b4412412231cd19cb5aa30c5e62b21914 Mon Sep 17 00:00:00 2001 From: Determinant Date: Mon, 20 Aug 2018 23:17:00 -0400 Subject: ... --- scripts/thr_hist.py | 5 ++++- src/hotstuff_client.cpp | 31 ++++++++++++++++++++----------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/scripts/thr_hist.py b/scripts/thr_hist.py index 6f6a43f..c5f2a72 100644 --- a/scripts/thr_hist.py +++ b/scripts/thr_hist.py @@ -24,17 +24,19 @@ if __name__ == '__main__': parser.add_argument('--interval', type=float, default=1, required=False) parser.add_argument('--output', type=str, default="hist.png", required=False) args = parser.parse_args() - commit_pat = re.compile('([^[].*) \[hotstuff info\].*got get_hash()).c_str()); +#endif waiting.insert(std::make_pair( cmd->get_hash(), Request(proposer, cmd))); if (max_iter_num > 0) @@ -70,6 +69,9 @@ void client_resp_cmd_handler(MsgRespCmd &&msg, MsgNetwork::Conn &) { HOTSTUFF_LOG_DEBUG("got %s", std::string(msg.fin).c_str()); const uint256_t &cmd_hash = fin.cmd_hash; auto it = waiting.find(cmd_hash); + auto &et = it->second.et; + if (it == waiting.end()) return; + et.stop(); if (fin.rid != proposer) { HOTSTUFF_LOG_INFO("reconnect to the new proposer"); @@ -79,20 +81,27 @@ void client_resp_cmd_handler(MsgRespCmd &&msg, MsgNetwork::Conn &) { { mn.send_msg(MsgReqCmd(*(waiting.find(cmd_hash)->second.cmd)), *conns.at(proposer)); +#ifndef HOTSTUFF_ENABLE_BENCHMARK HOTSTUFF_LOG_INFO("resend cmd %.10s", get_hex(cmd_hash).c_str()); - it->second.et.start(); +#endif + et.start(); it->second.rid = proposer; return; } - HOTSTUFF_LOG_INFO("got %s", std::string(fin).c_str()); - if (it == waiting.end()) return; +#ifndef HOTSTUFF_ENABLE_BENCHMARK + HOTSTUFF_LOG_INFO("got %s, wall: %.3f, cpu: %.3f", + std::string(fin).c_str(), + et.elapsed_sec, et.cpu_elapsed_sec); +#else + HOTSTUFF_LOG_INFO("%.6f %.6f", et.elapsed_sec, et.cpu_elapsed_sec); +#endif waiting.erase(it); try_send(); } std::pair split_ip_port_cport(const std::string &s) { - auto ret = trim_all(split(s, ";")); + auto ret = salticidae::trim_all(salticidae::split(s, ";")); return std::make_pair(ret[0], ret[1]); } @@ -117,7 +126,7 @@ int main(int argc, char **argv) { std::vector> raw; for (const auto &s: opt_replicas->get()) { - auto res = trim_all(split(s, ",")); + auto res = salticidae::trim_all(salticidae::split(s, ",")); if (res.size() != 2) throw HotStuffError("format error"); raw.push_back(std::make_pair(res[0], res[1])); -- cgit v1.2.3-70-g09d2 From 4022d928eae69ff60a4e22c25f8dad28a7a7a267 Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 21 Aug 2018 10:43:52 -0400 Subject: update readme --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3eeb61d..ad93287 100644 --- a/README.rst +++ b/README.rst @@ -41,8 +41,7 @@ section may be incomplete and subject to changes. # install from the repo git clone https://github.com/Determinant/hot-stuff.git cd hot-stuff/ - git submodule init - git submodule update --recursive + git submodule update --init --recursive # ensure openssl and libevent are installed on your machine cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED=ON -DHOTSTUFF_PROTO_LOG=ON -- cgit v1.2.3-70-g09d2 From 8222929b330c0cf3f8cce2c2d8b097e9713ecb24 Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 21 Aug 2018 11:57:14 -0400 Subject: ... --- salticidae | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salticidae b/salticidae index 3073f4a..2cc3eb4 160000 --- a/salticidae +++ b/salticidae @@ -1 +1 @@ -Subproject commit 3073f4a9470da0c7aec0aa0a3b527f25095b5c9f +Subproject commit 2cc3eb4fbb4ba50f08f4041f91e445c45306af66 -- cgit v1.2.3-70-g09d2 From 3b7801dcfd573858e9a8f27abe890def47ece2ad Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 21 Aug 2018 20:52:58 -0400 Subject: ... --- include/hotstuff/liveness.h | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/include/hotstuff/liveness.h b/include/hotstuff/liveness.h index 59306ab..488143d 100644 --- a/include/hotstuff/liveness.h +++ b/include/hotstuff/liveness.h @@ -117,6 +117,8 @@ class PMWaitQC: public virtual PaceMaker { std::queue pending_beats; block_t last_proposed; bool locked; + promise_t pm_qc_finish; + promise_t pm_wait_propose; protected: void schedule_next() { @@ -124,19 +126,23 @@ class PMWaitQC: public virtual PaceMaker { { auto pm = pending_beats.front(); pending_beats.pop(); - hsc->async_qc_finish(last_proposed).then([this, pm]() { - pm.resolve(get_proposer()); - }); + pm_qc_finish.reject(); + (pm_qc_finish = hsc->async_qc_finish(last_proposed)) + .then([this, pm]() { + pm.resolve(get_proposer()); + }); locked = true; } } void update_last_proposed() { - hsc->async_wait_proposal().then([this](const Proposal &prop) { - update_last_proposed(); + pm_wait_propose.reject(); + (pm_wait_propose = hsc->async_wait_proposal()).then( + [this](const Proposal &prop) { last_proposed = prop.blk; locked = false; schedule_next(); + update_last_proposed(); }); } @@ -155,7 +161,7 @@ class PMWaitQC: public virtual PaceMaker { promise_t pm; pending_beats.push(pm); schedule_next(); - return pm; + return std::move(pm); } promise_t beat_resp(ReplicaID last_proposer) override { @@ -245,7 +251,7 @@ class PMStickyProposer: virtual public PaceMaker { void reset_qc_timer() { timer.del(); timer.add_with_timeout(qc_timeout); - HOTSTUFF_LOG_PROTO("QC timer reset"); + HOTSTUFF_LOG_INFO("QC timer reset"); } void clear_promises() { @@ -273,11 +279,11 @@ class PMStickyProposer: virtual public PaceMaker { auto &qc_ref = prop.blk->get_qc_ref(); if (last_proposed && qc_ref != last_proposed) { - HOTSTUFF_LOG_PROTO("proposer misbehave"); + HOTSTUFF_LOG_INFO("proposer misbehave"); to_candidate(); /* proposer misbehave */ return; } - HOTSTUFF_LOG_PROTO("proposer emits new QC"); + HOTSTUFF_LOG_INFO("proposer emits new QC"); last_proposed = prop.blk; reset_qc_timer(); } @@ -332,7 +338,7 @@ class PMStickyProposer: virtual public PaceMaker { (pm_wait_propose = hsc->async_wait_proposal()).then([this](const Proposal &prop) { const auto &blk = prop.blk; (pm_qc_finish = hsc->async_qc_finish(blk)).then([this, blk]() { - HOTSTUFF_LOG_PROTO("collected QC for %s", std::string(*blk).c_str()); + HOTSTUFF_LOG_INFO("collected QC for %s", std::string(*blk).c_str()); /* managed to collect a QC */ to_proposer(); propose_elect_block(); @@ -341,7 +347,7 @@ class PMStickyProposer: virtual public PaceMaker { double t = salticidae::gen_rand_timeout(candidate_timeout); timer.del(); timer.add_with_timeout(t); - HOTSTUFF_LOG_PROTO("candidate next try in %.2fs", t); + HOTSTUFF_LOG_INFO("candidate next try in %.2fs", t); propose_elect_block(); } @@ -367,7 +373,7 @@ class PMStickyProposer: virtual public PaceMaker { /* role transitions */ void to_follower(ReplicaID new_proposer) { - HOTSTUFF_LOG_PROTO("new role: follower"); + HOTSTUFF_LOG_INFO("new role: follower"); clear_promises(); role = FOLLOWER; proposer = new_proposer; @@ -387,7 +393,7 @@ class PMStickyProposer: virtual public PaceMaker { } void to_proposer() { - HOTSTUFF_LOG_PROTO("new role: proposer"); + HOTSTUFF_LOG_INFO("new role: proposer"); clear_promises(); role = PROPOSER; proposer = hsc->get_id(); @@ -402,7 +408,7 @@ class PMStickyProposer: virtual public PaceMaker { } void to_candidate() { - HOTSTUFF_LOG_PROTO("new role: candidate"); + HOTSTUFF_LOG_INFO("new role: candidate"); clear_promises(); role = CANDIDATE; proposer = hsc->get_id(); -- cgit v1.2.3-70-g09d2 From 69208d4931f45911e401a97ba9b019a2ffdfe82c Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 21 Aug 2018 21:58:40 -0400 Subject: use randomized initial cnt to avoid duplicate command --- include/hotstuff/client.h | 5 ----- src/client.cpp | 2 -- src/hotstuff_client.cpp | 12 +++++++++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/hotstuff/client.h b/include/hotstuff/client.h index 92b4eec..614b654 100644 --- a/include/hotstuff/client.h +++ b/include/hotstuff/client.h @@ -26,7 +26,6 @@ struct MsgRespCmd { }; class CommandDummy: public Command { - static uint64_t cnt; uint64_t n; uint256_t hash; @@ -39,10 +38,6 @@ class CommandDummy: public Command { CommandDummy(uint64_t n): n(n), hash(salticidae::get_hash(*this)) {} - static command_t make_cmd() { - return new CommandDummy(cnt++); - } - void serialize(DataStream &s) const override { s << n; } diff --git a/src/client.cpp b/src/client.cpp index 562fab5..7827b7c 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -2,8 +2,6 @@ namespace hotstuff { -uint64_t CommandDummy::cnt = 0; - const opcode_t MsgReqCmd::opcode; MsgReqCmd::MsgReqCmd(const Command &cmd) { serialized << cmd; } void MsgReqCmd::postponed_parse(HotStuffCore *hsc) { diff --git a/src/hotstuff_client.cpp b/src/hotstuff_client.cpp index f925b73..97c5dc5 100644 --- a/src/hotstuff_client.cpp +++ b/src/hotstuff_client.cpp @@ -1,4 +1,5 @@ #include +#include #include "salticidae/type.h" #include "salticidae/netaddr.h" #include "salticidae/network.h" @@ -27,6 +28,8 @@ EventContext eb; ReplicaID proposer; size_t max_async_num; int max_iter_num; +uint64_t cnd_stride; +uint64_t cnt; struct Request { ReplicaID rid; @@ -51,7 +54,8 @@ void set_proposer(ReplicaID rid) { void try_send() { while (waiting.size() < max_async_num && max_iter_num) { - auto cmd = CommandDummy::make_cmd(); + auto cmd = new CommandDummy(cnt); + cnt += cnd_stride; mn.send_msg(MsgReqCmd(*cmd), *conns.at(proposer)); #ifndef HOTSTUFF_ENABLE_BENCHMARK HOTSTUFF_LOG_INFO("send new cmd %.10s", @@ -106,11 +110,15 @@ std::pair split_ip_port_cport(const std::string &s) { } int main(int argc, char **argv) { + cnt = std::random_device()(); + HOTSTUFF_LOG_INFO("init cnt = %lu", cnt); + Config config("hotstuff.conf"); auto opt_idx = Config::OptValInt::create(0); auto opt_replicas = Config::OptValStrVec::create(); auto opt_max_iter_num = Config::OptValInt::create(100); auto opt_max_async_num = Config::OptValInt::create(10); + auto opt_cnt_stride = Config::OptValInt::create(1000); mn.reg_handler(client_resp_cmd_handler); @@ -119,8 +127,10 @@ int main(int argc, char **argv) { config.add_opt("replica", opt_replicas, Config::APPEND); config.add_opt("iter", opt_max_iter_num, Config::SET_VAL); config.add_opt("max-async", opt_max_async_num, Config::SET_VAL); + config.add_opt("cnt-stride", opt_cnt_stride, Config::SET_VAL); config.parse(argc, argv); auto idx = opt_idx->get(); + cnd_stride = opt_cnt_stride->get(); max_iter_num = opt_max_iter_num->get(); max_async_num = opt_max_async_num->get(); std::vector> raw; -- cgit v1.2.3-70-g09d2 From 51796ba2e778001948f4cd5acb51e31fcaec0641 Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 21 Aug 2018 22:25:00 -0400 Subject: use cid instead of randomization --- include/hotstuff/client.h | 11 ++++++----- src/hotstuff_client.cpp | 16 ++++++---------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/include/hotstuff/client.h b/include/hotstuff/client.h index 614b654..447a9db 100644 --- a/include/hotstuff/client.h +++ b/include/hotstuff/client.h @@ -26,7 +26,8 @@ struct MsgRespCmd { }; class CommandDummy: public Command { - uint64_t n; + uint32_t cid; + uint32_t n; uint256_t hash; public: @@ -35,15 +36,15 @@ class CommandDummy: public Command { ~CommandDummy() override {} - CommandDummy(uint64_t n): - n(n), hash(salticidae::get_hash(*this)) {} + CommandDummy(uint32_t cid, uint32_t n): + cid(cid), n(n), hash(salticidae::get_hash(*this)) {} void serialize(DataStream &s) const override { - s << n; + s << cid << n; } void unserialize(DataStream &s) override { - s >> n; + s >> cid >> n; hash = salticidae::get_hash(*this); } diff --git a/src/hotstuff_client.cpp b/src/hotstuff_client.cpp index 97c5dc5..62b13ed 100644 --- a/src/hotstuff_client.cpp +++ b/src/hotstuff_client.cpp @@ -28,8 +28,8 @@ EventContext eb; ReplicaID proposer; size_t max_async_num; int max_iter_num; -uint64_t cnd_stride; -uint64_t cnt; +uint32_t cid; +uint32_t cnt = 0; struct Request { ReplicaID rid; @@ -54,8 +54,7 @@ void set_proposer(ReplicaID rid) { void try_send() { while (waiting.size() < max_async_num && max_iter_num) { - auto cmd = new CommandDummy(cnt); - cnt += cnd_stride; + auto cmd = new CommandDummy(cid, cnt++); mn.send_msg(MsgReqCmd(*cmd), *conns.at(proposer)); #ifndef HOTSTUFF_ENABLE_BENCHMARK HOTSTUFF_LOG_INFO("send new cmd %.10s", @@ -110,27 +109,23 @@ std::pair split_ip_port_cport(const std::string &s) { } int main(int argc, char **argv) { - cnt = std::random_device()(); - HOTSTUFF_LOG_INFO("init cnt = %lu", cnt); - Config config("hotstuff.conf"); auto opt_idx = Config::OptValInt::create(0); auto opt_replicas = Config::OptValStrVec::create(); auto opt_max_iter_num = Config::OptValInt::create(100); auto opt_max_async_num = Config::OptValInt::create(10); - auto opt_cnt_stride = Config::OptValInt::create(1000); + auto opt_cid = Config::OptValInt::create(-1); mn.reg_handler(client_resp_cmd_handler); try { config.add_opt("idx", opt_idx, Config::SET_VAL); + config.add_opt("cid", opt_cid, Config::SET_VAL); config.add_opt("replica", opt_replicas, Config::APPEND); config.add_opt("iter", opt_max_iter_num, Config::SET_VAL); config.add_opt("max-async", opt_max_async_num, Config::SET_VAL); - config.add_opt("cnt-stride", opt_cnt_stride, Config::SET_VAL); config.parse(argc, argv); auto idx = opt_idx->get(); - cnd_stride = opt_cnt_stride->get(); max_iter_num = opt_max_iter_num->get(); max_async_num = opt_max_async_num->get(); std::vector> raw; @@ -144,6 +139,7 @@ int main(int argc, char **argv) { if (!(0 <= idx && (size_t)idx < raw.size() && raw.size() > 0)) throw std::invalid_argument("out of range"); + cid = opt_cid->get() != -1 ? opt_cid->get() : idx; for (const auto &p: raw) { auto _p = split_ip_port_cport(p.first); -- cgit v1.2.3-70-g09d2 From 78f0a3ee23e39f318336a99b088eb18cccb33983 Mon Sep 17 00:00:00 2001 From: Determinant Date: Wed, 22 Aug 2018 15:51:41 -0400 Subject: ... --- include/hotstuff/hotstuff.h | 7 ------- 1 file changed, 7 deletions(-) diff --git a/include/hotstuff/hotstuff.h b/include/hotstuff/hotstuff.h index b0a6827..f3f3f18 100644 --- a/include/hotstuff/hotstuff.h +++ b/include/hotstuff/hotstuff.h @@ -23,13 +23,6 @@ using salticidae::_2; const double ent_waiting_timeout = 10; const double double_inf = 1e10; -enum { - PROPOSE = 0x0, - VOTE = 0x1, - QUERY_FETCH_BLK = 0x2, - RESP_FETCH_BLK = 0x3, -}; - /** Network message format for HotStuff. */ struct MsgPropose { static const opcode_t opcode = 0x0; -- cgit v1.2.3-70-g09d2 From 4388716ccf1cea6f691b96f0873f4d40312e3037 Mon Sep 17 00:00:00 2001 From: Determinant Date: Thu, 23 Aug 2018 11:41:20 -0400 Subject: add options for the arguments to pace makers --- src/hotstuff_app.cpp | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/hotstuff_app.cpp b/src/hotstuff_app.cpp index ead4e0b..a917976 100644 --- a/src/hotstuff_app.cpp +++ b/src/hotstuff_app.cpp @@ -124,15 +124,19 @@ int main(int argc, char **argv) { auto opt_privkey = Config::OptValStr::create(); auto opt_help = Config::OptValFlag::create(false); auto opt_pace_maker = Config::OptValStr::create("dummy"); + auto opt_fixed_proposer = Config::OptValInt::create(1); + auto opt_qc_timeout = Config::OptValDouble::create(0.5); config.add_opt("block-size", opt_blk_size, Config::SET_VAL); config.add_opt("parent-limit", opt_parent_limit, Config::SET_VAL); config.add_opt("stat-period", opt_stat_period, Config::SET_VAL); - config.add_opt("replica", opt_replicas, Config::APPEND); - config.add_opt("idx", opt_idx, Config::SET_VAL); - config.add_opt("cport", opt_client_port, Config::SET_VAL); + config.add_opt("replica", opt_replicas, Config::APPEND, 'a', "add an replica to the list"); + config.add_opt("idx", opt_idx, Config::SET_VAL, 'i', "specify the index in the replica list"); + config.add_opt("cport", opt_client_port, Config::SET_VAL, 'c', "specify the port listening for clients"); config.add_opt("privkey", opt_privkey, Config::SET_VAL); config.add_opt("pace-maker", opt_pace_maker, Config::SET_VAL, 'p', "specify pace maker (sticky, dummy)"); + config.add_opt("proposer", opt_fixed_proposer, Config::SET_VAL, 'l', "set the fixed proposer (for dummy)"); + config.add_opt("qc-timeout", opt_qc_timeout, Config::SET_VAL, 't', "set QC timeout (for sticky)"); config.add_opt("help", opt_help, Config::SWITCH_ON, 'h', "show this help info"); EventContext ec; @@ -175,9 +179,9 @@ int main(int argc, char **argv) { auto parent_limit = opt_parent_limit->get(); hotstuff::pacemaker_bt pmaker; if (opt_pace_maker->get() == "sticky") - pmaker = new hotstuff::PaceMakerSticky(parent_limit, 0.5, ec); + pmaker = new hotstuff::PaceMakerSticky(parent_limit, opt_qc_timeout->get(), ec); else - pmaker = new hotstuff::PaceMakerDummyFixed(1, parent_limit); + pmaker = new hotstuff::PaceMakerDummyFixed(opt_fixed_proposer->get(), parent_limit); papp = new HotStuffApp(opt_blk_size->get(), opt_stat_period->get(), -- cgit v1.2.3-70-g09d2 From 1dcd22c91ebe63337d1c3a99fad5d7936b9dbab3 Mon Sep 17 00:00:00 2001 From: Determinant Date: Thu, 23 Aug 2018 11:49:50 -0400 Subject: ... --- salticidae | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salticidae b/salticidae index 2cc3eb4..d2c471c 160000 --- a/salticidae +++ b/salticidae @@ -1 +1 @@ -Subproject commit 2cc3eb4fbb4ba50f08f4041f91e445c45306af66 +Subproject commit d2c471c2300141529983be425f7ad75d371fcdf7 -- cgit v1.2.3-70-g09d2 From b5e207ed22ca7fcd924ee9545283cb2431fe8ef8 Mon Sep 17 00:00:00 2001 From: Determinant Date: Thu, 23 Aug 2018 14:21:54 -0400 Subject: add impeach() --- include/hotstuff/hotstuff.h | 1 + include/hotstuff/liveness.h | 27 +++++++++++++++++------ src/hotstuff_app.cpp | 52 +++++++++++++++++++++++++++------------------ 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/include/hotstuff/hotstuff.h b/include/hotstuff/hotstuff.h index f3f3f18..f9aad3d 100644 --- a/include/hotstuff/hotstuff.h +++ b/include/hotstuff/hotstuff.h @@ -195,6 +195,7 @@ class HotStuffBase: public HotStuffCore { void start(bool eb_loop = false); size_t size() const { return pn.all_peers().size(); } + PaceMaker &get_pace_maker() { return *pmaker; } void print_stat() const; /* Helper functions */ diff --git a/include/hotstuff/liveness.h b/include/hotstuff/liveness.h index 488143d..77d4232 100644 --- a/include/hotstuff/liveness.h +++ b/include/hotstuff/liveness.h @@ -32,6 +32,8 @@ class PaceMaker { * to vote for a block. The promise is resolved with the next proposer's ID * */ virtual promise_t beat_resp(ReplicaID last_proposer) = 0; + /** Impeach the current proposer. */ + virtual void impeach() {} }; using pacemaker_bt = BoxObj; @@ -285,7 +287,7 @@ class PMStickyProposer: virtual public PaceMaker { } HOTSTUFF_LOG_INFO("proposer emits new QC"); last_proposed = prop.blk; - reset_qc_timer(); + //reset_qc_timer(); } reg_follower_receive_proposal(); } @@ -300,7 +302,7 @@ class PMStickyProposer: virtual public PaceMaker { pm_qc_finish.reject(); (pm_qc_finish = hsc->async_qc_finish(last_proposed)) .then([this, pm]() { - reset_qc_timer(); + timer.del(); pm.resolve(proposer); }); locked = true; @@ -315,6 +317,7 @@ class PMStickyProposer: virtual public PaceMaker { } void proposer_propose(const Proposal &prop) { + reset_qc_timer(); last_proposed = prop.blk; locked = false; proposer_schedule_next(); @@ -379,10 +382,11 @@ class PMStickyProposer: virtual public PaceMaker { proposer = new_proposer; last_proposed = nullptr; hsc->set_neg_vote(false); - timer = Event(ec, -1, 0, [this](int, short) { - /* unable to get a QC in time */ - to_candidate(); - }); + timer.clear(); + //timer = Event(ec, -1, 0, [this](int, short) { + // /* unable to get a QC in time */ + // to_candidate(); + //}); /* redirect all pending cmds to the new proposer */ while (!pending_beats.empty()) { @@ -404,7 +408,6 @@ class PMStickyProposer: virtual public PaceMaker { to_candidate(); }); proposer_propose(Proposal(-1, uint256_t(), hsc->get_genesis(), nullptr)); - reset_qc_timer(); } void to_candidate() { @@ -422,6 +425,16 @@ class PMStickyProposer: virtual public PaceMaker { candidate_qc_timeout(); } + protected: + void impeach() override { + if (role == CANDIDATE) return; + timer = Event(ec, -1, 0, [this](int, short) { + to_candidate(); + }); + timer.add_with_timeout(0); + HOTSTUFF_LOG_INFO("schedule to impeach the proposer"); + } + public: PMStickyProposer(double qc_timeout, const EventContext &ec): qc_timeout(qc_timeout), ec(ec) {} diff --git a/src/hotstuff_app.cpp b/src/hotstuff_app.cpp index a917976..210badc 100644 --- a/src/hotstuff_app.cpp +++ b/src/hotstuff_app.cpp @@ -48,25 +48,22 @@ using hotstuff::promise_t; using HotStuff = hotstuff::HotStuffSecp256k1; -#define LOG_INFO HOTSTUFF_LOG_INFO -#define LOG_DEBUG HOTSTUFF_LOG_DEBUG -#define LOG_WARN HOTSTUFF_LOG_WARN -#define LOG_ERROR HOTSTUFF_LOG_ERROR - class HotStuffApp: public HotStuff { double stat_period; + double impeach_timeout; EventContext ec; /** Network messaging between a replica and its client. */ ClientNetwork cn; /** Timer object to schedule a periodic printing of system statistics */ Event ev_stat_timer; + /** Timer object to monitor the progress for simple impeachment */ + Event impeach_timer; /** The listen address for client RPC */ NetAddr clisten_addr; using Conn = ClientNetwork::Conn; void client_request_cmd_handler(MsgReqCmd &&, Conn &); - void print_stat_cb(evutil_socket_t, short); command_t parse_cmd(DataStream &s) override { auto cmd = new CommandDummy(); @@ -74,15 +71,22 @@ class HotStuffApp: public HotStuff { return cmd; } + void reset_imp_timer() { + impeach_timer.del(); + impeach_timer.add_with_timeout(impeach_timeout); + } + void state_machine_execute(const Finality &fin) override { + reset_imp_timer(); #ifndef HOTSTUFF_ENABLE_BENCHMARK - LOG_INFO("replicated %s", std::string(fin).c_str()); + HOTSTUFF_LOG_INFO("replicated %s", std::string(fin).c_str()); #endif } public: HotStuffApp(uint32_t blk_size, double stat_period, + double impeach_timeout, ReplicaID idx, const bytearray_t &raw_privkey, NetAddr plisten_addr, @@ -126,6 +130,7 @@ int main(int argc, char **argv) { auto opt_pace_maker = Config::OptValStr::create("dummy"); auto opt_fixed_proposer = Config::OptValInt::create(1); auto opt_qc_timeout = Config::OptValDouble::create(0.5); + auto opt_imp_timeout = Config::OptValDouble::create(11); config.add_opt("block-size", opt_blk_size, Config::SET_VAL); config.add_opt("parent-limit", opt_parent_limit, Config::SET_VAL); @@ -137,6 +142,7 @@ int main(int argc, char **argv) { config.add_opt("pace-maker", opt_pace_maker, Config::SET_VAL, 'p', "specify pace maker (sticky, dummy)"); config.add_opt("proposer", opt_fixed_proposer, Config::SET_VAL, 'l', "set the fixed proposer (for dummy)"); config.add_opt("qc-timeout", opt_qc_timeout, Config::SET_VAL, 't', "set QC timeout (for sticky)"); + config.add_opt("imp-timeout", opt_imp_timeout, Config::SET_VAL, 'u', "set impeachment timeout (for sticky)"); config.add_opt("help", opt_help, Config::SWITCH_ON, 'h', "show this help info"); EventContext ec; @@ -185,6 +191,7 @@ int main(int argc, char **argv) { papp = new HotStuffApp(opt_blk_size->get(), opt_stat_period->get(), + opt_imp_timeout->get(), idx, hotstuff::from_hex(opt_privkey->get()), plisten_addr, @@ -209,6 +216,7 @@ int main(int argc, char **argv) { HotStuffApp::HotStuffApp(uint32_t blk_size, double stat_period, + double impeach_timeout, ReplicaID idx, const bytearray_t &raw_privkey, NetAddr plisten_addr, @@ -218,6 +226,7 @@ HotStuffApp::HotStuffApp(uint32_t blk_size, HotStuff(blk_size, idx, raw_privkey, plisten_addr, std::move(pmaker), ec), stat_period(stat_period), + impeach_timeout(impeach_timeout), ec(ec), cn(ec), clisten_addr(clisten_addr) { @@ -231,28 +240,29 @@ void HotStuffApp::client_request_cmd_handler(MsgReqCmd &&msg, Conn &conn) { msg.postponed_parse(this); auto cmd = msg.cmd; std::vector pms; - LOG_DEBUG("processing %s", std::string(*cmd).c_str()); + HOTSTUFF_LOG_DEBUG("processing %s", std::string(*cmd).c_str()); exec_command(cmd).then([this, addr](Finality fin) { cn.send_msg(MsgRespCmd(fin), addr); }); } void HotStuffApp::start() { - ev_stat_timer = Event(ec, -1, 0, - std::bind(&HotStuffApp::print_stat_cb, this, _1, _2)); + ev_stat_timer = Event(ec, -1, 0, [this](int, short) { + HotStuff::print_stat(); + //HotStuffCore::prune(100); + ev_stat_timer.add_with_timeout(stat_period); + }); ev_stat_timer.add_with_timeout(stat_period); - LOG_INFO("** starting the system with parameters **"); - LOG_INFO("blk_size = %lu", blk_size); - LOG_INFO("conns = %lu", HotStuff::size()); - LOG_INFO("** starting the event loop..."); + impeach_timer = Event(ec, -1, 0, [this](int, short) { + get_pace_maker().impeach(); + reset_imp_timer(); + }); + impeach_timer.add_with_timeout(impeach_timeout); + HOTSTUFF_LOG_INFO("** starting the system with parameters **"); + HOTSTUFF_LOG_INFO("blk_size = %lu", blk_size); + HOTSTUFF_LOG_INFO("conns = %lu", HotStuff::size()); + HOTSTUFF_LOG_INFO("** starting the event loop..."); HotStuff::start(); /* enter the event main loop */ ec.dispatch(); } - - -void HotStuffApp::print_stat_cb(evutil_socket_t, short) { - HotStuff::print_stat(); - //HotStuffCore::prune(100); - ev_stat_timer.add_with_timeout(stat_period); -} -- cgit v1.2.3-70-g09d2 From d781144f5edd0cacc436d04295ca68cff9f22036 Mon Sep 17 00:00:00 2001 From: Determinant Date: Thu, 23 Aug 2018 22:31:38 -0400 Subject: ... --- include/hotstuff/liveness.h | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/include/hotstuff/liveness.h b/include/hotstuff/liveness.h index 77d4232..404972d 100644 --- a/include/hotstuff/liveness.h +++ b/include/hotstuff/liveness.h @@ -235,6 +235,7 @@ class PMStickyProposer: virtual public PaceMaker { EventContext ec; /** QC timer or randomized timeout */ Event timer; + Event ev_imp; block_t last_proposed; /** the proposer it believes */ ReplicaID proposer; @@ -250,12 +251,6 @@ class PMStickyProposer: virtual public PaceMaker { promise_t pm_wait_propose; promise_t pm_qc_finish; - void reset_qc_timer() { - timer.del(); - timer.add_with_timeout(qc_timeout); - HOTSTUFF_LOG_INFO("QC timer reset"); - } - void clear_promises() { pm_wait_receive_proposal.reject(); pm_wait_propose.reject(); @@ -285,9 +280,8 @@ class PMStickyProposer: virtual public PaceMaker { to_candidate(); /* proposer misbehave */ return; } - HOTSTUFF_LOG_INFO("proposer emits new QC"); + HOTSTUFF_LOG_PROTO("proposer emits new QC"); last_proposed = prop.blk; - //reset_qc_timer(); } reg_follower_receive_proposal(); } @@ -304,6 +298,8 @@ class PMStickyProposer: virtual public PaceMaker { .then([this, pm]() { timer.del(); pm.resolve(proposer); + timer.add_with_timeout(qc_timeout); + HOTSTUFF_LOG_PROTO("QC timer reset"); }); locked = true; } @@ -317,7 +313,6 @@ class PMStickyProposer: virtual public PaceMaker { } void proposer_propose(const Proposal &prop) { - reset_qc_timer(); last_proposed = prop.blk; locked = false; proposer_schedule_next(); @@ -354,23 +349,24 @@ class PMStickyProposer: virtual public PaceMaker { propose_elect_block(); } - void reg_candidate_receive_proposal() { + void reg_cp_receive_proposal() { pm_wait_receive_proposal.reject(); (pm_wait_receive_proposal = hsc->async_wait_receive_proposal()) .then( salticidae::generic_bind( - &PMStickyProposer::candidate_receive_proposal, this, _1)); + &PMStickyProposer::cp_receive_proposal, this, _1)); } - void candidate_receive_proposal(const Proposal &prop) { + void cp_receive_proposal(const Proposal &prop) { auto _proposer = prop.proposer; auto &p = last_proposed_by[_proposer]; HOTSTUFF_LOG_PROTO("got block %s from %d", std::string(*prop.blk).c_str(), _proposer); p.reject(); - (p = hsc->async_qc_finish(prop.blk)).then([this, _proposer]() { - to_follower(_proposer); + (p = hsc->async_qc_finish(prop.blk)).then([this, blk=prop.blk, _proposer]() { + if (hsc->get_bqc()->get_qc_ref() == blk) + to_follower(_proposer); }); - reg_candidate_receive_proposal(); + reg_cp_receive_proposal(); } /* role transitions */ @@ -383,10 +379,6 @@ class PMStickyProposer: virtual public PaceMaker { last_proposed = nullptr; hsc->set_neg_vote(false); timer.clear(); - //timer = Event(ec, -1, 0, [this](int, short) { - // /* unable to get a QC in time */ - // to_candidate(); - //}); /* redirect all pending cmds to the new proposer */ while (!pending_beats.empty()) { @@ -407,6 +399,7 @@ class PMStickyProposer: virtual public PaceMaker { /* proposer unable to get a QC in time */ to_candidate(); }); + reg_cp_receive_proposal(); proposer_propose(Proposal(-1, uint256_t(), hsc->get_genesis(), nullptr)); } @@ -421,17 +414,17 @@ class PMStickyProposer: virtual public PaceMaker { candidate_qc_timeout(); }); candidate_timeout = qc_timeout; - reg_candidate_receive_proposal(); + reg_cp_receive_proposal(); candidate_qc_timeout(); } protected: void impeach() override { if (role == CANDIDATE) return; - timer = Event(ec, -1, 0, [this](int, short) { + ev_imp = Event(ec, -1, 0, [this](int, short) { to_candidate(); }); - timer.add_with_timeout(0); + ev_imp.add_with_timeout(0); HOTSTUFF_LOG_INFO("schedule to impeach the proposer"); } -- cgit v1.2.3-70-g09d2 From 75b5b9c9fe6a847510dfad452104fc6987567453 Mon Sep 17 00:00:00 2001 From: Determinant Date: Fri, 24 Aug 2018 09:18:54 -0400 Subject: WIP: RR pacemaker --- include/hotstuff/liveness.h | 269 +++++++++++++++++++++++++++++++++++++++++++- src/hotstuff_app.cpp | 2 + 2 files changed, 266 insertions(+), 5 deletions(-) diff --git a/include/hotstuff/liveness.h b/include/hotstuff/liveness.h index 404972d..8c9c9ab 100644 --- a/include/hotstuff/liveness.h +++ b/include/hotstuff/liveness.h @@ -206,23 +206,29 @@ class PaceMakerDummyFixed: public PaceMakerDummy { }; /** - * Simple long-standing proposer liveness gadget. + * Simple long-standing proposer liveness gadget (with randomization). * There are three roles for each replica: proposer, candidate and follower. * * For a proposer, it proposes a new block and refrains itself from proposing * the next block unless it receives the QC for the previous block. It will - * give up the leadership and turn into a candidate when it hasn't seen such QC - * for a while. + * give up the leadership and turn into a candidate when it sees QC for a + * higher block or being impeached. * * For a follower, it never proposes any block, but keeps a timer for the QC * for the block last proposed by the proposer (the proposer it believes to - * be). When it times out without seeing such QC, the follower turns into a - * candidate. + * be). When it times out without seeing such QC or the proposer is impeached, + * the follower turns into a candidate. * * For a candidate, it periodically proposes empty blocks to synchronize the * preferred branch, with randomized timeout, and check for any new QC. Once it * sees such new QC, if the QC is given by itself, it becomes the proposer, * otherwise yields to the creator of the QC as a follower. + * + * CAUTIONS: This pace maker does not guarantee liveness when a Byzantine node + * tries to contend with correct nodes and always proposes higher blocks to + * grab the leadership. If you want to use this for your system, please make + * sure you introduce mechanism to detect and ban such behavior, or use the + * round-robin style pace maker instead. */ class PMStickyProposer: virtual public PaceMaker { enum { @@ -471,6 +477,259 @@ struct PaceMakerSticky: public PMAllParents, public PMStickyProposer { } }; +/** + * Simple long-standing round-robin style proposer liveness gadget. + */ +class PMRoundRobinProposer: virtual public PaceMaker { + enum { + PROPOSER, + FOLLOWER, + CANDIDATE /* rotating */ + } role; + double qc_timeout; + double candidate_timeout; + EventContext ec; + /** QC timer or randomized timeout */ + Event timer; + Event ev_imp; + block_t last_proposed; + /** the proposer it believes */ + ReplicaID proposer; + + /* extra state needed for a proposer */ + std::queue pending_beats; + bool locked; + + /* extra state needed for a candidate */ + std::unordered_map last_proposed_by; + + promise_t pm_wait_receive_proposal; + promise_t pm_wait_propose; + promise_t pm_qc_finish; + + void clear_promises() { + pm_wait_receive_proposal.reject(); + pm_wait_propose.reject(); + pm_qc_finish.reject(); + for (auto &p: last_proposed_by) + p.second.reject(); + last_proposed_by.clear(); + } + + /* helper functions for a follower */ + + void reg_follower_receive_proposal() { + pm_wait_receive_proposal.reject(); + (pm_wait_receive_proposal = hsc->async_wait_receive_proposal()) + .then( + salticidae::generic_bind( + &PMRoundRobinProposer::follower_receive_proposal, this, _1)); + } + + void follower_receive_proposal(const Proposal &prop) { + if (prop.proposer == proposer) + { + auto &qc_ref = prop.blk->get_qc_ref(); + if (last_proposed && qc_ref != last_proposed) + { + HOTSTUFF_LOG_INFO("proposer misbehave"); + to_candidate(); /* proposer misbehave */ + return; + } + HOTSTUFF_LOG_PROTO("proposer emits new QC"); + last_proposed = prop.blk; + } + reg_follower_receive_proposal(); + } + + /* helper functions for a proposer */ + + void proposer_schedule_next() { + if (!pending_beats.empty() && !locked) + { + auto pm = pending_beats.front(); + pending_beats.pop(); + pm_qc_finish.reject(); + (pm_qc_finish = hsc->async_qc_finish(last_proposed)) + .then([this, pm]() { + timer.del(); + pm.resolve(proposer); + timer.add_with_timeout(qc_timeout); + HOTSTUFF_LOG_PROTO("QC timer reset"); + }); + locked = true; + } + } + + void reg_proposer_propose() { + pm_wait_propose.reject(); + (pm_wait_propose = hsc->async_wait_proposal()).then( + salticidae::generic_bind( + &PMRoundRobinProposer::proposer_propose, this, _1)); + } + + void proposer_propose(const Proposal &prop) { + last_proposed = prop.blk; + locked = false; + proposer_schedule_next(); + reg_proposer_propose(); + } + + void propose_elect_block() { + DataStream s; + /* FIXME: should extra data be the voter's id? */ + s << hsc->get_id(); + /* propose a block for leader election */ + hsc->on_propose(std::vector{}, + get_parents(), std::move(s)); + } + + /* helper functions for a candidate */ + + void reg_cp_receive_proposal() { + pm_wait_receive_proposal.reject(); + (pm_wait_receive_proposal = hsc->async_wait_receive_proposal()) + .then( + salticidae::generic_bind( + &PMRoundRobinProposer::cp_receive_proposal, this, _1)); + } + + void cp_receive_proposal(const Proposal &prop) { + auto _proposer = prop.proposer; + auto &p = last_proposed_by[_proposer]; + HOTSTUFF_LOG_PROTO("got block %s from %d", std::string(*prop.blk).c_str(), _proposer); + p.reject(); + (p = hsc->async_qc_finish(prop.blk)).then([this, blk=prop.blk, _proposer]() { + if (_proposer == proposer) + to_follower(); + }); + reg_cp_receive_proposal(); + } + + void candidate_qc_timeout() { + timer.del(); + timer.add_with_timeout(candidate_timeout); + candidate_timeout *= 1.01; + proposer = (proposer + 1) % hsc->get_config().nreplicas; + if (proposer == hsc->get_id()) + { + pm_qc_finish.reject(); + pm_wait_propose.reject(); + (pm_wait_propose = hsc->async_wait_proposal()).then([this](const Proposal &prop) { + const auto &blk = prop.blk; + (pm_qc_finish = hsc->async_qc_finish(blk)).then([this, blk]() { + HOTSTUFF_LOG_INFO("collected QC for %s", std::string(*blk).c_str()); + /* managed to collect a QC */ + to_proposer(); + propose_elect_block(); + }); + }); + propose_elect_block(); + } + HOTSTUFF_LOG_INFO("candidate rotates to %d, next try in %.2fs", + proposer, candidate_timeout); + } + + /* role transitions */ + + void to_follower() { + HOTSTUFF_LOG_INFO("new role: follower"); + clear_promises(); + role = FOLLOWER; + last_proposed = nullptr; + hsc->set_neg_vote(false); + timer.clear(); + /* redirect all pending cmds to the new proposer */ + while (!pending_beats.empty()) + { + pending_beats.front().resolve(proposer); + pending_beats.pop(); + } + reg_follower_receive_proposal(); + } + + void to_proposer() { + HOTSTUFF_LOG_INFO("new role: proposer"); + clear_promises(); + role = PROPOSER; + last_proposed = nullptr; + hsc->set_neg_vote(true); + timer = Event(ec, -1, 0, [this](int, short) { + /* proposer unable to get a QC in time */ + to_candidate(); + }); + proposer_propose(Proposal(-1, uint256_t(), hsc->get_genesis(), nullptr)); + } + + void to_candidate() { + HOTSTUFF_LOG_INFO("new role: candidate"); + clear_promises(); + role = CANDIDATE; + last_proposed = nullptr; + hsc->set_neg_vote(false); + timer = Event(ec, -1, 0, [this](int, short) { + candidate_qc_timeout(); + }); + candidate_timeout = qc_timeout * 0.1; + reg_cp_receive_proposal(); + candidate_qc_timeout(); + } + + protected: + void impeach() override { + if (role == CANDIDATE) return; + ev_imp = Event(ec, -1, 0, [this](int, short) { + to_candidate(); + }); + ev_imp.add_with_timeout(0); + HOTSTUFF_LOG_INFO("schedule to impeach the proposer"); + } + + public: + PMRoundRobinProposer(double qc_timeout, const EventContext &ec): + qc_timeout(qc_timeout), ec(ec), proposer(0) {} + + void init() { + to_candidate(); + } + + ReplicaID get_proposer() override { + return proposer; + } + + promise_t beat() override { + if (role != FOLLOWER) + { + promise_t pm; + pending_beats.push(pm); + if (role == PROPOSER) + proposer_schedule_next(); + return std::move(pm); + } + else + return promise_t([proposer=proposer](promise_t &pm) { + pm.resolve(proposer); + }); + } + + promise_t beat_resp(ReplicaID last_proposer) override { + return promise_t([this, last_proposer](promise_t &pm) { + pm.resolve(last_proposer); + }); + } +}; + +struct PaceMakerRR: public PMAllParents, public PMRoundRobinProposer { + PaceMakerRR(int32_t parent_limit, double qc_timeout, EventContext eb): + PMAllParents(parent_limit), PMRoundRobinProposer(qc_timeout, eb) {} + + void init(HotStuffCore *hsc) override { + PaceMaker::init(hsc); + PMAllParents::init(); + PMRoundRobinProposer::init(); + } +}; + } #endif diff --git a/src/hotstuff_app.cpp b/src/hotstuff_app.cpp index 210badc..768e81e 100644 --- a/src/hotstuff_app.cpp +++ b/src/hotstuff_app.cpp @@ -186,6 +186,8 @@ int main(int argc, char **argv) { hotstuff::pacemaker_bt pmaker; if (opt_pace_maker->get() == "sticky") pmaker = new hotstuff::PaceMakerSticky(parent_limit, opt_qc_timeout->get(), ec); + else if (opt_pace_maker->get() == "rr") + pmaker = new hotstuff::PaceMakerRR(parent_limit, opt_qc_timeout->get(), ec); else pmaker = new hotstuff::PaceMakerDummyFixed(opt_fixed_proposer->get(), parent_limit); -- cgit v1.2.3-70-g09d2 From 889d42f00f7f546322965a48e627e0ac76766002 Mon Sep 17 00:00:00 2001 From: Determinant Date: Mon, 27 Aug 2018 22:21:48 -0400 Subject: ... --- salticidae | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/salticidae b/salticidae index d2c471c..6b86aad 160000 --- a/salticidae +++ b/salticidae @@ -1 +1 @@ -Subproject commit d2c471c2300141529983be425f7ad75d371fcdf7 +Subproject commit 6b86aade15895d205f504ba70da93979bd6ee663 -- cgit v1.2.3-70-g09d2 From 17f7fd821cf71717a158e2c38699baa6ab2f2af8 Mon Sep 17 00:00:00 2001 From: Determinant Date: Tue, 28 Aug 2018 22:25:28 -0400 Subject: ... --- salticidae | 2 +- src/hotstuff.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/salticidae b/salticidae index 6b86aad..52e051c 160000 --- a/salticidae +++ b/salticidae @@ -1 +1 @@ -Subproject commit 6b86aade15895d205f504ba70da93979bd6ee663 +Subproject commit 52e051cb6924266003ef21814b5ebaf2b7b45241 diff --git a/src/hotstuff.cpp b/src/hotstuff.cpp index e1e2f81..e235bd8 100644 --- a/src/hotstuff.cpp +++ b/src/hotstuff.cpp @@ -25,7 +25,7 @@ void MsgVote::postponed_parse(HotStuffCore *hsc) { const opcode_t MsgReqBlock::opcode; MsgReqBlock::MsgReqBlock(const std::vector &blk_hashes) { - serialized << (uint32_t)htole(blk_hashes.size()); + serialized << htole((uint32_t)blk_hashes.size()); for (const auto &h: blk_hashes) serialized << h; } -- cgit v1.2.3-70-g09d2