Merge branch 'master' of github.com:Determinant/hot-stuff

6 files changed, 831 insertions, 7 deletions

diff --git a/scripts/gen_conf.py b/scripts/gen_conf.py
index bc45540..391e0d6 100644
--- a/scripts/gen_conf.py
+++ b/scripts/gen_conf.py
@@ -6,33 +6,43 @@ import argparse
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Generate configuration file for a batch of replicas')
     parser.add_argument('--prefix', type=str, default='hotstuff.gen')
-    parser.add_argument('--iplist', type=str, default=None)
+    parser.add_argument('--ips', type=str, default=None)
     parser.add_argument('--iter', type=int, default=10)
     parser.add_argument('--pport', type=int, default=10000)
     parser.add_argument('--cport', type=int, default=20000)
     parser.add_argument('--keygen', type=str, default='./hotstuff-keygen')
+    parser.add_argument('--nodes', type=str, default='nodes.txt')
+    parser.add_argument('--block-size', type=int, default=1)
+    parser.add_argument('--pace-maker', type=str, default='dummy')
     args = parser.parse_args()
 
 
-    if args.iplist is None:
+    if args.ips is None:
         ips = ['127.0.0.1']
     else:
-        ips = [l.strip() for l in open(args.iplist, 'r').readlines()]
+        ips = [l.strip() for l in open(args.ips, 'r').readlines()]
     prefix = args.prefix
     iter = args.iter
     base_pport = args.pport
     base_cport = args.cport
-    keygen_bin= args.keygen
+    keygen_bin = args.keygen
 
     main_conf = open("{}.conf".format(prefix), 'w')
+    nodes = open(args.nodes, 'w')
     replicas = ["{}:{};{}".format(ip, base_pport + i, base_cport + i)
                 for ip in ips
                 for i in range(iter)]
     p = subprocess.Popen([keygen_bin, '--num', str(len(replicas))],
                         stdout=subprocess.PIPE, stderr=open(os.devnull, 'w'))
     keys = [[t[4:] for t in l.decode('ascii').split()] for l in p.stdout]
+    if not (args.block_size is None):
+        main_conf.write("block-size = {}\n".format(args.block_size))
+    if not (args.pace_maker is None):
+        main_conf.write("pace-maker = {}\n".format(args.pace_maker))
     for r in zip(replicas, keys, itertools.count(0)):
         main_conf.write("replica = {}, {}\n".format(r[0], r[1][0]))
-        r_conf = open("{}-sec{}.conf".format(prefix, r[2]), 'w')
+        r_conf_name = "{}-sec{}.conf".format(prefix, r[2])
+        nodes.write("{}:{}\t{}\n".format(r[2], r[0], r_conf_name))
+        r_conf = open(r_conf_name, 'w')
         r_conf.write("privkey = {}\n".format(r[1][1]))
         r_conf.write("idx = {}\n".format(r[2]))
diff --git a/scripts/run.sh b/scripts/run.sh
new file mode 100755
index 0000000..53d9923
--- /dev/null
+++ b/scripts/run.sh
@@ -0,0 +1,457 @@
+#!/bin/bash
+
+proj_server_bin="hotstuff-app"
+proj_server_path="/home/ted/hot-stuff/$proj_server_bin"
+proj_conf_name="hotstuff.conf"
+
+peer_list="./nodes.txt"     # the list of nodes
+conf_src="./hotstuff.gen.conf"
+server_map="./server_map.txt"         # optional mapping from node ip to server ip
+template_dir="template"     # the dir that keeps the content shared among all nodes
+remote_base="/home/ted/testbed"  # remote dir used to keep files for the experiment
+#remote_base="/tmp/"  # remote dir used to keep files for the experiment
+remote_log="log"   # log filename
+remote_user="ted"
+copy_to_remote_pat="rsync -avz <local_path> <remote_user>@<remote_ip>:<remote_path>"
+copy_from_remote_pat="rsync -avz <remote_user>@<remote_ip>:<remote_path> <local_path>"
+exe_remote_pat="ssh <remote_user>@<remote_ip> bash"
+run_remote_pat="cd \"<rworkdir>\"; gdb -ex r -ex bt -ex generate-core-file -ex q --args '$proj_server_path' --conf \"hotstuff.gen-sec<node_id>.conf\""
+reset_remote_pat="pgrep -f '$proj_server_bin' | xargs kill -9"
+
+fin_keyword="error:"  # the keyword indicating completion of execution
+fin_chk_period=1
+fin_chk_skip_pat='^([A-O][0-9]*)|(_ctl)$'
+force_peer_list=0
+
+function join { local IFS="$1"; shift; echo "$*"; }
+function split {
+    local IFS="$1"
+    local arr=($2)
+    echo "${arr[@]}"
+}
+
+function die { echo "$1"; exit 1; }
+
+declare -A nodes
+declare -A node_confs
+nodes_cnt=0
+function get_node_info {
+    pl="$1"
+    if [[ "$force_peer_list" == 1 ]]; then
+        pl="$peer_list"
+    fi
+    OIFS="$IFS"
+    IFS=$'\n'
+    node_list=($(cat "$pl"))
+    IFS="$OIFS"
+    for tuple in "${node_list[@]}"; do
+        tup0=($(split $'\t' "$tuple"))
+        tup=($(split : "${tup0[0]}"))
+        nodes[${tup[0]}]="${tup[1]}:${tup[2]}"
+        node_confs[${tup[0]}]="${tup0[@]:1}"
+        echo "${tup[0]} => ${nodes[${tup[0]}]} & ${node_confs[${tup[0]}]}"
+        let nodes_cnt++
+    done
+}
+
+declare -A server_map
+function get_server_map {
+    {
+        IFS=$'\n'
+        map_list=($(cat "$1"))
+    }
+    IFS=$'\n \t'
+    for pair in "${map_list[@]}"; do
+        p=($pair)
+        server_map[${p[0]}]="${p[1]}"
+        echo "mapping ${p[0]} => ${p[1]}"
+    done
+}
+
+
+function get_addr {
+    tup=($(split ';' $1))
+    echo "${tup[0]}"
+}
+
+function get_ip {
+    tup=($(split : $1))
+    echo "${tup[0]}"
+}
+
+function get_peer_port {
+    tup=($(split : $1))
+    tup2=($(split ';' ${tup[1]}))
+    echo "${tup2[0]}"
+}
+
+
+function get_client_port {
+    tup=($(split : $1))
+    tup2=($(split ';' ${tup[1]}))
+    echo "${tup2[1]}"
+}
+
+
+function get_ip_by_id {
+    get_ip "${nodes[$1]}"
+}
+
+function get_peer_port_by_id {
+    get_peer_port "${nodes[$1]}"
+}
+
+
+function get_client_port_by_id {
+    get_client_port "${nodes[$1]}"
+}
+
+function copy_file {
+    local pat="$1"
+    local cmd="${pat//<local_path>/$2}"
+    cmd="${cmd//<remote_ip>/$3}"
+    cmd="${cmd//<remote_user>/$remote_user}"
+    cmd="${cmd//<remote_path>/$4}"
+    echo $cmd
+    eval "$cmd"
+} >> log 2>&1
+
+function execute_remote_cmd_pid {
+    local node_ip="$1"
+    local c="$2"
+    local l="$3"
+    local cmd="${exe_remote_pat//<remote_ip>/$node_ip}"
+    cmd="${cmd//<remote_user>/$remote_user}"
+    eval $cmd << EOF
+$c > $l 2>&1 & echo \$!
+EOF
+}
+
+
+
+function execute_remote_cmd_stat {
+    local node_ip="$1"
+    local c="$2"
+    local l="$3"
+    local cmd="${exe_remote_pat//<remote_ip>/$node_ip}"
+    cmd="${cmd//<remote_user>/$remote_user}"
+    eval $cmd << EOF
+$c > $l 2>&1 ; echo \$?
+EOF
+}
+
+
+function _remote_load {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_ip="$3"
+    local rid="$4"
+    local extra_conf=($5)
+    local tmpldir="$workdir/$template_dir/"
+    local node_tmpldir="$workdir/$rid"
+    [[ $(execute_remote_cmd_stat "$node_ip" \
+        "mkdir -p \"$rworkdir\"" \
+        /dev/null) == 0 ]] || die "failed to create directory $rworkdir"
+    copy_file "$copy_to_remote_pat" "$tmpldir" "$node_ip" "$rworkdir"
+    for conf in "${extra_conf[@]}"; do
+        copy_file "$copy_to_remote_pat" "$node_tmpldir/$conf" "$node_ip" "$rworkdir"
+    done
+}
+
+function _remote_start {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_id="$3"
+    local node_ip="$4"
+    local client_port="$5"
+    local cmd="${run_remote_pat//<rworkdir>/$rworkdir}"
+    cmd="${cmd//<node_id>/$node_id}"
+    cmd="${cmd//<cport>/$client_port}"
+    execute_remote_cmd_pid "$node_ip" "$cmd" \
+        "\"$rworkdir/$remote_log\"" > "$workdir/${node_id}.pid"
+}
+
+function _remote_exec {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_ip="$3"
+    local cmd="$4"
+    [[ $(execute_remote_cmd_stat "$node_ip" "$cmd" /dev/null) == 0 ]]
+}
+
+function _remote_stop {
+    local node_pid="$4"
+    _remote_exec "$1" "$2" "$3" "kill $node_pid"
+}
+
+function _remote_status {
+    local node_pid="$4"
+    _remote_exec "$1" "$2" "$3" "kill -0 $node_pid"
+}
+
+function _remote_finished {
+    _remote_exec "$1" "$2" "$3" "grep \"$fin_keyword\" \"$rworkdir/$remote_log\""
+}
+
+function _remote_fetch {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_id="$3"
+    local node_ip="$4"
+    copy_file "$copy_from_remote_pat" "$workdir/${node_id}.log" "$node_ip" "$rworkdir/$remote_log"
+}
+
+function start_all {
+    local workdir="$1"
+    local tmpldir="$workdir/$template_dir/"
+    mkdir "$workdir" > /dev/null 2>&1 || die "workdir already exists"
+    rm -rf "$tmpldir"
+    mkdir "$tmpldir"
+    cp "$peer_list" "$workdir/peer_list.txt"
+    cp "$server_map" "$workdir/server_map.txt"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    echo "copying configuration file"
+    cp "$conf_src" "$tmpldir/$proj_conf_name"
+    for rid in "${!nodes[@]}"; do
+        local node_tmpldir="$workdir/$rid"
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local pport="$(get_peer_port_by_id $rid)"
+        local cport="$(get_client_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        local extra_conf_=(${node_confs[$rid]})
+        rm -rf "$node_tmpldir"
+        mkdir "$node_tmpldir"
+        (
+        local extra_conf=()
+        for conf in "${extra_conf_[@]}"; do
+            cp "$conf" "$node_tmpldir/"
+            extra_conf+=($(basename "$conf"))
+            copy_file "$copy_to_remote_pat" "$tmpldir/$conf" "$node_ip" "$rworkdir"
+        done
+        echo "Starting $rid @ $ip, $pport and $cport"
+        _remote_load "$workdir" "$rworkdir" "$ip" "$rid" "${extra_conf[@]}"
+        echo "$rid loaded"
+        ) &
+    done
+    wait
+    for rid in "${!nodes[@]}"; do
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local pport="$(get_peer_port_by_id $rid)"
+        local cport="$(get_client_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        (
+        echo "Starting $rid @ $ip, $pport and $cport"
+        _remote_start "$workdir" "$rworkdir" "$rid" "$ip" "$cport"
+        echo "$rid started"
+        ) &
+    done
+    wait
+}
+
+function fetch_all {
+    local workdir="$1"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    for rid in "${!nodes[@]}"; do
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local port="$(get_peer_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        local pid="$(cat $workdir/${rid}.pid)"
+        local msg="Fetching $rid @ $ip, $port "
+        _remote_fetch "$workdir" "$rworkdir" "$rid" "$ip" && echo "$msg: copied" || echo "$msg: failed" &
+    done
+    wait
+}
+
+function exec_all {
+    local workdir="$1"
+    local cmd="$2"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    for rid in "${!nodes[@]}"; do
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local port="$(get_peer_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        local msg="Executing $rid @ $ip, $port "
+        _remote_exec "$workdir" "$rworkdir" "$ip" "$cmd" && echo "$msg: succeeded" || echo "$msg: failed" &
+    done
+    wait
+}
+
+function reset_all {
+    exec_all "$1" "$reset_remote_pat"
+}
+
+function stop_all {
+    local workdir="$1"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    for rid in "${!nodes[@]}"; do
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local port="$(get_peer_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        local pid="$(cat $workdir/${rid}.pid)"
+        local msg="Killing $rid @ $ip, $port "
+        _remote_stop "$workdir" "$rworkdir" "$ip" "$pid" && echo "$msg: stopped" || echo "$msg: failed" &
+    done
+    wait
+}
+
+function status_all {
+    local workdir="$1"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    for rid in "${!nodes[@]}"; do
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local port="$(get_peer_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        local pid="$(cat $workdir/${rid}.pid)"
+        local msg="$rid @ $ip, $port "
+        _remote_status "$workdir" "$rworkdir" "$ip" "$pid" && echo "$msg: running" || echo "$msg: dead" &
+    done
+    wait
+}
+
+function finished_all {
+    local workdir="$1"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    for rid in "${!nodes[@]}"; do
+        local ip="$(get_ip_by_id $rid)"
+        ip="${server_map[$ip]:-$ip}"
+        local port="$(get_peer_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${rid}"
+        if [[ "$rid" =~ $fin_chk_skip_pat ]]; then
+            continue
+        fi
+        printf "$rid @ $ip, $port "
+        _remote_finished "$workdir" "$rworkdir" "$ip" && echo "finished" || echo "in-progress"
+    done
+}
+
+function wait_all {
+    local workdir="$1"
+    get_node_info "$workdir/peer_list.txt"
+    get_server_map "$workdir/server_map.txt"
+    while true; do
+        finished=1
+        printf "checking the nodes..."
+        for rid in "${!nodes[@]}"; do
+            local ip="$(get_ip_by_id $rid)"
+            ip="${server_map[$ip]:-$ip}"
+            local port="$(get_peer_port_by_id $rid)"
+            local rworkdir="$remote_base/$workdir/${rid}"
+            if [[ "$rid" =~ $fin_chk_skip_pat ]]; then
+                continue
+            fi
+            if ! _remote_finished "$workdir" "$rworkdir" "$ip"; then
+                finished=0
+                break
+            fi
+        done
+        if [[ $finished == 1 ]]; then
+            break
+        fi
+        echo "not finished yet, wait for $fin_chk_period secs"
+        sleep "$fin_chk_period"
+    done
+    echo "finished"
+}
+
+function check_all {
+    status_all "$1" | grep dead -q
+    [[ "$?" -eq 0 ]] && die "some nodes are dead"
+    echo "ok"
+}
+
+function print_help {
+echo "Usage: $0 [--bin] [--path] [--conf] [--conf-src] [--peer-list] [--server-map] [--user] [--force-peer-list] [--help] COMMAND WORKDIR
+
+    --help                      show this help and exit
+    --bin                       name of binary executable
+    --path                      path to the binary
+    --conf                      shared configuration filename
+    --conf-src                  shared configuration source file
+    --peer-list FILE            read peer list from FILE (default: $peer_list)
+    --server-map FILE           read server map from FILE (default: $server_map)
+    --user      USER            the username to login the remote machines
+    --force-peer-list           force the use of FILE specified by --peer-list
+                                instead of the peer list in WORKDIR"
+    exit 0
+}
+
+function check_argnum {
+    argnum=$(($# - 1))
+    [[ "$1" -eq "$argnum" ]] || die "incorrect argnum: got $argnum, $1 expected"
+}
+
+getopt --test > /dev/null
+[[ $? -ne 4 ]] && die "getopt unsupported"
+
+SHORT=
+LONG='\
+bin:,path:,conf:,conf-src:,\
+peer-list:,\
+server-map:,\
+remote-base:,\
+remote-user:,\
+copy-to-remote-pat:,\
+copy-from-remote-pat:,\
+exe-remote-pat:,\
+run-remote-pat:,\
+reset-remote-pat:,\
+fin-keyword:,\
+fin-chk-period:,\
+fin-chk-skip-pat:,\
+force-peer-list,\
+help'
+
+PARSED=$(getopt --options "$SHORT" --longoptions "$LONG" --name "$0" -- "$@")
+[[ $? -ne 0 ]] && exit 1
+eval set -- "$PARSED"
+
+while true; do
+    case "$1" in
+        --bin) proj_server_bin="$2"; shift 2;;
+        --path) proj_server_path="$2"; shift 2;;
+        --conf) proj_conf_name="$2"; shift 2;;
+        --conf-src) conf_src="$2"; shift 2;;
+        --peer-list) peer_list="$2"; shift 2;;
+        --server-map) server_map="$2"; shift 2;;
+        --remote-base) remote_base="$2"; shift 2;;
+        --remote-user) remote_user="$2"; shift 2;;
+        --copy-to-remote-pat) copy_to_remote_pat="$2"; shift 2;;
+        --copy-from-remote-pat) copy_from_remote_pat="$2"; shift 2;;
+        --exe-remote-pat) exe_remote_pat="$2"; shift 2;;
+        --run-remote-pat) run_remote_pat="$2"; shift 2;;
+        --reset-remote-pat) reset_remote_pat="$2"; shift 2;;
+        --fin-keyword) fin_keyword="$2"; shift 2;;
+        --fin-chk-period) fin_chk_period="$2"; shift 2;;
+        --fin-chk-skip-pat) fin_chk_skip_pat="$2"; shift 2;;
+        --force-peer-list) force_peer_list=1; shift 1;;
+        --help) print_help; shift 1;;
+        --) shift; break;;
+        *) die "internal error";;
+    esac
+done
+cmd="$1"
+shift 1
+case "$cmd" in
+    start) check_argnum 1 "$@" && start_all "$1" ;;
+    stop) check_argnum 1 "$@" && stop_all "$1" ;;
+    status) check_argnum 1 "$@" && status_all "$1" ;;
+    check) check_argnum 1 "$@" && check_all "$1" ;;
+    finished) check_argnum 1 "$@" && finished_all "$1" ;;
+    fetch) check_argnum 1 "$@" && fetch_all "$1" ;;
+    wait) check_argnum 1 "$@" && wait_all "$1" ;;
+    reset) check_argnum 1 "$@" && reset_all "$1" ;;
+    exec) check_argnum 2 "$@" && exec_all "$1" "$2" ;;
+    *) print_help ;;
+esac
diff --git a/scripts/run_client.sh b/scripts/run_client.sh
index 93a9148..090dce1 100755
--- a/scripts/run_client.sh
+++ b/scripts/run_client.sh
@@ -1,2 +1,354 @@
 #!/bin/bash
-./hotstuff-client --idx 0 --iter -1 --max-async 3
+
+proj_client_bin="hotstuff-client"
+proj_client_path="/home/ted/hot-stuff/$proj_client_bin"
+proj_conf_name="hotstuff.conf"
+
+peer_list="./nodes.txt"     # the list of nodes
+client_list="./clients.txt"  # the list of clients
+conf_src="./hotstuff.gen.conf"
+template_dir="template"     # the dir that keeps the content shared among all nodes
+remote_base="/home/ted/testbed"  # remote dir used to keep files for the experiment
+#remote_base="/tmp/"  # remote dir used to keep files for the experiment
+remote_log="log"   # log filename
+remote_user="ted"
+copy_to_remote_pat="rsync -avz <local_path> <remote_user>@<remote_ip>:<remote_path>"
+copy_from_remote_pat="rsync -avz <remote_user>@<remote_ip>:<remote_path> <local_path>"
+exe_remote_pat="ssh <remote_user>@<remote_ip> bash"
+run_remote_pat="cd \"<rworkdir>\"; '$proj_client_path' --idx \"<node_id>\" --iter -1 --max-async 3"
+reset_remote_pat="pgrep -f '$proj_client_bin' | xargs kill -9"
+
+function join { local IFS="$1"; shift; echo "$*"; }
+function split {
+    local IFS="$1"
+    local arr=($2)
+    echo "${arr[@]}"
+}
+
+function die { echo "$1"; exit 1; }
+
+declare -A nodes
+nodes_cnt=0
+function get_node_info {
+    pl="$1"
+    if [[ "$force_peer_list" == 1 ]]; then
+        pl="$peer_list"
+    fi
+    OIFS="$IFS"
+    IFS=$'\n'
+    node_list=($(cat "$pl"))
+    IFS="$OIFS"
+    for tuple in "${node_list[@]}"; do
+        tup0=($(split $'\t' "$tuple"))
+        tup=($(split : "${tup0[0]}"))
+        nodes[${tup[0]}]="${tup[1]}:${tup[2]}"
+        echo "${tup[0]} => ${nodes[${tup[0]}]}"
+        let nodes_cnt++
+    done
+}
+
+function get_client_info {
+    cip_list=($(cat "$1"))
+}
+
+
+function get_addr {
+    tup=($(split ';' $1))
+    echo "${tup[0]}"
+}
+
+function get_ip {
+    tup=($(split : $1))
+    echo "${tup[0]}"
+}
+
+function get_peer_port {
+    tup=($(split : $1))
+    tup2=($(split ';' ${tup[1]}))
+    echo "${tup2[0]}"
+}
+
+
+function get_client_port {
+    tup=($(split : $1))
+    tup2=($(split ';' ${tup[1]}))
+    echo "${tup2[1]}"
+}
+
+
+function get_ip_by_id {
+    get_ip "${nodes[$1]}"
+}
+
+function get_peer_port_by_id {
+    get_peer_port "${nodes[$1]}"
+}
+
+
+function get_client_port_by_id {
+    get_client_port "${nodes[$1]}"
+}
+
+function copy_file {
+    local pat="$1"
+    local cmd="${pat//<local_path>/$2}"
+    cmd="${cmd//<remote_ip>/$3}"
+    cmd="${cmd//<remote_user>/$remote_user}"
+    cmd="${cmd//<remote_path>/$4}"
+    echo $cmd
+    eval "$cmd"
+} >> log 2>&1
+
+function execute_remote_cmd_pid {
+    local node_ip="$1"
+    local c="$2"
+    local l="$3"
+    local cmd="${exe_remote_pat//<remote_ip>/$node_ip}"
+    cmd="${cmd//<remote_user>/$remote_user}"
+    eval $cmd << EOF
+$c > $l 2>&1 & echo \$!
+EOF
+}
+
+
+
+function execute_remote_cmd_stat {
+    local node_ip="$1"
+    local c="$2"
+    local l="$3"
+    local cmd="${exe_remote_pat//<remote_ip>/$node_ip}"
+    cmd="${cmd//<remote_user>/$remote_user}"
+    eval $cmd << EOF
+$c > $l 2>&1 ; echo \$?
+EOF
+}
+
+
+function _remote_load {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_ip="$3"
+    local tmpldir="$workdir/$template_dir/"
+    [[ $(execute_remote_cmd_stat "$node_ip" \
+        "mkdir -p \"$rworkdir\"" \
+        /dev/null) == 0 ]] || die "failed to create directory $rworkdir"
+    copy_file "$copy_to_remote_pat" "$tmpldir" "$node_ip" "$rworkdir"
+}
+
+function _remote_start {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_id="$3"
+    local node_ip="$4"
+    local client_port="$5"
+    local client_ip="$6"
+    local cmd="${run_remote_pat//<rworkdir>/$rworkdir}"
+    cmd="${cmd//<node_id>/$node_id}"
+    cmd="${cmd//<server>/$node_ip:$client_port}"
+    execute_remote_cmd_pid "$client_ip" "$cmd" \
+        "\"$rworkdir/$remote_log\"" > "$workdir/${node_id}.pid"
+}
+
+function _remote_exec {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_ip="$3"
+    local cmd="$4"
+    [[ $(execute_remote_cmd_stat "$node_ip" "$cmd" /dev/null) == 0 ]]
+}
+
+function _remote_stop {
+    local node_pid="$4"
+    _remote_exec "$1" "$2" "$3" "kill $node_pid"
+}
+
+function _remote_status {
+    local node_pid="$4"
+    _remote_exec "$1" "$2" "$3" "kill -0 $node_pid"
+}
+
+function _remote_fetch {
+    local workdir="$1"
+    local rworkdir="$2"
+    local node_id="$3"
+    local node_ip="$4"
+    copy_file "$copy_from_remote_pat" "$workdir/${node_id}.log" "$node_ip" "$rworkdir/$remote_log"
+}
+
+function start_all {
+    local workdir="$1"
+    local tmpldir="$workdir/$template_dir/"
+    mkdir "$workdir" > /dev/null 2>&1 || die "workdir already exists"
+    rm -rf "$tmpldir"
+    mkdir "$tmpldir"
+    cp "$peer_list" "$workdir/peer_list.txt"
+    cp "$client_list" "$workdir/client_list.txt"
+    get_node_info "$workdir/peer_list.txt"
+    get_client_info "$workdir/client_list.txt"
+    echo "coyping configuration file"
+    cp "$conf_src" "$tmpldir/$proj_conf_name"
+    local nclient="${#cip_list[@]}"
+    local i=0
+    for tuple in "${node_list[@]}"; do
+        local cip="${cip_list[$i]}"
+        local tup=($(split : "$tuple"))
+        local rid="${tup[0]}"
+        local ip="$(get_ip_by_id $rid)"
+        local pport="$(get_peer_port_by_id $rid)"
+        local cport="$(get_client_port_by_id $rid)"
+        local rworkdir="$remote_base/$workdir/${i}"
+        (
+        echo "Starting a client @ $cip, connecting to server #$rid @ $ip:$cport"
+        _remote_load "$workdir" "$rworkdir" "$cip"
+        _remote_start "$workdir" "$rworkdir" "$i" "$ip" "$cport" "$cip"
+        echo "client #$i started"
+        ) &
+        let i++
+        if [[ "$i" -eq "$nclient" ]]; then
+            break
+        fi
+    done
+    wait
+}
+
+function fetch_all {
+    local workdir="$1"
+    get_client_info "$workdir/client_list.txt"
+    local i=0
+    for cip in "${cip_list[@]}"; do
+        local rworkdir="$remote_base/$workdir/${i}"
+        local pid="$(cat $workdir/${i}.pid)"
+        local msg="Fetching $i @ $cip"
+        _remote_fetch "$workdir" "$rworkdir" "$i" "$cip" && echo "$msg: copied" || echo "$msg: failed" &
+        let i++
+    done
+    wait
+}
+
+function exec_all {
+    local workdir="$1"
+    local cmd="$2"
+    get_client_info "$workdir/client_list.txt"
+    local i=0
+    for cip in "${cip_list[@]}"; do
+        local rworkdir="$remote_base/$workdir/${i}"
+        local msg="Executing $i @ $cip"
+        _remote_exec "$workdir" "$rworkdir" "$cip" "$cmd" && echo "$msg: succeeded" || echo "$msg: failed" &
+        let i++
+    done
+    wait
+}
+
+function reset_all {
+    exec_all "$1" "$reset_remote_pat"
+}
+
+function stop_all {
+    local workdir="$1"
+    get_client_info "$workdir/client_list.txt"
+    local i=0
+    for cip in "${cip_list[@]}"; do
+        local rworkdir="$remote_base/$workdir/${i}"
+        local pid="$(cat $workdir/${i}.pid)"
+        local msg="Killing $i @ $cip"
+        _remote_stop "$workdir" "$rworkdir" "$cip" "$pid" && echo "$msg: stopped" || echo "$msg: failed" &
+        let i++
+    done
+    wait
+}
+
+function status_all {
+    local workdir="$1"
+    get_client_info "$workdir/client_list.txt"
+    local i=0
+    for cip in "${cip_list[@]}"; do
+        local rworkdir="$remote_base/$workdir/${i}"
+        local pid="$(cat $workdir/${i}.pid)"
+        local msg="$i @ $cip"
+        _remote_status "$workdir" "$rworkdir" "$cip" "$pid" && echo "$msg: running" || echo "$msg: dead" &
+        let i++
+    done
+    wait
+}
+
+function check_all {
+    status_all "$1" | grep dead -q
+    [[ "$?" -eq 0 ]] && die "some nodes are dead"
+    echo "ok"
+}
+
+function print_help {
+echo "Usage: $0 [--bin] [--path] [--conf] [--conf-src] [--peer-list] [--client-list] [--user] [--force-peer-list] [--help] COMMAND WORKDIR
+
+    --help                      show this help and exit
+    --bin                       name of binary executable
+    --path                      path to the binary
+    --conf                      shared configuration filename
+    --conf-src                  shared configuration source file
+    --peer-list FILE            read peer list from FILE (default: $peer_list)
+    --client-list FILE          read client list from FILE (default: $client_list)
+    --user      USER            the username to login the remote machines
+    --force-peer-list           force the use of FILE specified by --peer-list
+                                instead of the peer list in WORKDIR"
+    exit 0
+}
+
+function check_argnum {
+    argnum=$(($# - 1))
+    [[ "$1" -eq "$argnum" ]] || die "incorrect argnum: got $argnum, $1 expected"
+}
+
+getopt --test > /dev/null
+[[ $? -ne 4 ]] && die "getopt unsupported"
+
+SHORT=
+LONG='\
+bin:,path:,conf:,conf-src:,\
+peer-list:,\
+client-list:,\
+remote-base:,\
+remote-user:,\
+copy-to-remote-pat:,\
+copy-from-remote-pat:,\
+exe-remote-pat:,\
+run-remote-pat:,\
+reset-remote-pat:,\
+force-peer-list,\
+help'
+
+PARSED=$(getopt --options "$SHORT" --longoptions "$LONG" --name "$0" -- "$@")
+[[ $? -ne 0 ]] && exit 1
+eval set -- "$PARSED"
+
+while true; do
+    case "$1" in
+        --bin) proj_client_bin="$2"; shift 2;;
+        --path) proj_client_path="$2"; shift 2;;
+        --conf) proj_conf_name="$2"; shift 2;;
+        --conf-src) conf_src="$2"; shift 2;;
+        --peer-list) peer_list="$2"; shift 2;;
+        --client-list) client_list="$2"; shift 2;;
+        --remote-base) remote_base="$2"; shift 2;;
+        --remote-user) remote_user="$2"; shift 2;;
+        --copy-to-remote-pat) copy_to_remote_pat="$2"; shift 2;;
+        --copy-from-remote-pat) copy_from_remote_pat="$2"; shift 2;;
+        --exe-remote-pat) exe_remote_pat="$2"; shift 2;;
+        --run-remote-pat) run_remote_pat="$2"; shift 2;;
+        --reset-remote-pat) reset_remote_pat="$2"; shift 2;;
+        --help) print_help; shift 1;;
+        --) shift; break;;
+        *) die "internal error";;
+    esac
+done
+cmd="$1"
+shift 1
+case "$cmd" in
+    start) check_argnum 1 "$@" && start_all "$1" ;;
+    stop) check_argnum 1 "$@" && stop_all "$1" ;;
+    status) check_argnum 1 "$@" && status_all "$1" ;;
+    check) check_argnum 1 "$@" && check_all "$1" ;;
+    fetch) check_argnum 1 "$@" && fetch_all "$1" ;;
+    reset) check_argnum 1 "$@" && reset_all "$1" ;;
+    exec) check_argnum 2 "$@" && exec_all "$1" "$2" ;;
+    *) print_help ;;
+esac
diff --git a/scripts/run_replicas.sh b/scripts/run_demo.sh
index 5f54787..5f54787 100755
--- a/scripts/run_replicas.sh
+++ b/scripts/run_demo.sh
diff --git a/scripts/run_demo_client.sh b/scripts/run_demo_client.sh
new file mode 100755
index 0000000..93a9148
--- /dev/null
+++ b/scripts/run_demo_client.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+./hotstuff-client --idx 0 --iter -1 --max-async 3
diff --git a/scripts/thr_hist.py b/scripts/thr_hist.py
index 6f6a43f..c5f2a72 100644
--- a/scripts/thr_hist.py
+++ b/scripts/thr_hist.py
@@ -24,17 +24,19 @@ if __name__ == '__main__':
     parser.add_argument('--interval', type=float, default=1, required=False)
     parser.add_argument('--output', type=str, default="hist.png", required=False)
     args = parser.parse_args()
-    commit_pat = re.compile('([^[].*) \[hotstuff info\].*got <fin decision=1')
+    commit_pat = re.compile('([^[].*) \[hotstuff info\] ([0-9.]*) [0-9.]*$')
     interval = args.interval
     begin_time = None
     next_begin_time = None
     cnt = 0
+    lat = 0
     timestamps = []
     values = []
     for line in sys.stdin:
         m = commit_pat.match(line)
         if m:
             timestamps.append(str2datetime(m.group(1)))
+            lat += float(m.group(2))
     timestamps.sort()
     for timestamp in timestamps:
         if begin_time and timestamp < next_begin_time:
@@ -47,4 +49,5 @@ if __name__ == '__main__':
             cnt = 1
     values.append(cnt)
     print(values)
+    print("lat = {:.3f}ms".format(lat / len(timestamps) * 1e3))
     plot_thr(args.output)