adamocomp/broker.sh

#!/usr/bin/env bash

# WARNING: functions do not test for permissions (mostly).

print_help () {
	echo "Usage: ./broker.sh [<options>] <command> [<args>]"
	echo ""
	echo "Options: "
	echo " -t, --testing    Report operations without executing them."
	echo " -e, --env        Set the environment variables from data."
	echo ""
	echo "Commands: "
	print_help_item\
		"info <deal name>"\
		"Collects all information for deal and prints it."
	print_help_item\
		"complete_local <deal name>"\
		"Updates and installs e.g. gitlab"
	print_help_item\
		"stop_container <deal name>"\
		"Stops a single container, e.g. 'gitlab'"
	print_help_item\
		"restart_all_containers [<root dir>]"\
		"Restarts all containers or all containers under root dir."
	print_help_item\
		"stop_all_containers [<root dir>]"\
		"Stops all containers or all containers under root dir."
	print_help_item\
		"setup_local"\
		"Installs environment data and software requirements locally."
	print_help_item\
		"install_on_nodes <cluster> <deal>"\
		"Installs fresh deal across nodes in range."
	print_help_item\
		"remote_node_setup <cluster>"\
		"Remotely updates adamocomp on nodes."
}

print_help_item () {
	printf ' %-69.69s \n\t\t %-61.61s \n\n' "$1" "$2"
}

# These positional arguments get run at the end of the script.
POSITIONAL=()
# These options get passed to brokers on remote systems.
PASS_OPTS=()
while [[ $# -gt 0 ]]; do
	key="$1"
	
	case $key in
	    -t|--testing)
	    TESTING_MODE="TRUE"
	    PASS_OPTS+=("-t")
	    shift # past option
	    ;;
	    # Set environment variables in data when running commands.
	    -e|--env)
		SET_ENV="TRUE"
		PASS_OPTS+=("-e")
		shift
		;;
	    -h|--help)
		print_help
		exit 0
	    ;;
	    -l|--lib)
	    LIBPATH="$2"
	    shift # past argument
	    shift # past value
	    ;;
	    # Don't use.
	    -d|--debug)
	    DEBUG=TRUE
	    shift # past argument
	    ;;
	    *)    # unknown option
	    POSITIONAL+=("$1") # save it in an array for later
	    shift # past argument
	    ;;
	esac
done

######################################################################
# Directory and config structure

program_root="$(\
	cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd\
)"
self=${program_root}/broker.sh
stage_root="${program_root}/staging"
target_root=${ADAMOCOMP_HOME:-"/"}
config_dir=${ADAMOCOMP_CONFDIR:-"${program_root}"}

if [[ "$TESTING_MODE" == "TRUE" ]]; then
	mkdir -p "${program_root}/errors"
	errfile="${program_root}/errors/`date '+%D.%H:%M:%S'|
		sed 's|/|.|g'`"
else
	errfile="/dev/null"
fi


# Non-public configurations; eventually should be off-loaded to Vault.
data_file="${config_dir}/data.json"
data_json="$(jq '.' ${data_file} 2>>$errfile)"

if [[ -r "${program_root}/data.json" ]]; then
	data_file="${program_root}/data.json"
	data_json="$(jq '.' ${data_file} 2>>$errfile)"
fi

######################################################################
# Data interface

datatool="${program_root}/data-tool.js -p ${data_file}"
editor="/usr/bin/nano"

if [[ $DEBUG == "TRUE" ]]; then
	datatool="${datatool} -d"
fi

user_edit () {
	$editor "$@"
}

get_filemode () {
	stat -c '%a' $1
}

option_str_from_arr () {
	opt_str=""
	flag="--publish"
	options=("443:443" "80:80" "22:22")	
	for option in ${options[@]}; do
		opt_str="${opt_str} ${flag} ${option} "
	done
}

jq_array_size () {
	json=`cat`
	size=$(echo $json|jq 'length' 2>>$errfile)
	if [[ size == "" ]]; then
		echo 0
		return 1
	fi
	echo $size
	return
}

json_to_bash_array () {
	local json=`cat`
	if jq -e . >/dev/null 2>&1 <<<"$json"; then
		echo $json|jq -cr '.[]'
	else
		echo ""
	fi
	return
}

data_array_size () {
	get_data $*|jq_array_size
}

data_array_seq_end () {
	echo $(( $(data_array_size $*) - 1 ))
}

get_data_array () {
	get_data "$*"|jq -cr '.[]'
}

get_data () {
	${datatool} get "$*"|jq -r '.[0]'
}

get_keys () {
	${datatool} get "$*"|jq -r '.[0]|keys'
}

get_key () {
	get_keys "${@:2}"|jq -r ".[$1]"
}

count_keys () {
	get_keys "$*"|jq_array_size
}

get_path () {
	${datatool} path "$*"|jq -r '.[0]'
}

get_cluster_size () {
	get_hosts "$*"|jq_array_size
}

get_hosts () {
	${datatool} hosts "$*"|jq '.[0]'
}

get_hosts_array () {
	get_hosts "$*"|json_to_bash_array
}


		#for i in `seq 1 $cluster_size`; do
			#local host_name=$(get_key $((i-1)) $cluster_name)

deal_exists () {
	${datatool} valid_deal "$*"
}

cluster_exists () {
	${datatool} valid_cluster "$*"
}

host_exists () {
	${datatool} valid_host "$*"
}

get_remote_host () {
	if host_exists $*; then
		get_data $* remote host
	fi
}

get_remote_ssh_port () {
	if host_exists $*; then
		get_data $* remote ports ssh
	fi
}

get_remote_user () {
	local user=$(get_data $* remote user)
	if [[ -z $user ]] || [[ $user == "null" ]]; then
		echo $dealer
	else
		echo $user
	fi
	return
}

######################################################################
# Formatted printing and debug.

# Prints a table of variable names and values to stdout.
pr_vars () {
	printf "%s \n" "----------------------------"
	pr_vars_no_break "$@"
}

pr_vars_no_break () {
	for k in "$@"; do
		len=$(echo "${!k}"|wc -m)
		if [[ len -gt 50 ]]; then
			begin=$(( len - 51 ))
			val=$(expr substr "${!k}" $begin $len )
		else
			val="${!k}"
		fi
		printf '%-15.15s : %.51s \n' "$k" "${val[*]}"
	done
}

# Debug that prints relevant environment information.
print_env () {
	pr_vars TESTING_MODE errfile
	pr_vars program_root config_dir logger datatool
	pr_vars stage_root target_root
	pr_vars data_file
	pr_vars dealer dealermail
	pr_vars gitlab_url gitlab_ssh_port gitlab_http_port gitlab_owner
}

# Takes a json array and prints it as a flat string.
strip_json_formatting () {
	echo $@|
		sed 's/[",\]//g'|
		sed 's/\[//g'|
		sed 's/\]//g'|
		sed 's/{//g'|
		sed 's/}//g'
}

format_container_name () {
	if [[ $2 =~ [0-9] ]]; then
		echo "${1}-${2}"
	else
		echo "${2}"
	fi
	return
}

concat_paths () {
	if [[ "${2}" == "null" ]] || [[ -z "${2}" ]]; then
		echo "${1}"
	else
		local result="${1}/${2}"
		local roots=$(
			awk -F"\\\.\\\." '{print NF-1}' <<< "${2}")
		if [[ "$roots" -gt 0 ]]; then
			result=$(dirname "$result")
			for i in `seq ${roots}`; do
				result=$(dirname "$result")
			done
		fi
		echo "$result"|sed 's|/./\+|/|g'|sed 's|//\+|/|g'
	fi
	return
}

######################################################################
# Environment Setup

dealer='caes'
dealermail='caes@adamonet'

logtool="${syslog_dir}/bash-logger/syslog.sh"

export_env () {
	local lines=()
	readarray -t lines < <(get_data_array etc environment)
	for line in "${lines[@]}"; do
		#if [[ $TESTING_MODE == "TRUE" ]]; then
		#	echo export $line
		#fi
		export $line
	done
}

install_environment_vars () {
	:
	# Should check /etc/environment and add lines
	# only if they don't already exist.
	# Need to write a function to do this for a file
	# in the general case to proceed.
}

set_npm_proxy () {
	# Not relying on this at the moment.
	:
	#npm config set proxy http://some.proxy:83
	#npm config set https-proxy http://some.proxy:83
}

export_completions () {
	:
}

update_data () {
	git_update_self $1
}

install_logger () {
	echo "Logger not currently installed".
#	rm -rf\
#		ADAMONET-bash-logger-*\
#		bash-logger\
#		bash-logger.tar\
#		2> /dev/null
#	get_release bash-logger
#	tar xf bash-logger.tar
#	rm bash-logger.tar
#	mv ADAMONET-bash-logger-* bash-logger
#	return
}

syslog () {
	${logtool} "[adamocomp] ""$@"
}

if [[ "$TESTING_MODE" == "TRUE" ]]; then
	syslog_local="TRUE"
fi

apt_update () {
	sudo apt update -y
}

install_docker () {
	curl -sSL https://get.docker.com/ | sh
}

install_prereqs () {
	if [[ ! -x /usr/bin/jq ]]; then
		sudo apt install jq
	fi
	if [[ ! -x /usr/bin/logger ]]; then
		sudo apt install logger
	fi
	if [[ ! -x /usr/bin/curl ]]; then
		sudo apt install curl
	fi
	if [[ ! -x /usr/bin/curl ]]; then
		sudo apt install curl
	fi
	if [[ ! -x /usr/bin/node ]]; then
		curl -sL https://deb.nodesource.com/setup_14.x | bash -
		sudo apt install -y nodejs npm
	fi
}

setup_env () {
	owd=`pwd`
	cd $program_root
	#set_npm_proxy
	update_data $1
	install_logger
	npm up
	cd $owd
}

setup_local () {
	setup_env $*
}

######################################################################
# Security

# $1: string to generate key
# $2: fully-qualified domain name of host
cert_gen_self_signed_crt () {
	local pass_str="$1"
	local fqdn="$2"
	local dir="ssl.$fqdn"
	local owd=`pwd`
	mkdir -p $dir
	cd $dir
	if [[ $(basename $(pwd)) == "$dir" ]]; then
		openssl genrsa -des3 \
			-passout pass:$pass_str \
			-out $fqdn.pass.key 2048
		openssl rsa \
			-passin pass:$pass_str \
			-in $fqdn.pass.key \
			-out $fqdn.key
		rm $fqdn.pass.key
		# writing RSA key
		openssl req -new \
			-key $fqdn.key \
			-out $fqdn.csr
		# ...
		openssl x509 -req -sha256 -days 365 \
			-in $fqdn.csr \
			-signkey $fqdn.key \
			-out server.crt
		rm $fqdn.csr
	else
		echo "Problem accessing subdir."
	fi
	cd $owd
	return
}

cert_gen_self_signed_pem () {
	openssl req -newkey rsa:2048 \
		-new -nodes -x509 -days 3650 \
		-keyout key.pem -out cert.pem
}

######################################################################
# Global constructions.

dofor_deal_info () {
	local command=""
	case "$*" in 
		"u"*) command="unset" ;;
		"p"*) command="pr_vars" ;;
	esac
	if [[ $command != "" ]]; then
		${command} deal_valid deal_node parent deal_name issue_path
		${command} use_tools toolset missing_tools 
		${command} deal_repo deal_branch deal_tag use_npm
		${command} docker_image docker_options
		${command} tool_path tool_src
		${command} stage_node close_node stage_path close_path
		${command} tool_target tool_stage
	fi
}

clear_deal_info () {
	dofor_deal_info "unset"
}

pr_deal_info () {
	dofor_deal_info "pr_vars"
}

# might as well run this at runtime

clear_deal_info

construct_deal_info () {
	clear_deal_info
	if deal_exists "$*"; then
		deal_node=$(get_path $*)
		parent=$(basename $(dirname $deal_node))
		deal_name=$(basename $deal_node)
		issue_path=$(get_data "$parent $deal_name issue_path")

		toolset=$(get_data "$parent $deal_name toolset")
		tool_path=$(get_data "$parent $deal_name tool_path")
		tool_src="${program_root}/toolsets/${toolset}"

		docker_image=$(get_data "$parent $deal_name docker image")
		docker_options=$(get_data "$parent $deal_name docker options")

		deal_repo=$(get_data "$parent $deal_name repo"|
			sed 's/.git$//'|sed 's\^ADAMONET/\\'
		)
		deal_branch=$(get_data "$parent $deal_name branch")
		deal_tag=$(get_data "$parent $deal_name tag")
		use_npm=$(get_data "$parent $deal_name npm_setup")
		
		
		stage_node=$(concat_paths $stage_root $deal_node)
		close_node=$(concat_paths $target_root $deal_node)
		stage_path=$(concat_paths $stage_node $issue_path)
		close_path=$(concat_paths $close_node $issue_path)
		tool_target=$(concat_paths $close_node $tool_path)
		tool_stage=$(concat_paths $stage_node $tool_path)
		
		if [[ $tool_path == "null" ]] || [[ -z $tool_path ]]; then
			unset use_tools missing_tools
		else
			use_tools="TRUE"
			if ls ${tool_target}/*.sh >> /dev/null 2> /dev/null;
			then
				unset missing_tools
			else
				missing_tools="TRUE"
			fi
		fi
		deal_valid="TRUE"
	else
		return 2
	fi
	return 0
}

clear_host_list () {
	unset cluster_name cluster_size
	host_list=()
}

push_host () {
	host=$1
	host_list+=("$host")
}

pr_host_list () {
	pr_vars cluster_name
	for host in "${host_list[@]}"; do
		local fqdn=$(get_remote_host $host)
		local ssh_port=$(get_remote_ssh_port $host)
		pr_vars host fqdn ssh_port 
	done
}

# This fills in fqdn_list using arguments.
construst_cluster_info () {
	clear_host_list

	cluster_name=$1;shift

	if [[ ! $cluster_name =~ ^[a-zA-Z]+$ ]]; then
		>&2 echo "Cluster argument needs to be string."
		return 12
	fi

	if cluster_exists $cluster_name; then
		cluster_size=$(get_cluster_size $cluster_name)
		local hosts=()
		readarray -t hosts < <(get_hosts_array $cluster_name)
		if [[ ${#hosts[@]} -lt 2 ]] && [[ -z ${hosts[0]} ]]; then
			push_host "$cluster_name"
		else 
			for host in "${hosts[@]}"; do
				push_host "$host"
			done
		fi
		return 0
	else
		>&2 echo "Invalid cluster."
		return 2
	fi
}

######################################################################
# Informatics

info () {
	construct_deal_info $*
	pr_deal_info
	return
}

cluster_info () {
	construst_cluster_info $*
	pr_host_list
}

######################################################################
# Git operations

# new gitlab
gitlab_http_port="45080"
gitlab_ssh_port="45022"

gitlab_owner="$dealer"
gitlab_url="gitlab.adamo.network"
gitlab_http_url="http://${gitlab_url}:${gitlab_http_port}"
gitlab_ssh_url="ssh://git@${gitlab_url}:${gitlab_ssh_port}"

git_identify_global () {
	git config --global user.name "${dealer}"
	git config --global user.email "${dealermail}"
}

git_update_self () {
	owd=`pwd`
	cd ${program_root}
	git_identify_global
	git stash
	git fetch
	if [[ ! -z $1 ]]; then
		git checkout $1
	else
		git checkout master
	fi
	git merge
	cd ${owd}
}

git_pull_all () {
	owd=`pwd`
	for gitdir in `git_list_dirs_under $1`; do
		local dir=$(dirname $gitdir)
		if [[ $TESTING_MODE ]]; then
			pr_vars dir owd
		else
			cd $dir
			git pull
			cd $owd
		fi
	done
}

git_list_dirs_under () {
	find $1 -type d -a -name '.git'
}

######################################################################
# Gitlab operations

# $1: gitlab token
gitlab_runner_register_on_host () {
	#docker run --rm -t -i -v \
	#	/srv/gitlab-runner/config:/etc/gitlab-runner \
	docker run --rm -v /srv/gitlab-runner/config:/etc/gitlab-runner \
		gitlab/gitlab-runner register \
		--non-interactive \
		--executor "docker" \
		--docker-image alpine:latest \
		--url "https://gitlab.adamo.network" \
		--registration-token "$1" \
		--description "adamocomp-runner" \
		--tag-list "docker,adamocomp" \
		--run-untagged="true" \
		--locked="false" \
		--access-level="not_protected"
}

gitlab_download_backups () {
	scp rancher@gitlab.adamo.network:~/* .
}

gitlab_container_backup () {
	docker exec -it gitlab gitlab-rake gitlab:backup:create
	sudo tar cf gitlab-secrets.tar /opt/gitlab/config/
}

gitlab_get_token () {
	cat ${program_root}/gitlab_read.pat
}
gitlab_token=$(gitlab_get_token)

# $1: directory
# $2: gitlab namespace path (this better exist!)
gitlab_create_project_from_repo () {
	owd=`pwd`
	if [[ -x $1 ]]; then
		cd $1
	else
		return 2
	fi
	shift
	local path=$(echo "$1"|sed 's/caes/adamo/')
	local name="$(git rev-parse --show-toplevel | xargs basename)"
	local branch="$(git rev-parse --abbrev-ref HEAD)"
	if [[ $path ]]; then
		local uri="${gitlab_ssh_url}/$path"
	else
		local uri="${gitlab_ssh_url}"
	fi
	pr_vars uri name branch
	if [[ $TESTING_MODE ]]; then
		echo git push --set-upstream ${uri}.git $branch
	else
		git push --set-upstream ${uri}.git $branch
	fi
	cd $owd
	return 0
}

gitlab_create_projects_for_all_in_dir () {
	local root=$1
	for gitdir in `git_list_dirs_under $root`; do
		local dir=$(dirname $gitdir)
		local path=$(echo $dir|sed "s|${root}/*||")
		gitlab_create_project_from_repo $dir $path
	done
}

gitlab_clone_all_projects () {
	for repo_path in $(gitlab_list_project_paths); do
		local dest="gitlab_scrape/$(dirname $repo_path)"
		local name="$(basename $repo_path)"
		local repo_url="${gitlab_ssh_url}/${repo_path}.git"
		pr_vars repo_path dest name
		if [[ $TESTING_MODE ]]; then
			echo mkdir -p $dest
			echo git clone ${repo_url} $dest/$name
		else
			mkdir -p $dest
			git clone ${repo_url} $dest/$name
		fi
	done
}

gitlab_list_projects () {
	gitlab_api_get_projects|
		jq -r '.[] | .["name"]'
}

gitlab_list_project_paths () {
	gitlab_api_get_projects|
		jq -r '.[] | .["path_with_namespace"]'
}

gitlab_count_projects () {
	gitlab_list_projects|wc -l
}

gitlab_api_get_projects () {
	gitlab_api_request "api/v4/projects" "per_page=100"
}

gitlab_api_request () {
	local path="$1"
	local opts="$2"
	local uri="${gitlab_http_url}/${path}"
	local header="Authorization: Bearer $gitlab_read_token"
	#pr_vars path uri header opts
	if [[ $opts ]]; then
		curl -s -H "$header" "${uri}?${opts}"
	else
		curl -s -H "$header" "${uri}"
	fi
}

# Prints json object of repo release.
get_release_data () {
	local repo="${1}"
	local release="${2}"
	local owner_path="${gitlab_url}/repos/${gitlab_owner}"
	local repo_path="${owner_path}/${repo}/releases/${release}"
	curl -s \
		-H "Authorization: token ${gitlab_token}"\
		 "${repo_path}"
		#|jq -r
}

# Retrieves latest release of deal.
# Usage: get_release <repo name> [<tag>]
# Downloads release tarball as "<repo name>.tar".
get_release () {
	local repo="${1}"
	local release="${2:-latest}"
	local release_data=$(get_release_data "$repo" "$release")
	local tarball_url=$(echo "$release_data"|jq -r '.tarball_url')

	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		pr_vars repo release tarball_url release_data
	else
		curl -s \
			-H "Authorization: token ${gitlab_token}" \
			-L -o "${1}.tar" $tarball_url
	fi
}

# Retrieves clone of branch in repo.
# Usage: get_clone <repo name> <branch>
# Extracts clone to directory of same name, I guess.
# Needs to be changed to use token authentication.
get_clone () {
	local repo="${1}"
	local branch="${2:-master}"
	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		pr_vars repo branch
		return 0
	else
		github_root="${gitlab_token}@gitlab.adamonet"
		git clone \
			--single-branch \
			--branch $branch \
			"https://${github_root}/${gitlab_owner}/${repo}.git"
		return
	fi
}

######################################################################
# Container operations.

docker_options_string () {
	num_opt=$(data_array_seq_end ${deal_name} docker options)
	for i in `seq 0 $num_opt`; do
		num_args=$(data_array_seq_end ${deal_name} docker options $i)
		for j in `seq 1 $num_args`; do
			printf "%s "\
				"$(get_data ${deal_name} docker options $i 0)"
			printf "%s "\
				"$(get_data ${deal_name} docker options $i $j)"
		done
	done
	return
}

build_container () {
	if [[ ! -x $target_root ]]; then
		>&2 echo "Cannot access root at ${target_root}."
		return 2
	fi
	if deal_exists "$*"; then
		local deal_node=`${datatool} path "$*"|jq -r '.[0]'`
		local parent=$(basename $(dirname $deal_node))
		local node=$(basename $deal_node)
		local issue_path=$(get_data "$parent $node issue_path")
		local name=$(format_container_name $parent $node|sed 's/-/:/')
		local build_node="${target_root}/${deal_node}"
		local build_dir=$(concat_paths $build_node $issue_path)
		local repo=$(get_data "$parent $name repo"|jq -r '.[0]')

		pr_vars name repo build_dir
		
		if [[ "$TESTING_MODE" == "TRUE" ]]; then
			pr_vars deal_node parent node issue_path \
				name build_node build_dir repo
			return 0
		fi

	    if DOCKER_BUILDKIT=1 docker build \
	        --ssh id_rsa=~/.ssh/id_rsa --no-cache \
	        --tag ${name} ${build_dir}
	    then
	    	syslog "Image $name rebuilt from $deal_repo."
	    else
	    	syslog "Error rebuilding image $name."
	    fi
		return
    else 
    	>&2 echo "Deal not found."
    	return 10
    fi
}

# start_container <directory containing a dockerfile> [<script name>]
start_container () {
	local container_dir=$1
	local script_name=$2

	if [ -z $script_name ]; then script_name="prod"; fi

	container_parent_dir=$(
		cd "$(dirname $container_dir)" >/dev/null && pwd \
	)

	local parent=$(basename "$container_parent_dir")
	local node=$(basename "$container_dir")
	local container_name="$parent-$node"
	local container_logs_dir=$container_dir/logs
	local run_name=$(format_container_name $parent $node)
	local name=$(echo $run_name|sed 's/-/:/')
	local port=10000

	if [ ! -d "$container_logs_dir" ]; then
		mkdir -p $container_logs_dir
	fi

	pr_vars name port

    docker run -d \
        -p "${publish_string}" \
        --name "$run_name" \
        --cidfile "${1}/${run_name}.cid" \
        -v "$container_logs_dir:/workdir/logs/:rw" \
        -e "ADAMO_HOST=$(hostname)" \
        --dns=19.13.0.246 \
        --dns=19.69.0.246 \
        --dns-search=some.dns \
        --dns-search=another.dns \
        --restart=on-failure:10 \
        --memory 100M \
        --memory-reservation 20M \
        "$name"

    return
}

stop_container () {
	# Use either a direct path to the CID file or a directory
	# If directory, all CID files found under it will be stopped.
	local cid_path=$1

	#Array of CID files to stop
	local cid_files=()

	#printf "Stopping container(s) at path specified: $cid_path\n"

	if [ -f "$cid_path" ] ; then # if file, add file
			cid_files+=("$cid_path")
	elif [ -d "$cid_path" ]; then # if dir, add all *.cid files
		for f in $(find "$cid_path" -type f -name "*.cid"); do
			#echo "Found CID file: $f"
			cid_files+=($f)
		done
	fi

	if [ ${#cid_files[@]} -gt 0 ]; then
		for file in ${cid_files[*]} ; do
			#printf "Stopping container"
			#prinft " [$(basename $file)]...\n"
			cid=`cat "$file"`
			docker kill $cid
			docker rm $cid
			rm $file
		done
	else
		printf "No CID files found (no containers stopped).\n"
	fi
}

stop_all_containers() {
	if [[ -z $1 ]]; then
		stop_container "$program_root"
	else
		stop_container "$1"
	fi
	return
}

restart_all_containers () {
	local container_root
	if [[ -z $1 ]]; then
		container_root=$(dirname "$0")
		syslog "Restarting all containers."
	else
		container_root=$1
		syslog "Restarting all containers under $container_root."
	fi
	for f in $( \
		find $container_root -type f -name "restart_container.sh" \
	); do
			$f
	done
	return
}

######################################################################
# Local dealing

run_apt () {
	if construct_deal_info $*; then
		local lines=()
		readarray -t lines < <(get_data_array $* apt)
		for line in "${lines[@]}"; do
			if [[ $TESTING_MODE == "TRUE" ]]; then
				echo sudo apt $line
			else
				sudo apt $line
			fi
		done
	fi
}

docker_run_from_data () {
	local dealkeypath="$*"
	# env exported here since this is for bootstrapping
	export_env
	if construct_deal_info $dealkeypath; then
		local docker_opts="$(docker_options_string $dealkeypath)"
		pr_vars deal_name docker_image docker_opts
		if [[ $TESTING_MODE == "TRUE" ]]; then
			echo docker run -d $docker_opts \
				--name $deal_name $docker_image
		else
			docker run -d $docker_opts \
				--name $deal_name $docker_image
		fi
	fi
}

docker_start_from_data () {
	dealkeypath="$*"
	# env exported here since this is for bootstrapping
	export_env
	if construct_deal_info $dealkeypath; then
		pr_vars deal_name docker_image
		if [[ $TESTING_MODE == "TRUE" ]]; then
			echo docker start $deal_name
		else
			docker start $deal_name
		fi
	fi
}

docker_rm_from_data () {
	dealkeypath="$*"
	# env exported here since this is for bootstrapping
	export_env
	if construct_deal_info $dealkeypath; then
		pr_vars deal_name docker_image
		if [[ $TESTING_MODE == "TRUE" ]]; then
			echo docker stop $deal_name
			echo docker rm $deal_name
		else
			docker stop $deal_name
			docker rm $deal_name
		fi
	fi
}

prepare_issue_path () {
	if $deal_valid; then
		if [[ -x $issue_path && -w $issue_path ]]; then
			return 0
		else
			if [[ "$TESTING_MODE" == "TRUE" ]]; then
				perms=$(get_filemode $issue_path)
				pr_vars issue_path perms
			else
				sudo mkdir -p $issue_path
				sudo chmod u=wrx $issue_path
			fi
		fi
	fi
}

clear_stage () {
	if [[ $TESTING_MODE == "TRUE" ]]; then
		pr_vars stage_root
		return 0
	fi
	if [[ -x $stage_root ]]; then
		rm -rf $stage_root
		return 0
	else
		return 0
	fi
}

install_stage () {
	mkdir -p $target_root
	if [[ ! -x $target_root ]] || [[ ! -w $target_root ]]; then
		>&2 echo "Cannot access root at ${target_root}."
		return 1
	fi
	
	if [[ -x $stage_root ]]; then
		pr_vars stage_root target_root 
		#cp -R --preserve=mode $stage_root/* $target_root
		rsync -a $stage_root/ $target_root
		return 0
	else
		return 0
	fi
}

apply_stage () {
	install_stage
	clear_stage
}

stage_release () {
	if [[ $deal_valid ]]; then
		if [[ -z $issue_path ]] || [[ -z $deal_repo ]]\
		|| [[ -z $deal_branch ]] || [[ -z $deal_node ]]; then
			>&2 echo "Error in data or API not found."
			return 2
		fi

		pr_vars stage_path 
		if [[ "$TESTING_MODE" == "TRUE" ]]; then
			pr_vars stage_node issue_path
			if [[ "$deal_tag" == "null" ]]; then
				if [[ "$deal_branch" == "null" ]]; then
					echo "No valid release data."
				else
					echo ""
					echo "I want to clone using: "
					get_clone ${deal_repo}
				fi
			else
				echo ""
				echo "I want to release using: "
				get_release ${deal_repo} ${deal_tag}
			fi
			return 0			
		fi

		mkdir -p $stage_path

		if [[ ! -e ${stage_path} ]]; then
			>&2 echo "Problem accessing release path ${stage_path}."
			return 5
		fi

		if [[ ! -w $stage_path ]]; then
			>&2 echo "Cannot write to release path."
			return 1
		fi

		# Perfect world: should go into container,
		# npm update, npm test, if npm tests pass, then 

		if [[ "$deal_tag" == "null" ]]; then
			if [[ "$deal_branch" == "null" ]]; then
				>&2 echo "No valid update source in data."
				return 8
			else
				>&2 echo "Deal by git clone currently disabled."
				return 9
				#get_clone ${deal_repo}
				#mv "${deal_repo}/*" "${stage_path}/"
			fi
		else
			get_release ${deal_repo} ${deal_tag}
			rm -rf "${gitlab_owner}-${deal_repo}-*"
			tar xf "${deal_repo}.tar"
			cp -R $(echo ${gitlab_owner}-${deal_repo}-*/*)\
				"${stage_path}/"
			rm -rf "$(echo ${gitlab_owner}-${deal_repo}-*)"
			rm "${deal_repo}.tar"
		fi
		return 0
	else
		>&2 echo "Cannot stage invalid deal."
		return 6
	fi
}

stage_scripts () {
	# Currently disabled and toolset must be deployed manually.
	return 0
	if [[ $deal_valid ]]; then
		pr_vars tool_src tool_path
		if [[ "$TESTING_MODE" == "TRUE" ]]; then
			pr_vars tool_stage
		else
			if [[ ! -z $tool_stage ]]; then
				mkdir -p $tool_stage
				if [[ ! -w $tool_stage ]]; then
					>&2 echo "Cannot write to tools path."
					return 2
				fi
			fi
	
			if [[ ! -x $tool_src ]]; then
				>&2 echo "Cannot access toolset from $tool_src."
				return 3
			fi

			if [[ ! -r $tool_src ]] ; then
				>&2 echo "Cannot read toolset source dir."
				return 4
			fi

			cp --preserve=mode \
				$tool_src/*.sh $tool_stage
			cp --preserve=mode \
				${program_root}/broker.sh $tool_stage
		fi
	else
		>&2 echo "Cannot stage tools for invalid deal."
		return 5
	fi
	return
}

######################################################################
# Remote dealing

reboot_hosts () {
	local clstr_name=$*
	if construst_cluster_info $clstr_name; then
		pr_host_list
		if [[ -z $TESTING_MODE ]]; then
			execute_on_nodes "sudo reboot"
		fi
	fi
}

ssh_tty_to_host () {
	local host=$1
	shift
	if host_exists $host; then
		local fqdn=$(get_remote_host $host)
		local port=$(get_remote_ssh_port $host)
		local user=$(get_remote_user $host)
		local cmd="$*"
		pr_vars fqdn port user cmd
		ssh -tt -o port=${port} ${user}@${fqdn} "$*"
	fi
}

scp_contents_to_host () {
	local host=$1
	local source_dir=$2
	local target_dir=$3
	if host_exists $1; then
		local fqdn=$(get_remote_host $host)
		local port=$(get_remote_ssh_port $host)
		local user=$(get_remote_user $host)
		pr_vars host fqdn port source_dir target_dir
		if [[ $TESTING_MODE == "TRUE" ]]; then
			:
		else
			ssh_tty_to_host $host "mkdir -p ${target_dir}"
			scp -r -o port=${port} $source_dir/* \
				"${user}@${fqdn}:${target_dir}/"
		fi
	fi
}

scp_to_host () {
	local host=$1
	local source_file=$2
	local target_dir=$3
	if host_exists $1; then
		local fqdn=$(get_remote_host $host)
		local port=$(get_remote_ssh_port $host)
		local user=$(get_remote_user $host)
		pr_vars host fqdn port source_file target_dir
		if [[ $TESTING_MODE == "TRUE" ]]; then
			:
		else
			scp -o port=${port} $source_file\
				"${user}@${fqdn}:${target_dir}/"
		fi
	fi
	
}

docker_ps () {
	if construst_cluster_info $1; then
		shift
		if [[ -z $1 ]]; then
			execute_on_nodes "docker ps"
		else
			execute_on_nodes "docker ps|grep $1"
		fi
	fi
}

download_file () {
	local filename=$1
	pr_vars filename
	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		pr_vars ${fqdn_list[0]} filename
	else
		local user=$(get_remote_user $host)
		pr_vars	host
		scp -o port="${ssh_port_list[0]}" \
			"${user}@${fqdn_list[0]}:${filename}" ./
	fi
	return
}

distribute_files_in_dir () {
	local source_dir=$1
	local target_dir=$2
	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		for host in ${host_list[@]}; do
			pr_vars host source_dir target_dir
		done
	else
		for host in "${host_list[@]}"; do
			pr_vars	host source_dir target_dir
			ssh_tty_to_host "$host" "mkdir -p ${target_dir}"
			scp_contents_to_host "$host" "$source_dir" "$target_dir"
			ssh_tty_to_host "$host" "ls -lh ${target_dir}"
		done
	fi
	return
}

distribute_file () {
	local source_file=$1
	local target_dir=$2
	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		for host in ${host_list[@]}; do
			pr_vars host source_file target_dir
		done
	else
		for host in ${host_list[@]}; do
			pr_vars	host source_file target_dir
			ssh_tty_to_host "$host" "mkdir -p ${target_dir}"
			scp_to_host
		done
	fi
	return
}

remote_broker () {
	execute_on_nodes "cd /opt/adamocomp; ${rem_broker_cmd} $*"
}

execute_on_nodes () {
	command=$1
	for host in ${host_list[@]}; do
		ssh_tty_to_host "$host" $command
	done
}

# $1: host deal
# $2: command
# $3: docker deal
docker_exec_on_host () {
	local host="$1"; shift
	local cmd="$1"; shift
	local dealkeypath="$*"
	pr_vars host cmd dealkeypath
	if construct_deal_info $dealkeypath; then
		ssh_tty_to_host $host "docker exec -it $deal_name \"${cmd}\""
	fi
}

# $1: host deal
# $2: docker deal
docker_bash_on_host () {
	docker_exec_on_host $1 "/bin/bash" "${@:2}"
}

docker_run_from_data_on_host () {
	local host=$1
	shift
	local dealkeypath="$*"
	# env exported here since this is for bootstrapping
	export_env
	if construct_deal_info $dealkeypath; then
		local docker_opts="$(docker_options_string $dealkeypath)"
		pr_vars deal_name docker_image docker_opts
		if [[ $TESTING_MODE == "TRUE" ]]; then
			ssh_tty_to_host $host "echo docker run -d $docker_opts \
				--name $deal_name $docker_image"
		else
			ssh_tty_to_host $host "docker run -d $docker_opts \
				--name $deal_name $docker_image"
		fi
	fi
}

docker_destroy_from_data_on_host () {
	local host=$1
	shift
	local dealkeypath="$*"
	# env exported here since this is for bootstrapping
	export_env
	if construct_deal_info $dealkeypath; then
		pr_vars deal_name docker_image
		if [[ $TESTING_MODE == "TRUE" ]]; then
			ssh_tty_to_host $host "echo docker stop $deal_name;
				echo docker rm $deal_name"
		else
			ssh_tty_to_host $host "docker stop $deal_name;
				docker rm $deal_name"
		fi
	fi
}

######################################################################
# Local automation

local_node_setup () {
	if [[ $TESTING_MODE == "true" ]]; then
		pr_vars 1
	else
		setup_env $1
		clear_stage
	fi
}

# Selfishly performs chmod and chown operations on
# existing container structure to ensure proper permissions
# under the deploy user used during remote deployments.
local_node_repair () {
	local user=`whoami`

	executables=()

	if [[ $TESTING_MODE == "true" ]]; then
		pr_vars rootdirs userdirs executables
	else
		for dir in ${rootdirs[@]}; do
			sudo chown -R root:root $dir
			sudo chmod 755 $dir
			ls -lhd $dir
			ls -lh $dir
		done

		for file in ${locallinkedconfs[@]}; do
			sudo rm $file
			sudo ln -s ${program_root}/$(basename $file) $file
			ls -lhd $file
		done
	
		for dir in ${userdirs[@]}; do
			sudo chown -R ${user}:users $dir
			ls -lhd $dir
			ls -lh $dir
		done

		sudo chmod 755 ${executables[*]}
	fi
}

install_local_scripts () {
	if construct_deal_info $*; then
		pr_vars tool_src tool_target
		if [[ -x $tool_src ]] && [[ -r $tool_src ]]; then
			mkdir -p $tool_target
			if [[ -x $tool_target ]] && [[ -w $tool_target ]]
			then
				if [[ $TESTING_MODE == "TRUE" ]]; then
					echo cp ${tool_src}/* $tool_target/
				else
					cp ${tool_src}/* $tool_target/
				fi
			else
				>&2 echo "Cannot write to destination."
				return 2
			fi
		fi
	else
		>&2 echo "Cannot access toolset."
		return 3
	fi
	return
}

complete_local () {
	if construct_deal_info $@;then
		if [[ $TESTING_MODE	 == "TRUE" ]]; then
			pr_vars parent deal_name repo close_path tool_target\
			use_tools use_npm
		else
			pr_vars parent deal_name repo close_path tool_target\
				use_tools use_npm
			clear_stage
			if [[ $use_tools ]]; then
				${tool_target}/stop_container.sh
				if [[ $missing_tools ]]; then
					stage_scripts
				fi
			fi
			stage_release
			apply_stage
			if [[ $use_npm ]]; then
				owd=`pwd`
				cd ${close_path}
				npm update
				cd $owd
			fi
			if [[ $use_tools ]]; then
				${tool_target}/start_container.sh
			fi
			# Note this assumes latest since the latest
			# tag is currently baked into the code.
			report_version="Latest tagged release of ${deal_repo}"
			syslog "${report_version} installed at '${close_path}'."
		fi
	else
		>&2 echo "Invalid deal search string: $*."
		return 2
	fi
	return 
}


update_local () {
	if construct_deal_info $@;then
		if [[ $TESTING_MODE	 == "TRUE" ]]; then
			pr_vars parent deal_name repo deal_node close_path use_npm
		else
			pr_vars parent deal_name repo deal_node close_path use_npm
			clear_stage
			stage_release
			install_stage
			if [[ $use_tools ]]; then
				${tool_target}/stop_container.sh
			fi
			if [[ $use_npm ]]; then
				owd=`pwd`
				cd ${close_path}
				npm update
				cd $owd
			fi
			if [[ $use_tools ]]; then
				${tool_target}/start_container.sh
			fi
			# Note this assumes latest since the latest
			# tag is currently baked into the code.
			report_version="Latest tagged release of ${deal_repo}"
			syslog "${report_version} updated at '${close_path}'."
		fi
	else
		>&2 echo "Invalid deal search string: $*."
		return 2
	fi
	return 
}

update_api_container () {
	construct_deal_info $@
	if [[ $deal_valid ]]; then
		pr_vars parent deal_name repo close_path tool_target use_npm
		if [[ "$TESTING_MODE" == "TRUE" ]]; then
			pr_vars close_node close_path
			return 0
		else
			#clear_stage
			#stage_deployment "$*"
			#install_stage
			#clear_stage
			#build_container "$*"
			#${build_dir}/restart_container.sh
			:
		fi
	else
		>&2 echo "Invalid deal identifier."
	fi
	return
}

######################################################################
# Remote automation
# Recommended working ssh key

rem_broker_cmd="/opt/adamocomp/broker.sh"
qa_source="1"
prod_source="4"

for opt in ${PASS_OPTS[@]}; do
	rem_broker_cmd="${rem_broker_cmd} ${opt}"
done

# Runs docker_run_from_data on remote hosts for some deal.
# This is essentially meant for bootstrapping.
remote_docker_run () {
	if construst_cluster_info $1; then
		shift
		remote_broker "docker_run_from_data $*"
	fi
}

remote_docker_start () {
	if construst_cluster_info $1; then
		shift
		remote_broker "docker_start_from_data $*"
	fi
}

remote_docker_replace () {
	if construst_cluster_info $1; then
		shift
		remote_broker "docker_rm_from_data $*"
		remote_broker "docker_run_from_data $*"
		execute_on_nodes "docker ps"
	fi	
}

remote_node_bootstrap () {
	remote_node_init "$@"
	remote_copy_self "$@"
	remote_prereq_install "$@"
	remote_prereq_install "$@"
	remote_node_setup "$@"
}

# Prepares nodes for dealing by installing software.
remote_node_init () {
	construst_cluster_info $1
	local user=$(get_remote_user $host)
	execute_on_nodes \
		"sudo mkdir -p $ADAMOCOMP_HOME;
		 sudo mkdir -p $ADAMOCOMP_CONFDIR;
		 sudo mkdir -p $GITLAB_HOME;
		 sudo chown -R ${user}:${user} $ADAMOCOMP_HOME;
		 sudo chown -R ${user}:${user} $ADAMOCOMP_CONFDIR;
		 sudo chown -R ${user}:${user} $GITLAB_HOME;
		 ls -lhd $ADAMOCOMP_HOME;
		 ls -lhd $ADAMOCOMP_CONFDIR;
		 ls -lhd $GITLAB_HOME;"
}

remote_copy_self () {
	if construst_cluster_info $1; then
		distribute_files_in_dir "${program_root}" "$ADAMOCOMP_HOME"
		distribute_file "${data_file}" "$ADAMOCOMP_CONFDIR"
	fi
}

remote_prereq_install () {
	if construst_cluster_info $1; then
		remote_broker apt_update
		remote_broker install_prereqs
	fi
}

remote_node_setup () {
	construst_cluster_info $1
	remote_broker "local_node_setup $2"
}

# Runs repair operations on each node.
remote_node_repair () {
	construst_cluster_info $*
	execute_on_nodes \
		"${rem_broker_cmd} local_node_repair"
}

remote_sudo_edit () {
	if construst_cluster_info $1; then
		shift
		execute_on_nodes "sudo $editor $*"
	fi	
}

remote_edit () {
	if construst_cluster_info $1; then
		shift
		execute_on_nodes "$editor $*"
	fi	
}

cluster_run () {
	local cluster=$1; shift
	construst_cluster_info $cluster
	execute_on_nodes \
		"$*"
}

# Only one node number instead of seq range.
cluster_get () {
	local cluster=$1;shift;
	if construst_cluster_info $cluster; then
			download_file $1
	else
		>&2 echo "Invalid node identification."
	fi	
}

#if construst_cluster_info $cluster $source_node $source_node; then
	#	download_file $file

# Sends file to nodes.
cluster_send () {
	local cluster=$1;shift
	if construst_cluster_info $cluster; then
			distribute_file $1 $2
	else
		>&2 echo "Invalid node identification."
	fi
}

# This should only be used if you know what you are doing,
# and prompts for a password at each node.
distribute_env_file () {
	local cluster=$1; shift

	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		pr_vars path
	fi

	local filename=$(basename $1)

	construst_cluster_info $cluster
	distribute_file "$1" "~"
	execute_on_nodes "chmod 644 $filename;
		sudo chown root:root $filename;
		sudo mv $filename ${config_dir};
		ls -lh ${config_dir};
		"
}

cluster_edit () {
	local cluster=$1; shift
	if cluster_exists $cluster; then
		local file=$1;shift
		local source_file=$(basename $file)
		local target_dir=$(dirname $file)
		cluster_get $cluster $file
		user_edit $source_file
		read -p "Do you want to distribute this file? (y/n)"\
			-n 1 -r
		echo ""
		if [[ $REPLY =~ ^[Yy]$ ]]; then
			cluster_send $cluster $source_file $target_dir
		fi
		rm $source_file
	fi
}

# Used to install toolset before a new deal is installed across
# the cluster.
install_scripts_on_nodes () {
	local cluster=$1; shift
	if construct_deal_info $*; then
		pr_vars tool_src tool_target
		if [[ -x $tool_src ]] && [[ -r $tool_src ]]; then
			if construst_cluster_info $cluster; then
				distribute_files_in_dir $tool_src $tool_target
			else
				>&2 echo "Invalid node identification."
				return 2
			fi
		fi
	else
		>&2 echo "Cannot access toolset."
		return 3
	fi
	return
}

# Update a deal across all nodes between 
# and incl. numbers.
# ./broker.sh <start num> <end num>
# DOES NOT VALIDATE YET.
install_on_nodes () {
	local loc=$1;shift;local frontnode=$1;shift;local endnode=$1;shift
	
	uses_dockerfile=$(
		${datatool} get "$* use_dockerfile"|jq -r '.[0]')

	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		pr_vars uses_dockerfile master
	fi

	# Gating against dockerfile usage due to a bug.
	unset uses_dockerfile

	construst_cluster_info $loc $frontnode $endnode
	if [[ "$uses_dockerfile" == "true" ]]; then
		execute_on_nodes \
			"${rem_broker_cmd} update_api_container $*"
	else
		execute_on_nodes \
			"${rem_broker_cmd} complete_local $*"
	fi
}

# Runs an npm update on a container and restarts it.
update_on_nodes () {
	local loc=$1;shift;local frontnode=$1;shift;local endnode=$1;shift
	construst_cluster_info $loc $frontnode $endnode
	execute_on_nodes \
		"${rem_broker_cmd} update_local $*"
}

# Restart a container on a series of nodes.
# ./broker.sh <start num> <end num>
# DOES NOT VALIDATE YET.
restart_on_nodes () {
	local loc=$1;shift;local frontnode=$1;shift;local endnode=$1;shift

	path=$(${datatool} path "$*"|jq -r '.[0]')

	if [[ "$TESTING_MODE" == "TRUE" ]]; then
		pr_vars path
	fi

	construst_cluster_info $loc $frontnode $endnode
	execute_on_nodes "if \
		[[ -x \${ADAMOCOMP_HOME}/${path}/restart_container.sh ]]; then
			\${ADAMOCOMP_HOME}/${path}/restart_container.sh;
		fi
		"
}

# This part checks that the data.json file is
# accessible, and finally executes the command given at the
# shell prompt.
if [[ ! -r "$data_file" ]]; then
	if [[ ! -e "$data_file" ]]; then
		update_data
	fi
	if [[ ! -r "$data_file" ]]; then
		>&2 echo "Could not locate or update data.json."
	fi
fi

# Always export environment variables from data, for now.
export_env

# Run command if one exists.
if [[ ! -z ${POSITIONAL[0]} ]]; then
	${POSITIONAL[0]} ${POSITIONAL[@]:1}
fi