#!/bin/bash

#
# 
# Copyright (c) 2007-2012,
#   Lingxiao Jiang         <lxjiang@ucdavis.edu>
#   Ghassan Misherghi      <ghassanm@ucdavis.edu>
#   Zhendong Su            <su@ucdavis.edu>
#   Stephane Glondu        <steph@glondu.net>
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in the
#       documentation and/or other materials provided with the distribution.
#     * Neither the name of the University of California nor the
#       names of its contributors may be used to endorse or promote products
#       derived from this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# 
#

# To save disk space, this script does everything (clustering and
# post-processing) for each MIN_TOKENS, each STRIDE, and each SIMILARITY at
# once, and delete intermediate files (cluster_*, post_*, ) before starting
# processing the next parameter combination (thus the name of the scripts).
# Feel free to adjust the workflows in the scripts to fit your needs.

TOOVERWRITE=
coverage=`dirname $0`/cd_coverage
GROUPING_H=100000

# Delete intermediate files, but leave times/*, cluster/coverage*, cluster/post_* files there:
clean()
{
      echo "Cleaning up all intermediate files..."
      echo -n "...deleting vdb_*_*_g*_*_*, including parameter files ..."
      find "$VECTOR_DIR" -type f -name "vdb_*_*_g[0-9]*_*_*" -print0 | xargs -0 -r --max-args=500 rm -f
#      find "$VECTOR_DIR" -type f -name "vdb_*_*_g[0-9]*_*_${GROUPING_S}" -print0 | xargs -0 -r --max-args=500 rm -f
# also delete parameter files, coz. the number of files is too big to be handled by certain OS:
#      find "$VECTOR_DIR" -type f -name "vdb_*_*_g[0-9]*_*_${GROUPING_S}.param" -print0 | xargs -0 -r --max-args=500 rm -f
      echo "Done"
      echo -n "...deleting cluster_vdb_*_*_g*_*_* ..."
      find "$CLUSTER_DIR" -type f -name "cluster_vdb_*_*_g[0-9]*_*_*" -print0 | xargs -0 -r --max-args=500 rm -f
      echo "Done"
}

# Delete log files and result files: if no argv, delete all files
clean_log()
{
	echo "Cleaning up all results and log files..."
	echo -n "...deleting cluster_vdb_*_*_allg_*_* ..."
	find "$CLUSTER_DIR" -type f -name "cluster_vdb_*_*_allg_*_*" -print0 | xargs -0 -r rm -f
	echo "Done"
	echo -n "...deleting post_cluster_vdb_*_*_allg_*_* ..."
	find "$CLUSTER_DIR" -type f -name "post_cluster_vdb_*_*_allg_*_*" -print0 | xargs -0 -r rm -f
	echo "Done"
	echo -n "...deleting ranges_*_* ..."
	find "$VECTOR_DIR" -type f -name "ranges_*_*" -print0 | xargs -0 -r rm -f
	echo "Done"
	echo -n "...deleting logs..."
	find "$TIME_DIR" -type f -name "grouping_*" -print0 | xargs -0 -r --max-args=500 rm -f 
	find "$TIME_DIR" -type f -name "paramsetting_*" -print0 | xargs -0 -r --max-args=500 rm -f 
	find "$TIME_DIR" -type f -name "merging_*" -print0 | xargs -0 -r --max-args=500 rm -f 
	find "$TIME_DIR" -type f -name "cluster_*" -print0 | xargs -0 -r --max-args=500 rm -f 
	find "$TIME_DIR" -type f -name "post_*" -print0 | xargs -0 -r --max-args=500 rm -f 
	find "$TIME_DIR" -type f -name "coverage_*" -print0 | xargs -0 -r --max-args=500 rm -f 
	echo "Done"
}

# vector grouping: $1=<token size>, $2=<stride>, $3=<distance>
# group file names generated by rundispatchonefile: $VECTOR_DIR/vdb_$1_$2_g<gid>_$3_${GROUPING_S}_${GROUPING_H}
grouping()
{
    echo -n "$FUNCNAME: $VECTOR_DIR/vdb_$1_$2 with distance=$3..." | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}"
    # note for bash: must have the space before and after '!=' to make it work correctly; otherwise, silent errors.
    if [[ "$TOOVERWRITE" != "true" ]]; then
	togroup=
	grpnumber=`find "$VECTOR_DIR" -type f -regex ".*vdb_$1_$2_g[0-9]+_$3_${GROUPING_S}.*" | wc -l`
	if [[ $grpnumber -le 0 ]]; then
		togroup="true"
	else
	    while read grpf; do
		if [[ ! -s "$grpf" || "$grpf" -ot "$VECTOR_DIR/vdb_$1_$2" ]]; then
			togroup="true"
			break
		fi
	    done < <( find "$VECTOR_DIR" -regex ".*vdb_$1_$2_g[0-9]+_$3_${GROUPING_S}.*" )
        fi
	if [[ "$togroup" != "true" ]]; then
		echo "Warning: $VECTOR_DIR/vdb_$1_$2 has already been grouped for distance=$3 and starting range ${GROUPING_S}. Delete $VECTOR_DIR/vdb_$1_$2_g[0-9]* manually or use '$0 overwrite'." | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}"
		echo "$FUNCNAME: $1 $2 $3 ...Skip." | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}"
		return 0
	fi
    fi

	# delete old vector groups, in case some files may not be overwritten:
	find "$VECTOR_DIR" -regex ".*vdb_$1_$2_g[0-9]+_$3_${GROUPING_S}.*" -print0 | xargs -0 -r --max-args=500 rm -f
	( time \
          "`dirname ${GROUPING_EXEC}`/rundispatchonefile" "$VECTOR_DIR/vdb_$1_$2" $3 -l $GROUPING_S -h $GROUPING_H -o "$VECTOR_DIR" -p "vdb_$1_$2" || (echo "Error: $FUNCNAME failure: $1 $2 $3"; exit 1) \
	) 1>>"$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" 2>&1
	errcode=$?
	if [[ $errcode -ne 0 ]]; then
		echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}" | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}"
	else
		echo "Done $FUNCNAME $1 $2 $3. See groups in $VECTOR_DIR/vdb_$1_$2_g[0-9]*_$3_${GROUPING_S}*" | tee -a "$TIME_DIR/grouping_$1_$2_$3_${GROUPING_S}"
	fi
	return $errcode
}

# set parameter files: $1=<token size>, $2=<stride>, $3=<sim>
paramsetting()
{
    t=$1
    s=$2
    sim=$3
    echo -n "paramsetting: $t $s $sim ..." | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
# convert SIMILARITY to DISTANCE:
    i=`echo "$sim ${GROUPING_S}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'`
# create the range file if not exist:
    if [[ "$TOOVERWRITE" = "true" || ! -s "${VECTOR_DIR}/ranges_${i}_${GROUPING_S}" ]]; then
        "`dirname $GROUPING_EXEC`/computeranges" $i $GROUPING_S $GROUPING_H > "${VECTOR_DIR}/ranges_${i}_${GROUPING_S}"
    fi
    if [[ ! -s "${VECTOR_DIR}/ranges_${i}_${GROUPING_S}" ]]; then
        echo "Error: $FUNCNAME failure: Can't generate the range file: ranges_${i}_${GROUPING_S}" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
        exit 87
    fi
# use the largest group file for paramter tuning:
# "head -n 1" may break the pipe coming from "ls", causing signal 13 (no more reader, while writer's still writing), but should not matter here:
    groupfortuning=`find "$VECTOR_DIR" -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" -not -name '*.param' -print0 | xargs -0 ls -S | head -n 1`
    grptuningid=`echo ${groupfortuning} | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}.*/\1/"`
    grpal=0
    grpdist=0
# if $grptuningid is empty, error occurs...
    if [[ $grptuningid -le 0 ]]; then
        echo "Error: $FUNCNAME: invalid group id: $grptuningid: $t $s $sim $i" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
        exit 127
    elif [[ $grptuningid -eq 1 ]]; then
	grpal=`head -n $(expr $grptuningid + 1) ${VECTOR_DIR}/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'`
	grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'`
    else
	grpal=`head -n $(expr $grptuningid + 1) ${VECTOR_DIR}/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'`
	grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'`
    fi
    echo -n "Looking for optimal parameters by " | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
    cluster $groupfortuning $grpdist -c | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
    errcode=$?

    if [[ $errcode -ne 0 || ! -s "${groupfortuning}.param" ]]; then
	echo "Error: $FUNCNAME failure...exit." | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
	exit 3
    fi

    echo -n "Setting Parameters for all other groups (may take hours on cygwin but only minutes on Linux...why?)..." | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
    # TODO: performance improvements to reduce file I/O; replace head/tail/awk/generateparam
    ( time \
	find "$VECTOR_DIR" -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" -not -name '*.param' | while read vdb;
	do
		echo "$FUNCNAME: ${vdb}.param"
		grpfileid=`echo "$vdb" | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}.*/\1/"`
		grpal=0
		grpdist=0
		if [[ ${grptuningid} -eq $grpfileid ]]; then
			continue
		elif [[ $grpfileid -eq 1 ]]; then
			grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'`
			grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'`
		else
			grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'`
			grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'`
		fi
		lineno=`wc -l "$vdb" | awk '{print $1}'`
		lineno=$(($lineno / 2))
		"`dirname $0`/generateparam" "${groupfortuning}.param" $lineno $grpdist > "${vdb}.param"
	done \
    ) 1>>"${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" 2>&1
    errcode=$?

    if [[ $errcode -ne 0 ]]; then
	    echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
    else
	    echo "Done $FUNCNAME $t $s $sim. Log: $TIME_DIR/paramsetting_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "${TIME_DIR}/paramsetting_${t}_${s}_${sim}_${GROUPING_S}"
    fi
    return $errcode
}

# clustering: $1=<vec file>, $2=<distance>, $3=<flag to computer parameter>
cluster()
{
	vdb="$1"
	dist=$2
	vfile=`basename "$vdb"`
	flag=$3

	if [[ $3 != "-c" ]]; then
	    flag=
	fi

	echo -n "Clustering '${vdb}' ${grpdist} ..." | tee -a "$TIME_DIR/cluster_${vfile}"

	if [[ "$TOOVERWRITE" != "true" &&
	      -s "$CLUSTER_DIR/cluster_${vfile}_$2" &&
	      "$CLUSTER_DIR/cluster_${vfile}_$2" -nt "$vdb" ]];
        then
		echo "Warning: $CLUSTER_DIR/cluster_${vfile}_$2 exists...Skip." | tee -a "$TIME_DIR/cluster_${vfile}"
		return 0
	fi

	# dumb (not flexible) memory limit setting
	mem=`ls -l "$vdb" | awk '{printf("%.0f", $5/1024/1024+0.5)}'`
	if [ $mem -lt 2 ]; then
		mem=10000000
	elif [ $mem -lt 5 ]; then
		mem=20000000
	elif [ $mem -lt 10 ]; then
		mem=30000000
	elif [ $mem -lt 20 ]; then
		mem=60000000
	elif [ $mem -lt 50 ]; then
		mem=150000000
	elif [ $mem -lt 100 ]; then
		mem=300000000
	elif [ $mem -lt 200 ]; then
		mem=600000000
	elif [ $mem -lt 500 ]; then
		mem=900000000
	elif [ $mem -lt 1024 ]; then
		mem=1900000000
	elif [ $mem -lt 2048 ]; then
		mem=3800000000
	else
		echo "Error: Size of $vdb > 2G. I don't want to do it before you think of any optimization." | tee -a "$TIME_DIR/cluster_${vfile}"
		exit 1;
	fi

	# group vector files should have already <dist> in their names; parameter files should also exist:
	if [[ $3 != "-c" && -s "${vdb}" &&
	      ! -s "${vdb}.param" ]]; then
		echo "Error: Parameter File missing for '$vdb' $dist...Exit" | tee -a "$TIME_DIR/cluster_${vfile}"
		exit 65
	fi
	( time \
	  "$CLUSTER_EXEC" -R $dist -M $mem -b 2 -A -f "$vdb" ${flag} -p "${vdb}.param" \
	  > "$CLUSTER_DIR/cluster_${vfile}" \
	) 1>>"$TIME_DIR/cluster_${vfile}" 2>&1
	errcode=$?

	if [[ $errcode -ne 0 ]]; then
		echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/cluster_${vfile}" | tee -a "$TIME_DIR/cluster_${vfile}"
	else
		echo "Done clustering '${vdb}' ${grpdist}. Log: $TIME_DIR/cluster_${vfile}" | tee -a "$TIME_DIR/cluster_${vfile}"
	fi
	return $errcode
}

# Post-processing and count clone line #:
pcluster()
{
	cdb="$1"
	cfile=`basename "$cdb"`
# Count clone line # before post-processing:
#        echo "Count clone lines for '$cdb' ..."
#        ( time \
#          "$coverage" "$cdb" > "${CLUSTER_DIR}/coverage_${cfile}" \
#        ) >& "${TIME_DIR}/coverage_${cfile}"
# Post-processing:
	echo -n "Post processing $cdb ..." | tee -a "${TIME_DIR}/post_${cfile}"
	if [[ $(wc -l "$cdb" | awk '{print $1}') -lt 2 ]];
	then
		echo "Warning: clone cluster file is too small to be right: $cdb...Skip." | tee -a "${TIME_DIR}/post_${cfile}"
		return 1
	fi

	if [[ "$TOOVERWRITE" != "true" &&
	      -s "$CLUSTER_DIR/post_${cfile}" &&
	      "$CLUSTER_DIR/post_${cfile}" -nt "$cdb" ]];
        then
		echo "Warning: $CLUSTER_DIR/post_${cfile} exists...Skip. May manually delete it or set 'overwrite'" | tee -a "${TIME_DIR}/post_${cfile}"
		return 0
	else
		( time \
		  "$POSTPRO_EXEC" "$cdb" > "${CLUSTER_DIR}/post_${cfile}" \
		) 1>>"${TIME_DIR}/post_${cfile}" 2>&1
		errcode=$?
		if [[ $errcode -ne 0 ]]; then
			echo "$FUNCNAME: Possible errors occurred. Check log: $TIME_DIR/post_${cfile}" | tee -a "${TIME_DIR}/post_${cfile}"
		else
			echo "Done post-processing $cdb. Log: $TIME_DIR/post_${cfile}" | tee -a "${TIME_DIR}/post_${cfile}"
		fi
		return $errcode
	fi
# Count clone line # after post-processing:
#	echo "Count clone lines for $CLUSTER_DIR/post_${cfile} ..."
#	( time \
#	  "$coverage" "${CLUSTER_DIR}/post_${cfile}" > "${CLUSTER_DIR}/coverage_post_${cfile}" \
#	) 2& "${TIME_DIR}/coverage_post_${cfile}"

	return 0
}


. `dirname $0`/configure

if [[ $# -ge 1 ]]; then
	case "$1" in
		clean )
			clean
			exit $?
			;;
		clean_all )
			clean
			clean_log
			exit $?
		       	;;
		overwrite )
			TOOVERWRITE="true"
			;;
		* )
			echo "Usage: $0 [overwrite | clean | clean_all]"
			exit 1
			;;
	esac
fi

errcode=0
for t in ${MIN_TOKENS};
do
  for s in ${STRIDE};
  do
    for sim in $SIMILARITY;
    do
	echo "Vector clustering w/ MIN_TOKENS=$t, STRIDE=$s, SIMILARITY=$sim ..."
	echo

# convert SIMILARITY to DISTANCE used by LSH:
#       (1) always round up; (2) different for different code sizes; (3) require 'awk' for sqrt
	i=`echo "$sim ${GROUPING_S}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'`
# Grouping:
	grouping $t $s $i
	errcode=$?
	# NOTE:
	#   range file name:  ranges_${i}_${GROUPING_S}
	#   group file names: vdb_${t}_${s}_g[0-9]+_${i}_${GROUPING_S}
# Set parameter files for LSH:
	paramsetting $t $s $sim
	errcode=$?
# Clustering:
	echo "Cluster every vector groups..." | tee "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}"
	find "$VECTOR_DIR" -type f -name "vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" -not -name '*.param' | while read vdb;
	do
	  grpfileid=`echo "$vdb" | sed "s/.*vdb_${t}_${s}_g\([0-9]*\)_${i}_${GROUPING_S}.*/\1/"`
	  grpal=0
	  grpdist=0
	  # TODO: what are better distance parameters for the groups?
          if [[ $grpfileid -le 0 ]]; then
              echo "Warning: invalid group id: $grpfileid in group $vdb...Skip." | tee -a "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}"
              continue
	  elif [[ $grpfileid -eq 1 ]]; then
		grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $3}'`
	  	grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*$2) )}'`
	  else
		grpal=`head -n $(expr $grpfileid + 1) $(dirname "$vdb")/ranges_${i}_${GROUPING_S} | tail -n 1 | awk '{print $2}'`
	  	grpdist=`echo "$sim ${grpal}" | awk '{printf( "%.7g\n", sqrt((1-$1)*2*$2) )}'`
	  fi
	  cluster "$vdb" $grpdist 1>>"$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}" 2>&1
	  errcode=$?
	  if [[ $errcode -ne 0 ]]; then
		  echo "Clustering: Possible errors for vector group: $vdb $grpdist. Check log: $TIME_DIR/cluster_$(basename "${vdb}")" | tee -a "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}"
	  fi
	done
	echo "Done clustering. Check log: $TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S} and $TIME_DIR/cluster_vdb_${t}_${s}_g[0-9]+_${i}_${GROUPING_S}" | tee -a "$TIME_DIR/cluster_vdb_${t}_${s}_allg_${i}_${GROUPING_S}"
# Merging:
	echo -n "Merging all clone reports into $CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S} ..." | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}"
	tomerge=
	if [[ "$TOOVERWRITE" = "true" ]]; then
		tomerge="true"
	elif [[ ! -s "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" ]]; then
		tomerge="true"
	else
		while read grpf; do
			if [[ "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}" -ot "$grpf" ]]; then
				tomerge="true"
				break
			fi
		done < <( find "$CLUSTER_DIR" -type f -name "cluster_vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" )
	fi
	if [[ "$tomerge" != "true" ]]; then
		echo "Warning: '$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}' exists...Skip." | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}"
	else
		# Continue to merge:
		echo -n > "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}"
		( time \
		  find "$CLUSTER_DIR" -type f -name "cluster_vdb_${t}_${s}_g[0-9]*_${i}_${GROUPING_S}*" | while read grpf;
		  do
			# remove some output lines in each file outputted by LSH/enumBuckets:
			lineno=`wc -l $grpf | awk '{print $1;}'`
			head -n `expr $lineno - 2` $grpf | tail -n `expr $lineno - 11` >> "$CLUSTER_DIR/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}"
		  done \
		) 1>>"${TIME_DIR}/merging_${t}_${s}_${sim}_${GROUPING_S}" 2>&1
		errcode=$?
		if [[ $errcode -ne 0 ]]; then
			echo "Merging: Possible errors occurred. Check log: $TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}"
		else
			echo "Done merging $CLUSTER_DIR/cluster_vdb_${t}_${s}_all_${sim}_${GROUPING_S}. Log: $TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}" | tee -a "$TIME_DIR/merging_${t}_${s}_${sim}_${GROUPING_S}"
		fi
	fi
# Post-processing:
	pcluster "${CLUSTER_DIR}/cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}"
	errcode=$?
# Clean up intermediate files:
	clean
# Bug Finding: done in main.sh instead.

	echo
	echo "Done clustering: ${t} ${s} $sim. See clone reports in $CLUSTER_DIR/post_cluster_vdb_${t}_${s}_allg_${sim}_${GROUPING_S}"
	echo
    done
  done
done


