render

#!/bin/bash

# usage: src/render <copy_of_parameters.jl>

# git-pull will complain if parameters.jl is modified

# to monitor progress:
#   watch -n 120 tail -n 40 <destination>/monitor.log

# data are temporarily logged in <logfile_scratch>/{director,monitor,squatter1-N,merge1-M}.log,
# where N=throttle_leaf_njobs from parameters.jl, and M~throttle_octree_njobs, and then
# tar'd into <destination>/logs.tar.gz.  manual queries into these log files are informative:
#   tar xzfO <destination>/logs.tar.gz render.log 
#   tar xzfO <destination>/logs.tar.gz | grep "ERROR"
#   tar xzfO <destination>/logs.tar.gz | grep "reading input tile" | wc

# also put in <destination> are
#   set_parameters.jl: a copy of <parameters.jl> specified on the command line
#   calculated_parameters.jl
#   tilebase.cache.yml
#   transform.txt
#   render.log

parameters_file=$1

export RENDER_PATH=/groups/mousebrainmicro/mousebrainmicro/Software/barycentric6
export JULIA=/groups/mousebrainmicro/mousebrainmicro/Software/barycentric6/julia-1.8.3/bin/julia

export JULIA_PROJECT=$RENDER_PATH/src/render 
export JULIA_DEPOT_PATH=$(dirname $(dirname $JULIA))/local/share/julia

umask 002

# get vars from parameters.jl file
expr="include(\"${RENDER_PATH}/src/render/src/admin.jl\");
      tiles = TileBaseOpen(source);
      tile_shape = TileShape(TileBaseIndex(tiles,1));
      nchannels = tile_shape[4];
      print(join([destination,shared_scratch,logfile_scratch,delete_scratch,notify_addr,bill_userid,
            typeof(which_cluster) <: String ? which_cluster : join(which_cluster, ','),
            throttle_octree_njobs,throttle_octree_njobs_per_machine,octree_ncores_per_job,
            nchannels,short_queue,
            overall_time_limit, leaf_time_limit, octree_time_limit, cleanup_time_limit,
            source,downsample_from_existing_leaves],' '))"

read -a tmp <<< $($JULIA -L $parameters_file -e "$expr")
destination=${tmp[0]}
shared_scratch=${tmp[1]}
logfile_scratch=${tmp[2]}
delete_scratch=${tmp[3]}
notify_addr=${tmp[4]}
bill_userid=${tmp[5]}
which_cluster=${tmp[6]}
throttle_octree_njobs=${tmp[7]}
throttle_octree_njobs_per_machine=${tmp[8]}
octree_ncores_per_job=${tmp[9]}
nchannels=${tmp[10]}
short_queue=${tmp[11]}
overall_time_limit=${tmp[12]}
leaf_time_limit=${tmp[13]}
octree_time_limit=${tmp[14]}
cleanup_time_limit=${tmp[15]}
source_path=${tmp[16]}
downsample_from_existing_leaves=${tmp[17]}

# delete <logfile_scratch> and <destination> if they exist
if [ -d $logfile_scratch ] ; then
  echo "deleting logfile_scratch = $logfile_scratch" >> $logfile_scratch/render.log
  rm -rf $logfile_scratch
fi
mkdir -p $logfile_scratch
if [ -d $destination ] ; then
  echo "deleting destination = $destination" >> $logfile_scratch/render.log
  rm -rf $destination
fi
mkdir -p $destination

date >> $logfile_scratch/render.log
hostname >> $logfile_scratch/render.log

# copy parameters to <destination>
cp $parameters_file $destination/set_parameters.jl
chmod g+rw $destination/set_parameters.jl

# create a probably-unique job name
jobname=$(cat /dev/urandom | tr -dc 'a-zA-Z' | head -c 7)
echo jobname = $jobname >> $logfile_scratch/render.log

# copy files
if [ $downsample_from_existing_leaves == 'true' ]; then
  cmd="cp ${source_path}/{calculated_parameters.jl,tilebase.cache.yml,tilebase.cache.jld2,transform.txt} $destination"
  echo $cmd &>> $logfile_scratch/render.log
  eval $cmd &>> $logfile_scratch/render.log
fi

sync

# create the leaf nodes
hold=''
if [ $downsample_from_existing_leaves == 'false' ]; then
  cmd="umask 002;
       $JULIA ${RENDER_PATH}/src/render/src/director.jl $destination/set_parameters.jl $jobname"
  echo $cmd &>> $logfile_scratch/render.log
  if [ $which_cluster == 'janelia' ]; then
    echo $cmd | bsub -P $bill_userid -J $jobname -W $overall_time_limit \
          -o $logfile_scratch/director.log
    hold="-w done($jobname)"
  else
    eval $cmd &> $logfile_scratch/director.log
  fi
fi

# get nlevels from calculated_parameters.jl file
while [ ! -f ${destination}/calculated_parameters.jl ]; do sleep 2; done
expr="include(joinpath(destination,\"calculated_parameters.jl\"));
      print(nlevels)"
read -a tmp <<< $($JULIA -L $parameters_file -e "$expr")
nlevels=${tmp[0]}

# link to the leaf nodes
if [ $downsample_from_existing_leaves == 'true' ]; then
  echo symlinking leaf nodes &>> $logfile_scratch/render.log
  cd $source_path
  for i in `seq 1 8` ; do
    find $i -mindepth $nlevels -name '*.tif' | xargs -n 1 -I{} bash -c "foo={}; source=$source_path; destination=$destination;"' mkdir -p ${destination}/${foo%/*}; ln -s ${source}/${foo} ${destination}/${foo}' &
  done
  wait
  cd -
fi

# downsample the octree
if [ "$nlevels" -gt "2" ]; then
  expr="using Distributed;
        p=addprocs(${octree_ncores_per_job});
        @everywhere include(joinpath(dirname(dirname(\"$JULIA\")),\"etc\",\"julia\",\"startup.jl\"));
        @everywhere include(\"${destination}/set_parameters.jl\");
        @everywhere include(\"${destination}/calculated_parameters.jl\");
        @everywhere include(\"${RENDER_PATH}/src/render/src/admin.jl\");
        t=time();
        id=parse(Int,ENV[\"LSB_JOBINDEX\"]);
        oct1,oct2,oct3 = ((id-1)&0x1c0)>>6+1, ((id-1)&0x38)>>3+1, ((id-1)&0x7)+1;
        frompath = downsample_from_existing_leaves ? destination : shared_scratch;
        delete_flag = !downsample_from_existing_leaves && delete_scratch==\"as-you-go\";
        if isdir(joinpath(frompath,string(oct1),string(oct2),string(oct3)))
          merge_output_tiles(frompath, destination, \"default\", file_format_save,
                string(oct1)*\"/\"*string(oct2)*\"/\"*string(oct3), true, true, delete_flag);
          @info string(\"merging took \",string(round(Int,time()-t)),\" sec\");
        end;
        rmprocs(p);"
  cmd="umask 002;
       date;
       hostname;
       $JULIA -e '$expr';
       date"
  echo $cmd &>> $logfile_scratch/render.log
  if [ $which_cluster == 'janelia' ]; then
    echo $cmd | bsub -P $bill_userid -J ${jobname}2[1-512]%${throttle_octree_njobs} \
          -n $octree_ncores_per_job \
          $hold -W $octree_time_limit -o $logfile_scratch/merge%I.log
  else
    pcmd="export RENDER_PATH=$RENDER_PATH;
          export JULIA=$JULIA;
          export JULIA_PROJECT=$JULIA_PROJECT;
          export JULIA_DEPOT_PATH=$JULIA_DEPOT_PATH;
          export HOSTNAME=$HOSTNAME;
          export LSB_JOBINDEX={};
          $cmd &> ${logfile_scratch}/merge{}.log"
    parallel -S $which_cluster -j $throttle_octree_njobs_per_machine $pcmd ::: `seq 1 512` &>> $logfile_scratch/render.log
  fi
  hold="-w done(${jobname}2)"
fi

expr="using Distributed;
      p=addprocs(${octree_ncores_per_job});
      @everywhere include(joinpath(dirname(dirname(\"$JULIA\")),\"etc\",\"julia\",\"startup.jl\"));
      @everywhere include(\"${destination}/set_parameters.jl\");
      @everywhere include(\"${destination}/calculated_parameters.jl\");
      @everywhere include(\"${RENDER_PATH}/src/render/src/admin.jl\");
      t=time();
      frompath = (nlevels>2 || downsample_from_existing_leaves) ? destination : shared_scratch;
      delete_flag = !downsample_from_existing_leaves && delete_scratch==\"as-you-go\";
      merge_output_tiles(frompath, destination, \"default\", file_format_save, \"\", true, true, delete_flag);
      @info string(\"merging took \",string(round(Int,time()-t)),\" sec\");
      rmprocs(p);"
cmd="umask 002;
     date;
     hostname;
     $JULIA -e '$expr';
     date"
echo $cmd &>> $logfile_scratch/render.log
if [ $which_cluster == 'janelia' ]; then
  echo $cmd | bsub -P $bill_userid -J ${jobname}3 \
        -n $octree_ncores_per_job \
        $hold -W $octree_time_limit -o $logfile_scratch/merge513.log
else
  pcmd="export RENDER_PATH=$RENDER_PATH;
        export JULIA=$JULIA;
        export JULIA_PROJECT=$JULIA_PROJECT;
        export JULIA_DEPOT_PATH=$JULIA_DEPOT_PATH;
        export HOSTNAME=$HOSTNAME;
        $cmd &> ${logfile_scratch}/merge513.log"
  ssh ${which_cluster%%,*} $pcmd &>> $logfile_scratch/render.log
fi

# email user
echo watch -n 600 tail -n '$((LINES-2))' $logfile_scratch/monitor.log | mail -s "job $jobname started" ${notify_addr}@janelia.hhmi.org

# start the monitor process
nohup sh -c "sleep 600; ${RENDER_PATH}/src/render/src/monitor $jobname $destination/set_parameters.jl &> ${logfile_scratch}/monitor.log" 2>/dev/null &

date >> $logfile_scratch/render.log

# delete shared_scratch
if [ $delete_scratch != 'never' ]; then
  cmd="date;
       hostname;
       df -h $shared_scratch;
       echo deleting shared_scratch = $shared_scratch;
       rm -rf $shared_scratch;
       date"
  echo $cmd &>> $logfile_scratch/render.log
  if [ $which_cluster == 'janelia' ]; then
    echo $cmd | bsub -P $bill_userid -J ${jobname}4 \
          -w "done(${jobname}3)" -W $cleanup_time_limit \
          -o ${destination}/delete.log
  else
    eval $cmd &> ${destination}/delete.log
  fi
else
  touch ${destination}/delete.log
fi

# tar log files, profile
cmd="date;
     hostname;
     echo tar\'ing log files in $logfile_scratch;
     cd $logfile_scratch;
     tar czf $destination/logs.tar.gz *log;
     echo deleting logfile_scratch = $logfile_scratch;
     rm -rf $logfile_scratch;
     echo running beancounter;
     $JULIA ${RENDER_PATH}/src/render/src/beancounter.jl $destination;
     date"
echo $cmd &>> $logfile_scratch/render.log
if [ $which_cluster == 'janelia' ]; then
  echo $cmd | bsub -P $bill_userid -J ${jobname}5 \
        -w "done(${jobname}3)" -W $cleanup_time_limit \
        -o ${destination}/tar-profile.log
else
  eval $cmd &> ${destination}/tar-profile.log
fi