#!/bin/bash

###############################################
# SCRIPT CONSTANTS DECLARATION
###############################################
DEFAULT_SC_CFG="default"

# Next default values need to be consistent with runcompss
DEFAULT_LANGUAGE=java
DEFAULT_LIBRARY_PATH=$(pwd)
DEFAULT_APPDIR=$(pwd)
DEFAULT_CLASSPATH=$(pwd)
DEFAULT_PYTHONPATH=$(pwd)
DEFAULT_DEBUG=off
DEFAULT_LOG_LEVEL_ARG=debug
DEFAULT_COMMUNICATION_ADAPTOR=es.bsc.compss.nio.master.NIOAdaptor
DEFAULT_TRACING=false
DEFAULT_TRACING_ARG=true
DEFAULT_CUSTOM_EXTRAE_FILE="null"
DEFAULT_MASTER_PORT_BASE=43000
DEFAULT_MASTER_PORT_RAND_RANGE=1000
DEFAULT_CPU_AFFINITY="automatic"
DEFAULT_GPU_AFFINITY="automatic"
DEFAULT_TASK_EXECUTION=compss
DEFAULT_STORAGE_CONF=null
DEFAULT_PERSISTENT_WORKER_C=false

###############################################
# ERROR CONSTANTS DECLARATION
###############################################
ERROR_CFG_SC="SuperComputer CFG file doesn't exist"
ERROR_CFG_Q="Queue system CFG file doesn't exist"
ERROR_MASTER_NODE="Missing master node parameter"
ERROR_WORKER_NODES="Missing worker nodes parameter"
ERROR_NUM_CPUS="Invalid number of CPUS per node"
ERROR_WORKER_WD="Invalid Worker Working Dir option"
ERROR_NETWORK="Invalid network option"
ERROR_WORKER_IN_MASTER_CPUS="Parameter worker_in_master_cpus is bigger than the maximum number of cpus_per_node"
ERROR_WORKER_IN_MASTER_MEMORY="Incorrect worker_in_master_memory parameter. Only disabled or <int> allowed. I.e. 33000, 66000"
ERROR_WORKER_IN_MASTER_MEMORY_TOO_HIGH="Parameter worker_in_master_memory exceeds the node_memory limit"
ERROR_WORKER_IN_MASTER_MEMORY_NOT_SPECIFIED="Parameter worker_in_master_memory is mandatory if worker_in_master_cpus is not 0"
ERROR_PROLOG_ACTION="Exception executing prolog action"
ERROR_EPILOG_ACTION="Exception executing epilog action"


#---------------------------------------------------------------------------------------
# HELPER FUNCTIONS
#---------------------------------------------------------------------------------------

###############################################
# Displays usage
###############################################
usage() {
  local exitValue=$1

  cat <<EOT
Usage: $0 [options] application_name application_arguments

* Options:
  General:
    --help, -h                              Print this help message

    --opts                                  Show available options

    --version, -v                           Print COMPSs version
    
    --sc_cfg=<name>                         SuperComputer configuration file to use. Must exist inside queues/cfgs/
                                            Mandatory
                                            Default: ${DEFAULT_SC_CFG}

    --master_node=<string>                  Node where to run the COMPSs Master
                                            Mandatory

    --worker_nodes="<string string...>"     Space separated nodes where to run the COMPSs Workers (Notice the quotes)
                                            Mandatory
    
  Launch configuration:
EOT

  show_opts "$exitValue"
}

###############################################
# Show Options
###############################################
show_opts() {
  local exitValue=$1

  # Load default CFG for default values
  local defaultSC_cfg=${SCRIPT_DIR}/../queues/cfgs/${DEFAULT_SC_CFG}.cfg
  source "${defaultSC_cfg}"
  local defaultQS_cfg=${SCRIPT_DIR}/../queues/${QUEUE_SYSTEM}/${QUEUE_SYSTEM}.cfg
  source "${defaultQS_cfg}"
  
  # Show usage
  cat <<EOT
    --cpus_per_node=<int>                   Available CPU computing units on each node
                                            Default: ${DEFAULT_CPUS_PER_NODE}
    --gpus_per_node=<int>                   Available GPU computing units on each node
                                            Default: ${DEFAULT_GPUS_PER_NODE}
    --max_tasks_per_node=<int>              Maximum number of simultaneous tasks running on a node
                                            Default: ${DEFAULT_MAX_TASKS_PER_NODE}
    --node_memory=<MB>                      Maximum node memory: disabled | <int> (MB)
                                            Default: ${DEFAULT_NODE_MEMORY}
    --network=<name>                        Communication network for transfers: default | ethernet | infiniband | data.
                                            Default: ${DEFAULT_NETWORK}
                                              
    --prolog="<string>"                     Task to execute before launching COMPSs (Notice the quotes)
                                            If the task has arguments split them by "," rather than spaces.
                                            This argument can appear multiple times for more than one prolog action
                                            Default: Empty
    --epilog="<string>"                     Task to execute after executing the COMPSs application (Notice the quotes)
                                            If the task has arguments split them by "," rather than spaces.
                                            This argument can appear multiple times for more than one epilog action
                                            Default: Empty

    --master_working_dir=<path>             Working directory of the application
                                            Default: ${DEFAULT_MASTER_WORKING_DIR}
    --worker_working_dir=<name | path>      Worker directory. Use: scratch | gpfs | <path>
                                            Default: ${DEFAULT_WORKER_WORKING_DIR}
                                              
    --worker_in_master_cpus=<int>           Maximum number of CPU computing units that the master node can run as worker. Cannot exceed cpus_per_node.
                                            Default: ${DEFAULT_WORKER_IN_MASTER_CPUS}
    --worker_in_master_memory=<int> MB      Maximum memory in master node assigned to the worker. Cannot exceed the node_memory.
                                            Mandatory if worker_in_master_cpus is specified.
                                            Default: ${DEFAULT_WORKER_IN_MASTER_MEMORY}
    --jvm_worker_in_master_opts="<string>"  Extra options for the JVM of the COMPSs Worker in the Master Node. 
                                            Each option separed by "," and without blank spaces (Notice the quotes)
                                            Default: ${DEFAULT_JVM_WORKER_IN_MASTER}
    --container_image=<path>                Runs the application by means of a container engine image
                                            Default: Empty
    --container_compss_path=<path>          Path where compss is installed in the container image
                                            Default: /opt/COMPSs
    --container_opts="<string>"             Options to pass to the container engine
					    Default: empty
    --elasticity=<max_extra_nodes>	    Activate elasticity specifiying the maximum extra nodes (ONLY AVAILABLE FORM SLURM CLUSTERS WITH NIO ADAPTOR)
				            Default: 0

  Runcompss configuration:

EOT
    "${SCRIPT_DIR}"/runcompss --opts 

  exit "$exitValue"
}
  
###############################################
# Displays version
###############################################
display_version() {
  local exitValue=$1

  "${SCRIPT_DIR}"/runcompss --version

  exit "$exitValue"
}

###############################################
# Displays errors when treating arguments
###############################################
display_error() {
  local error_msg=$1
  
  echo "$error_msg"
  echo " "
  
  usage 1
}

###############################################
# Displays errors when executing actions
###############################################
action_error() {
  local error_msg=$1
  
  echo "$error_msg"
  echo " "
  
  exit 1
}

###############################################
# Loads the tracing environment
###############################################
load_tracing_env() {
  local module_tmp=$(mktemp)
  module list 2> "${module_tmp}"

  # Look for openmpi / impi / none
  impi=$(cat "${module_tmp}" | grep -i "impi")
  openmpi=$(cat "${module_tmp}" | grep -i "openmpi")
  
  if [ ! -z "$impi" ]; then
    # Load Extrae IMPI
    export EXTRAE_HOME=${COMPSS_HOME}/Dependencies/extrae-impi/
  elif [ ! -z "$openmpi" ]; then
    # Load Extrae OpenMPI
    export EXTRAE_HOME=${COMPSS_HOME}/Dependencies/extrae-openmpi/
  else 
    # Load sequential extrae
    export EXTRAE_HOME=${COMPSS_HOME}/Dependencies/extrae/
  fi

  # Clean tmp file
  rm -f "${module_tmp}"
}

###############################################
# Creates Worker CMD
###############################################
worker_cmd() {
  # WARNING: SETS GLOBAL SCRIPT VARIABLE WCMD
  local nodeId=$1
  local nodeName=$2
  local jvm_opts_size=$3
  local jvm_opts_str=$4
  local cusCPU=$5
  local cusGPU=$6
  local cpuMap=$7
  local gpuMap=$8
  local lot=$9

  local sandbox_worker_working_dir=${worker_working_dir}/${uuid}/${nodeName}
  local maxSend=5
  local maxReceive=5
  local worker_port=43001
  if [ -z "${container_image}" ]; then
     WCMD="${COMPSS_HOME}/Runtime/scripts/system/adaptors/nio/persistent_worker_starter.sh"
  else 
     WCMD="singularity exec $container_opts $container_image ${COMPSS_HOME}/Runtime/scripts/system/adaptors/nio/persistent_worker_starter.sh"
  fi
  WCMD="$WCMD ${library_path} \
            ${appdir} \
            ${cp} \
            ${jvm_opts_size} \
            ${jvm_opts_str} \
            ${debug} \
            ${maxSend} \
            ${maxReceive} \
            ${nodeName} \
            ${worker_port} \
            ${master_port} \
            ${cusCPU} \
            ${cusGPU} \
            ${cpuMap} \
            ${gpuMap} \
            ${lot} \
            ${uuid} \
            ${lang} \
            ${sandbox_worker_working_dir} \
            ${worker_install_dir} \
            ${appdir} \
            ${library_path} \
            ${cp} \
            ${pythonpath} \
            ${w_tracing} \
            ${custom_extrae_file} \
            ${nodeId} \
            ${storage_conf} \
            ${taskExecution} \
	    ${persistent_worker_c}"
}

###############################################
# Create Master CMD
###############################################
master_cmd() {
  local master_name=$1

  # WARNING: SETS GLOBAL SCRIPT VARIABLE MCMD
  if [ -z "${container_image}" ]; then
    MCMD="${COMPSS_HOME}/Runtime/scripts/user/runcompss"
  else
    MCMD="singularity exec $container_opts $container_image ${COMPSS_HOME}/Runtime/scripts/user/runcompss"
  fi
  MCMD="$MCMD --project=${PROJECT_FILE} \
            --resources=${RESOURCES_FILE} \
            --uuid=${uuid} \
            --specific_log_dir=${specific_log_dir} \
            --master_name=${master_name} \
            --master_port=${master_port} \
            --jvm_master_opts=\"${jvm_master_opts}\" \
            --jvm_workers_opts=\"${jvm_workers_opts}\""
}  


#---------------------------------------------------
# MAIN FUNCTIONS DECLARATION
#---------------------------------------------------

###############################################
# Function to get the arguments
###############################################
get_args() {
  # Avoid enqueue if there is no application
  if [ $# -eq 0 ]; then
    usage 1
  fi
  
  # Parse COMPSs Options
  while getopts hvgtmd-: flag; do 
    # Treat the argument
    case "$flag" in
      h)
        # Display help
        usage 0
        ;;
      v)
        # Display version
        display_version 0
        ;;
      d)
        log_level=${DEFAULT_LOG_LEVEL_ARG}
        #Keep it for runcompss (to add them to master)
        args_pass="$args_pass -$flag"
        ;;
      t)
        tracing=${DEFAULT_TRACING_ARG}
        #Keep it for runcompss (to add them to master)
        args_pass="$args_pass -$flag"
        ;;
      -)
      # Check more complex arguments 
      case "$OPTARG" in
        help)
          # Display help
          usage 0
          ;;
        version)
          # Display compss version
          display_version 0
          ;;
        opts)
          # Display options
          show_opts 0
          ;;
        master_node=*)
          master_node=${OPTARG//master_node=/}
          ;;
        worker_nodes=*)
          worker_nodes=${OPTARG//worker_nodes=/}
          ;;
        sc_cfg=*)
          sc_cfg=${OPTARG//sc_cfg=/}
          ;;
        cpus_per_node=*)
          cpus_per_node=${OPTARG//cpus_per_node=/}
          ;;
        gpus_per_node=*)
          gpus_per_node=${OPTARG//gpus_per_node=/}
          ;;
        max_tasks_per_node=*)
          max_tasks_per_node=${OPTARG//max_tasks_per_node=}
          ;;
        cpu_affinity=*)
          cpu_affinity=${OPTARG//cpu_affinity=}
          args_pass="$args_pass --$OPTARG"
          ;;
        gpu_affinity=*)
          gpu_affinity=${OPTARG//gpu_affinity=/}
          args_pass="$args_pass --$OPTARG"
          ;;
        master_working_dir=*)
          master_working_dir=${OPTARG//master_working_dir=/}
          ;;
        worker_working_dir=*)
          worker_working_dir=${OPTARG//worker_working_dir=/}
          ;; 
        worker_in_master_cpus=*)
          worker_in_master_cpus=${OPTARG//worker_in_master_cpus=/}
          ;;
        worker_in_master_memory=*)
          worker_in_master_memory=${OPTARG//worker_in_master_memory=/}
          ;;
        node_memory=*)
          node_memory=${OPTARG//node_memory=/}
          ;;
        network=*)
          network=${OPTARG//network=/}
          ;;
        lang=*)
          lang=${OPTARG//lang=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        library_path=*)
          library_path=${OPTARG//library_path=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        classpath=*)
          cp=${OPTARG//classpath=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        pythonpath=*)
          pythonpath=${OPTARG//pythonpath=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
	appdir=*)
	  appdir=${OPTARG//appdir=/}
	  #Keep it for runcompss (to add them to master)
	  args_pass="$args_pass --$OPTARG"
	  ;;
        log_level=*)
          log_level=${OPTARG///log_level=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        debug)
          log_level=${DEFAULT_LOG_LEVEL_ARG}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        tracing=*)
          tracing=${OPTARG//tracing=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        tracing)
          tracing=${DEFAULT_TRACING_ARG}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        extrae_config_file=*)
          custom_extrae_file=${OPTARG//extrae_config_file=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        comm=*)
          comm=${OPTARG//comm=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        jvm_master_opts=*)
          jvm_master_opts=${OPTARG//jvm_master_opts=/}
          ;;
        jvm_workers_opts=*)
          jvm_workers_opts=${OPTARG//jvm_workers_opts=/}
          ;;
        jvm_worker_in_master_opts=*)
          jvm_worker_in_master_opts=${OPTARG//jvm_worker_in_master_opts=/}
          ;;
        storage_conf=*)
          storage_conf=${OPTARG//storage_conf=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        task_execution=*)
          taskExecution=${OPTARG//task_execution=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        persistent_worker_c=*)
          persistent_worker_c=${OPTARG//persistent_worker_c=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        prolog=*)
          action=${OPTARG//prolog=/}
          prologActions[numPrologActions]="$action"
          numPrologActions=$((numPrologActions + 1))
          ;;
        epilog=*)
          action=${OPTARG//epilog=/}
          epilogActions[numEpilogActions]="$action"
          numEpilogActions=$((numEpilogActions + 1))
          ;; 
        uuid=*)
          # UUID will be generated by launch.sh remove it from COMPSs flags
          echo "WARNING: UUID is automatically generated. Omitting parameter"
          ;;
        specific_log_dir=*)
          # Specific log dir is automatically generated by launch_compss.sh. Remove it from COMPSs flags
          echo "WARNING: specific_log_dir is automatically generated. Omitting parameter. Define base_log_dir instead."
          ;;
        base_log_dir=*)
          # Base log dir is automatically generated by launch_compss.sh. Remove it from COMPSs flags
          # echo "WARNING: base_log_dir is automatically generated. Omitting parameter"
	  base_log_dir=${OPTARG//base_log_dir=/}
          ;;
        master_name=*)
          # Master name is automatically generated by launch_compss.sh. Remove it from COMPSs flags
          echo "WARNING: master_name is automatically generated. Omitting parameter"
          ;;
        master_port=*)
          # Remove from runcompss since launcher will add it
          master_port=${OPTARG//master_port=/}
          ;;
	container_image=*)
          container_image=${OPTARG//container_image=/}
          ;;
        container_compss_path=*)
          container_compss_path=${OPTARG//container_compss_path=/}
          ;;
	container_opts=*)
          container_opts=${OPTARG//jvm_master_opts=/}
          ;;
        elasticity=*)
          elasticity=${OPTARG//elasticity=/}
	  args_pass="$args_pass --conn=es.bsc.compss.connectors.DefaultNoSSHConnector"
          ;;
	*)
          # Flag didn't match any patern. Add to COMPSs 
          args_pass="$args_pass --$OPTARG"
          ;;
      esac
      ;;
    *)
      # Flag didn't match any patern. End of COMPSs flags
      args_pass="$args_pass -$flag"
      ;; 
    esac
  done
  
  # Shift COMPSs arguments
  shift $((OPTIND-1))

  # Pass application name and args
  args_pass="$args_pass $@" 
}

###############################################
# Function to check the arguments
###############################################
check_args() {
  ###############################################################
  # SC Configuration checks
  ###############################################################
  # Check sc configuration argument
  if [ -z "${sc_cfg}" ]; then
    sc_cfg=${DEFAULT_SC_CFG}
  fi
  if [[ ${sc_cfg} != *cfg ]]; then
    # Add cfg suffix
    sc_cfg=${sc_cfg}.cfg
  fi
   
  local scCfgFullPath=${SCRIPT_DIR}/../queues/cfgs/${sc_cfg}

  if [ ! -f "${scCfgFullPath}" ]; then
    # CFG file doesn't exist
    display_error "${ERROR_CFG_SC}"   
  fi
 
  # Source SC CFG env
  source "${scCfgFullPath}"

  # Check queue configuration env
  local queueCfgFullPath=${SCRIPT_DIR}/../queues/${QUEUE_SYSTEM}/${QUEUE_SYSTEM}.cfg
  if [ ! -f "${queueCfgFullPath}" ]; then
    # CFG file doesn't exist
    display_error "${ERROR_CFG_Q}" 
  fi

  # Source queue system CFG env
  source "${queueCfgFullPath}"
  
  ###############################################################
  # Infrastructure checks
  ###############################################################
  if [ -z "${master_node}" ]; then
    display_error "${ERROR_MASTER_NODE}"
  fi
  
  if [ -z "${worker_nodes}" ] && [ "${DEFAULT_WORKER_IN_MASTER_CPUS}" -eq "0" ]; then
    if [ -z "${worker_in_master_cpus}" ]; then
      display_error "${ERROR_WORKER_NODES}"
    else
      worker_nodes=""
    fi
  fi

  if [ -z "${network}" ]; then
    network=${DEFAULT_NETWORK}
  elif [ "${network}" == "default" ]; then
    network=${DEFAULT_NETWORK}
  elif [ "${network}" != "ethernet" ] && [ "${network}" != "infiniband" ] && [ "${network}" != "data" ]; then
    display_error "${ERROR_NETWORK}"
  fi
  
  if [ -z "${storage_conf}" ]; then
    storage_conf=${DEFAULT_STORAGE_CONF}
  fi

  if [ -z "${taskExecution}" ]; then
    taskExecution=${DEFAULT_TASK_EXECUTION}
  fi
  
  if [ -z "${persistent_worker_c}" ]; then
    persistent_worker_c=${DEFAULT_PERSISTENT_WORKER_C}
  fi

  ###############################################################
  # Node checks
  ###############################################################
  if [ -z "${max_tasks_per_node}" ]; then
    max_tasks_per_node=${DEFAULT_MAX_TASKS_PER_NODE}
  fi

  if [ -z "${cpus_per_node}" ]; then
    cpus_per_node=${DEFAULT_CPUS_PER_NODE}
  fi
  
  if [ "${cpus_per_node}" -lt "${MINIMUM_CPUS_PER_NODE}" ]; then
    display_error "${ERROR_NUM_CPUS}"                                                                                                                                                                                                       
  fi

  if [ -z "${gpus_per_node}" ]; then
    gpus_per_node=${DEFAULT_GPUS_PER_NODE}
  fi

  if [ -z "${cpu_affinity}" ]; then
    cpu_affinity=${DEFAULT_CPU_AFFINITY}
  fi

  if [ -z "${gpu_affinity}" ]; then
    gpu_affinity=${DEFAULT_GPU_AFFINITY}
  fi

  if [ -z "${node_memory}" ]; then
    node_memory=${DEFAULT_NODE_MEMORY}
  fi 
 
  if [ -z "${worker_in_master_cpus}" ]; then
    worker_in_master_cpus=${DEFAULT_WORKER_IN_MASTER_CPUS}
  fi
  if [ "${worker_in_master_cpus}" -gt "${cpus_per_node}" ]; then
    display_error "${ERROR_WORKER_IN_MASTER_CPUS}"
  fi

  if [ -z "${worker_in_master_memory}" ]; then
    worker_in_master_memory=${DEFAULT_WORKER_IN_MASTER_MEMORY}
  elif [ "${worker_in_master_memory}" != "disabled" ] && ! [[ "${worker_in_master_memory}" =~ ^[0-9]+$ ]]; then
    display_error "${ERROR_WORKER_IN_MASTER_MEMORY}"
  fi
  if [ "${worker_in_master_memory}" != "${DEFAULT_WORKER_IN_MASTER_MEMORY}" ] && [ "${node_memory}" != "${DEFAULT_NODE_MEMORY}" ]; then
    if [ "${worker_in_master_memory}" -gt "${node_memory}" ]; then
      display_error "${ERROR_WORKER_IN_MASTER_MEMORY_TOO_HIGH} (${worker_in_master_memory} < ${node_memory} "
    fi 
  fi

  if [ "${worker_in_master_cpus}" -gt 0 ] && [ "${worker_in_master_memory}" -le 0 ]; then
    display_error "${ERROR_WORKER_IN_MASTER_MEMORY_NOT_SPECIFIED}"
  fi

  if [ -z "${master_port}" ]; then
    rand_num=$RANDOM
    offset=$((rand_num % DEFAULT_MASTER_PORT_RAND_RANGE))
    master_port=$((DEFAULT_MASTER_PORT_BASE + offset))
  fi

  ###############################################################
  # Working Directory Checks
  ###############################################################
  if [ -z "${master_working_dir}" ]; then
    master_working_dir=${DEFAULT_MASTER_WORKING_DIR}
  fi
  
  if [ -z "${worker_working_dir}" ]; then
    worker_working_dir=${DEFAULT_WORKER_WORKING_DIR}
  elif [ "${worker_working_dir}" != "scratch" ] && [ "${worker_working_dir}" != "gpfs" ] && [[ ${worker_working_dir} != /* ]]; then
    display_error "${ERROR_WORKER_WD}"
  fi

  ###############################################################
  # JVM Checks
  ###############################################################
  if [ -z "${jvm_master_opts}" ]; then
    jvm_master_opts=${DEFAULT_JVM_MASTER}
  fi

  if [ -z "${jvm_workers_opts}" ]; then
    jvm_workers_opts=${DEFAULT_JVM_WORKERS}
  fi

  ###############################################################
  # Runtime and Tools Checks
  ###############################################################
  if [ -z "${log_level}" ]; then
    log_level=${DEFAULT_DEBUG}
  fi

  if [ -z "${comm}" ]; then
    comm=${DEFAULT_COMMUNICATION_ADAPTOR}
  fi

  if [ -z "${tracing}" ]; then
    tracing=${DEFAULT_TRACING}
  fi

  if [ -z "${custom_extrae_file}" ]; then
    custom_extrae_file=${DEFAULT_CUSTOM_EXTRAE_FILE}
  fi

  ###############################################################
  # Application Checks
  ###############################################################
  # Lang
  if [ -z "$lang" ]; then
    lang=${DEFAULT_LANGUAGE}
  fi

  # Library path
  if [ -z "${library_path}" ]; then
    library_path=${DEFAULT_LIBRARY_PATH}
  fi
  
  # Classpath
  if [ -z "$cp" ]; then
    cp=${DEFAULT_CLASSPATH}
  else
    fcp=""
    for currcp in ${cp//:/$'\n'}; do
    if [ ! "${currcp:0:1}" == '/' ]; then              # Relative paths to abs
      if [ -d "$currcp" ] || [ -f "$currcp" ]; then  # If the dir/file exists
        absdir="$(cd "$(dirname "$currcp")" && pwd)" # Get absolute dir
        file="$(basename "$currcp")"
        currcp="$absdir/$file"
      else
        echo "[ WARNING ]: Classpath \"$currcp\" does not exist..."
      fi
    fi
    fcp="${fcp}:$currcp"
    done
    cp="$(echo "$fcp" | cut -c2-)"
  fi

  # Pythonpath
  if [ -z "$pythonpath" ]; then
    pythonpath=${DEFAULT_PYTHONPATH}
  fi

  # AppDir
  if [ -z "$appdir" ]; then
    appdir=${DEFAULT_APPDIR}
  fi
}


###############################################
# Sets job variables
###############################################
set_variables() { 
  # Set script variables
  if [ -z "${container_image}" ]; then
      export COMPSS_HOME=${COMPSS_HOME}
      export GAT_LOCATION=${COMPSS_HOME}/Dependencies/JAVA_GAT
      worker_install_dir=${COMPSS_HOME}
  else
      if [ -z "${container_compss_path}" ]; then
              export COMPSS_HOME=/opt/COMPSs/
              export GAT_LOCATION=/opt/COMPSs/Dependencies/JAVA_GAT
              worker_install_dir=/opt/COMPSs
      else
              export COMPSS_HOME=${container_compss_path}/
              export GAT_LOCATION=${container_compss_path}/Dependencies/JAVA_GAT
              worker_install_dir=${container_compss_path}/
      fi
  fi

  # Create .COMPSs log dir for application execution
  if [ -z "${base_log_dir}" ]; then
        base_log_dir=$HOME
  fi
  specific_log_dir=${base_log_dir}/.COMPSs/${!ENV_VAR_JOB_ID}/
  mkdir -p "${specific_log_dir}"
  
  # SharedDisk variables
  if [ "${worker_working_dir}" == "gpfs" ]; then
    worker_working_dir=$(mktemp -d -p "${GPFS_PREFIX}${HOME}")
  elif [ "${worker_working_dir}" == "scratch" ]; then
    worker_working_dir=$TMPDIR
  else 
    # The working dir is a custom absolute path, create tmp
    worker_working_dir=$(mktemp -d -p "${worker_working_dir}")
  fi

  # Network variables
  if [ "${network}" == "ethernet" ]; then
    network=""
  elif [ "${network}" == "infiniband" ]; then
    network=${NETWORK_INFINIBAND_SUFFIX}
  elif [ "${network}" == "data" ]; then
    network=${NETWORK_DATA_SUFFIX}
  fi

  # Memory variables
  if [ "${node_memory}" == "${DEFAULT_NODE_MEMORY}" ]; then
    # Default value
    node_memory=${DEFAULT_NODE_MEMORY_SIZE}
  else
    # Change from MB to GB
    node_memory=$(( node_memory / 1024 - 4))
  fi

  if [ "${worker_in_master_memory}" == "${DEFAULT_WORKER_IN_MASTER_MEMORY}" ]; then
    # Default value
    worker_in_master_memory=${DEFAULT_NODE_MEMORY_SIZE}
  else
    # Change from MB to GB
    worker_in_master_memory=$(( worker_in_master_memory / 1024 - 4))
  fi

  # Load tracing and debug only for NIO
  if [ "${comm/NIO}" != "${comm}" ]; then
    # Adapting tracing flag to worker tracing level
    if [ -z "$tracing" ]; then
        w_tracing=0
    elif [ $tracing == "false" ]; then
        w_tracing=0
    elif [ $tracing == "basic" ] || [ $tracing == "true" ]; then
        w_tracing=1
        load_tracing_env
    elif [ $tracing == "advanced" ]; then
        w_tracing=2
        load_tracing_env
    fi

    # Adapt debug flag to worker script
    if [ "${log_level}" == "debug" ]; then
      debug="true"
    else
      debug="false"
    fi
  fi
  
  # Generate a UUID for workers and runcompss
  uuid=$(cat /proc/sys/kernel/random/uuid)
}

###############################################
# Creates XML Files
###############################################
log_variables() {
  echo "-------- Launch arguments --------"
  echo "Master:                    ${master_node}"
  echo "Workers:                   ${worker_nodes}"
  echo "Tasks per Node:            ${max_tasks_per_node}"
  echo "CPUs per Node:             ${cpus_per_node}"
  echo "GPUs per Node:             ${gpus_per_node}"
  echo "CPU Affinity:              ${cpu_affinity}"
  echo "GPU Affinity:              ${gpu_affinity}"
  echo "Network:                   ${network}"
  echo "Worker in Master CPUs:     ${worker_in_master_cpus}"
  echo "Worker in Master Memory:   ${worker_in_master_memory}"
  echo "Master Port:               ${master_port}"
  echo "Master WD:                 ${master_working_dir}"
  echo "Worker WD:                 ${worker_working_dir}"
  echo "Master JVM Opts:           ${jvm_master_opts}"
  echo "Workers JVM Opts:          ${jvm_workers_opts}"
  echo "Worker in Master JVM Opts: ${jvm_worker_in_master_opts}"
  echo "Library Path:              ${library_path}"
  echo "Classpath:                 ${cp}" 
  echo "Pythonpath:                ${pythonpath}"
  echo "Appdir:			   ${appdir}"
  echo "Lang:                      ${lang}"
  echo "COMM:                      ${comm}"
  echo "Prolog:                    ${prologActions[*]}"
  echo "Epilog:                    ${epilogActions[*]}"
  echo "Storage conf:              ${storage_conf}"
  echo "Task execution:            ${taskExecution}"
  echo "To COMPSs:                 ${args_pass}"
  echo "-----------------------------------"
  echo " "
}

###############################################
# Creates XML Files
###############################################
create_xml_files() {
  # Resources.xml and project.xml filenames
  local sec=$(date +%s)
  RESOURCES_FILE=${worker_working_dir}/resources_$sec.xml
  PROJECT_FILE=${worker_working_dir}/project_mn_$sec.xml

  # Begin creating the resources file and the project file
  insert_xml_headers

  # Log Master Node
  echo "Master will run in ${master_node}"

  # Log List of Workers
  echo "List of workers: ${worker_nodes}"

  # Add worker slots on master if needed
  if [ "${worker_in_master_cpus}" -ne 0 ]; then
    # add_compute_node name cpus gpus lot memory
    add_compute_node "${master_node}" "${worker_in_master_cpus}" "${gpus_per_node}" "${max_tasks_per_node}" "${worker_in_master_memory}"
  fi

  # Find the number of tasks to be executed on each node
  for node in ${worker_nodes}; do
    # add_compute_node name cpus gpus lot memory
    add_compute_node "$node" "${cpus_per_node}" "${gpus_per_node}" "${max_tasks_per_node}" "${node_memory}"
  done
  if [ ! -z "${elasticity}" ]; then
  	insert_xml_elasticity "${elasticity}" "${cpus_per_node}" "${gpus_per_node}" "${max_tasks_per_node}" "${node_memory}"
  fi
  # Finish the resources file and the project file 
  insert_xml_footers

  echo "Generation of resources and project file finished"
  echo "Project.xml:   ${PROJECT_FILE}"
  echo "Resources.xml: ${RESOURCES_FILE}"
}

###############################################
# Launches the application
###############################################
launch() {
  echo "------ Launching application ------"

  # Launch workers separately if they are persistent
  if [ "${comm/NIO}" != "${comm}" ]; then
    # Start workers' processes
    local hostid=1
    if [ "${worker_in_master_cpus}" -ne 0 ]; then
      # Worker in master node
      local jvm_worker_in_master_opts_str=$(echo "${jvm_worker_in_master_opts}" | tr "," " ")
      local jvm_worker_in_master_opts_size=$(echo "${jvm_worker_in_master_opts_str}" | wc -w)
      # worker_cmd id name jvmOptsSize jvmOpts cpus gpus cpuMap gpuMap lot
      worker_cmd $hostid "${master_node}${network}" "${jvm_worker_in_master_opts_size}" "${jvm_worker_in_master_opts_str}" "${worker_in_master_cpus}" "${gpus_per_node}" "${cpu_affinity}" "${gpu_affinity}" "${max_tasks_per_node}"
      WCMD="${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${master_node} ${WCMD}"
      echo "CMD Worker $hostid launcher: $WCMD"
      $WCMD&
      hostid=$((hostid+1))
    fi

    local jvm_workers_opts_str=$(echo "${jvm_workers_opts}" | tr "," " ")
    local jvm_workers_opts_size=$(echo "${jvm_workers_opts_str}" | wc -w)
    if [ ! -z "${worker_nodes}" ]; then
      for node in ${worker_nodes}; do
        # worker_cmd id name jvmOptsSize jvmOpts cpus gpus cpuMap gpuMap lot
        worker_cmd $hostid "$node${network}" "${jvm_workers_opts_size}" "${jvm_workers_opts_str}" "${cpus_per_node}" "${gpus_per_node}" "${cpu_affinity}" "${gpu_affinity}" "${max_tasks_per_node}"
        WCMD="${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${node} ${WCMD}"
        echo "CMD Worker $hostid launcher: $WCMD"
        $WCMD&
        hostid=$((hostid+1))
      done
    fi

    # Sleep a little before launching master
    sleep 10s
  fi

  # Launch master
  master_cmd "${master_node}${network}"
  #Old master launch with launch_command (removed maybe not necessary)
  #MCMD="${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${master_node} ${MCMD} ${args_pass}"
  MCMD="${MCMD} ${args_pass}"
  echo "CMD Master: $MCMD"
  #Old launch master do not need to run in background
  #$MCMD&
  $MCMD
}

###############################################
# Prolog actions
###############################################
prolog() {
  # Execute user prolog actions
  echo "---- Executing Prolog actions ----"
  for action in "${prologActions[@]}"; do
    realAction=$(echo "${action}" | tr "," " ")
    echo "- Prolog: $realAction"
    $realAction
    local actionExitValue=$?
    if [ $actionExitValue -ne 0 ]; then
      action_error "$ERROR_PROLOG_ACTION"
    fi
  done
}

###############################################
# Epilog actions
###############################################
epilog() {
  # Execute user epilog actions
  echo "---- Executing Epilog actions ----"
  for action in "${epilogActions[@]}"; do
    realAction=$(echo "${action}" | tr "," " ")
    echo "- Epilog: $realAction"
    $realAction
    local actionExitValue=$?
    if [ $actionExitValue -ne 0 ]; then
      action_error "$ERROR_EPILOG_ACTION"
    fi
  done
}

###############################################
# Wait for execution end
###############################################
wait_for_completion() {
  # Wait for Master and Workers to finish
  echo "Waiting for application completion"
  wait
}

###############################################
# Clean function for trap
###############################################
cleanup() {
  # Cleanup
  echo "Cleanup Worker TMP files"
  for node in ${worker_nodes}; do
    local sandboxWD=${worker_working_dir}/${uuid}
    echo "- Removing ${sandboxWD}"
    ${LAUNCH_CMD} "${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}$node" "${CMD_SEPARATOR}"rm -rf "${sandboxWD}${CMD_SEPARATOR}"
  done

  echo "Cleanup TMP files"
  rm -f "${PROJECT_FILE}"
  rm -f "${RESOURCES_FILE}"
}
  
  
#---------------------------------------------------------------------------------------
# XML SPECIFIC FUNCTIONS
#---------------------------------------------------------------------------------------

###############################################
# Insert XML Headers
###############################################
insert_xml_headers() {
  if [ "${GPFS2_PREFIX}" != "" ] && [ "${GPFS2_PREFIX}" != "${GPFS_PREFIX}" ]; then
    cat > "${RESOURCES_FILE}" << EOT
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ResourcesList>
  <SharedDisk Name="gpfs" />
  <SharedDisk Name="gpfs2" />

EOT
  else
    cat > "${RESOURCES_FILE}" << EOT
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<ResourcesList>
  <SharedDisk Name="gpfs" />

EOT
  fi

  cat > "${PROJECT_FILE}" << EOT
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<Project>
  <MasterNode>
      <SharedDisks>
          <AttachedDisk Name="gpfs">
              <MountPoint>${GPFS_PREFIX}</MountPoint>
          </AttachedDisk>
EOT
  if [ "${GPFS2_PREFIX}" != "" ] && [ "${GPFS2_PREFIX}" != "${GPFS_PREFIX}" ]; then
    cat >> "${PROJECT_FILE}" << EOT
          <AttachedDisk Name="gpfs2">
              <MountPoint>${GPFS2_PREFIX}</MountPoint>
          </AttachedDisk>
EOT
  fi 

  cat >> "${PROJECT_FILE}" << EOT
      </SharedDisks>
  </MasterNode>

EOT
}

###############################################
# Insert XML Footers
###############################################
insert_xml_footers() {
  cat >> "${RESOURCES_FILE}" << EOT
</ResourcesList>
EOT

  cat >> "${PROJECT_FILE}" << EOT
</Project>
EOT
}


###############################################
# Insert XML Elasticity 
###############################################
insert_xml_elasticity() {
  local max_vms=$1
  local cus=$2
  local gpus=$3
  local lot=$4
  local memory=$5
  local jvm_workers_opts_str=$(echo "${jvm_workers_opts}" | tr "," " ")
  local jvm_workers_opts_size=$(echo "${jvm_workers_opts_str}" | wc -w)
  cat >> "${RESOURCES_FILE}" << EOT
   <CloudProvider Name="SLURM-Cluster">
        <Endpoint>
            <Server></Server>
            <ConnectorJar>slurm-conn.jar</ConnectorJar>
            <ConnectorClass>es.bsc.conn.slurm.SLURMConnector</ConnectorClass>
        </Endpoint>
        <Images>
EOT
  if [ -z "${container_image}" ]; then
	cat >> "${RESOURCES_FILE}" << EOT
            <Image Name="None">
EOT
  else
       cat >> "${RESOURCES_FILE}" << EOT
            <Image Name="${container_image}">
EOT
  fi
  cat >> "${RESOURCES_FILE}" << EOT
                <CreationTime>10</CreationTime>
      		<OperatingSystem>
          		<Type>Linux</Type>
          		<Distribution>SMP</Distribution>
          		<Version>3.0.101-0.35-default</Version>
      		</OperatingSystem>
      		<Software>
          		<Application>JAVA</Application>
          		<Application>PYTHON</Application>
          		<Application>EXTRAE</Application>
          		<Application>COMPSS</Application>
      		</Software>
      		<Adaptors>
          		<Adaptor Name="es.bsc.compss.nio.master.NIOAdaptor">
              			<SubmissionSystem>
                  			<Interactive/>
              			</SubmissionSystem>
              			<Ports>
                  			<MinPort>43001</MinPort>
                  			<MaxPort>43002</MaxPort>
                  			<RemoteExecutionCommand>${REMOTE_EXECUTOR}</RemoteExecutionCommand>
              			</Ports>
          		</Adaptor>
      		</Adaptors>
      		<SharedDisks>
          		<AttachedDisk Name="gpfs">
              			<MountPoint>${GPFS_PREFIX}</MountPoint>
          		</AttachedDisk>
EOT
  if [ "${GPFS2_PREFIX}" != "" ] && [ "${GPFS2_PREFIX}" != "${GPFS_PREFIX}" ]; then
    cat >> "${RESOURCES_FILE}" << EOT
          		<AttachedDisk Name="gpfs2">
              			<MountPoint>${GPFS2_PREFIX}</MountPoint>
          		</AttachedDisk>
EOT
  fi

  cat >> "${RESOURCES_FILE}" << EOT
      		</SharedDisks>
            </Image>
        </Images>
        <InstanceTypes>
            <InstanceType Name="Resource-Type-1">
		<Processor Name="MainProcessor">
          		<ComputingUnits>${cus}</ComputingUnits>
          		<Architecture>Intel</Architecture>
          		<Speed>2.6</Speed>
      		</Processor>
EOT
  if [ "${gpus}" != "$DEFAULT_GPUS_PER_NODE" ]; then
    cat >> "${RESOURCES_FILE}" << EOT
        	<Processor Name="GPU">
            		<Type>GPU</Type>
            		<ComputingUnits>${gpus}</ComputingUnits>
            		<Architecture>k80</Architecture>
            		<Speed>2.6</Speed>
        	</Processor>
EOT
  fi
cat >> "${RESOURCES_FILE}" << EOT
		<Memory>
          		<Size>${memory}</Size>
      		</Memory>		
                <Price>
                    <TimeUnit>1</TimeUnit>
                    <PricePerUnit>0.085</PricePerUnit>
                </Price>
            </InstanceType>
        </InstanceTypes>
    </CloudProvider>

EOT


  cat >> "${PROJECT_FILE}" << EOT
  <Cloud>
        <InitialVMs>0</InitialVMs>
        <MinimumVMs>0</MinimumVMs>
        <MaximumVMs>${max_vms}</MaximumVMs>
        <CloudProvider Name="SLURM-Cluster">
		<LimitOfVMs>${max_vms}</LimitOfVMs>
            		<Properties>
                		<Property>
                    			<Name>master_name</Name>
                    			<Value>${master_node}</Value>
                		</Property>
EOT
  if [ -z "${container_image}" ]; then
          cat >> "${PROJECT_FILE}" << EOT
                        <Property>
                    			<Name>slurm_over_ssh</Name>
                    			<Value>false</Value>
                		</Property>
EOT
  else
  	     cat >> "${PROJECT_FILE}" << EOT
                        <Property>
                    			<Name>slurm_over_ssh</Name>
                    			<Value>true</Value>
                		</Property>
EOT
  fi
  cat >> "${PROJECT_FILE}" << EOT
				<Property>
                                        <Name>master_port</Name>
                                        <Value>${master_port}</Value>
                                </Property>
				<Property>
                                        <Name>worker_debug</Name>
                                        <Value>${debug}</Value>
                                </Property>
				<Property>
                                        <Name>jvm_opts_size</Name>
                                        <Value>${jvm_workers_opts_size}</Value>
                                </Property>
				<Property>
                                        <Name>jvm_opts_str</Name>
                                        <Value>${jvm_workers_opts_str}</Value>
                                </Property>
			 	<Property>
                                        <Name>network</Name>
                                        <Value>${network}</Value>
                                </Property>
                		<Property>
                    			<Name>estimated-creation-time</Name>
                    			<Value>10</Value>
                		</Property>
            		</Properties>
		<Images>
EOT
  if [ -z "${container_image}" ]; then
        cat >> "${PROJECT_FILE}" << EOT
            		<Image Name="None">
EOT
  else
	cat >> "${PROJECT_FILE}" << EOT
                        <Image Name="${container_image}">
EOT
  fi
  cat >> "${PROJECT_FILE}" <<EOT
				<InstallDir>${worker_install_dir}</InstallDir>
      				<WorkingDir>${worker_working_dir}</WorkingDir>
      				<Application>
					    <AppDir>${appdir}</AppDir>
          				<LibraryPath>${library_path}</LibraryPath>
					<Classpath>${cp}</Classpath>
					<Pythonpath>${pythonpath}</Pythonpath>
      				</Application>
EOT
  if [ "$lot" != "" ] && [ "$lot" -ge 0 ]; then
    cat >> "${PROJECT_FILE}" <<EOT
      				<LimitOfTasks>${lot}</LimitOfTasks>
      
EOT
  fi

  cat >> "${PROJECT_FILE}" << EOT
  			</Image>
            </Images>
            <InstanceTypes>
                <InstanceType Name="Resource-Type-1"/>
            </InstanceTypes>
        </CloudProvider>
    </Cloud>
EOT

}



###############################################
# Adds a compute node to Resources/Project
###############################################
add_compute_node() {
  local nodeName=$1
  local cus=$2
  local gpus=$3
  local lot=$4
  local memory=$5

  echo "Worker WD mkdir: ${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${nodeName} ${CMD_SEPARATOR}mkdir -p ${worker_working_dir}${CMD_SEPARATOR}"
  ${LAUNCH_CMD} "${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${nodeName}" "${CMD_SEPARATOR}"mkdir -p "${worker_working_dir}${CMD_SEPARATOR}"

  nodeName=${nodeName}${network}

  cat >> "${RESOURCES_FILE}" << EOT
  <ComputeNode Name="${nodeName}">
      <Processor Name="MainProcessor">
          <ComputingUnits>${cus}</ComputingUnits>
          <Architecture>Intel</Architecture>
          <Speed>2.6</Speed>
      </Processor>
EOT

  if [ "${gpus}" != "$DEFAULT_GPUS_PER_NODE" ]; then
    cat >> "${RESOURCES_FILE}" << EOT
        <Processor Name="GPU">
            <Type>GPU</Type>
            <ComputingUnits>${gpus}</ComputingUnits>
            <Architecture>k80</Architecture>
            <Speed>2.6</Speed>
        </Processor>
EOT
  fi

  cat >> "${RESOURCES_FILE}" << EOT
      <OperatingSystem>
          <Type>Linux</Type>
          <Distribution>SMP</Distribution>
          <Version>3.0.101-0.35-default</Version>
      </OperatingSystem>
      <Memory>
          <Size>${memory}</Size>
      </Memory>
      <Software>
          <Application>JAVA</Application>
          <Application>PYTHON</Application>
          <Application>EXTRAE</Application>
          <Application>COMPSS</Application>
      </Software>
      <Adaptors>
          <Adaptor Name="es.bsc.compss.nio.master.NIOAdaptor">
              <SubmissionSystem>
                  <Interactive/>
              </SubmissionSystem>
              <Ports>
                  <MinPort>43001</MinPort>
                  <MaxPort>43002</MaxPort>
                  <RemoteExecutionCommand>${REMOTE_EXECUTOR}</RemoteExecutionCommand>
              </Ports>
          </Adaptor>
          <Adaptor Name="es.bsc.compss.gat.master.GATAdaptor">
              <SubmissionSystem>
                  <Interactive/>
              </SubmissionSystem>
              <BrokerAdaptor>sshtrilead</BrokerAdaptor>
          </Adaptor>
      </Adaptors>
      <SharedDisks>
          <AttachedDisk Name="gpfs">
              <MountPoint>${GPFS_PREFIX}</MountPoint>
          </AttachedDisk>
EOT
  if [ "${GPFS2_PREFIX}" != "" ] && [ "${GPFS2_PREFIX}" != "${GPFS_PREFIX}" ]; then
    cat >> "${RESOURCES_FILE}" << EOT
          <AttachedDisk Name="gpfs2">
              <MountPoint>${GPFS2_PREFIX}</MountPoint>
          </AttachedDisk>
EOT
  fi

  cat >> "${RESOURCES_FILE}" << EOT
      </SharedDisks>
  </ComputeNode>

EOT

  cat >> "${PROJECT_FILE}" << EOT
  <ComputeNode Name="${nodeName}">
      <InstallDir>${worker_install_dir}</InstallDir>
      <WorkingDir>${worker_working_dir}</WorkingDir>
      <Application>
          <LibraryPath>${library_path}</LibraryPath>
      </Application>
EOT
  if [ "$lot" != "" ] && [ "$lot" -ge 0 ]; then
    cat >> "${PROJECT_FILE}" <<EOT
      <LimitOfTasks>${lot}</LimitOfTasks>
EOT
  fi

  cat >> "${PROJECT_FILE}" << EOT
  </ComputeNode>

EOT
}


#---------------------------------------------------------------------------------------
# MAIN EXECUTION
#---------------------------------------------------------------------------------------
  
  # Set script variables
  SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
  COMPSS_HOME=${SCRIPT_DIR}/../../../
  
  numPrologActions=0
  declare -a prologActions
  numEpilogActions=0
  declare -a epilogActions
  
  # Get command args
  get_args "$@"

  # Check other command args
  check_args

  # Set job variables
  set_variables
 
  # Log variables
  log_variables
 
  # Add clean up for execution end
  trap cleanup EXIT

  # Create XML files
  create_xml_files
  
  # Prolog
  prolog

  # Launch execution
  launch

  # Wait 
  wait_for_completion
    
  # Epilog
  epilog
