#!/bin/bash

###############################################
# SCRIPT CONSTANTS DECLARATION
###############################################
DEFAULT_SC_CFG="default"

# Next default values need to be consistent with runcompss
DEFAULT_LANGUAGE=java
DEFAULT_LIBRARY_PATH=$(pwd)
DEFAULT_APPDIR=$(pwd)
DEFAULT_CLASSPATH=$(pwd)
DEFAULT_PYTHONPATH=$(pwd)
DEFAULT_DEBUG=off
DEFAULT_LOG_LEVEL_ARG=debug
DEFAULT_COMMUNICATION_ADAPTOR=es.bsc.compss.nio.master.NIOAdaptor
DEFAULT_TRACING=false
DEFAULT_TRACING_ARG=true
DEFAULT_CUSTOM_EXTRAE_FILE="null"
DEFAULT_MASTER_PORT_BASE=43000
DEFAULT_MASTER_PORT_RAND_RANGE=1000
DEFAULT_CPU_AFFINITY="automatic"
DEFAULT_GPU_AFFINITY="automatic"
DEFAULT_FPGA_AFFINITY="automatic"
DEFAULT_FPGA_REPROGRAM=""
DEFAULT_TASK_EXECUTION=compss
DEFAULT_STORAGE_CONF=null
DEFAULT_PERSISTENT_WORKER_C=false
DEFAULT_PYTHON_INTERPRETER=python
DEFAULT_PYTHON_VERSION=$( ${DEFAULT_PYTHON_INTERPRETER} -c "import sys; print(sys.version_info[:][0])" )
if [ -z "${VIRTUAL_ENV}" ]; then
  DEFAULT_PYTHON_VIRTUAL_ENVIRONMENT="null"
else
  DEFAULT_PYTHON_VIRTUAL_ENVIRONMENT="${VIRTUAL_ENV}"
fi
DEFAULT_PYTHON_PROPAGATE_VIRTUAL_ENVIRONMENT=true
DEFAULT_JUPYTER_NOTEBOOK=false

###############################################
# ERROR CONSTANTS DECLARATION
###############################################
ERROR_CFG_SC="SuperComputer CFG file doesn't exist"
ERROR_CFG_Q="Queue system CFG file doesn't exist"
ERROR_MASTER_NODE="Missing master node parameter"
ERROR_WORKER_NODES="Missing worker nodes parameter"
ERROR_NUM_CPUS="Invalid number of CPUS per node"
ERROR_WORKER_WD="Invalid Worker Working Dir option"
ERROR_NETWORK="Invalid network option"
ERROR_WORKER_IN_MASTER_CPUS="Parameter worker_in_master_cpus is bigger than the maximum number of cpus_per_node"
ERROR_WORKER_IN_MASTER_MEMORY="Incorrect worker_in_master_memory parameter. Only disabled or <int> allowed. I.e. 33000, 66000"
ERROR_WORKER_IN_MASTER_MEMORY_TOO_HIGH="Parameter worker_in_master_memory exceeds the node_memory limit"
ERROR_WORKER_IN_MASTER_MEMORY_NOT_SPECIFIED="Parameter worker_in_master_memory is mandatory if worker_in_master_cpus is not 0"
ERROR_PROLOG_ACTION="Exception executing prolog action"
ERROR_EPILOG_ACTION="Exception executing epilog action"
ERROR_LANGUAGE="Value of option --lang must be: java, c or python"

#---------------------------------------------------------------------------------------
# HELPER FUNCTIONS
#---------------------------------------------------------------------------------------

###############################################
# Displays usage
###############################################
usage() {
  local exitValue=$1

  cat <<EOT
Usage: $0 [options] application_name application_arguments

* Options:
  General:
    --help, -h                              Print this help message

    --opts                                  Show available options

    --version, -v                           Print COMPSs version

    --sc_cfg=<name>                         SuperComputer configuration file to use. Must exist inside queues/cfgs/
                                            Mandatory
                                            Default: ${DEFAULT_SC_CFG}

    --master_node=<string>                  Node where to run the COMPSs Master
                                            Mandatory

    --worker_nodes="<string string...>"     Space separated nodes where to run the COMPSs Workers (Notice the quotes)
                                            Mandatory

  Launch configuration:
EOT

  show_opts "$exitValue"
}

###############################################
# Show Options
###############################################
show_opts() {
  local exitValue=$1

  # Load default CFG for default values
  local defaultSC_cfg=${SCRIPT_DIR}/../queues/cfgs/${DEFAULT_SC_CFG}.cfg
  #shellcheck source=../queues/cfgs/default.cfg
  source "${defaultSC_cfg}"
  local defaultQS_cfg=${SCRIPT_DIR}/../queues/${QUEUE_SYSTEM}/${QUEUE_SYSTEM}.cfg
  #shellcheck source=../queues/slurm/slurm.cfg
  source "${defaultQS_cfg}"

  # Show usage
  cat <<EOT
    --cpus_per_node=<int>                   Available CPU computing units on each node
                                            Default: ${DEFAULT_CPUS_PER_NODE}
    --gpus_per_node=<int>                   Available GPU computing units on each node
                                            Default: ${DEFAULT_GPUS_PER_NODE}
    --fpgas_per_node=<int>                  Available FPGA computing units on each node
                                            Default: ${DEFAULT_FPGAS_PER_NODE}
    --fpga_reprogram="<string>              Specify the full command that needs to be executed to reprogram the FPGA with
                                            the desired bitstream. The location must be an absolute path.
                                            Default: ${DEFAULT_FPGA_REPROGRAM}
    --max_tasks_per_node=<int>              Maximum number of simultaneous tasks running on a node
                                            Default: ${DEFAULT_MAX_TASKS_PER_NODE}
    --node_memory=<MB>                      Maximum node memory: disabled | <int> (MB)
                                            Default: ${DEFAULT_NODE_MEMORY}
    --network=<name>                        Communication network for transfers: default | ethernet | infiniband | data.
                                            Default: ${DEFAULT_NETWORK}

    --prolog="<string>"                     Task to execute before launching COMPSs (Notice the quotes)
                                            If the task has arguments split them by "," rather than spaces.
                                            This argument can appear multiple times for more than one prolog action
                                            Default: Empty
    --epilog="<string>"                     Task to execute after executing the COMPSs application (Notice the quotes)
                                            If the task has arguments split them by "," rather than spaces.
                                            This argument can appear multiple times for more than one epilog action
                                            Default: Empty

    --master_working_dir=<path>             Working directory of the application
                                            Default: ${DEFAULT_MASTER_WORKING_DIR}
    --worker_working_dir=<name | path>      Worker directory. Use: scratch | gpfs | <path>
                                            Default: ${DEFAULT_WORKER_WORKING_DIR}

    --worker_in_master_cpus=<int>           Maximum number of CPU computing units that the master node can run as worker. Cannot exceed cpus_per_node.
                                            Default: ${DEFAULT_WORKER_IN_MASTER_CPUS}
    --worker_in_master_memory=<int> MB      Maximum memory in master node assigned to the worker. Cannot exceed the node_memory.
                                            Mandatory if worker_in_master_cpus is specified.
                                            Default: ${DEFAULT_WORKER_IN_MASTER_MEMORY}
    --jvm_worker_in_master_opts="<string>"  Extra options for the JVM of the COMPSs Worker in the Master Node.
                                            Each option separed by "," and without blank spaces (Notice the quotes)
                                            Default: ${DEFAULT_JVM_WORKER_IN_MASTER}
    --container_image=<path>                Runs the application by means of a container engine image
                                            Default: Empty
    --container_compss_path=<path>          Path where compss is installed in the container image
                                            Default: /opt/COMPSs
    --container_opts="<string>"             Options to pass to the container engine
                                            Default: empty
    --elasticity=<max_extra_nodes>          Activate elasticity specifiying the maximum extra nodes (ONLY AVAILABLE FORM SLURM CLUSTERS WITH NIO ADAPTOR)
                                            Default: 0

    --jupyter_notebook=<bool>,              Swap the COMPSs master initialization with jupyter notebook. Use: true | false
    --jupyter_notebook                      Default: ${DEFAULT_JUPYTER_NOTEBOOK}

  Runcompss configuration:

EOT
    "${SCRIPT_DIR}"/runcompss --opts

  exit "$exitValue"
}

###############################################
# Displays version
###############################################
display_version() {
  local exitValue=$1

  "${SCRIPT_DIR}"/runcompss --version

  exit "$exitValue"
}

###############################################
# Displays errors when treating arguments
###############################################
display_error() {
  local error_msg=$1

  echo "$error_msg"
  echo " "

  usage 1
}

###############################################
# Displays errors when executing actions
###############################################
action_error() {
  local error_msg=$1

  echo "$error_msg"
  echo " "

  exit 1
}

###############################################
# Infers the language from the Application Path
# WARN: Sets global lang variable
###############################################
infer_language() {
  local fullApp
  local fileName
  local extension
  local isPython
  local isPythonCompiled
  local isC

  fullApp=$1
  fileName=$(basename "$fullApp")
  extension=${fileName##*.}
  if [ "$extension" == "py" ] || [ "$extension" == "pyc" ]; then
    echo "[  INFO] Inferred PYTHON language"
    lang=python
  else
    # Try to infer language by file command
    isPython=$(file "$fullAppPath" | grep "Python" | cat)
    isPythonCompiled=$(file "$fullAppPath" | grep "python" | cat)
    isC=$(file "$fullAppPath" | grep "executable" | cat)
    if [ -n "$isPython" ] || [ -n "$isPythonCompiled" ]; then
      echo "[  INFO] Inferred PYTHON language"
      lang=python
    elif [ -n "$isC" ]; then
      echo "[  INFO] Inferred C/C++ language"
      lang=c
    else
      # Lang cannot be infered or it's the default JAVA
      echo "[  INFO] Using default language: ${DEFAULT_LANGUAGE}"
      lang=${DEFAULT_LANGUAGE}
    fi
  fi
}

###############################################
# Loads the tracing environment
###############################################
load_tracing_env() {
  local module_tmp
  module_tmp=$(mktemp)
  module list 2> "${module_tmp}"

  # Look for openmpi / impi / none
  impi=$(grep -i "impi" "${module_tmp}" | cat)
  openmpi=$(grep -i "openmpi" "${module_tmp}" | cat)

  if [ ! -z "$impi" ]; then
    # Load Extrae IMPI
    export EXTRAE_HOME=${COMPSS_HOME}/Dependencies/extrae-impi/
  elif [ ! -z "$openmpi" ]; then
    # Load Extrae OpenMPI
    export EXTRAE_HOME=${COMPSS_HOME}/Dependencies/extrae-openmpi/
  else
    # Load sequential extrae
    export EXTRAE_HOME=${COMPSS_HOME}/Dependencies/extrae/
  fi

  # Clean tmp file
  rm -f "${module_tmp}"
}

###############################################
# Creates Worker CMD
###############################################
worker_cmd() {
  # WARNING: SETS GLOBAL SCRIPT VARIABLE WCMD
  local nodeId=$1
  local nodeName=$2
  local jvm_opts_size=$3
  local jvm_opts_str=$4
  local fpga_reprog_size=$5
  local fpga_reprog_str=$6
  local cusCPU=$7
  local cusGPU=$8
  local cusFPGA=$9
  local cpuMap=${10}
  local gpuMap=${11}
  local fpgaMap=${12}
  local lot=${13}

  local sandbox_worker_working_dir=${worker_working_dir}/${uuid}/${nodeName}
  local maxSend=5
  local maxReceive=5
  local worker_port=43001
  local mName=${master_name}
  if [ -z "${container_image}" ]; then
     WCMD="${COMPSS_HOME}/Runtime/scripts/system/adaptors/nio/persistent_worker_starter.sh"
  else
     WCMD="singularity exec $container_opts $container_image ${COMPSS_HOME}/Runtime/scripts/system/adaptors/nio/persistent_worker_starter.sh"
  fi
  if [ -z "${master_name}" ]; then
     mName="null"
  fi
  WCMD="$WCMD ${library_path} \
            ${appdir} \
            ${cp} \
            ${jvm_opts_size} \
            ${jvm_opts_str} \
            ${fpga_reprog_size} \
            ${fpga_reprog_str} \
            ${debug} \
            ${maxSend} \
            ${maxReceive} \
            ${nodeName} \
            ${worker_port} \
            ${mName} \
            ${master_port} \
            ${cusCPU} \
            ${cusGPU} \
            ${cusFPGA} \
            ${cpuMap} \
            ${gpuMap} \
            ${fpgaMap} \
            ${lot} \
            ${uuid} \
            ${lang} \
            ${sandbox_worker_working_dir} \
            ${worker_install_dir} \
            ${appdir} \
            ${library_path} \
            ${cp} \
            ${pythonpath} \
            ${w_tracing} \
            ${custom_extrae_file} \
            ${nodeId} \
            ${storage_conf} \
            ${taskExecution} \
            ${persistent_worker_c} \
            ${python_interpreter} \
            ${python_version} \
            ${DEFAULT_PYTHON_VIRTUAL_ENVIRONMENT} \
            ${python_propagate_virtual_environment}"
}

###############################################
# Create Master CMD
###############################################
master_cmd() {
  local master_name=$1

  if [ -z "${container_image}" ]; then
    MCMD="${COMPSS_HOME}/Runtime/scripts/user/runcompss"
  else
    MCMD="singularity exec $container_opts $container_image ${COMPSS_HOME}/Runtime/scripts/user/runcompss"
  fi

  # WARNING: SETS GLOBAL SCRIPT VARIABLE MCMD
  MCMD="$MCMD --project=${PROJECT_FILE} \
            --resources=${RESOURCES_FILE} \
            --uuid=${uuid} \
            --specific_log_dir=${specific_log_dir} \
            --master_name=${master_name} \
            --master_port=${master_port} \
            --jvm_master_opts=${jvm_master_opts} \
            --jvm_workers_opts=${jvm_workers_opts}"
}


#---------------------------------------------------
# MAIN FUNCTIONS DECLARATION
#---------------------------------------------------

###############################################
# Function to get the arguments
###############################################
get_args() {
  # Avoid enqueue if there is no application
  if [ $# -eq 0 ]; then
    usage 1
  fi

  # Parse COMPSs Options
  while getopts hvgtmd-: flag; do
    # Treat the argument
    case "$flag" in
      h)
        # Display help
        usage 0
        ;;
      v)
        # Display version
        display_version 0
        ;;
      d)
        log_level=${DEFAULT_LOG_LEVEL_ARG}
        #Keep it for runcompss (to add them to master)
        args_pass="$args_pass -$flag"
        ;;
      t)
        tracing=${DEFAULT_TRACING_ARG}
        #Keep it for runcompss (to add them to master)
        args_pass="$args_pass -$flag"
        ;;
      -)
      # Check more complex arguments
      case "$OPTARG" in
        help)
          # Display help
          usage 0
          ;;
        version)
          # Display compss version
          display_version 0
          ;;
        opts)
          # Display options
          show_opts 0
          ;;
        master_node=*)
          master_node=${OPTARG//master_node=/}
          ;;
        worker_nodes=*)
          worker_nodes=${OPTARG//worker_nodes=/}
          ;;
        resources=*)
          resources=${OPTARG//resources=/}
          ;;
        sc_cfg=*)
          sc_cfg=${OPTARG//sc_cfg=/}
          ;;
        cpus_per_node=*)
          cpus_per_node=${OPTARG//cpus_per_node=/}
          ;;
        gpus_per_node=*)
          gpus_per_node=${OPTARG//gpus_per_node=/}
          ;;
        fpgas_per_node=*)
          fpgas_per_node=${OPTARG//fpgas_per_node=/}
          ;;
        max_tasks_per_node=*)
          max_tasks_per_node=${OPTARG//max_tasks_per_node=}
          ;;
        cpu_affinity=*)
          cpu_affinity=${OPTARG//cpu_affinity=}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        gpu_affinity=*)
          gpu_affinity=${OPTARG//gpu_affinity=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        fpga_affinity=*)
          fpga_affinity=${OPTARG//fpga_affinity=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        fpga_reprogram=*)
          fpga_prog=${OPTARG//fpga_reprogram=/}
          ;;
        master_working_dir=*)
          master_working_dir=${OPTARG//master_working_dir=/}
          ;;
        worker_working_dir=*)
          worker_working_dir=${OPTARG//worker_working_dir=/}
          ;;
        worker_in_master_cpus=*)
          worker_in_master_cpus=${OPTARG//worker_in_master_cpus=/}
          ;;
        worker_in_master_memory=*)
          worker_in_master_memory=${OPTARG//worker_in_master_memory=/}
          ;;
        node_memory=*)
          node_memory=${OPTARG//node_memory=/}
          ;;
        network=*)
          network=${OPTARG//network=/}
          ;;
        lang=*)
          lang=${OPTARG//lang=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        library_path=*)
          library_path=${OPTARG//library_path=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        classpath=*)
          cp=${OPTARG//classpath=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        pythonpath=*)
          pythonpath=${OPTARG//pythonpath=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        appdir=*)
          appdir=${OPTARG//appdir=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        log_level=*)
          log_level=${OPTARG//log_level=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        debug)
          log_level=${DEFAULT_LOG_LEVEL_ARG}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        tracing=*)
          tracing=${OPTARG//tracing=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        tracing)
          tracing=${DEFAULT_TRACING_ARG}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        extrae_config_file=*)
          custom_extrae_file=${OPTARG//extrae_config_file=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        comm=*)
          comm=${OPTARG//comm=/}
          #Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        jvm_master_opts=*)
          jvm_master_opts=${OPTARG//jvm_master_opts=/}
          ;;
        jvm_workers_opts=*)
          jvm_workers_opts=${OPTARG//jvm_workers_opts=/}
          ;;
        jvm_worker_in_master_opts=*)
          jvm_worker_in_master_opts=${OPTARG//jvm_worker_in_master_opts=/}
          ;;
        storage_conf=*)
          storage_conf=${OPTARG//storage_conf=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        task_execution=*)
          taskExecution=${OPTARG//task_execution=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        persistent_worker_c=*)
          persistent_worker_c=${OPTARG//persistent_worker_c=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        python_interpreter=*)
          python_interpreter=${OPTARG//python_interpreter=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        python_propagate_virtual_environment=*)
          python_propagate_virtual_environment=${OPTARG//python_propagate_virtual_environment=/}
          # Keep it for runcompss (to add them to master)
          args_pass="$args_pass --$OPTARG"
          ;;
        prolog=*)
          action=${OPTARG//prolog=/}
          prologActions[numPrologActions]="$action"
          numPrologActions=$((numPrologActions + 1))
          ;;
        epilog=*)
          action=${OPTARG//epilog=/}
          epilogActions[numEpilogActions]="$action"
          numEpilogActions=$((numEpilogActions + 1))
          ;;
        uuid=*)
          #Modified for heterogeneity
          uuid=${OPTARG//uuid=/}
          ;;
        specific_log_dir=*)
          # Specific log dir is automatically generated by launch_compss.sh. Remove it from COMPSs flags
          echo "WARNING: specific_log_dir is automatically generated. Omitting parameter. Define base_log_dir instead."
          ;;
        base_log_dir=*)
          # Base log dir is automatically generated by launch_compss.sh. Remove it from COMPSs flags
          # echo "WARNING: base_log_dir is automatically generated. Omitting parameter"
        base_log_dir=${OPTARG//base_log_dir=/}
          ;;
        master_name=*)
          # Master name is automatically generated by launch_compss.sh. Remove it from COMPSs flags
          echo "WARNING: master_name is automatically generated. Omitting parameter"
          ;;
        master_port=*)
          # Remove from runcompss since launcher will add it
          master_port=${OPTARG//master_port=/}
          ;;
        container_image=*)
          container_image=${OPTARG//container_image=/}
          ;;
        container_compss_path=*)
          container_compss_path=${OPTARG//container_compss_path=/}
          ;;
        container_opts=*)
          container_opts=${OPTARG//container_opts=/}
          ;;
        elasticity=*)
          elasticity=${OPTARG//elasticity=/}
          args_pass="$args_pass --conn=es.bsc.compss.connectors.DefaultNoSSHConnector"
          ;;
        queue=*)
          # Added for elasticity
          queue=${OPTARG//queue=/}
          ;;
        reservation=*)
          # Added for elasticity
          reservation=${OPTARG//reservation=/}
          ;;
        qos=*)
          # Added for elasticity
          qos=${OPTARG//qos=/}
          ;;
        constraints=*)
          #Added for elasticity
          constraints=${OPTARG//constraints=/}
          ;;
        jupyter_notebook)
          jupyter_notebook=true
          ;;
        jupyter_notebook=*)
          jupyter_notebook=${OPTARG//jupyter_notebook=/}
          ;;
        xmls_phase=*)
          #Added for heterogeneity
          xmls_phase=${OPTARG//xmls_phase=/}
          ;;
        xmls_suffix=*)
          #Added for heterogeneity
          xmls_suffix=${OPTARG//xmls_suffix=/}
          ;;
        initial_hostid=*)
          #Added for tracing in heterogeneity
          initial_hostid=${OPTARG//initial_hostid=/}
          ;;
        *)
          # Flag didn't match any patern. Add to COMPSs
          args_pass="$args_pass --$OPTARG"
          ;;
      esac
      ;;
    *)
      # Flag didn't match any patern. End of COMPSs flags
      args_pass="$args_pass -$flag"
      ;;
    esac
  done

  # Shift COMPSs arguments
  shift $((OPTIND-1))

  # Wrap full app path for lang inference
  fullAppPath=$1

  # Pass application name and args
  args_pass="$args_pass $*"
}

###############################################
# Function to check the arguments
###############################################
check_args() {
  ###############################################################
  # SC Configuration checks
  ###############################################################
  # Check sc configuration argument
  if [ -z "${sc_cfg}" ]; then
    sc_cfg=${DEFAULT_SC_CFG}
  fi
  if [[ ${sc_cfg} != *cfg ]]; then
    # Add cfg suffix
    sc_cfg=${sc_cfg}.cfg
  fi

  local scCfgFullPath=${SCRIPT_DIR}/../queues/cfgs/${sc_cfg}

  if [ ! -f "${scCfgFullPath}" ]; then
    # CFG file doesn't exist
    display_error "${ERROR_CFG_SC}"
  fi

  # Source SC CFG env
  # shellcheck source=../queues/cfgs/default.cfg
  source "${scCfgFullPath}"

  # Check queue configuration env
  local queueCfgFullPath=${SCRIPT_DIR}/../queues/${QUEUE_SYSTEM}/${QUEUE_SYSTEM}.cfg
  if [ ! -f "${queueCfgFullPath}" ]; then
    # CFG file doesn't exist
    display_error "${ERROR_CFG_Q}"
  fi

  # Source queue system CFG env
  # shellcheck source=../queues/slurm/slurm.cfg
  source "${queueCfgFullPath}"

  ###############################################################
  # Infrastructure checks
  ###############################################################
  # Changed error by warning because in heterogeneous there can be a launch of just workers 
  if [ -z "${master_node}" ]; then
    echo "${ERROR_MASTER_NODE}"
  fi

  if [ -z "${worker_nodes}" ] && [ "${DEFAULT_WORKER_IN_MASTER_CPUS}" -eq "0" ]; then
    if [ -z "${worker_in_master_cpus}" ]; then
      display_error "${ERROR_WORKER_NODES}"
    else
      worker_nodes=""
    fi
  fi

  if [ -z "${network}" ]; then
    network=${DEFAULT_NETWORK}
  elif [ "${network}" == "default" ]; then
    network=${DEFAULT_NETWORK}
  elif [ "${network}" != "ethernet" ] && [ "${network}" != "infiniband" ] && [ "${network}" != "data" ]; then
    display_error "${ERROR_NETWORK}"
  fi

  if [ -z "${storage_conf}" ]; then
    storage_conf=${DEFAULT_STORAGE_CONF}
  fi

  if [ -z "${taskExecution}" ]; then
    taskExecution=${DEFAULT_TASK_EXECUTION}
  fi

  if [ -z "${persistent_worker_c}" ]; then
    persistent_worker_c=${DEFAULT_PERSISTENT_WORKER_C}
  fi

  if [ -z "${python_interpreter}" ]; then
    python_interpreter=${DEFAULT_PYTHON_INTERPRETER}
    python_version=${DEFAULT_PYTHON_VERSION}
  else
    python_version=$( ${python_interpreter} -c "import sys; print(sys.version_info[:][0])" )
  fi

  if [ -z "${python_propagate_virtual_environment}" ]; then
    python_propagate_virtual_environment=${DEFAULT_PYTHON_PROPAGATE_VIRTUAL_ENVIRONMENT}
  fi

  ###############################################################
  # Node checks
  ###############################################################
  if [ -z "${max_tasks_per_node}" ]; then
    max_tasks_per_node=${DEFAULT_MAX_TASKS_PER_NODE}
  fi

  if [ -z "${cpus_per_node}" ]; then
    cpus_per_node=${DEFAULT_CPUS_PER_NODE}
  fi

  if [ "${cpus_per_node}" -lt "${MINIMUM_CPUS_PER_NODE}" ]; then
    display_error "${ERROR_NUM_CPUS}"
  fi

  if [ -z "${gpus_per_node}" ]; then
    gpus_per_node=${DEFAULT_GPUS_PER_NODE}
  fi

  if [ -z "${fpgas_per_node}" ]; then
    fpgas_per_node=${DEFAULT_FPGAS_PER_NODE}
  fi

  if [ -z "${cpu_affinity}" ]; then
    cpu_affinity=${DEFAULT_CPU_AFFINITY}
  fi

  if [ -z "${gpu_affinity}" ]; then
    gpu_affinity=${DEFAULT_GPU_AFFINITY}
  fi

  if [ -z "${fpga_affinity}" ]; then
    fpga_affinity=${DEFAULT_FPGA_AFFINITY}
  fi

  if [ -z "${fpga_prog}" ]; then
    fpga_prog=${DEFAULT_FPGA_REPROGRAM}
  fi

  if [ -z "${node_memory}" ]; then
    node_memory=${DEFAULT_NODE_MEMORY}
  fi

  if [ -z "${worker_in_master_cpus}" ]; then
    worker_in_master_cpus=${DEFAULT_WORKER_IN_MASTER_CPUS}
  fi
  if [ "${worker_in_master_cpus}" -gt "${cpus_per_node}" ]; then
    display_error "${ERROR_WORKER_IN_MASTER_CPUS}"
  fi

  if [ -z "${worker_in_master_memory}" ]; then
    worker_in_master_memory=${DEFAULT_WORKER_IN_MASTER_MEMORY}
  elif [ "${worker_in_master_memory}" != "disabled" ] && ! [[ "${worker_in_master_memory}" =~ ^[0-9]+$ ]]; then
    display_error "${ERROR_WORKER_IN_MASTER_MEMORY}"
  fi
  if [ "${worker_in_master_memory}" != "${DEFAULT_WORKER_IN_MASTER_MEMORY}" ] && [ "${node_memory}" != "${DEFAULT_NODE_MEMORY}" ]; then
    if [ "${worker_in_master_memory}" -gt "${node_memory}" ]; then
      display_error "${ERROR_WORKER_IN_MASTER_MEMORY_TOO_HIGH} ${worker_in_master_memory} < ${node_memory} "
    fi
  fi

  if [ "${worker_in_master_cpus}" -gt 0 ] && [ "${worker_in_master_memory}" -le 0 ]; then
    display_error "${ERROR_WORKER_IN_MASTER_MEMORY_NOT_SPECIFIED}"
  fi

  if [ -z "${master_port}" ]; then
    rand_num=$RANDOM
    offset=$((rand_num % DEFAULT_MASTER_PORT_RAND_RANGE))
    master_port=$((DEFAULT_MASTER_PORT_BASE + offset))
  fi

  while lsof -i :${master_port}; do
    echo "Port ${master_port} is already in use, incrementing port by 1"
    master_port=$((master_port+1))
  done

  ###############################################################
  # Working Directory Checks
  ###############################################################
  if [ -z "${master_working_dir}" ]; then
    master_working_dir=${DEFAULT_MASTER_WORKING_DIR}
  fi

  if [ -z "${worker_working_dir}" ]; then
    worker_working_dir=${DEFAULT_WORKER_WORKING_DIR}
  elif [ "${worker_working_dir}" != "scratch" ] && [ "${worker_working_dir}" != "gpfs" ] && [[ ${worker_working_dir} != /* ]]; then
    display_error "${ERROR_WORKER_WD}"
  fi

  ###############################################################
  # JVM Checks
  ###############################################################
  if [ -z "${jvm_master_opts}" ]; then
    jvm_master_opts=${DEFAULT_JVM_MASTER}
  fi
  jvm_master_opts=${jvm_master_opts//\"/}

  if [ -z "${jvm_workers_opts}" ]; then
    jvm_workers_opts=${DEFAULT_JVM_WORKERS}
  fi
  jvm_workers_opts=${jvm_workers_opts//\"/}

  if [ -n "${worker_in_master_cpus}" ]; then
    jvm_worker_in_master_opts=${DEFAULT_JVM_WORKER_IN_MASTER}
  fi
  jvm_worker_in_master_opts=${jvm_worker_in_master_opts//\"/}

  ###############################################################
  # Runtime and Tools Checks
  ###############################################################
  if [ -z "${log_level}" ]; then
    log_level=${DEFAULT_DEBUG}
  fi

  if [ -z "${comm}" ]; then
    comm=${DEFAULT_COMMUNICATION_ADAPTOR}
  fi

  if [ -z "${tracing}" ]; then
    tracing=${DEFAULT_TRACING}
  fi

  if [ -z "${custom_extrae_file}" ]; then
    custom_extrae_file=${DEFAULT_CUSTOM_EXTRAE_FILE}
  fi

  if [ -z "${jupyter_notebook}" ]; then
    jupyter_notebook=${DEFAULT_JUPYTER_NOTEBOOK}
  fi

  ###############################################################
  # Application Checks
  ###############################################################
  # Lang
  if [ -z "$lang" ]; then
    # Try to infer language
    infer_language "$fullAppPath"
  elif [ "$lang" = "java" ]; then
    lang=java
  elif [ "$lang" = "c" ]; then
    lang=c
  elif [ "$lang" = "python" ]; then
    lang=python
  else
    display_error "${ERROR_LANGUAGE}"
  fi

  # Library path
  if [ -z "${library_path}" ]; then
    library_path=${DEFAULT_LIBRARY_PATH}
  fi

  # Classpath
  if [ -z "$cp" ]; then
    cp=${DEFAULT_CLASSPATH}
  else
    fcp=""
    for currcp in ${cp//:/$'\n'}; do
    if [ ! "${currcp:0:1}" == '/' ]; then              # Relative paths to abs
      if [ -d "$currcp" ] || [ -f "$currcp" ]; then  # If the dir/file exists
        absdir="$(cd "$(dirname "$currcp")" && pwd)" # Get absolute dir
        file="$(basename "$currcp")"
        currcp="$absdir/$file"
      else
        echo "[ WARNING ]: Classpath \"$currcp\" does not exist..."
      fi
    fi
    fcp="${fcp}:$currcp"
    done
    cp="$(echo "$fcp" | cut -c2-)"
  fi

  # Pythonpath
  if [ -z "$pythonpath" ]; then
    pythonpath=${DEFAULT_PYTHONPATH}
  fi

  # AppDir
  if [ -z "$appdir" ]; then
    appdir=${DEFAULT_APPDIR}
  fi
}


###############################################
# Sets job variables
###############################################
set_variables() {
  # Set script variables
  if [ -z "${container_image}" ]; then
    export COMPSS_HOME=${COMPSS_HOME}
    export GAT_LOCATION=${COMPSS_HOME}/Dependencies/JAVA_GAT
    worker_install_dir=${COMPSS_HOME}
  else
    if [ -z "${container_compss_path}" ]; then
      export COMPSS_HOME=/opt/COMPSs/
      export GAT_LOCATION=/opt/COMPSs/Dependencies/JAVA_GAT
      worker_install_dir=/opt/COMPSs
    else
      export COMPSS_HOME=${container_compss_path}/
      export GAT_LOCATION=${container_compss_path}/Dependencies/JAVA_GAT
      worker_install_dir=${container_compss_path}/
    fi
  fi

  # Create .COMPSs log dir for application execution
  if [ -z "${base_log_dir}" ]; then
        base_log_dir=$HOME
  fi
  specific_log_dir=${base_log_dir}/.COMPSs/${!ENV_VAR_JOB_ID}/
  mkdir -p "${specific_log_dir}"

  # SharedDisk variables
  if [ "${worker_working_dir}" == "gpfs" ]; then
    worker_working_dir=$(mktemp -d -p "${GPFS_PREFIX}${HOME}")
  elif [ "${worker_working_dir}" == "scratch" ]; then
    worker_working_dir=$TMPDIR/${!ENV_VAR_JOB_ID}
  else
    # The working dir is a custom absolute path, create tmp
    worker_working_dir=$(mktemp -d -p "${worker_working_dir}")
  fi

  # Network variables
  if [ "${network}" == "ethernet" ]; then
    network=""
  elif [ "${network}" == "infiniband" ]; then
    network=${NETWORK_INFINIBAND_SUFFIX}
  elif [ "${network}" == "data" ]; then
    network=${NETWORK_DATA_SUFFIX}
  fi

  # Memory variables
  if [ "${node_memory}" == "${DEFAULT_NODE_MEMORY}" ]; then
    # Default value
    node_memory=${DEFAULT_NODE_MEMORY_SIZE}
  else
    # Change from MB to GB
    node_memory=$(( node_memory / 1024 - 4))
  fi

  if [ "${worker_in_master_memory}" == "${DEFAULT_WORKER_IN_MASTER_MEMORY}" ]; then
    # Default value
    worker_in_master_memory=${DEFAULT_NODE_MEMORY_SIZE}
  else
    # Change from MB to GB
    worker_in_master_memory=${DEFAULT_NODE_MEMORY_SIZE}
  fi

  # Load tracing and debug only for NIO
  if [ "${comm/NIO}" != "${comm}" ]; then
    # Adapting tracing flag to worker tracing level
    if [ -z "$tracing" ]; then
        w_tracing=0
    elif [ $tracing == "false" ]; then
        w_tracing=0
    elif [ $tracing == "basic" ] || [ $tracing == "true" ]; then
        w_tracing=1
        load_tracing_env
    elif [ $tracing == "advanced" ]; then
        w_tracing=2
        load_tracing_env
    fi

    # Adapt debug flag to worker script
    if [ "${log_level}" == "debug" ]; then
      debug="true"
    else
      debug="false"
    fi
  fi

  if [ -z "${uuid}" ]; then
    # Generate a UUID for workers and runcompss
    uuid=$(cat /proc/sys/kernel/random/uuid)
  fi
}

###############################################
# Log execution variables
###############################################
log_variables() {
  echo "-------- Launch arguments --------"
  echo "Master:                    ${master_node}"
  echo "Workers:                   ${worker_nodes}"
  echo "Tasks per Node:            ${max_tasks_per_node}"
  echo "CPUs per Node:             ${cpus_per_node}"
  echo "GPUs per Node:             ${gpus_per_node}"
  echo "FPGAs per Node:            ${fpgas_per_node}"
  echo "CPU Affinity:              ${cpu_affinity}"
  echo "GPU Affinity:              ${gpu_affinity}"
  echo "FPGA Affinity:             ${fpga_affinity}"
  echo "FPGA reprogram command:    ${fpga_prog}"
  echo "Network:                   ${network}"
  echo "Worker in Master CPUs:     ${worker_in_master_cpus}"
  echo "Worker in Master Memory:   ${worker_in_master_memory}"
  echo "Master Port:               ${master_port}"
  echo "Master WD:                 ${master_working_dir}"
  echo "Worker WD:                 ${worker_working_dir}"
  echo "Master JVM Opts:           ${jvm_master_opts}"
  echo "Workers JVM Opts:          ${jvm_workers_opts}"
  echo "Worker in Master JVM Opts: ${jvm_worker_in_master_opts}"
  echo "Library Path:              ${library_path}"
  echo "Classpath:                 ${cp}"
  echo "Pythonpath:                ${pythonpath}"
  echo "Appdir:			   ${appdir}"
  echo "Lang:                      ${lang}"
  echo "Python Interpreter:        ${python_interpreter}"
  echo "Python version:            ${python_version}"
  echo "Python virtual env:        ${DEFAULT_PYTHON_VIRTUAL_ENVIRONMENT}"
  echo "Python propagate virt env.:${python_propagate_virtual_environment}"
  echo "COMM:                      ${comm}"
  echo "Prolog:                    ${prologActions[*]}"
  echo "Epilog:                    ${epilogActions[*]}"
  echo "Storage conf:              ${storage_conf}"
  echo "Task execution:            ${taskExecution}"
  echo "To COMPSs:                 ${args_pass}"
  echo "-----------------------------------"
  echo " "
}


###############################################
# Write usage to log file
###############################################
write_log_usage() {
  # Create file with header if required
  local log_usage_file="${COMPSS_HOME}/usage.log"
  if [ ! -f "${log_usage_file}" ]; then
    echo -e "USAGE DATE\\t\\t\\tUSER" > "${log_usage_file}"
  fi

  # Write usage message
  msg="$(date)\\t$(whoami)"
  echo -e "$msg" >> "${log_usage_file}"

  # Ensure file is public
  chmod 777 "${log_usage_file}"
}


###############################################
# Launches the application
###############################################
launch() {
  echo "------ Launching application ------"

  # Launch workers separately if they are persistent
  if [ "${comm/NIO}" != "${comm}" ]; then
    # Start workers' processes
    local hostid=1
    if [ ! -z "${initial_hostid}" ]; then
      echo "Host id set to ${initial_hostid}"
      hostid=${initial_hostid}
    fi
    if [ ! -z "${master_node}" ]; then
      if [ "${worker_in_master_cpus}" -ne 0 ]; then
        # Worker in master node
        local jvm_worker_in_master_opts_str
        local jvm_worker_in_master_opts_size
        local fpga_reprogram_str
        local fpga_reprogram_size
        jvm_worker_in_master_opts_str=$(echo "${jvm_worker_in_master_opts}" | tr "," " ")
        jvm_worker_in_master_opts_size=$(echo "${jvm_worker_in_master_opts_str}" | wc -w)
        fpga_reprogram_str=$(echo "${fpga_prog}" | tr "," " ")
        fpga_reprogram_size=$(echo "${fpga_prog}" | wc -w)
        # worker_cmd id name jvmOptsSize jvmOpts cpus gpus cpuMap gpuMap lot
        worker_cmd $hostid "${master_node}${network}" "${jvm_worker_in_master_opts_size}" "${jvm_worker_in_master_opts_str}" "${fpga_reprogram_size}" "${fpga_reprogram_str}" "${worker_in_master_cpus}" "${gpus_per_node}" "${fpgas_per_node}" "${cpu_affinity}" "${gpu_affinity}" "${fpga_affinity}" "${max_tasks_per_node}"

        if [ ! -z "${NODE_NAME_QUEUE}" ]; then
          master_node=$(${NODE_NAME_QUEUE} "${master_node}")
        fi

        WCMD="${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${master_node} ${WCMD}"
        echo "CMD Worker $hostid launcher: $WCMD"
        $WCMD&
        hostid=$((hostid+1))
      fi
    fi
    if [ ! -z "${worker_nodes}" ]; then
      local jvm_workers_opts_str
      local jvm_workers_opts_size
      jvm_workers_opts_str=$(echo "${jvm_workers_opts}" | tr "," " ")
      jvm_workers_opts_size=$(echo "${jvm_workers_opts_str}" | wc -w)
      fpga_reprogram_str=$(echo "${fpga_prog}" | tr "," " ")
      fpga_reprogram_size=$(echo "${fpga_prog}" | wc -w)
      for node in ${worker_nodes}; do
        # worker_cmd id name jvmOptsSize jvmOpts cpus gpus cpuMap gpuMap lot
        worker_cmd $hostid "$node${network}" "${jvm_workers_opts_size}" "${jvm_workers_opts_str}" "${fpga_reprogram_size}" "${fpga_reprogram_str}" "${cpus_per_node}" "${gpus_per_node}" "${fpgas_per_node}" "${cpu_affinity}" "${gpu_affinity}" "${fpga_affinity}" "${max_tasks_per_node}"
        if [ ! -z "${NODE_NAME_QUEUE}" ]; then
            node=$(${NODE_NAME_QUEUE} ${node})
        fi
        WCMD="${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${node} ${WCMD}"
        echo "CMD Worker $hostid launcher: $WCMD"
        $WCMD&
        hostid=$((hostid+1))
      done
    fi

    # Sleep a little before launching master
    sleep 10s
  fi
  if [ ! -z "${master_node}" ]; then
    # Launch master
    if [ "${jupyter_notebook}" = false ]; then
      master_cmd "${master_node}${network}"
      MCMD="${MCMD} ${args_pass}"
      echo "CMD Master: $MCMD"
      $MCMD
    else
      # Prepare needed environment variables for starting the runtime from the PyCOMPSs API (in normal executions it is done by runcompss).
      export PYTHONPATH=${COMPSS_HOME}/Bindings/python/${python_version}/:${COMPSS_HOME}/Bindings/bindings-common/lib/:$PYTHONPATH
      # JAVA_HOME variable must exist
      export LD_LIBRARY_PATH=${COMPSS_HOME}/Bindings/bindings-common/lib/:${COMPSS_HOME}/Runtime/compss-engine.jar:${JAVA_HOME}/jre/lib/amd64/server/:$LD_LIBRARY_PATH
      unset XDG_RUNTIME_DIR  # Required for Jupyter in MN4. Supposed to be necessary also in other supercomputers. Otherwise, move this line to the supercomputer "module"
      export COMPSS_LOG_LEVEL=${log_level}
      export COMPSS_TRACING=${tracing}
      if [ -v "$tracing" ] && [ $tracing != "false" ]; then
          load_tracing_env
      fi
      # Export project and resources xmls as environment variables to be used from PyCOMPSs binding
      # Also export the master name, master port, and worker nodes to be used from PyCOMPSs binding
      export COMPSS_PROJECT_XML="${PROJECT_FILE}"
      export COMPSS_RESOURCES_XML="${RESOURCES_FILE}"
      export COMPSS_MASTER_NODE="${master_node}"
      export COMPSS_MASTER_PORT="${master_port}"
      export COMPSS_WORKER_NODES="${worker_nodes}"
      export COMPSS_UUID="${uuid}"
      export COMPSS_BASE_LOG_DIR="${base_log_dir}"
      export COMPSS_SPECIFIC_LOG_DIR="${specific_log_dir}"
      export COMPSS_STORAGE_CONF="${storage_conf}"
      # Run jupyter notebook instead of the master
      # The master will be started when the user wants later from the notebook (the workers have already been launched)
      jupyter notebook --no-browser --ip=127.0.0.1 --port=8888
    fi
  fi
}

###############################################
# Prolog actions
###############################################
prolog() {
  # Execute user prolog actions
  echo "---- Executing Prolog actions ----"
  for action in "${prologActions[@]}"; do
    realAction=$(echo "${action}" | tr "," " ")
    echo "- Prolog: $realAction"
    $realAction
    local actionExitValue=$?
    if [ $actionExitValue -ne 0 ]; then
      action_error "$ERROR_PROLOG_ACTION"
    fi
  done
}

###############################################
# Epilog actions
###############################################
epilog() {
  # Execute user epilog actions
  echo "---- Executing Epilog actions ----"
  for action in "${epilogActions[@]}"; do
    realAction=$(echo "${action}" | tr "," " ")
    echo "- Epilog: $realAction"
    $realAction
    local actionExitValue=$?
    if [ $actionExitValue -ne 0 ]; then
      action_error "$ERROR_EPILOG_ACTION"
    fi
  done
}

###############################################
# Wait for execution end
###############################################
wait_for_completion() {
  # Wait for Master and Workers to finish
  echo "Waiting for application completion"
  wait
}

###############################################
# Clean function for trap
###############################################
cleanup() {
  # Avoid clean up if removeWD=flase worker flag is set
  if [[ "$jvm_workers_opts" == *"-Dcompss.worker.removeWD=false"* ]]; then
    echo "[WARN] Flag removeWD set to false. Not cleaning Workers WD"
    echo "[WARN] Flag removeWD set to false. Not cleaning XML files"
  else
    # Cleanup
    echo "Cleanup Worker TMP files"
    for node in ${worker_nodes}; do
      local sandboxWD=${worker_working_dir}/${uuid}
      if [ ! -z "${NODE_NAME_QUEUE}" ]; then
        node=$(${NODE_NAME_QUEUE} ${node})
      fi
      echo "- Removing ${sandboxWD}"
      # shellcheck disable=SC2086
      ${LAUNCH_CMD} ${LAUNCH_PARAMS}${LAUNCH_SEPARATOR}${node} ${CMD_SEPARATOR}rm -rf ${sandboxWD}${CMD_SEPARATOR}
    done

    echo "Cleanup TMP files"
    if [ -z "${xmls_phase}" ] || [ "${xmls_phase}" = "all" ] || [ "${xmls_phase}" = "fini" ]; then
    	rm -f "${PROJECT_FILE}"
    	rm -f "${RESOURCES_FILE}"
    fi
  fi
}


#---------------------------------------------------------------------------------------
# MAIN EXECUTION
#---------------------------------------------------------------------------------------

  # Set script variables
  if [ -z "$COMPSS_HOME" ]; then
     SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
     COMPSS_HOME=${SCRIPT_DIR}/../../../
  else
     SCRIPT_DIR="${COMPSS_HOME}/Runtime/scripts/user"
  fi

  numPrologActions=0
  declare -a prologActions
  numEpilogActions=0
  declare -a epilogActions

  # Get command args
  get_args "$@"

  # Check other command args
  check_args

  # Set job variables
  set_variables

  # Log variables
  log_variables

  # Write log usage
  write_log_usage

  # Add clean up for execution end
  trap cleanup EXIT

  # Create XML files
  source ${SCRIPT_DIR}/../system/xmls/xmls_utils.sh
  CREATE_WORKING_DIRS=true
  echo " Creating xml phase: '${xmls_phase}' "
  if [ -z "${xmls_phase}" ]; then
      create_xml_files
  elif [ "${xmls_phase}" == "all" ]; then
      create_xml_files
  elif [ "${xmls_phase}" == "init" ]; then
      init_het_xml_files ${xmls_suffix}
  elif [ "${xmls_phase}" == "add" ]; then
      add_het_xml_files ${xmls_suffix}
  elif [ "${xmls_phase}" == "fini" ]; then
      fini_het_xml_files ${xmls_suffix}
  fi

  # Prolog
  prolog

  # Launch execution
  launch

  # Wait
  wait_for_completion

  # Epilog
  epilog
