#!/bin/bash

#---------------------------------------------------
# SCRIPT CONSTANTS DECLARATION
#---------------------------------------------------
DEFAULT_LANGUAGE=java

DEFAULT_SUMMARY=false

DEFAULT_LOG_LEVEL=off
DEFAULT_LOG_LEVEL_ARGUMENT=debug
LOG_LEVEL_DEBUG=debug
LOG_LEVEL_INFO=info
LOG_LEVEL_OFF=off
DEFAULT_EXTRAE_CONFIG_FILE="null"

DEFAULT_TASK_EXECUTION=compss
DEFAULT_GRAPH=false
DEFAULT_GRAPH_ARGUMENT=true

DEFAULT_TRACING=false
DEFAULT_TRACING_ARGUMENT=true # 0: activated, 1: basic, 2:advanced

DEFAULT_MONITORING_INTERVAL=0
DEFAULT_MONITORING_INTERVAL_ARGUMENT=2000

DEFAULT_TASK_COUNT=50
DEFAULT_CLASSPATH=$(pwd)
DEFAULT_PYTHONPATH=$(pwd)
DEFAULT_LIBRARY_PATH=$(pwd)
DEFAULT_APPDIR=$(pwd)
DEFAULT_STORAGE_CONF="None"
DEFAULT_PyOBJECT_SERIALIZE=false

DEFAULT_COMMUNICATION_ADAPTOR=integratedtoolkit.nio.master.NIOAdaptor
#DEFAULT_COMMUNICATION_ADAPTOR=integratedtoolkit.gat.master.GATAdaptor
DEFFAULT_CONNECTOR=integratedtoolkit.connectors.DefaultSSHConnector
DEFAULT_MASTER_PORT="[43000,44000]"
DEFAULT_MASTER_NAME=""
#DEFAULT_SCHEDULER=integratedtoolkit.scheduler.fullGraphScheduler.FullGraphScheduler
#DEFAULT_SCHEDULER=integratedtoolkit.scheduler.fifoScheduler.FIFOScheduler
#DEFAULT_SCHEDULER=integratedtoolkit.scheduler.fifoDataScheduler.FIFODataScheduler
#DEFAULT_SCHEDULER=integratedtoolkit.components.impl.TaskScheduler
DEFAULT_SCHEDULER=integratedtoolkit.scheduler.loadBalancingScheduler.LoadBalancingScheduler
RUNTIME_LOADER=integratedtoolkit.loader.ITAppLoader
DEFAULT_JVM_MASTER=""
DEFAULT_JVM_WORKERS="-Xms1024m,-Xmx1024m,-Xmn400m"
DEFAULT_CPU_AFFINITY="automatic" # disabled, automatic, user string
DEFAULT_GPU_AFFINITY="automatic" # disabled, automatic, user string

DEFAULT_DEBUGGER=false
DEFAULT_DEBUGGER_ARGUMENT=true
DEFAULT_DEBUGGER_PORT=9999

APPLICATION_ERROR="Error: Application name not provided"
LANGUAGE_ERROR="Value of option --lang must be: java, c or python"
JAVA_HOME_ERROR="Error: Cannot find Java JRE installation. Please set JAVA_HOME."
JAVA_JRE_ERROR="Error: Can't find JVM libraries in JAVA_HOME. Please check your Java JRE Installation."
RUNTIME_ERROR="Error running application"
TMP_FILE_JVM_ERROR="Error: Can't create temporary file for JVM options."
TRACING_ERROR="Value of option --tracing must be: false, true, basic or advanced"

#---------------------------------------------------
# FUNCTIONS DECLARATION
#---------------------------------------------------
show_opts() {
  cat <<EOT
  Tools enablers:
    --graph=<bool>, --graph, -g             Generation of the complete graph (true/false)
                                            When no value is provided it is set to ${DEFAULT_GRAPH_ARGUMENT}
                                            Default: ${DEFAULT_GRAPH}
    --tracing=<level>, --tracing, -t        Set generation of traces and/or tracing level ( [ true | basic ] | advanced | false)
                                            True and basic levels will produce the same traces.
                                            When no value is provided it is set to ${DEFAULT_TRACING_ARGUMENT}
                                            Default: ${DEFAULT_TRACING}
    --monitoring=<int>, --monitoring, -m    Period between monitoring samples (milliseconds)
                                            When no value is provided it is set to ${DEFAULT_MONITORING_INTERVAL_ARGUMENT}
                                            Default: ${DEFAULT_MONITORING_INTERVAL}
    --external_debugger=<int>,
    --external_debugger		            Enables external debugger connection on the specified port (or ${DEFAULT_DEBUGGER_PORT} if empty)
                                            Default: ${DEFAULT_DEBUGGER}

  Runtime configuration options:
    --task_execution=<compss|storage>	    Task execution under COMPSs or Storage.
                                            Default: ${DEFAULT_TASK_EXECUTION}
    --storage_conf=<path>		    Path to the storage configuration file
					    Default: ${DEFAULT_STORAGE_CONF}
    --project=<path>                        Path to the project XML file
                                            Default: ${DEFAULT_PROJECT}
    --resources=<path>                      Path to the resources XML file
                                            Default: ${DEFAULT_RESOURCES}
    --lang=<name>                           Language of the application (java/c/python)
                                            Default: Inferred is possible. Otherwise: ${DEFAULT_LANGUAGE}
    --summary                               Displays a task execution summary at the end of the application execution
                                            Default: ${DEFAULT_SUMMARY}
    --log_level=<level>, --debug, -d        Set the debug level: ${LOG_LEVEL_OFF} | ${LOG_LEVEL_INFO} | ${LOG_LEVEL_DEBUG}
                                            Default: ${DEFAULT_LOG_LEVEL}

  Advanced options:
    --extrae_config_file=<path>             Sets a custom extrae config file. Must be in a shared disk between all COMPSs workers.
                                            Default: ${DEFAULT_EXTRAE_CONFIG_FILE}
    --comm=<ClassName>                      Class that implements the adaptor for communications
                                            Supported adaptors: integratedtoolkit.nio.master.NIOAdaptor | integratedtoolkit.gat.master.GATAdaptor
                                            Default: ${DEFAULT_COMMUNICATION_ADAPTOR}
    --conn=<className>                      Class that implements the runtime connector for the cloud
                                            Supported connectors: integratedtoolkit.connectors.DefaultSSHConnector
								  | integratedtoolkit.connectors.DefaultNoSSHConnector
                                            Default: ${DEFAULT_CONNECTOR}
    --scheduler=<className>	            Class that implements the Scheduler for COMPSs
                                            Supported schedulers: integratedtoolkit.scheduler.fullGraphScheduler.FullGraphScheduler
                                                                  | integratedtoolkit.scheduler.fifoScheduler.FIFOScheduler
                                                                  | integratedtoolkit.scheduler.resourceEmptyScheduler.ResourceEmptyScheduler
                                            Default: ${DEFAULT_SCHEDULER}
    --library_path=<path>                   Non-standard directories to search for libraries (e.g. Java JVM library, Python library, C binding library)
                                            Default: Working Directory
    --classpath=<path>                      Path for the application classes / modules
                                            Default: Working Directory
    --appdir=<path>			    Path for the application class folder.
					    Default: ${DEFAULT_APPDIR}
    --base_log_dir=<path>                   Base directory to store COMPSs log files (a .COMPSs/ folder will be created inside this location)
                                            Default: User home
    --specific_log_dir=<path>               Use a specific directory to store COMPSs log files (the folder MUST exist and no sandbox is created)
					    Warning: Overwrites --base_log_dir option
                                            Default: Disabled
    --uuid=<int>                            Preset an application UUID
                                            Default: Automatic random generation
    --master_name=<string>                  Hostname of the node to run the COMPSs master
                                            Default: ${DEFAULT_MASTER_NAME}
    --master_port=<int>                     Port to run the COMPSs master communications.
                                            Only for NIO adaptor
                                            Default: ${DEFAULT_MASTER_PORT}
    --jvm_master_opts="<string>"            Extra options for the COMPSs Master JVM. Each option separed by "," and without blank spaces (Notice the quotes)
                                            Default: ${DEFAULT_JVM_MASTER}
    --jvm_workers_opts="<string>"	    Extra options for the COMPSs Workers JVMs. Each option separed by "," and without blank spaces (Notice the quotes)
                                            Default: ${DEFAULT_JVM_WORKERS}
    --cpu_affinity="<string>"               Sets the CPU affinity for the workers
                                            Supported options: disabled, automatic, user defined map of the form "0-8/9,10,11/12-14,15,16"
                                            Default: ${DEFAULT_CPU_AFFINITY}
    --gpu_affinity="<string>"               Sets the GPU affinity for the workers
                                            Supported options: disabled, automatic, user defined map of the form "0-8/9,10,11/12-14,15,16"
                                            Default: ${DEFAULT_GPU_AFFINITY}
    --task_count=<int>                      Only for C/Python Bindings. Maximum number of different functions/methods, invoked from the application, that have been selected as tasks
                                            Default: ${DEFAULT_TASK_COUNT}
    --pythonpath=<path>                     Additional folders or paths to add to the PYTHONPATH
                                            Default: ${DEFAULT_PYTHONPATH}
    --PyObject_serialize=<bool>             Only for Python Binding. Enable the object serialization to string when possible (true/false).
                                            Default: $DEFAULT_PyOBJECT_SERIALIZE


* Application name:
    For Java applications:   Fully qualified name of the application
    For C applications:      Path to the master binary
    For Python applications: Path to the .py file containing the main program

* Application arguments:
    Command line arguments to pass to the application. Can be empty.

EOT
}

usage() {
  exitValue=$1

  cat <<EOT
Usage: $0 [options] application_name application_arguments

* Options:
  General:
    --help, -h                              Print this help message

    --opts                                  Show available options

    --version, -v 			    Print COMPSs version
EOT
  show_opts
  exit $exitValue
}

show_version() {
  echo "COMPSs version 2.1 Bougainvillea"
  echo " "
}

show_full_version() {
  cat << EOF
        .-~~-.--.            ____       __
       :         )          |___ \     /  |
 .~ ~ -.\       /.- ~~ .      __) |     | |
 >       \`.   .'       <     / __/   _  | |
(         .- -.         )   |_____| |_| |_|
 \`- -.-~  \`- -'  ~-.- -'
   (        :        )           _ _ .-:
    ~--.    :    .--~        .-~  .-~  }
        ~-.-^-.-~ \_      .~  .-~   .~
                 \ \'     \ '_ _ -~
                  \`.\`.    //
         . - ~ ~-.__\`.\`-.//
     .-~   . - ~  }~ ~ ~-.~-.
   .' .-~      .-~       :/~-.~-./:
  /_~_ _ . - ~                 ~-.~-._
                                   ~-.<
EOF
  show_version
}

show_flower() {
  cat << EOF

Bougainvillea (Bougainvillea spectabilis) - Flower

Bougainvillea, also known as buganvilla in Spain, is a genus of thorny 
ornamental vines, bushes, and trees with flower-like spring leaves near
its flowers. They are native plants of South America from Brazil west
to Ecuador and south to southern Argentina (Chubut Province). 

Different authors accept between 4 and 18 species in the genus. The 
vine species grow anywhere from 1 to 12 m (3 to 40 ft.) tall, 
scrambling over other plants with their spiky thorns. The thorns are
tipped with a black, waxy substance. They are evergreen where rainfall 
occurs all year, or deciduous if there is a dry season. The leaves are
alternate, simple ovate-acuminate, 4–13 cm long and 2–6 cm broad. The 
actual flower of the plant is small and generally white, but each 
cluster of three flowers is surrounded by three or six bracts with the
bright colours associated with the plant, including pink, magenta, 
purple, red, orange, white, or yellow. 

Check more information at:
   https://en.wikipedia.org/wiki/Bougainvillea

EOF
}

show_recipe() {
  cat << EOF

Bougainvillea - Cocktail

  100 ml pink, sparkling wine
  100 ml fresh-squeezed orange juice 

  75 ml pink, sparkling wine 
  75 ml fresh-squeezed orange juice (preferably blood orange)
  75 ml freshly-brewed, then cooled hibiscus tea
  25 ml homemade grenadine:
    1/4 cup sugar
    1 cup unsweetened organic pomegranate juice
    1/2 lemon
  10 ml elderflower liqueur
  10 ml maraschino liqueur
  5 dashes cardamom bitters
  bougainvillea blossoms for garnish

EOF
}

# Displays parsing arguments errors
display_error() {
  local error_msg=$1

  echo $error_msg
  echo " "

  usage 1
}

# Displays runtime/application errors
error_msg() {
  local error_msg=$1

  # Display error
  echo
  echo $error_msg
  echo

  # Exit
  exit 1
}

check_env() {
  # Added for SGE queue systems which do not allow to copy LD_LIBRARY_PATH
  if [ -z "$LD_LIBRARY_PATH" ]; then
     if [ -n "$LIBRARY_PATH" ]; then
         export LD_LIBRARY_PATH=$LIBRARY_PATH
         echo "[  INFO] LD_LIBRARY_PATH not defined set to LIBRARY_PATH"
     fi
  fi
  
  # Command to simulate realpath
  IT_HOME=$(cd $(dirname $0)/../../.. ; pwd)
  export IT_HOME=${IT_HOME}

  if [ -z "$DEFAULT_PROJECT" ]; then
    DEFAULT_PROJECT=${IT_HOME}/Runtime/configuration/xml/projects/default_project.xml
  fi

  if [ -z "$DEFAULT_RESOURCES" ]; then
    DEFAULT_RESOURCES=${IT_HOME}/Runtime/configuration/xml/resources/default_resources.xml
  fi

  if [ -z "${GAT_LOCATION}" ]; then
    GAT_LOCATION=${IT_HOME}/Dependencies/JAVA_GAT
  fi

  if [ -z "${EXTRAE_HOME}" ]; then
    EXTRAE_HOME=${IT_HOME}/Dependencies/extrae
  fi

  if [ -z "${EXTRAE_LIB}" ]; then
    EXTRAE_LIB=${EXTRAE_HOME}/lib
  fi

  if [[ -z "$JAVA_HOME" ]]; then
    JAVA=java
  elif [ -f $JAVA_HOME/jre/bin/java ]; then
    JAVA=$JAVA_HOME/jre/bin/java
  elif [ -f $JAVA_HOME/bin/java ]; then
    JAVA=$JAVA_HOME/bin/java
  else
    display_error "${JAVA_HOME_ERROR}"
  fi
}

get_args() {
  # Parse COMPSs Options
  while getopts hvgtmd-: flag; do
    # Treat the argument
    case "$flag" in
      h)
	# Display help
	usage 0
	;;
      v)
        # Display version
        show_version
        exit
        ;;
      g)
	# Enable graph generation at the end of the execution
	graph=${DEFAULT_GRAPH_ARGUMENT}
	;;
      t)
	# Enable tracing
	tracing=${DEFAULT_TRACING_ARGUMENT}
	;;
      m)
	# Enable monitoring with default value
	monitoring=${DEFAULT_MONITORING_INTERVAL_ARGUMENT}
	;;
      d)
        # Enable debug in log level
        log_level=${DEFAULT_LOG_LEVEL_ARGUMENT}
        ;;
      -)
	# Check more complex arguments
	case "$OPTARG" in
	  help)
	    # Display help
	    usage 0
	    ;;
          version)
            # Show version
            show_full_version
            exit 0
            ;;
          opts)
            # Display help
            show_opts
            exit 0
            ;;
          flower)
            # Display flower
            show_flower
            exit 0
            ;;
          recipe)
            # Display recipe
            show_recipe
            exit 0
            ;;
	  project=*)
	    # Custom project file
	    projFile=$(echo $OPTARG | sed -e 's/project=//g')
	    if [ ! -f $projFile ]; then
	      echo "[ WARNING ]: Project XML file '$projFile' could not be found."
	    fi
	    ;;
	  resources=*)
	    # Custom resources file
	    resFile=$(echo $OPTARG | sed -e 's/resources=//g')
	    if [ ! -f $resFile ]; then
	      echo "[ WARNING ]: Resources XML file '$resFile' could not be found."
	    fi
            ;;
          summary)
            summary=true
            ;;
	  storage_conf=*)
            storageConf=`echo $OPTARG | sed -e 's/storage_conf=//g'`;
            ;;
	  task_execution=*)
	    # Task execution under COMPSs or Storage
	    taskExecution=$(echo $OPTARG | sed -e 's/task_execution=//g')
	    ;;
	  lang=*)
	    # Language selection
	    lang=$(echo $OPTARG | sed -e 's/lang=//g')
	    ;;
          log_level=*)
            # Enable different log_levels by user selection
            log_level=$(echo $OPTARG | sed -e 's/log_level=//g')
            ;;
          debug*)
            # Enable debug in log level
            log_level=${DEFAULT_LOG_LEVEL_ARGUMENT}
            ;;
	  graph=*)
	    # Graph generation at the end of the execution with user option
	    graph=$(echo $OPTARG | sed -e 's/graph=//g')
	    ;;
	  graph)
	    # Graph generation at the end of the execution by default arg
	    graph=${DEFAULT_GRAPH_ARGUMENT}
	    ;;
          tracing=*)
            # Tracing system
            tracing=$(echo $OPTARG | sed -e 's/tracing=//g')
            ;;
          tracing)
            # Tracing system
            tracing=${DEFAULT_TRACING_ARGUMENT}
            ;;
          extrae_config_file=*)
            # Custom extrae config file
            custom_extrae_config_file=$(echo $OPTARG | sed -e 's/extrae_config_file=//g')
            ;;
	  monitoring=*)
	    # Value between monitor steps (ms)
	    monitoring=$(echo $OPTARG | sed -e 's/monitoring=//g')
	    ;;
	  monitoring)
	    # Value between monitor steps (ms)
	    monitoring=${DEFAULT_MONITORING_INTERVAL_ARGUMENT}
	    ;;
	  comm=*)
	    # Communication adaptor main class
	    comm=$(echo $OPTARG | sed -e 's/comm=//g')
	    ;;
	  conn=*)
	    # Communication connector main class
	    conn=$(echo $OPTARG | sed -e 's/conn=//g')
	    ;;
          scheduler=*)
            # Scheduler main class
            scheduler=$(echo $OPTARG | sed -e 's/scheduler=//g')
            ;;
	  classpath=*)
	    # Additional classpath
	    cp=$(echo $OPTARG | sed -e 's/classpath=//g')
	    ;;
	  library_path=*)
	    # Additional library path
	    library_path=$(echo $OPTARG | sed -e 's/library_path=//g')
	    ;;
          jvm_master_opts=*)
            # Master JVM option
            jvm_master_opts=$(echo $OPTARG | sed -e 's/jvm_master_opts=//g')
            ;;
          jvm_workers_opts=*)
            # Workers JVMs option
            jvm_workers_opts=$(echo $OPTARG | sed -e 's/jvm_workers_opts=//g')
            ;;
          cpu_affinity=*)
            # Worker CPU affinity
            worker_cpu_affinity=$(echo $OPTARG | sed -e 's/cpu_affinity=//g')
            ;;
          gpu_affinity=*)
            # Worker GPU affinity
            worker_gpu_affinity=$(echo $OPTARG | sed -e 's/gpu_affinity=//g')
            ;;
          external_debugger=*)
            # Enable external debugger on specific port
            external_debugger=${DEFAULT_DEBUGGER_ARGUMENT}
            external_debugger_port=$(echo $OPTARG | sed -e 's/external-debugger=//g')
            ;;
          external_debugger)
            # Enable default external debugger
            external_debugger=${DEFAULT_DEBUGGER_ARGUMENT}
            external_debugger_port=${DEFAULT_DEBUGGER_PORT}
            ;;
          base_log_dir=*)
            # Set a custom base log dir
            base_log_dir=$(echo $OPTARG | sed -e 's/base_log_dir=//g')
            ;;
          specific_log_dir=*)
            # Set a custom specific log dir
            specific_log_dir=$(echo $OPTARG | sed -e 's/specific_log_dir=//g')
            ;;
	  uuid=*)
            # Preset an Application UUID
            uuid=$(echo $OPTARG | sed -e 's/uuid=//g')
            ;;
          master_name=*)
            # Preset a Master hostname
            master_name=$(echo $OPTARG | sed -e 's/master_name=//g')
            ;;
          master_port=*)
            # Preset a Master port
            master_port=$(echo $OPTARG | sed -e 's/master_port=//g')
            ;;
          task_count=*)
            # Maximum task count. Only for bindings
            task_count=$(echo $OPTARG | sed -e 's/task_count=//g')
            ;;
    	  appdir=*)
	    # Main binary directory (only for C/C++ applications)
	    task_count=$(echo $OPTARG | sed -e 's/appdir=//g')
	    ;;
          pythonpath=*)
            # Additional pythonpath
            pythonpath=$(echo $OPTARG | sed -e 's/pythonpath=//g')
            ;;
          PyObject_serialize=*)
            # Enable the object to string serialization (only for PyCOMPSs applications)
            PyObject_serialize=$(echo $OPTARG | sed -e 's/PyObject_serialize=//g')
            ;;
	  *)
	    # Flag didn't match any patern. Raise exception
	    display_error "Bad argument: $OPTARG"
	    ;;
	esac
	;;
      *)
	# Flag didn't match any patern. End of COMPSs flags
	break
	;;
    esac
  done
  # Shift COMPSs arguments
  shift $((OPTIND-1))

  # Parse application name
  if [[ $# -eq 0 ]]; then
    display_error "${APPLICATION_ERROR}"
  else
    fullAppPath=$1
    if [ -z "$fullAppPath" ]; then
      display_error "${APPLICATION_ERROR}"
    else
      shift 1
    fi
  fi

  # Parse application arguments
  application_args=$*
}

check_args() {
  if [ -z "${taskExecution}" ]; then
    echo "[  INFO] Using default execution type: ${DEFAULT_TASK_EXECUTION}"
    taskExecution=${DEFAULT_TASK_EXECUTION}
  fi

  if [ -z "$projFile" ]; then
    echo "[  INFO] Using default location for project file: ${DEFAULT_PROJECT}"
    projFile=${DEFAULT_PROJECT}
  fi

  if [ -z "$resFile" ]; then
    echo "[  INFO] Using default location for resources file: ${DEFAULT_RESOURCES}"
    resFile=${DEFAULT_RESOURCES}
  fi

  if [ -z "${summary}" ]; then
    summary=${DEFAULT_SUMMARY}
  fi

  if [ -z "$lang" ]; then
    # Try to infer language by extension
    fileName=$(basename $fullAppPath)
    extension=${fileName##*.}
    if [ "$extension" == "py" ] || [ "$extension" == "pyc" ]; then
      echo "[  INFO] Inferred PYTHON language"
      lang=python
    else
      # Try to infer language by file command
      isPython=$(file $fullAppPath | grep "Python" | cat)
      isPythonCompiled=$(file $fullAppPath | grep "python" | cat)
      isC=$(file $fullAppPath | grep "executable" | cat)
      if [ -n "$isPython" ] || [ -n "$isPythonCompiled" ]; then
        echo "[  INFO] Inferred PYTHON language"
        lang=python
      elif [ -n "$isC" ]; then
        echo "[  INFO] Inferred C/C++ language"
        lang=c
      else
        # Lang cannot be infered or it's the default JAVA
        echo "[  INFO] Using default language: ${DEFAULT_LANGUAGE}"
        lang=${DEFAULT_LANGUAGE}
      fi
    fi
  elif [ "$lang" = "java" ]; then
    lang=java
  elif [ "$lang" = "c" ]; then
    lang=c
  elif [ "$lang" = "python" ]; then
    lang=python
  else
    display_error "${LANGUAGE_ERROR}"
  fi

  if [ -z "${log_level}" ]; then
    log_level=${DEFAULT_LOG_LEVEL}
  fi

  if [ -z "$graph" ]; then
    graph=${DEFAULT_GRAPH}
  fi

  # TRACING file option
  if [ -z "${custom_extrae_config_file}" ]; then
    custom_extrae_config_file=${DEFAULT_EXTRAE_CONFIG_FILE}
  fi

  # Tracing level
  if [ -z "$tracing" ]; then
    tracing=0
  elif [ $tracing == "false" ]; then
     tracing=0
  elif [ $tracing == "basic" ] || [ $tracing == "true" ]; then
    tracing=1
  elif [ $tracing == "advanced" ]; then
    tracing=2
  else
    display_error "${TRACING_ERROR}"
  fi
  if [ $tracing -eq 1 ]; then
    extraeFile=${IT_HOME}/Runtime/configuration/xml/tracing/extrae_basic.xml
  elif [ $tracing -eq 2 ]; then
    extraeFile=${IT_HOME}/Runtime/configuration/xml/tracing/extrae_advanced.xml
  fi

  # Overwrite extraeFile if already defined
  if [ "${custom_extrae_config_file}" != "${DEFAULT_EXTRAE_CONFIG_FILE}" ]; then
    extraeFile=${custom_extrae_config_file}
  fi

  # Set tracing env
  if [ $tracing -gt 0 ]; then
    export EXTRAE_HOME=${EXTRAE_HOME}
    export LD_LIBRARY_PATH=${EXTRAE_LIB}:${LD_LIBRARY_PATH}
    export EXTRAE_CONFIG_FILE=${extraeFile}
  fi

  if [ -z "$monitoring" ]; then
    monitoring=${DEFAULT_MONITORING_INTERVAL}
  else
    # If monitor as been activated trigger final graph generation and log_level = at least info
    graph=${DEFAULT_GRAPH_ARGUMENT}
    if [ "${log_level}" == "${DEFAULT_LOG_LEVEL}" ] || [ "${log_level}" == "${LOG_LEVEL_OFF}" ]; then
       log_level=${LOG_LEVEL_INFO}
    fi
  fi

  # Storage configuration
  if [ -z "$storageConf" ]; then
    storageConf="null"
  fi 

  # Master log level
  if [ "${log_level}" == "${DEFAULT_LOG_LEVEL}" ]; then
    itlog4j_file="COMPSsMaster-log4j"
  else
    itlog4j_file="COMPSsMaster-log4j.${log_level}"
  fi

  # Adaptor
  if [ -z "$comm" ]; then
    comm=${DEFAULT_COMMUNICATION_ADAPTOR}
  fi
  
  # Connector
  if [ -z "$conn" ]; then
    conn=${DEFAULT_CONNECTOR}
  fi

  # Scheduler
  if [ -z "$scheduler" ]; then
    scheduler=${DEFAULT_SCHEDULER}
  fi

  # Classpath
  if [ -z "$cp" ]; then
    cp=${DEFAULT_CLASSPATH}
    for jar in $(ls ${DEFAULT_CLASSPATH} | grep *.jar); do
       cp=$cp:${DEFAULT_CLASSPATH}/$jar
    done
  else
    fcp=""
    for currcp in ${cp//:/$'\n'}; do
      if [ ! ${currcp:0:1} == '/' ]; then # Relative paths to absolute
        if [ -d "$currcp" ] || [ -f "$currcp" ]; then   # If the dir/file exists locally
	  absdir="$(cd "$(dirname "$currcp")" && pwd)" # Get absolute dir
	  file="$(basename "$currcp")"
	  fcp="${fcp}$absdir/$file:"
	else
	  echo "[ WARNING ]: Classpath \"$currcp\" does not exist..."
	fi
      else
   	fcp="${fcp}$currcp:"
      fi
    done
    cp="$fcp"
    echo "[  INFO] Relative Classpath resolved: $cp"
  fi

  if [ -z "$task_count" ]; then
    task_count=${DEFAULT_TASK_COUNT}
  fi

  if [ -z "$library_path" ]; then
    library_path=${DEFAULT_LIBRARY_PATH}
  fi

  if [ -z "$uuid" ]; then
    uuid=$(uuidgen)
    if [ -z "$uuid" ]; then
      uuid=$(cat /proc/sys/kernel/random/uuid)
    fi
  fi

  # MASTER
  if [ -z "${master_name}" ]; then
    master_name=${DEFAULT_MASTER_NAME}
  fi

  # JVM
  if [ -z "${jvm_master_opts}" ]; then
    jvm_master_opts=${DEFAULT_JVM_MASTER}
  fi
  # Change jvm master opts separation character "," by " "
  jvm_master_opts=$(echo $jvm_master_opts | tr "," "\n")

  if [ -z "${jvm_workers_opts}" ]; then
    jvm_workers_opts=${DEFAULT_JVM_WORKERS}
  fi

  # WORKER THREAD AFFINITY
  if [ -z "${worker_cpu_affinity}" ]; then
    worker_cpu_affinity=${DEFAULT_CPU_AFFINITY}
  fi
  if [ -z "${worker_gpu_affinity}" ]; then
    worker_gpu_affinity=${DEFAULT_GPU_AFFINITY}
  fi

  # Debugger
  if [ -z "$external_debugger" ]; then
    external_debugger=${DEFAULT_DEBUGGER}
  fi
  if [ "${external_debugger}" == "true" ]; then
    jvm_debugger_opts="-Xdebug -agentlib:jdwp=transport=dt_socket,address=${external_debugger_port},server=y,suspend=y"
  else
    jvm_debugger_opts=""
  fi

  # Options ONLY for PYTHON
  if [ "$lang" = "python" ]; then
    if [ -z "$pythonpath" ]; then
      pythonpath=${DEFAULT_PYTHONPATH}
    else
      # Adds execution dir by default to pythonpath
      pythonpath=$pythonpath":${DEFAULT_PYTHONPATH}"
    fi
    # Adds all given Pythonpaths to the global variable
    PYTHONPATH=$PYTHONPATH":$pythonpath"
    export PYTHONPATH

    if [ -z "$PyObject_serialize" ]; then
      PyObject_serialize=$DEFAULT_PyOBJECT_SERIALIZE
    fi
  fi

  # Options ONLY for C
  if [ "$lang" = "c" ]; then
    if [ -z "$appdir" ]; then
      appdir=$DEFAULT_APPDIR
    fi
  fi
}

execute_runtime() {
  # Create tmp dir for initial loggers configuration
  mkdir /tmp/$uuid

  # Create JVM Options file
  appName=$(basename "$fullAppPath")
  generate_jvm_opts_file

  # Init COMPSs
  echo -e "\n----------------- Executing $appName --------------------------\n"
  if [ $lang = java ]; then
    add_jvm_opts_java
    exec_java
  elif [ $lang = c ]; then
    add_jvm_opts_c
    exec_c
  elif [ $lang = python ]; then
    add_jvm_opts_python
    exec_python
  fi
  echo
  echo ------------------------------------------------------------
}

generate_jvm_opts_file() {
  jvm_options_file=$(mktemp)
  if [ $? -ne 0 ]; then
    display_error "${TMP_FILE_JVM_ERROR}"
  fi

  /bin/cat >> ${jvm_options_file} << EOT
${jvm_master_opts}
${jvm_debugger_opts}
-XX:+PerfDisableSharedMem
-XX:-UsePerfData
-XX:+UseG1GC
-XX:+UseThreadPriorities
-XX:ThreadPriorityPolicy=42
-Dlog4j.configurationFile=${IT_HOME}/Runtime/configuration/log/${itlog4j_file}
-Dit.to.file=false
-Dit.project.file=${projFile}
-Dit.resources.file=${resFile}
-Dit.project.schema=${IT_HOME}/Runtime/configuration/xml/projects/project_schema.xsd
-Dit.resources.schema=${IT_HOME}/Runtime/configuration/xml/resources/resources_schema.xsd
-Dit.lang=${lang}
-Dit.summary=${summary}
-Dit.task.execution=$taskExecution
-Dit.storage.conf=$storageConf
-Dit.core.count=${task_count}
-Dit.appName=${appName}
-Dit.uuid=${uuid}
-Dit.baseLogDir=${base_log_dir}
-Dit.specificLogDir=${specific_log_dir}
-Dit.appLogDir=/tmp/${uuid}/
-Dit.graph=${graph}
-Dit.monitor=${monitoring}
-Dit.tracing=${tracing}
-Dit.extrae.file=${custom_extrae_config_file}
-Dit.comm=${comm}
-Dit.conn=${conn}
-Dit.masterName=${master_name}
-Dit.masterPort=${master_port}
-Dit.scheduler=${scheduler}
-Dgat.adaptor.path=${GAT_LOCATION}/lib/adaptors
-Dit.gat.broker.adaptor=sshtrilead
-Dit.gat.file.adaptor=sshtrilead
-Dit.worker.cp=${cp}:${CLASSPATH}
-Dit.worker.jvm_opts=${jvm_workers_opts}
-Dit.worker.cpu_affinity=${worker_cpu_affinity}
-Dit.worker.gpu_affinity=${worker_gpu_affinity}
EOT
}

add_jvm_opts_java() {
  /bin/cat >> ${jvm_options_file} << EOT
-noverify
-classpath ${cp}:${IT_HOME}/Runtime/compss-engine.jar:${CLASSPATH}
EOT
}

add_jvm_opts_c() {
  /bin/cat >> ${jvm_options_file} << EOT
-Djava.class.path=${cp}:${IT_HOME}/Runtime/compss-engine.jar:${CLASSPATH}
-Dit.constraints.file=$fullAppPath.idl
EOT
}

add_jvm_opts_python() {
  /bin/cat >> ${jvm_options_file} << EOT
-Djava.class.path=${cp}:${IT_HOME}/Runtime/compss-engine.jar:${CLASSPATH}
-Dit.worker.pythonpath=${PYTHONPATH}
EOT
}

exec_java() {
  # Export needed variables
  export LD_LIBRARY_PATH=$library_path:$LD_LIBRARY_PATH

  # Define command
  local java_opts=$(cat ${jvm_options_file} | tr '\n' ' ')
  local JAVACMD=$JAVA" ${java_opts}"

  if [ $tracing -gt 0 ]; then
    export LD_PRELOAD=${EXTRAE_LIB}/libpttrace.so
  fi

  # Launch application
  $JAVACMD ${RUNTIME_LOADER} total $fullAppPath $application_args
  endCode=$?
  if [ $tracing -gt 0 ]; then
      unset LD_PRELOAD  # no tracing check because if not set does nothing
  fi
  if [ $endCode -ne 0 ]; then
    error_msg "${RUNTIME_ERROR}"
  fi
}

exec_c() {
  # Export needed variables
  if [ -d ${IT_HOME}/Bindings/c ]; then
    local CPP_COMPSS_HOME=${IT_HOME}/Bindings/c
    export CPP_PATH=${CPP_COMPSS_HOME}:$cp
  else
    export CPP_PATH=$cp
  fi

  # Look for the JVM Library
  libjava=$(find ${JAVA_HOME}/jre/lib/ -name libjvm.so | head -n 1)
  if [ -z "$libjava" ]; then
    libjava=$(find ${JAVA_HOME}/jre/lib/ -name libjvm.dylib | head -n 1)
    if [ -z "$libjava" ]; then
      display_error "${JAVA_JRE_ERROR}"
    fi
  fi
  libjavafolder=$(dirname $libjava)

  export LD_LIBRARY_PATH=$library_path:${LD_LIBRARY_PATH}:$libjavafolder:${IT_HOME}/Bindings/bindings-common/lib:${IT_HOME}/Bindings/c/lib

  export JVM_OPTIONS_FILE=${jvm_options_file}

  # Launch application
  echo "JVM_OPTIONS_FILE: $JVM_OPTIONS_FILE"
  echo "IT_HOME: $IT_HOME"
  echo "Args: $application_args"
  echo " "

  if [ $tracing -gt 0 ]; then
    export LD_PRELOAD=${EXTRAE_LIB}/libpttrace.so
  fi
  $fullAppPath $application_args
  endCode=$?
  if [ $tracing -gt 0 ]; then
      unset LD_PRELOAD  # no tracing check because if not set does nothing
  fi
  if [ $endCode -ne 0 ]; then
    error_msg "${RUNTIME_ERROR}"
  fi
}

exec_python() {
  # Python Storage API
  storageApi=${IT_HOME}/Dependencies/
  cp=$cp:$storageApi

  # Export needed variables
  if [ -d $IT_HOME/Bindings/python ]; then
    PYCOMPSS_HOME=${IT_HOME}/Bindings/python
    export PYTHONPATH=${PYCOMPSS_HOME}:$cp:$PYTHONPATH
  else
    export PYTHONPATH=$cp:$PYTHONPATH
  fi

  # Look for the JVM Library
  libjava=$(find ${JAVA_HOME}/jre/lib/ -name libjvm.so | head -n 1)
  if [ -z "$libjava" ]; then
    libjava=$(find ${JAVA_HOME}/jre/lib/ -name libjvm.dylib | head -n 1)
    if [ -z "$libjava" ]; then
      display_error "${JAVA_JRE_ERROR}"
    fi
  fi     
  libjavafolder=$(dirname $libjava)

  export LD_LIBRARY_PATH=$library_path:${LD_LIBRARY_PATH}:$libjavafolder:${IT_HOME}/Bindings/bindings-common/lib

  export JVM_OPTIONS_FILE=${jvm_options_file}

  # Launch application
  if [ $tracing -gt 0 ]; then
    export LD_PRELOAD=${EXTRAE_LIB}/libpttrace.so
  fi
  python $PYCOMPSS_HOME/pycompss/runtime/launch.py $log_level $PyObject_serialize $storageConf $fullAppPath $application_args
  endCode=$?
  if [ $tracing -gt 0 ]; then
      unset LD_PRELOAD  # no tracing check because if not set does nothing
  fi
  if [ $endCode -ne 0 ]; then
    error_msg "${RUNTIME_ERROR}"
  fi
}

clean_env() {
  # Delete JVM options file
  rm -f ${jvm_options_file}

  # Delete tmp dir for initial loggers
  rm -rf /tmp/$uuid
}

#---------------------------------------------------
# MAIN EXECUTION
#---------------------------------------------------
  # Environment and arguments
  check_env
  get_args $@
  check_args

  # Trap to ensure clean end
  trap clean_env EXIT

  # Real runtime execution
  execute_runtime
