#****************************************************************************
#*  SCALASCA    http://www.scalasca.org/                                   **
#****************************************************************************
#*  Copyright (c) 1998-2013                                                **
#*  Forschungszentrum Juelich GmbH, Juelich Supercomputing Centre          **
#*                                                                         **
#*  See the file COPYRIGHT in the package base directory for details       **
#***************************************************************************/


#--- OpenMP-related performance patterns ------------------------------------

PROLOG {
#if defined(_OPENMP)
namespace {
    int omp_nest_level = 0;
}   // unnamed namespace
#endif
}


PATTERN "OMP_IDLE_THREADS" = [
  PARENT    = "TIME"
  NAME      = "OMP"
  DOCNAME   = "OpenMP Idle threads Time"
  CLASS     = "PatternOMP_IdleThreads"
  INFO      = "Unused CPU reservation time"
  DESCR     = {
    Idle time on CPUs that may be reserved for teams of threads when the
    process is executing sequentially before and after OpenMP parallel
    regions, or with less than the full team within OpenMP parallel
    regions.
    </dd><p><dd>
    @img(OMPIdle.png)
  }
  DIAGNOSIS = {
    On shared compute resources, unused threads may simply sleep and allow
    the resources to be used by other applications, however, on dedicated
    compute resources (or where unused threads busy-wait and thereby occupy
    the resources) their idle time is charged to the application. 
    According to Amdahl's Law, the fraction of inherently serial execution
    time limits the effectiveness of employing additional threads to reduce
    the execution time of parallel regions. Where the Idle Threads Time is
    significant, total @ref(TIME) (and wall-clock execution time) may be
    reduced by effective parallelization of sections of code which execute
    serially. Alternatively, the proportion of wasted Idle Threads Time
    will be reduced by running with fewer threads, albeit resulting in a
    longer wall-clock execution time but more effective usage of the
    allocated compute resources.
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_LIMITED_PARALLELISM" = [
  PARENT    = "OMP_IDLE_THREADS"
  NAME      = "OMP"
  DOCNAME   = "OpenMP Limited parallelism Time"
  CLASS     = "PatternOMP_LimitedParallelism"
  INFO      = "Unused CPU reservation time in parallel regions due to limited parallelism"
  DESCR     = {
    Idle time on CPUs that may be reserved for threads within OpenMP
    parallel regions where not all of the thread team participates.
    </dd><p><dd>
    @img(OMPLimitedParallelism.png)
  }
  DIAGNOSIS = {
    Code sections marked as OpenMP parallel regions which are executed
    serially (i.e., only by the master thread) or by less than the full
    team of threads, can result in allocated but unused compute resources
    being wasted. Typically this arises from insufficient work being
    available within the marked parallel region to productively employ all
    threads. This may be because the loop contains too few iterations or
    the OpenMP runtime has determined that additional threads would not be
    productive. Alternatively, the OpenMP <tt>omp_set_num_threads</tt> API
    or <tt>num_threads</tt> or <tt>if</tt> clauses may have been explicitly
    specified, e.g., to reduce parallel execution overheads such as
    @ref(OMP_MANAGEMENT) or @ref(OMP_SYNCHRONIZATION). If the proportion of
    OpenMP Limited parallelism Time is significant, it may be more
    efficient to run with fewer threads for that problem size.
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_TIME" = [
  PARENT    = "EXECUTION"
  NAME      = "OMP"
  DOCNAME   = "OpenMP Time"
  CLASS     = "PatternOMP"
  INFO      = "Time spent in the OpenMP runtime system and API"
  DESCR     = {
    Time spent in OpenMP API calls and code generated by the OpenMP compiler.
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_FLUSH" = [
  PARENT  = "OMP_TIME"
  NAME    = "Flush"
  DOCNAME = "OpenMP Flush Time"
  CLASS   = "PatternOMP_Flush"
  INFO    = "Time spent in OpenMP flush directives"
  DESCR   = {
    Time spent in OpenMP <tt>flush</tt> directives.
  }
  UNIT    = "sec"
  HIDDEN
]


PATTERN "OMP_MANAGEMENT" = [
  PARENT    = "OMP_TIME"
  NAME      = "Management"
  DOCNAME   = "OpenMP Management Time"
  TYPE      = "OMP"
  CLASS     = "PatternOMP_Mgmt"
  INFO      = "Time spent in OpenMP thread management"
  DESCR     = {
    Time spent managing teams of threads, creating and initializing them
    when forking a new parallel region and clearing up afterwards when
    joining.
    </dd><p><dd>
    @img(OMPThreadManagement.png)
  }
  DIAGNOSIS = {
    Management overhead for an OpenMP parallel region depends on the number
    of threads to be employed and the number of variables to be initialized
    and saved for each thread, each time the parallel region is executed.
    Typically a pool of threads is used by the OpenMP runtime system to
    avoid forking and joining threads in each parallel region, however, 
    threads from the pool still need to be added to the team and assigned
    tasks to perform according to the specified schedule. When the overhead
    is a significant proportion of the time for executing the parallel
    region, it is worth investigating whether several parallel regions can
    be combined to amortize thread management overheads. Alternatively, it
    may be appropriate to reduce the number of threads either for the
    entire execution or only for this parallel region (e.g., via
    <tt>num_threads</tt> or <tt>if</tt> clauses).
  }
  UNIT      = "sec"
  CALLBACKS = [
    "OMP_FORK" = {
      // Increment nesting level
      omp_nest_level++;
    }

    "OMP_JOIN" = {
      // Decrement nesting level
      omp_nest_level--;
    }

    "ENTER" = {
      // Are we entering an OpenMP parallel region?
      Region* region = event->get_region();
      if (!is_omp_parallel(region))
        return;

      // Determine timestamp of FORK event
      static timestamp_t forkTime;
      #pragma omp master
      {
        Event fork = event.prev();
        while (fork->get_type() != OMP_FORK)
          --fork;
        forkTime = fork->get_time();
      }

      // Calculate thread fork time
      #pragma omp barrier
      data->m_idle = event->get_time() - forkTime;
      if (data->m_idle > 0) {
        m_severity[event.get_cnode()] += data->m_idle;

        cbmanager.notify(OMP_MGMT_FORK, event, data);
      }
      #pragma omp barrier
    }

    "EXIT" = {
      // Are we leaving an OpenMP parallel region?
      Event   enter  = data->m_callstack.top();
      Region* region = enter->get_region();
      if (!is_omp_parallel(region))
        return;

      // Determine timestamp of JOIN event
      static timestamp_t joinTime;
      #pragma omp master
      {
        Event join = event.next();
        while (join->get_type() != OMP_JOIN)
          ++join;

        joinTime = join->get_time();
      }

      // Calculate thread join time
      #pragma omp barrier
      data->m_idle = joinTime - event->get_time();
      if (data->m_idle > 0) {
        m_severity[enter.get_cnode()] += data->m_idle;

        cbmanager.notify(OMP_MGMT_JOIN, event, data);
      }
      #pragma omp barrier
    }
  ]
]


PATTERN "OMP_FORK" = [
  PARENT    = "OMP_MANAGEMENT"
  NAME      = "Fork"
  DOCNAME   = "OpenMP Management Fork Time"
  TYPE      = "OMP"
  CLASS     = "PatternOMP_Mgmt_Fork"
  INFO      = "Time spent in OpenMP thread forking"
  DESCR     = {
    Time spent creating and initializing teams of threads.
    </dd><p><dd>
    @img(OMPThreadFork.png)
  }
  UNIT      = "sec"
  CALLBACKS = [
    "OMP_MGMT_FORK" = {
      m_severity[event.get_cnode()] += data->m_idle;
    }
  ]
]


PATTERN "OMP_SYNCHRONIZATION" = [
  PARENT  = "OMP_TIME"
  NAME    = "Synchronization"
  DOCNAME = "OpenMP Synchronization Time"
  CLASS   = "PatternOMP_Sync"
  INFO    = "Time spent in OpenMP synchronization"
  DESCR   = {
    Time spent in OpenMP synchronization, whether barriers or mutual exclusion
    via ordered sequentialization, critical sections, atomics or lock API calls.
  }
  UNIT    = "sec"
  HIDDEN
]


PATTERN "OMP_BARRIER" = [
  PARENT  = "OMP_SYNCHRONIZATION"
  NAME    = "Synchronization"
  DOCNAME = "OpenMP Barrier Synchronization Time"
  TYPE    = "OMP"
  CLASS   = "PatternOMP_Sync_Barrier"
  INFO    = "Time spent in OpenMP barrier synchronization"
  DESCR   = {
    Time spent in implicit (compiler-generated)
    or explicit (user-specified) OpenMP barrier synchronization. Note that
    during measurement implicit barriers are treated similar to explicit
    ones. The instrumentation procedure replaces an implicit barrier with an
    explicit barrier enclosed by the parallel construct. This is done by
    adding a nowait clause and a barrier directive as the last statement of
    the parallel construct. In cases where the implicit barrier cannot be
    removed (i.e., parallel region), the explicit barrier is executed in
    front of the implicit barrier, which will then be negligible because the
    team will already be synchronized when reaching it. The synthetic
    explicit barrier appears as a special implicit barrier construct.
  }
  UNIT    = "sec"
  HIDDEN
  CALLBACKS = [
    "OMP_COLLEXIT" = {
      Event   enter  = data->m_callstack.top();
      Region* region = enter->get_region();

      // Are we leaving an implicit or explicit OpenMP barrier?
      if (is_omp_ebarrier(region))
        cbmanager.notify(OMP_EBARRIER, event, data);
      else if (is_omp_ibarrier(region))
        cbmanager.notify(OMP_IBARRIER, event, data);
    }
  ]
]


PATTERN "OMP_EBARRIER" = [
  PARENT    = "OMP_BARRIER"
  NAME      = "Explicit"
  DOCNAME   = "OpenMP Explicit Barrier Synchronization Time"
  CLASS     = "PatternOMP_Sync_Ebarrier"
  TYPE      = "OMP"
  INFO      = "Time spent in explicit OpenMP barrier synchronization"
  DESCR     = {
    Time spent in explicit (i.e., user-specified) OpenMP <tt>barrier</tt>
    synchronization, both waiting for other threads @ref(OMP_EBARRIER_WAIT)
    and inherent barrier processing overhead.
  }
  DIAGNOSIS = {
    Locate the most costly barrier synchronizations and determine whether
    they are necessary to ensure correctness or could be safely removed
    (based on algorithm analysis). Consider replacing an explicit barrier
    with a potentially more efficient construct, such as a critical section
    or atomic, or use explicit locks. Examine the time that each thread
    spends waiting at each explicit barrier, and try to re-distribute
    preceding work to improve load balance.
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_EBARRIER_WAIT" = [
  PARENT    = "OMP_EBARRIER"
  NAME      = "Wait at Barrier"
  DOCNAME   = "Wait at Explicit OpenMP Barrier Time"
  CLASS     = "PatternOMP_Sync_Ebarrier_Wait"
  TYPE      = "OMP"
  INFO      = "Waiting time in front of explicit OpenMP barriers"
  DESCR     = {
    Time spent in explicit (i.e., user-specified) OpenMP <tt>barrier</tt>
    synchronization waiting for the last thread.
  }
  DIAGNOSIS = {
    A large amount of waiting time at barriers can be an indication of load
    imbalance. Examine the waiting times for each thread and try to
    distribute the preceding computation from threads with the shortest
    waiting times to those with the longest waiting times.
  }
  UNIT      = "sec"
  CALLBACKS = [
    "OMP_EBARRIER" = {
      // Skip OpenMP barrier waiting time calculation when running serially
      // (the barriers called in the algorithm cause ill side effects...)
      if (0 == omp_nest_level)
        return;

      Event       enter     = data->m_callstack.top();
      timestamp_t enterTime = enter->get_time();

      // Determine latest enter time
      // Implicitly shared timestamp for reduction
      static timestamp_t latestEnter;
      #pragma omp barrier
      #pragma omp master
      latestEnter = 0;
      #pragma omp barrier
      #pragma omp critical
      {
        if(latestEnter < enterTime)
          latestEnter = enterTime;
      }
      #pragma omp barrier
      
      data->m_idle = latestEnter - enterTime;
      if(data->m_idle > 0)
        m_severity[enter.get_cnode()] += data->m_idle;

      // There will always be waiting time at barriers; all processes need
      // to take part in most-severe instance tracking
      cbmanager.notify(OMP_EBARRIER_WAIT, event, data);
    }
  ]
]


PATTERN "OMP_IBARRIER" = [
  PARENT    = "OMP_BARRIER"
  NAME      = "Implicit"
  DOCNAME   = "OpenMP Implicit Barrier Synchronization Time"
  CLASS     = "PatternOMP_Sync_Ibarrier"
  TYPE      = "OMP"
  INFO      = "Time spent in implicit OpenMP barrier synchronization"
  DESCR     = {
    Time spent in implicit (i.e., compiler-generated) OpenMP barrier
    synchronization, both waiting for other threads @ref(OMP_IBARRIER_WAIT)
    and inherent barrier processing overhead.
  }
  DIAGNOSIS = {
    Examine the time that each thread spends waiting at each implicit
    barrier, and if there is a significant imbalance then investigate
    whether a <tt>schedule</tt> clause is appropriate. Note that
    <tt>dynamic</tt> and <tt>guided</tt> schedules may require more
    @ref(OMP_MANAGEMENT) than <tt>static</tt> schedules. Consider whether
    it is possible to employ the <tt>nowait</tt> clause to reduce the
    number of implicit barrier synchronizations.
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_IBARRIER_WAIT" = [
  PARENT    = "OMP_IBARRIER"
  NAME      = "Wait at Barrier"
  DOCNAME   = "Wait at Implicit OpenMP Barrier Time"
  CLASS     = "PatternOMP_Sync_Ibarrier_Wait"
  TYPE      = "OMP"
  INFO      = "Waiting time in front of implicit OpenMP barriers"
  DESCR     = {
    Time spent in implicit (i.e., compiler-generated) OpenMP barrier
    synchronization.
  }
  DIAGNOSIS = {
    A large amount of waiting time at barriers can be an indication of load
    imbalance. Examine the waiting times for each thread and try to
    distribute the preceding computation from threads with the shortest
    waiting times to those with the longest waiting times.
  }
  UNIT      = "sec"
  CALLBACKS = [
    "OMP_IBARRIER" = {
      // Skip OpenMP barrier waiting time calculation when running serially
      // (the barriers called in the algorithm cause ill side effects...)
      if (0 == omp_nest_level)
        return;

      Event       enter     = data->m_callstack.top();
      timestamp_t enterTime = enter->get_time();

      // Determine latest enter time
      // Implicitly shared timestamp for reduction
      static timestamp_t latestEnter;
      #pragma omp barrier
      #pragma omp master
      latestEnter = 0;
      #pragma omp barrier
      #pragma omp critical
      {
        if(latestEnter < enterTime)
          latestEnter = enterTime;
      }
      #pragma omp barrier
      
      data->m_idle = latestEnter - enterTime;
      if(data->m_idle > 0)
        m_severity[enter.get_cnode()] += data->m_idle;

      // There will always be waiting time at barriers; all processes need
      // to take part in most-severe instance tracking
      cbmanager.notify(OMP_IBARRIER_WAIT, event, data);
    }
  ]
]


PATTERN "OMP_CRITICAL" = [
  PARENT    = "OMP_SYNCHRONIZATION"
  NAME      = "Critical"
  DOCNAME   = "OpenMP Critical Synchronization Time"
  CLASS     = "PatternOMP_Sync_Critical"
  INFO      = "Time spent waiting at OpenMP critical section"
  DESCR     = {
    Time spent waiting to enter OpenMP critical sections and in atomics,
    where mutual exclusion restricts access to a single thread at a time.
  }
  DIAGNOSIS = {
    Locate the most costly critical sections and atomics and determine
    whether they are necessary to ensure correctness or could be safely
    removed (based on algorithm analysis).
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_LOCK_API" = [
  PARENT    = "OMP_SYNCHRONIZATION"
  NAME      = "Lock API"
  DOCNAME   = "OpenMP Lock API Synchronization Time"
  CLASS     = "PatternOMP_Sync_LockAPI"
  INFO      = "Time spent in OpenMP lock calls"
  DESCR     = {
    Time spent in OpenMP API calls dealing with locks.
  }
  DIAGNOSIS = {
    Locate the most costly usage of locks and determine whether they are
    necessary to ensure correctness or could be safely removed (based on
    algorithm analysis). Consider re-writing the algorithm to use lock-free
    data structures.
  }
  UNIT      = "sec"
  HIDDEN
]


PATTERN "OMP_ORDERED" = [
  PARENT    = "OMP_SYNCHRONIZATION"
  NAME      = "Ordered"
  DOCNAME   = "OpenMP Ordered Synchronization Time"
  CLASS     = "PatternOMP_Sync_Ordered"
  INFO      = "Time spent waiting at OpenMP ordered region"
  DESCR     = {
    Time spent waiting to enter OpenMP <tt>ordered</tt> regions due to enforced
    sequentialization of loop iteration execution order in the region.
  }
  DIAGNOSIS = {
    Locate the most costly <tt>ordered</tt> regions and determine
    whether they are necessary to ensure correctness or could be safely
    removed (based on algorithm analysis).
  }
  UNIT      = "sec"
  HIDDEN
]
