main_spec.tex

\documentclass[10pt,oneside]{book}

\input{utils/packages}

\input{utils/defs}

\begin{document}

\input{content/frontmatter}



\section{The OpenSHMEM Effort}\label{subsec:openshmem_effort}
\input{content/the_openshmem_effort}

\section{Programming Model Overview}\label{subsec:programming_model}
\input{content/programming_model_overview}

\section{Memory Model}\label{subsec:memory_model}
\input{content/memory_model}

\section{Execution Model}\label{subsec:execution_model}
\input{content/execution_model}

\section{Language Bindings and Conformance}\label{subsec:bindings}
\input{content/language_bindings_and_conformance}

\section{Library Constants}\label{subsec:library_constants}
\input{content/library_constants}

\section{Library Handles}\label{subsec:library_handles}
\input{content/library_handles}

\section{Environment Variables }\label{subsec:environment_variables}
\input{content/environment_variables}

\clearpage



\section{OpenSHMEM Library \acs{API}}\label{sec:openshmem_library_api}

\subsection{Library Setup, Exit, and Query Routines}
The library setup and query interfaces that initialize and monitor the parallel
environment of the \acp{PE}.

\subsubsection{\textbf{SHMEM\_INIT}}\label{subsec:shmem_init}
\input{content/shmem_init}

\subsubsection{\textbf{SHMEM\_MY\_PE}}\label{subsec:shmem_my_pe}
\input{content/shmem_my_pe}

\subsubsection{\textbf{SHMEM\_N\_PES}}\label{subsec:shmem_n_pes}
\input{content/shmem_n_pes}

\subsubsection{\textbf{SHMEM\_FINALIZE}}\label{subsec:shmem_finalize}
\input{content/shmem_finalize}

\subsubsection{\textbf{SHMEM\_GLOBAL\_EXIT}}\label{subsec:shmem_global_exit}
\input{content/shmem_global_exit}

\subsubsection{\textbf{SHMEM\_PE\_ACCESSIBLE}}\label{subsec:shmem_pe_accessible}
\input{content/shmem_pe_accessible}

\subsubsection{\textbf{SHMEM\_ADDR\_ACCESSIBLE}}\label{subsec:shmem_addr_accessible}
\input{content/shmem_addr_accessible}

\subsubsection{\textbf{SHMEM\_PTR}}\label{subsec:shmem_ptr}
\input{content/shmem_ptr}

\subsubsection{\textbf{SHMEM\_TEAM\_PTR}}\label{subsec:shmem_team_ptr}
\input{content/shmem_team_ptr}

\subsubsection{\textbf{SHMEM\_INFO\_GET\_VERSION}}\label{subsec:shmem_info_get_version}
\input{content/shmem_info_get_version}

\subsubsection{\textbf{SHMEM\_INFO\_GET\_NAME}}\label{subsec:shmem_info_get_name}
\input{content/shmem_info_get_name}

\subsubsection{\textbf{START\_PES}}\label{subsec:start_pes}
\input{content/start_pes}

\subsection{Thread Support}
\label{subsec:thread_support}
\input{content/threads_intro.tex}

\subsubsection{\textbf{SHMEM\_INIT\_THREAD}}
\label{subsec:shmem_init_thread}
\input{content/shmem_init_thread}

\subsubsection{\textbf{SHMEM\_QUERY\_THREAD}}
\label{subsec:shmem_query_thread}
\input{content/shmem_query_thread}


\subsection{Memory Management Routines}
\label{sec:memory_management}

\openshmem provides a set of \acp{API} for managing the symmetric heap. The
\acp{API} allow one to dynamically allocate, deallocate, reallocate and align
symmetric data objects in the symmetric heap.

\subsubsection{\textbf{SHMEM\_MALLOC, SHMEM\_FREE, SHMEM\_REALLOC, SHMEM\_ALIGN}}\label{subsec:shfree}
\input{content/shmem_malloc.tex}

\subsubsection{\textbf{SHMEM\_MALLOC\_WITH\_HINTS}}\label{subsec:shmmallochint}
\input{content/shmem_malloc_hints.tex}

\subsubsection{\textbf{SHMEM\_CALLOC}}\label{subsec:shmem_calloc}
\input{content/shmem_calloc.tex}



\subsection{Team Management Routines}\label{subsec:team}
\input{content/teams_intro.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_MY\_PE}}\label{subsec:shmem_team_my_pe}
\input{content/shmem_team_my_pe.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_N\_PES}}\label{subsec:shmem_team_n_pes}
\input{content/shmem_team_n_pes.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_CONFIG\_T}}
\label{subsec:shmem_team_config_t}
\input{content/shmem_team_config_t.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_GET\_CONFIG}}\label{subsec:shmem_team_get_config}
\input{content/shmem_team_get_config.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_TRANSLATE\_PE}}\label{subsec:shmem_team_translate_pe}
\input{content/shmem_team_translate_pe.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_SPLIT\_STRIDED}}\label{subsec:shmem_team_split_strided}
\input{content/shmem_team_split_strided.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_SPLIT\_2D}}\label{subsec:shmem_team_split_2d}
\input{content/shmem_team_split_2d.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_DESTROY}}\label{subsec:shmem_team_destroy}
\input{content/shmem_team_destroy.tex}



\subsection{Communication Management Routines}\label{sec:ctx}
\input{content/context_intro.tex}

\subsubsection{\textbf{SHMEM\_CTX\_CREATE}}
\label{subsec:shmem_ctx_create}
\input{content/shmem_ctx_create.tex}

\subsubsection{\textbf{SHMEM\_TEAM\_CREATE\_CTX}}
\label{subsec:shmem_team_create_ctx}
\input{content/shmem_team_create_ctx.tex}

\subsubsection{\textbf{SHMEM\_CTX\_DESTROY}}
\label{subsec:shmem_ctx_destroy}
\input{content/shmem_ctx_destroy.tex}

\subsubsection{\textbf{SHMEM\_CTX\_GET\_TEAM}}
\label{subsec:shmem_ctx_get_team}
\input{content/shmem_ctx_get_team.tex}


\subsection{Remote Memory Access Routines}\label{sec:rma}
\input{content/rma_intro.tex}

\subsubsection{Blocking Remote Memory Access Routines}\label{subsec:rma}
\subsubsubsection{\textbf{SHMEM\_PUT}}\label{subsec:shmem_put}
\input{content/shmem_put.tex}

\subsubsubsection{\textbf{SHMEM\_P}}\label{subsec:shmem_p}
\input{content/shmem_p.tex}

\subsubsubsection{\textbf{SHMEM\_IPUT}}\label{subsec:shmem_iput}
\input{content/shmem_iput.tex}

\subsubsubsection{\textbf{SHMEM\_GET}}\label{subsec:shmem_get}
\input{content/shmem_get.tex}

\subsubsubsection{\textbf{SHMEM\_G}}\label{subsec:shmem_g}
\input{content/shmem_g.tex}

\subsubsubsection{\textbf{SHMEM\_IGET}}\label{subsec:shmem_iget}
\input{content/shmem_iget.tex}

\subsubsection{Nonblocking Remote Memory Access Routines}\label{subsec:rma_nbi}

\subsubsubsection{\textbf{SHMEM\_PUT\_NBI}}\label{subsec:shmem_put_nbi}
\input{content/shmem_put_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_GET\_NBI}}\label{subsec:shmem_get_nbi}
\input{content/shmem_get_nbi.tex}



\subsection{Atomic Memory Operations}\label{sec:amo}
\input{content/atomics_intro}

\subsubsection{Blocking Atomic Memory Operations}\label{subsec:amo}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH}}
\label{subsec:shmem_atomic_fetch}
\input{content/shmem_atomic_fetch.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_SET}}
\label{subsec:shmem_atomic_set}
\input{content/shmem_atomic_set.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_COMPARE\_SWAP}}
\label{subsec:shmem_atomic_compare_swap}
\input{content/shmem_atomic_compare_swap.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_SWAP}}
\label{subsec:shmem_atomic_swap}
\input{content/shmem_atomic_swap.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_INC}}
\label{subsec:shmem_atomic_fetch_inc}
\input{content/shmem_atomic_fetch_inc.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_INC}}
\label{subsec:shmem_atomic_inc}
\input{content/shmem_atomic_inc.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_ADD}}
\label{subsec:shmem_atomic_fetch_add}
\input{content/shmem_atomic_fetch_add.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_ADD}}
\label{subsec:shmem_atomic_add}
\input{content/shmem_atomic_add.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_AND}}
\label{subsec:shmem_atomic_fetch_and}
\input{content/shmem_atomic_fetch_and.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_AND}}
\label{subsec:shmem_atomic_and}
\input{content/shmem_atomic_and.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_OR}}
\label{subsec:shmem_atomic_fetch_or}
\input{content/shmem_atomic_fetch_or.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_OR}}
\label{subsec:shmem_atomic_or}
\input{content/shmem_atomic_or.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_XOR}}
\label{subsec:shmem_atomic_fetch_xor}
\input{content/shmem_atomic_fetch_xor.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_XOR}}
\label{subsec:shmem_atomic_xor}
\input{content/shmem_atomic_xor.tex}

\subsubsection{Nonblocking Atomic Memory Operations}\label{subsec:amo_nbi}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_NBI}}
\label{subsec:shmem_atomic_fetch_nbi}
\input{content/shmem_atomic_fetch_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_COMPARE\_SWAP\_NBI}}
\label{subsec:shmem_atomic_compare_swap_nbi}
\input{content/shmem_atomic_compare_swap_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_SWAP\_NBI}}
\label{subsec:shmem_atomic_swap_nbi}
\input{content/shmem_atomic_swap_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_INC\_NBI}}
\label{subsec:shmem_atomic_fetch_inc_nbi}
\input{content/shmem_atomic_fetch_inc_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_ADD\_NBI}}
\label{subsec:shmem_atomic_fetch_add_nbi}
\input{content/shmem_atomic_fetch_add_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_AND\_NBI}}
\label{subsec:shmem_atomic_fetch_and_nbi}
\input{content/shmem_atomic_fetch_and_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_OR\_NBI}}
\label{subsec:shmem_atomic_fetch_or_nbi}
\input{content/shmem_atomic_fetch_or_nbi.tex}

\subsubsubsection{\textbf{SHMEM\_ATOMIC\_FETCH\_XOR\_NBI}}
\label{subsec:shmem_atomic_fetch_xor_nbi}
\input{content/shmem_atomic_fetch_xor_nbi.tex}



\subsection{Signaling Operations}\label{sec:shmem_signal}
This section specifies the OpenSHMEM support for \OPR{put-with-signal},
nonblocking \OPR{put-with-signal}, and \OPR{signal-fetch} routines. The
put-with-signal routines provide a method for copying data from a contiguous
local data object to a data object on a specified \ac{PE} and subsequently
updating a remote flag to signal completion. The signal-fetch routine provides
support for fetching a signal update operation.

\openshmem \OPR{put-with-signal} routines specified in this section have two
variants. In one of the variants, the context handle, \VAR{ctx}, is explicitly
passed as an argument. In this variant, the operation is performed on the
specified context. If the context handle \VAR{ctx} does not correspond to a
valid context, the behavior is undefined. In the other variant, the context
handle is not explicitly passed and thus, the operations are performed on the
default context.

\subsubsection{Atomicity Guarantees for Signaling Operations}
\label{subsec:signal_atomicity}
All signaling operations put-with-signal, nonblocking put-with-signal, and
signal-fetch are performed on a signal data object, a remotely accessible
symmetric object of type \VAR{uint64\_t}. A signal operator in the
put-with-signal routine is a \openshmem library constant that determines the
type of update to be performed as a signal on the signal data object.

All signaling operations on the signal data object completes as if performed
atomically with respect to the following:
\begin{itemize}
    \item other blocking or nonblocking variant of the put-with-signal routine
    that updates the signal data object using the same signal update operator;
    \item signal-fetch routine that fetches the signal data object; and
    \item any point-to-point synchronization routine that accesses the signal
    data object.
\end{itemize}

\subsubsection{Available Signal Operators}
\label{subsec:signal_operator}

With the atomicity guarantees as described in
Section~\ref{subsec:signal_atomicity}, the following options can be used as a
signal operator.

    \apitablerow{\LibConstRef{SHMEM\_SIGNAL\_SET}}{An update to signal data
    object is an atomic set operation. It writes an unsigned 64-bit value as a
    signal into the signal data object on a remote \VAR{PE} as an atomic
    operation.}

    \apitablerow{\LibConstRef{SHMEM\_SIGNAL\_ADD}}{An update to signal data
    object is an atomic add operation. It adds an unsigned 64-bit value as a
    signal into the signal data object on a remote \VAR{PE} as an atomic
    operation.}


\subsubsection{\textbf{SHMEM\_PUT\_SIGNAL}}\label{subsec:shmem_put_signal}
\input{content/shmem_put_signal.tex}

\subsubsection{\textbf{SHMEM\_PUT\_SIGNAL\_NBI}}\label{subsec:shmem_put_signal_nbi}
\input{content/shmem_put_signal_nbi.tex}

\subsubsection{\textbf{SHMEM\_SIGNAL\_FETCH}}\label{subsec:shmem_signal_fetch}
\input{content/shmem_signal_fetch.tex}



\subsection{Collective Routines}\label{subsec:coll}
\input{content/collective_intro.tex}

\subsubsection{\textbf{SHMEM\_BARRIER\_ALL}}\label{subsec:shmem_barrier_all}
\input{content/shmem_barrier_all.tex}

\subsubsection{\textbf{SHMEM\_BARRIER}}\label{subsec:shmem_barrier}
\input{content/shmem_barrier.tex}

\subsubsection{\textbf{SHMEM\_SYNC}}\label{subsec:shmem_sync}
\input{content/shmem_sync.tex}

\subsubsection{\textbf{SHMEM\_SYNC\_ALL}}\label{subsec:shmem_sync_all}
\input{content/shmem_sync_all.tex}

\subsubsection{\textbf{SHMEM\_ALLTOALL}}\label{subsec:shmem_alltoall}
\input{content/shmem_alltoall.tex}

\subsubsection{\textbf{SHMEM\_ALLTOALLS}}\label{subsec:shmem_alltoalls}
\input{content/shmem_alltoalls.tex}

\subsubsection{\textbf{SHMEM\_BROADCAST}}\label{subsec:shmem_broadcast}
\input{content/shmem_broadcast.tex}

\subsubsection{\textbf{SHMEM\_COLLECT, SHMEM\_FCOLLECT}}\label{subsec:shmem_collect}
\input{content/shmem_collect.tex}

\subsubsection{\textbf{SHMEM\_REDUCTIONS}}\label{subsec:shmem_reductions}
\input{content/shmem_reductions.tex}




\subsection{Point-To-Point Synchronization Routines}\label{subsec:p2p_intro}
\input{content/p2p_sync_intro.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL}}\label{subsec:shmem_wait_until}
\input{content/shmem_wait_until.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ALL}}\label{subsec:shmem_wait_until_all}
\input{content/shmem_wait_until_all.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ANY}}\label{subsec:shmem_wait_until_any}
\input{content/shmem_wait_until_any.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_SOME}}\label{subsec:shmem_wait_until_some}
\input{content/shmem_wait_until_some.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ALL\_VECTOR}}\label{subsec:shmem_wait_until_all_vector}
\input{content/shmem_wait_until_all_vector.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_ANY\_VECTOR}}\label{subsec:shmem_wait_until_any_vector}
\input{content/shmem_wait_until_any_vector.tex}

\subsubsection{\textbf{SHMEM\_WAIT\_UNTIL\_SOME\_VECTOR}}\label{subsec:shmem_wait_until_some_vector}
\input{content/shmem_wait_until_some_vector.tex}

\subsubsection{\textbf{SHMEM\_TEST}}\label{subsec:shmem_test}
\input{content/shmem_test.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ALL}}\label{subsec:shmem_test_all}
\input{content/shmem_test_all.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ANY}}\label{subsec:shmem_test_any}
\input{content/shmem_test_any.tex}

\subsubsection{\textbf{SHMEM\_TEST\_SOME}}\label{subsec:shmem_test_some}
\input{content/shmem_test_some.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ALL\_VECTOR}}\label{subsec:shmem_test_all_vector}
\input{content/shmem_test_all_vector.tex}

\subsubsection{\textbf{SHMEM\_TEST\_ANY\_VECTOR}}\label{subsec:shmem_test_any_vector}
\input{content/shmem_test_any_vector.tex}

\subsubsection{\textbf{SHMEM\_TEST\_SOME\_VECTOR}}\label{subsec:shmem_test_some_vector}
\input{content/shmem_test_some_vector.tex}

\subsubsection{\textbf{SHMEM\_SIGNAL\_WAIT\_UNTIL}}\label{subsec:shmem_signal_wait_until}
\input{content/shmem_signal_wait_until.tex}



\subsection{Memory Ordering Routines}\label{subsec:memory_order}
The following section discusses \openshmem \acp{API} that provide mechanisms to
ensure ordering and/or delivery of completion on memory store, blocking, and
nonblocking \openshmem routines. Table~\ref{mem-order} lists the operations
affected by \openshmem memory ordering routines.

\begin{longtable}{|l|c|c|}
 \hline
  Operations & Fence & Quiet \\ \hline
  Memory Store \footnote{Ordering and/or delivery of memory store operations are
  ensured only on contexts created with certain options. For details, refer the
  description of context options in Section~\ref{subsec:shmem_ctx_create}.}
                                     & X & X \\
  Blocking \OPR{Put}                 & X & X \\
  Blocking \OPR{Get}                 &   &   \\
  Blocking \OPR{\acs{AMO}}           & X & X \\
  Blocking \OPR{put-with-signal}     & X & X \\
  Nonblocking \OPR{Put}              & X & X \\
  Nonblocking \OPR{Get}              &   & X \\
  Nonblocking \OPR{\acs{AMO}}        & X\footnote{\openshmem fence routines does
  not guarantee order of delivery of values fetched by nonblocking \ac{AMO}
  routines.}
                                         & X \\
  Nonblocking \OPR{put-with-signal}  & X & X \\ \hline
\TableCaptionRef{List of operations affected by \openshmem Memory Ordering
routines}
\label{mem-order}
\end{longtable}



\subsubsection{\textbf{SHMEM\_FENCE}}\label{subsec:shmem_fence}
\input{content/shmem_fence.tex}

\subsubsection{\textbf{SHMEM\_QUIET}}\label{subsec:shmem_quiet}
\input{content/shmem_quiet.tex}

\subsubsection{Synchronization and Communication Ordering in OpenSHMEM}
\input{content/synchronization_model.tex}






\subsection{Distributed Locking Routines}
The following section discusses \openshmem locks as a mechanism to provide
mutual exclusion. Three routines are available for distributed locking,
\textit{set, test} and \textit{clear}.

\subsubsection{\textbf{SHMEM\_LOCK}}\label{subsec:shmem_lock}
\input{content/shmem_lock.tex}





\section{OpenSHMEM Profiling Interface}\label{sec:openshmem_profiling_interface}
\input{content/profiling_interface.tex}

\clearpage
\input{content/backmatter}

\chapter*{Glossary}
\addcontentsline{toc}{chapter}{Glossary}
\input{content/glossary.tex}

\clearpage
\phantomsection
\addcontentsline{toc}{chapter}{Index}
\printindex

\end{document}