From ccb9f0279e2113cf5860d61dfed32587677d3206 Mon Sep 17 00:00:00 2001 From: naveen-rn Date: Wed, 25 Jan 2017 17:13:19 -0600 Subject: [PATCH 1/2] Draft memory partition proposal --- content/backmatter.tex | 8 ++-- content/environment_variables.tex | 60 +++++++++++++----------- content/library_constants.tex | 18 +++++++- content/mem_partition_env.tex | 77 +++++++++++++++++++++++++++++++ content/memory_model.tex | 13 +++--- content/memory_partitions.tex | 14 ++++++ content/shmem_addr_accessible.tex | 4 +- content/shmem_kind.tex | 53 +++++++++++++++++++++ content/shmem_malloc.tex | 8 +++- main_spec.tex | 23 ++++++--- utils/defs.tex | 36 +++++++++++++++ 11 files changed, 264 insertions(+), 50 deletions(-) create mode 100644 content/mem_partition_env.tex create mode 100644 content/memory_partitions.tex create mode 100644 content/shmem_kind.tex diff --git a/content/backmatter.tex b/content/backmatter.tex index 62104bc54..8cded690b 100644 --- a/content/backmatter.tex +++ b/content/backmatter.tex @@ -237,10 +237,10 @@ \section{\ac{MPI} Interoperability} \Fortran{} common block or \Clang{} global variable, is symmetric between processes running from the same executable file, but is not symmetric between processes running from different executable files. Data allocated from the -symmetric heap (\FUNC{shmem\_malloc} or \FUNC{shpalloc}) is symmetric across the -same or different executable files. The routine \FUNC{shmem\_addr\_accessible} -can be used to determine if a local address is accessible via \openshmem -communication from a remote \ac{PE}. +symmetric heap (\FUNC{shmem\_malloc}, \newtext{\FUNC{shmem\_kind\_malloc}}, or +\FUNC{shpalloc}) is symmetric across the same or different executable files. The +routine \FUNC{shmem\_addr\_accessible} can be used to determine if a local +address is accessible via \openshmem communication from a remote \ac{PE}. Another important feature of these systems is that the \FUNC{shmem\_pe\_accessible} routine returns \CONST{TRUE} only if the remote diff --git a/content/environment_variables.tex b/content/environment_variables.tex index 689c6ca3f..193260911 100644 --- a/content/environment_variables.tex +++ b/content/environment_variables.tex @@ -1,28 +1,32 @@ -The \openshmem specification provides a set of environment variables that allows -users to configure the \openshmem implementation, and receive information about -the implementation. The implementations of the specification are free to define -additional variables. Currently, the specification defines four environment -variables. All environment variables that start with \VAR{SMA\_*} are -deprecated, but currently supported for backwards compatibility. - -\medskip{} - -\begin{tabular}{|l|l|l|} -\hline -Variable & Value & Purpose\tabularnewline -\hline -\hline -\texttt{SHMEM\_VERSION} & any & print the library version at -start-up\tabularnewline -\hline -\texttt{SHMEM\_INFO} & any & print helpful text about all these environment -variables\tabularnewline -\hline -\texttt{SHMEM\_SYMMETRIC\_SIZE} & non-negative integer & number of bytes to -allocate for symmetric heap\tabularnewline -\hline -\texttt{SHMEM\_DEBUG} & any & enable debugging messages\tabularnewline -\hline -\end{tabular} - -\medskip{} +The \openshmem specification provides a set of environment variables that allows +users to configure the \openshmem implementation, and receive information about +the implementation. The implementations of the specification are free to define +additional variables. Currently, the specification defines five environment +variables. All environment variables that start with \VAR{SMA\_*} are +deprecated, but currently supported for backwards compatibility. + +\medskip{} + +\begin{tabular}{|l|l|l|} +\hline +Variable & Value & Purpose\tabularnewline +\hline +\hline +\texttt{SHMEM\_VERSION} & any & print the library version at +start-up\tabularnewline +\hline +\texttt{SHMEM\_INFO} & any & print helpful text about all these environment +variables\tabularnewline +\hline +\texttt{SHMEM\_SYMMETRIC\_SIZE} & non-negative integer & number of bytes to +allocate for symmetric heap\tabularnewline +\hline +\texttt{\newtext{SHMEM\_SYMMETRIC\_PARTITION}} +& \newtext{Refer Section \ref{subsubsec:usr_defn_env}} +& \newtext{Refer Section \ref{subsubsec:usr_defn_env}}\tabularnewline +\hline +\texttt{SHMEM\_DEBUG} & any & enable debugging messages\tabularnewline +\hline +\end{tabular} + +\medskip{} \ No newline at end of file diff --git a/content/library_constants.tex b/content/library_constants.tex index 4e2346f0d..4626377e8 100644 --- a/content/library_constants.tex +++ b/content/library_constants.tex @@ -131,6 +131,22 @@ and whitespace padded. It can also be equal in length to \const{SHMEM\_MAX\_NAME\_LEN} since Fortran does not NULL terminate strings. \tabularnewline \hline - +\vspace{3mm} +\vtop{\hbox{\newtext{\CorCpp:}} +\hbox{\hspace*{12mm} \newtext{\const{SHMEM\_MAX\_PARTITIONS}}}} +&\newtext{ +Integer representing the implementations maximum for the number of partitions +that can be defined by the users.} %Implementations value must be at least 7 so +%that a portable program can assume at least that possible partitions} +\tabularnewline +\hline +\vspace{3mm} +\vtop{\hbox{\newtext{\CorCpp:}} +\hbox{\hspace*{12mm} \newtext{\const{SHMEM\_MAX\_PARTITION\_ID}}}} +&\newtext{ +Integer representing the implementations maximum number that can be used to +represent the partition ID.} %The value should be at least 127} +\tabularnewline +\hline \end{tabular} \color{black} diff --git a/content/mem_partition_env.tex b/content/mem_partition_env.tex new file mode 100644 index 000000000..0da453d2e --- /dev/null +++ b/content/mem_partition_env.tex @@ -0,0 +1,77 @@ +All OpenSHMEM implementations supporting the use of multiple Symmetric Memory +Partitions provide support for a special environment variable called +SHMEM\_SYMMETRIC\_PARTITION to define the properties of the user-defined memory +locations which can be used to create symmetric heaps. The properties of the +memory partitions including the size of the symmetric heap to be created are +passed as specifiers to the environment variable. + +\begin{envvardefinition} +\begin{envvarname} +SHMEM_SYMMETRIC_PARTITION=SIZE=[:PGSIZE=][:KIND=:POLICY=] +\end{envvarname} + +\begin{envvararguments} + \envvarargument{REQ}{ID}{Integer to represent the partition number} + \envvarargument{REQ}{SIZE}{Number of bytes to allocate for symmetric heap} + \envvarargument{OPT}{PGSIZE}{Number of bytes to specify the memory page size} + \envvarargument{OPT}{KIND}{String constants to specify the memory kind if + multiple memory kinds are supported by the system and identified by the + implementation} + \envvarargument{OPT}{POLICY}{String constants to specify the memory + allocation policy} +\end{envvararguments} + +One, two, or more partitions can be specified using this environment variable. +Each partition is represented using a number called \emph{Partition ID}. +\textless ID\textgreater is the user-specified part of the environment variable +to represent the Partition ID. A maximum of SHMEM\_MAX\_PARTITIONS can be +defined and the Partition ID may be any non-zero positive integer between 1 and +SHMEM\_MAX\_PARTITION\_ID. + +Each partition can take a maximum of four property specifiers as input. The +\emph{SIZE} specifier is the only required value. It is used to specify the +required symmetric heap size on the partition. The \emph{PGSIZE} specifier is +used to specify the page size of the memory kind. + +On systems supporting multiple memory kinds, each memory kinds which are +identified by the implementation should be documented and made available as +string constants to the users. The \emph{KIND} specifier is used to specify the +memory kind to determine location of the memory partition. + +The \emph{POLICY} is used to specify on how strictly the request optional +specifiers \emph{PGSIZE} and \emph{KIND} are to be honored. The following are +the accepted values for \emph{POLICY} specifier. +\begin{itemize} +\item \emph{MANDATORY} Any unavailability of the user requested optional +specifier would result in program termination with an informative error during +symmetric heap creation on the defined partition. +\item \emph{PREFERRED} Any unavailability of the user requested optional +specifier would allow the library to try to allocate the requested \emph{SIZE} +on the implementation defined location. +\item \emph{INTERLEAVED} Page allocations are interleaved across numa domains +associated with the memory kind. +\item \emph{SYSDEFAULT} System defined memory policy is used. +\end{itemize} + +\envvarnotes{ + SHMEM\_SYMMETRIC\_SIZE is supported with its current syntax and meaning. + But, it is an error to use both SHMEM\_SYMMETRIC\_SIZE and any form of + SHMEM\_SYMMETRIC\_PARTITION environment variables together. + + The total size of the symmetric heap is determined as the sum of sizes + of all \emph{SIZE} specifier in the defined memory partitions. + + All available page sizes and supported memory kinds be documented by the + implementation along with the \emph{SYSDEFAULT} values for the optional + properties. + + Partition ID 1 has a special usage. Symmetric data object allocations + identified using the partition ID (with \textit{shmem\_kind\_malloc} and + \textit{shmem\_kind\_align} routines) are allocated on the symmetric heap + created over the partition with requested ID. Any generic allocations + without the partition ID (using \textit{shmem\_malloc} and + \textit{shmem\_align}) would result in using the symmetric heap created over + partition with ID 1.} + +\end{envvardefinition} + diff --git a/content/memory_model.tex b/content/memory_model.tex index 4ed391c0c..330221040 100644 --- a/content/memory_model.tex +++ b/content/memory_model.tex @@ -38,9 +38,10 @@ \textit{shmem\_malloc}) allow collective allocation of \emph{Symmetric Data Objects} on a special memory region called the \emph{Symmetric Heap}. The Symmetric Heap is created during the execution of a program at a memory location -determined by the implementation. The Symmetric Heap may reside in different -memory regions on different \acp{PE}. Figure~\ref{fig:mem_model} shows how -\openshmem implements a \ac{PGAS} model using remotely accessible symmetric -objects and private data objects when executing an \openshmem program. -Symmetric data objects are stored on the symmetric heap or in the global/static -memory section of each \ac{PE}. +determined by the implementation \newtext{or on multiple user determined memory +locations called as the \emph{Symmetric Memory Partitions}}. The Symmetric +Heap may reside in different memory regions on different \acp{PE}. +Figure~\ref{fig:mem_model} shows how \openshmem implements a \ac{PGAS} model +using remotely accessible symmetric objects and private data objects when executing +an \openshmem program. Symmetric data objects are stored on the symmetric heap or +in the global/static memory section of each \ac{PE}. diff --git a/content/memory_partitions.tex b/content/memory_partitions.tex new file mode 100644 index 000000000..47d446ccf --- /dev/null +++ b/content/memory_partitions.tex @@ -0,0 +1,14 @@ +\emph{Symmetric Heaps} can be created on a single memory location determined by +the implementation or on multiple memory locations determined by the users. The +user-determined memory locations are called as the \emph{Symmetric Memory +Partitions}. Only a single symmetric heap can be created on each symmetric +memory partition. Each symmetric memory partitions is identified using its +\emph{Partition ID}, which is used on dynamic OpenSHMEM memory allocation +(\textit{shmem\_kind\_malloc} and \textit{shmem\_kind\_align}) routines. Apart +from the Partition ID, each symmetric memory partition have their own memory +properties. Refer to Section \ref{subsec:environment_variables} for the +complete list of properties which are available to characterize a memory +partition. Multiple symmetric heaps can be created by defining multiple +separate memory partitions. Apart from name, type and size attributes, +symmetric data objects stored on symmetric heaps created over a memory +partition have Partition ID as an extra attribute. \ No newline at end of file diff --git a/content/shmem_addr_accessible.tex b/content/shmem_addr_accessible.tex index ad84a36f8..8383a4cb6 100644 --- a/content/shmem_addr_accessible.tex +++ b/content/shmem_addr_accessible.tex @@ -35,8 +35,8 @@ \Fortran{} common block or \Clang{} global variable, is symmetric between processes running from the same executable file, but is not symmetric between processes running from different executable files. Data allocated from the - symmetric heap (\FUNC{shmem\_malloc} or \FUNC{shpalloc}) is symmetric across the - same or different executable files. + symmetric heap (\FUNC{shmem\_malloc}, \newtext{\FUNC{shmem\_kind\_malloc}} or + \FUNC{shpalloc}) is symmetric across the same or different executable files. } \apireturnvalues{ diff --git a/content/shmem_kind.tex b/content/shmem_kind.tex new file mode 100644 index 000000000..92284ec4d --- /dev/null +++ b/content/shmem_kind.tex @@ -0,0 +1,53 @@ +\apisummary{ Symmetric heap memory management routines identified using + symmetric memory partition ID. } + +\begin{apidefinition} + +\begin{Csynopsis} +void *shmem_kind_malloc(size_t size, int partition_id); +void *shmem_kind_align(size_t alignment, size_t size, int partition_id); +\end{Csynopsis} + +\begin{apiarguments} + \apiargument{IN}{size}{The size, in bytes, of a block to be + allocated from the symmetric heap. This argument is of type + \VAR{size\_t}} + \apiargument{IN}{alignment}{Byte alignment of the block allocated from the + symmetric heap.} + \apiargument{IN}{partition\_id}{Integer to identify the user-defined memory + partition on which the symemtric heap is created} +\end{apiarguments} + +\apidescription{ + The \FUNC{shmem\_kind\_malloc} routine returns a pointer to a block of at + least \VAR{size} bytes suitably aligned for any use. This space is + allocated from the symmetric heap created on the user-defined memory + partition with characteristics associated with the partition\_id(in + contrast to \FUNC{malloc}, which allocates from the private heap). + + The \FUNC{shmem\_kind\_align} routine allocates a block in the symmetric + heap heap created on the user-defined memory partition with characteristics + associated with the partition\_id that has a byte alignment specified by the + alignment argument. } + +\apireturnvalues{ + The \FUNC{shmem\_kind\_malloc} routine returns a pointer to the allocated + space; otherwise, it returns a \CONST{NULL} pointer. + + The \FUNC{shmem\_kind\_align} routine returns an aligned pointer to the + allocated space; otherwise, it returns a \CONST{NULL} pointer. } + +\apinotes{ Requirements for all PEs to use \FUNC{shmem\_malloc} and + \FUNC{shmem\_align} memory management routines are the same for + \FUNC{shmem\_kind\_malloc} and \FUNC{shmem\_kind\_align} routines. +} + +\apiimpnotes{ + In a quality implementation \FUNC{shmem\_realloc} and \FUNC{shmem\_free} + should also handle the symmetric memory partitions correctly. It is + assumed that the new memory address returned on \FUNC{shmem\_realloc} will + be in the same symmetric memory partition associated with the input + address. } + + +\end{apidefinition} diff --git a/content/shmem_malloc.tex b/content/shmem_malloc.tex index 74df3fb1b..dbfdc68e5 100644 --- a/content/shmem_malloc.tex +++ b/content/shmem_malloc.tex @@ -76,9 +76,13 @@ \FUNC{shmem\_realloc} instead. The behavior and signature of the routines remains unchanged from the deprecated versions. + \newtext{Usage of \FUNC{shmem\_malloc} and \FUNC{shmem\_align} along with + Symmetric Memory Partitions would result in the memory space to be + allocated on the symmetric heap created on partition with ID as 1.} + The total size of the symmetric heap is determined at job startup. One can - adjust the size of the heap using the \CONST{SHMEM\_SYMMETRIC\_SIZE} environment - variable (where available). + adjust the size of the heap using the \CONST{SHMEM\_SYMMETRIC\_SIZE} or + \CONST{SHMEM\_SYMMETRIC\_PARTITION} environment variable (where available). The \FUNC{shmem\_malloc}, \FUNC{shmem\_free}, and \FUNC{shmem\_realloc} routines differ from the private heap allocation routines in that all \ac{PE}s in a diff --git a/main_spec.tex b/main_spec.tex index 996317c1c..d44319fe5 100644 --- a/main_spec.tex +++ b/main_spec.tex @@ -20,7 +20,10 @@ \section{Programming Model Overview}\label{subsec:programming_model} \section{Memory Model}\label{subsec:memory_model} \input{content/memory_model} - +\newtext{ +\subsection{Symmetric Memory Partitions}\label{subsubsec:memory_partitions} +\input{content/memory_partitions} +} \section{Execution Model}\label{subsec:execution_model} \input{content/execution_model} @@ -32,9 +35,10 @@ \section{Library Constants}\label{subsec:library_constants} \section{Environment Variables }\label{subsec:environment_variables} \input{content/environment_variables} - - - +\color{Green} +\subsection{User-defined Environment Variables}\label{subsubsec:usr_defn_env} +\input{content/mem_partition_env} +\color{Black} \clearpage @@ -91,6 +95,11 @@ \subsection{Memory Management Routines} \subsubsection{\textbf{SHMEM\_MALLOC, SHMEM\_FREE, SHMEM\_REALLOC, SHMEM\_ALIGN}}\label{subsec:shfree} \input{content/shmem_malloc.tex} +\color{Green} +\subsubsection{\textbf{SHMEM\_KIND\_MALLOC, SHMEM\_KIND\_ALIGN}}\label{subsec:shmem_kind} +\input{content/shmem_kind.tex} +\color{Black} + \subsubsection{\textbf{SHPALLOC}}\label{subsec:shpalloc} \input{content/shpalloc.tex} @@ -143,7 +152,7 @@ \subsubsection{\textbf{SHMEM\_ADD}}\label{subsec:shmem_add} \input{content/shmem_add.tex} \subsubsection{\textbf{SHMEM\_CSWAP}}\label{subsec:shmem_cswap} -\input{content/shmem_cswap.tex} +\input{content/shmem_cswap.tex} \subsubsection{\textbf{SHMEM\_SWAP}}\label{subsec:shmem_swap} \input{content/shmem_swap.tex} @@ -198,7 +207,7 @@ \subsubsection{\textbf{SHMEM\_ALLTOALLS}}\label{subsec:shmem_alltoalls} \subsection{Point-To-Point Synchronization Routines} The following section discusses \openshmem \ac{API}s that provides a mechanism for synchronization between two \ac{PE}s based on the value of a symmetric data -object. +object. Where appropriate compiler support is available, \openshmem provides type-generic point-to-point synchronization interfaces via \Celev{} generic @@ -233,7 +242,7 @@ \subsubsection{\textbf{SHMEM\_WAIT}}\label{subsec:shmem_wait} \subsection{Memory Ordering Routines}\label{subsec:memory_order} The following section discusses \openshmem \ac{API}s that provide mechanisms to ensure ordering and/or delivery of \OPR{Put}, \ac{AMO}, and memory store -routines to symmetric data objects. +routines to symmetric data objects. \subsubsection{\textbf{SHMEM\_FENCE}}\label{subsec:shmem_fence} \input{content/shmem_fence.tex} diff --git a/utils/defs.tex b/utils/defs.tex index b8bee6ff9..722b2630e 100644 --- a/utils/defs.tex +++ b/utils/defs.tex @@ -363,6 +363,42 @@ deletekeywords=[3]{LOG}, aboveskip=0pt, belowskip=0pt}}{} + +\newenvironment{envvardefinition}{ +\begin{description} +\item[SYNOPSIS] \hfill \\ \\ +} +{ +\end{description} +} + +\lstnewenvironment{envvarname} +{ \textbf{Variable Name:} + \lstset{backgroundcolor=\color{gray}, lineskip=2pt, + morekeywords={ID, SIZE, KIND, POLICY, PGSIZE}, aboveskip=0pt, belowskip=0pt}}{} + +\newenvironment{envvararguments}{ +\newcommand{\envvarargument}[3]{ +\begin{tabular}{p{2cm} p{2cm} p{10cm}} +\textbf{##1} & \textit{##2} & {##3} \\ +\end{tabular} +} +\hfill + +\begin{description} +\item[Specifiers] \hfill \\ +} +{ +\hfill +\end{description} +} + +\newcommand{\envvarnotes}[1]{ +\item[Notes] \hfill \\ + #1 +\hfill \\ +} + \newenvironment{apiarguments}{ \newcommand{\apiargument}[3]{ \begin{tabular}{p{2cm} p{2cm} p{10cm}} From 4f2360a2df13eed1260b00245afb9bfb3e67f438 Mon Sep 17 00:00:00 2001 From: Naveen Ravichandrasekaran Date: Wed, 3 May 2017 15:33:58 -0500 Subject: [PATCH 2/2] Update mem-partition proposal --- content/backmatter.tex | 76 +++++++++++++------------- content/library_constants.tex | 83 ++++++++++++++-------------- content/mem_partition_env.tex | 91 ++++++++++++++++--------------- content/memory_model.tex | 26 ++++----- content/memory_partitions.tex | 22 ++++---- content/shmem_addr_accessible.tex | 18 +++--- content/shmem_kind.tex | 32 +++++------ content/shmem_malloc.tex | 31 ++++++----- main_spec.tex | 4 +- utils/defs.tex | 72 ++++++++++++------------ 10 files changed, 232 insertions(+), 223 deletions(-) diff --git a/content/backmatter.tex b/content/backmatter.tex index 8cded690b..3bbf7a84e 100644 --- a/content/backmatter.tex +++ b/content/backmatter.tex @@ -18,7 +18,7 @@ \chapter{Writing \openshmem Programs} \section*{Incorporating \openshmem{} into Programs}\label{sec:writing_programs} In this section, we describe how to write a ``Hello World" \openshmem program. -To write a ``Hello World" \openshmem program we need to: +To write a ``Hello World" \openshmem program we need to: \begin{itemize} \item Add the include file shmem.h (for \Clang) or shmem.fh (for \Fortran). @@ -151,38 +151,38 @@ \chapter{Undefined Behavior in \openshmem}\label{sec:undefined} various library routines. In cases where routines are improperly used or the input is not in accordance with the specification, undefined behavior may be observed. Depending on the implementation there are -many interpretations of undefined behavior. +many interpretations of undefined behavior. $\;$ $ $% \begin{tabular}{|>{\raggedright}p{0.3\textwidth}|>{\raggedright}p{0.6\textwidth}|} -\hline +\hline \textbf{Inappropriate Usage} & \textbf{Undefined Behavior}\tabularnewline -\hline -\hline +\hline +\hline Uninitialized library & If \openshmem is not initialized through a call to \FUNC{shmem\_init}, subsequent accesses to \openshmem routines have undefined results. An implementation may choose, for example, to try to continue or abort immediately upon the first call to an uninitialized routine.\tabularnewline -\hline +\hline Accessing non-existent \ac{PE}s & If a communications routine accesses a non-existent \ac{PE}, then the \openshmem library can choose to handle this situation in an implementation-defined way. For example, the library may issue an error message saying that the \ac{PE} accessed is outside the range of accessible \ac{PE}s, or may exit without a warning.\tabularnewline -\hline +\hline Use of non-symmetric variables & Some routines require remotely accessible variables to perform their function. A \PUT{} to a non-symmetric variable can be trapped where possible and the library can abort the program. Another implementation may choose to continue either with a warning or silently.\tabularnewline -\hline +\hline Non-symmetric variables & The symmetric memory management routines are collectives, which means that all \ac{PE}s in the program must issue the same \FUNC{shmem\_malloc} call with the same size request. Program behavior after a mismatched \FUNC{shmem\_malloc} call is undefined.\tabularnewline -\hline +\hline Use of NULL pointers with non-zero \VAR{len} specified & In any \openshmem routine that takes a pointer and \VAR{len} describing the number of elements in that pointer, NULL may not be specified for the pointer unless the corresponding \VAR{len} is also @@ -195,11 +195,11 @@ \chapter{Undefined Behavior in \openshmem}\label{sec:undefined} \item \VAR{len} is not 0, pointer is not NULL: supported. \end{itemize} \tabularnewline -\hline +\hline Multiple calls to \FUNC{shmem\_init} & In an OpenSHMEM program where \FUNC{shmem\_init} has already be called, any subsequent calls to \FUNC{shmem\_init} result in undefined behavior.\tabularnewline -\hline +\hline \end{tabular} @@ -237,9 +237,9 @@ \section{\ac{MPI} Interoperability} \Fortran{} common block or \Clang{} global variable, is symmetric between processes running from the same executable file, but is not symmetric between processes running from different executable files. Data allocated from the -symmetric heap (\FUNC{shmem\_malloc}, \newtext{\FUNC{shmem\_kind\_malloc}}, or -\FUNC{shpalloc}) is symmetric across the same or different executable files. The -routine \FUNC{shmem\_addr\_accessible} can be used to determine if a local +symmetric heap (\FUNC{shmem\_malloc}, \newtext{\FUNC{shmem\_partition\_malloc}}, +or \FUNC{shpalloc}) is symmetric across the same or different executable files. +The routine \FUNC{shmem\_addr\_accessible} can be used to determine if a local address is accessible via \openshmem communication from a remote \ac{PE}. Another important feature of these systems is that the @@ -297,7 +297,7 @@ \chapter{History of \openshmem}\label{sec:openshmem_history} \item University of Florida \end{itemize} \end{itemize} -\item OpenSHMEM Implementations +\item OpenSHMEM Implementations \begin{itemize} \item SGI \openshmem \item University of Houston - \openshmem Reference Implementation @@ -323,7 +323,7 @@ \section{Overview}\label{subsec:dep_overview} indicated as otherwise by the Specification. In this chapter we will record the API that has been deprecated, the \openshmem Specification that effected the deprecation, and if the feature is supported in the current version of the -specification. +specification. \begin{center} \scriptsize @@ -459,7 +459,7 @@ \section{Version 1.5} \FUNC{shmem\_global\_exit}. \\ See Section \ref{subsec:shmem_global_exit}. % -\item Clarified ordering semantics of memory ordering, point-to-point synchronization and collective +\item Clarified ordering semantics of memory ordering, point-to-point synchronization and collective synchronization routines. % \item Clarified deprecation overview and added deprecation rationale in Annex F. @@ -469,8 +469,8 @@ \section{Version 1.5} \section{Version 1.3} This section summarizes the changes from the \openshmem specification Version -1.2 to Version 1.3. Many major changes to the specification were introduced in Version 1.3. This includes non-blocking RMA operations, -generic interfaces for various OpenSHMEM interfaces, atomic \FUNC{Put} and \FUNC{Get} operations, and Alltoall interfaces. +1.2 to Version 1.3. Many major changes to the specification were introduced in Version 1.3. This includes non-blocking RMA operations, +generic interfaces for various OpenSHMEM interfaces, atomic \FUNC{Put} and \FUNC{Get} operations, and Alltoall interfaces. The following list describes the specific changes in 1.3: @@ -505,11 +505,11 @@ \section{Version 1.3} \FUNC{SHMEM\_SET}. \\See Sections \ref{subsec:shmem_fetch} and \ref{subsec:shmem_set} % -\item New alltoall data exchange operations, \FUNC{SHMEM\_ALLTOALL} +\item New alltoall data exchange operations, \FUNC{SHMEM\_ALLTOALL} and \FUNC{SHMEM\_ALLTOALLS}. \\See Sections \ref{subsec:shmem_alltoall} and \ref{subsec:shmem_alltoalls}. % -\item Added \textbf{volatile} to remotely accessible pointer argument in +\item Added \textbf{volatile} to remotely accessible pointer argument in \FUNC{SHMEM\_WAIT} and \FUNC{SHMEM\_LOCK}. \\See Sections \ref{subsec:shmem_wait} and \ref{subsec:shmem_lock}. % @@ -526,7 +526,7 @@ \section{Version 1.2} 1.1 to Version 1.2. A major change in this version is that it improves upon the execution model described in 1.1 by introducing an explicit \FUNC{shmem\_finalize} library call. This provides a collective mechanism of -exiting an \openshmem program and releasing resources used by the library. +exiting an \openshmem program and releasing resources used by the library. The following list describes the specific changes in 1.2: @@ -563,7 +563,7 @@ \section{Version 1.2} \FUNC{shmem\_ptr}. \\See Section \ref{subsec:shmem_ptr}. % -\item New API to query the version and name information. +\item New API to query the version and name information. \\See Section \ref{subsec:shmem_info_get_version} and \ref{subsec:shmem_info_get_name}. % \item \openshmem library API normalization. All C symmetric memory management @@ -574,7 +574,7 @@ \section{Version 1.2} \\See Section \ref{subsec:shfree}. % \item Deprecation of Fortran API routine \FUNC{SHMEM\_PUT}. -\\See Section \ref{subsec:shmem_put}. +\\See Section \ref{subsec:shmem_put}. % \item Clarification related to \FUNC{shmem\_wait}. \\See Section \ref{subsec:shmem_wait}. @@ -606,7 +606,7 @@ \section{Version 1.1} \begin{itemize} % -\item Clarifications of the completion semantics of memory synchronization +\item Clarifications of the completion semantics of memory synchronization interfaces. \\See Section \ref{subsec:memory_order}. % @@ -644,28 +644,28 @@ \section{Version 1.1} \\See Section \ref{subsec:library_constants} and \ref{subsec:shmem_wait}. % \item Added \ac{API} calls: \FUNC{shmem\_char\_p}, \FUNC{shmem\_char\_g}. -\\See Sections \ref{subsec:shmem_p} and \ref{subsec:shmem_g}. +\\See Sections \ref{subsec:shmem_p} and \ref{subsec:shmem_g}. % \item Removed \ac{API} calls: \FUNC{shmem\_char\_put}, \FUNC{shmem\_char\_get}. -\\See Sections \ref{subsec:shmem_put} and \ref{subsec:shmem_get}. +\\See Sections \ref{subsec:shmem_put} and \ref{subsec:shmem_get}. % \item The usage of \VAR{ptrdiff\_t}, \VAR{size\_t}, and \VAR{int} in the interface signature was made consistent with the description. \\See Sections \ref{subsec:coll}, \ref{subsec:shmem_iput}, and \ref{subsec:shmem_iget}. % \item Revised \FUNC{shmem\_barrier} example. -\\See Section \ref{subsec:shmem_barrier}. +\\See Section \ref{subsec:shmem_barrier}. % \item Clarification of the initial value of \VAR{pSync} work arrays for -\FUNC{shmem\_barrier}.\\ See Section \ref{subsec:shmem_barrier}. +\FUNC{shmem\_barrier}.\\ See Section \ref{subsec:shmem_barrier}. % \item Clarification of the expected behavior when multiple \FUNC{start\_pes} calls are encountered has been clarified. \\See Section \ref{subsec:start_pes}. % \item Corrected the definition of atomic increment operation. -\\See Section \ref{subsec:shmem_inc}. +\\See Section \ref{subsec:shmem_inc}. % \item Clarification of the size of the symmetric heap and when it is set. \\See Section \ref{subsec:shfree}. @@ -673,29 +673,29 @@ \section{Version 1.1} \item Clarification of the integer and real sizes for \Fortran{} \ac{API}. \\See Sections \ref{subsec:shmem_add}, \ref{subsec:shmem_cswap}, \ref{subsec:shmem_swap}, \ref{subsec:shmem_finc}, \ref{subsec:shmem_inc}, and - \ref{subsec:shmem_fadd}. + \ref{subsec:shmem_fadd}. % \item Clarification of the expected behavior on program \OPR{exit}. -\\See Section \ref{subsec:execution_model}, Execution Model. +\\See Section \ref{subsec:execution_model}, Execution Model. % \item More detailed description for the progress of \openshmem operations provided. -\\See Section \ref{subsec:progress}. +\\See Section \ref{subsec:progress}. % \item Clarification of naming convention for non-standard interfaces and their inclusion in \FUNC{shmemx.h}. -\\See Section \ref{subsec:bindings}. +\\See Section \ref{subsec:bindings}. % \item Various fixes to \openshmem code examples across the specification to -include appropriate header files. +include appropriate header files. % \item Removing requirement that implementations should detect size mismatch and return error information for \FUNC{shmalloc} and ensuring consistent language. -\\See Sections \ref{subsec:shfree} and Annex \ref{sec:undefined}. +\\See Sections \ref{subsec:shfree} and Annex \ref{sec:undefined}. % \item Fortran programming fixes for examples.\\ See Sections -\ref{subsec:shmem_reductions} and \ref{subsec:shmem_wait}. +\ref{subsec:shmem_reductions} and \ref{subsec:shmem_wait}. % \item Clarifications of the reuse \VAR{pSync} and \VAR{pWork} across collectives. @@ -703,7 +703,7 @@ \section{Version 1.1} \ref{subsec:shmem_collect} and \ref{subsec:shmem_reductions}. % \item Name changes for UV and ICE for SGI systems. -\\See Annex \ref{sec:openshmem_history}. +\\See Annex \ref{sec:openshmem_history}. % \end{itemize} diff --git a/content/library_constants.tex b/content/library_constants.tex index 4626377e8..e8cda3a7e 100644 --- a/content/library_constants.tex +++ b/content/library_constants.tex @@ -1,5 +1,5 @@ The constants that start with SHMEM\_* are for both \Fortran{} -and \CorCpp, and they are compile-time constants. +and \CorCpp, and they are compile-time constants. All constants that start with \_SHMEM\_* are deprecated and provided for backwards compatibility. \newline @@ -8,7 +8,7 @@ \hline \textbf{Constant} & \textbf{Description} \tabularnewline -\hline +\hline \hline %new \vspace{3mm} @@ -20,28 +20,28 @@ about the usage of this constant. Work arrays sized for specific operations may consume less memory.\tabularnewline %new -\hline +\hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_BCAST\_SYNC\_SIZE}}} -& +\vtop{\hbox{\CorCppFor:} +\hbox{\hspace*{12mm} \const{SHMEM\_BCAST\_SYNC\_SIZE}}} +& Length of the \VAR{pSync} arrays needed for broadcast routines. The value of this constant is implementation specific. Refer to the \hyperref[subsec:shmem_broadcast]{Broadcast Routines} section under \hyperref[sec:openshmem_library_api]{Library Routines} for more information about the usage of this constant. \tabularnewline -\hline +\hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_SYNC\_VALUE}}} -& +\vtop{\hbox{\CorCppFor:} +\hbox{\hspace*{12mm} \const{SHMEM\_SYNC\_VALUE}}} +& The value used to initialize the elements of \VAR{pSync} arrays. The value of this constant is implementation specific.\tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} +\vtop{\hbox{\CorCppFor:} \hbox{\hspace*{12mm} \const{SHMEM\_REDUCE\_SYNC\_SIZE}}} -& +& Length of the work arrays needed for reduction routines. The value of this constant is implementation specific. Refer to the \hyperref[subsec:shmem_reductions]{Reduction Routines} section under @@ -49,9 +49,9 @@ about the usage of this constant.\tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_BARRIER\_SYNC\_SIZE}}} -& +\vtop{\hbox{\CorCppFor:} +\hbox{\hspace*{12mm} \const{SHMEM\_BARRIER\_SYNC\_SIZE}}} +& Length of the work array needed for barrier routines. The value of this constant is implementation specific. Refer to the \hyperref[subsec:shmem_barrier]{Barrier Synchronization Routines} section under @@ -60,8 +60,8 @@ \hline \vspace{3mm} \vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_COLLECT\_SYNC\_SIZE}}} -& +\hbox{\hspace*{12mm} \const{SHMEM\_COLLECT\_SYNC\_SIZE}}} +& Length of the work array needed for collect routines. The value of this constant is implementation specific. Refer to the \hyperref[subsec:shmem_collect]{Collect Routines} section under @@ -70,8 +70,8 @@ \hline \vspace{3mm} \vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_ALLTOALL\_SYNC\_SIZE}}} -& +\hbox{\hspace*{12mm} \const{SHMEM\_ALLTOALL\_SYNC\_SIZE}}} +& Length of the work array needed for \FUNC{shmem\_alltoall} routines. The value of this constant is implementation specific. Refer to the \hyperref[subsec:shmem_alltoall]{Alltoall @@ -84,8 +84,8 @@ \hline \vspace{3mm} \vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_ALLTOALLS\_SYNC\_SIZE}}} -& +\hbox{\hspace*{12mm} \const{SHMEM\_ALLTOALLS\_SYNC\_SIZE}}} +& Length of the work array needed for \FUNC{shmem\_alltoalls} routines. The value of this constant is implementation specific. Refer to the \hyperref[subsec:shmem_alltoalls]{Alltoalls @@ -93,51 +93,52 @@ for more information about the usage of this constant.\tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_REDUCE\_MIN\_WRKDATA\_SIZE}}} +\vtop{\hbox{\CorCppFor:} +\hbox{\hspace*{12mm} \const{SHMEM\_REDUCE\_MIN\_WRKDATA\_SIZE}}} & Minimum length of work arrays used in various collective routines.\tabularnewline \hline \vspace{3mm} %\color{red} -%\vtop{\hbox{} -%\hbox{\hspace*{12mm} \const{}} -%\hbox{} -%\hbox{\hspace*{12mm} \const{}}} +%\vtop{\hbox{} +%\hbox{\hspace*{12mm} \const{}} +%\hbox{} +%\hbox{\hspace*{12mm} \const{}}} %& \color{red} %Ticket \#107 \tabularnewline -\vtop{\hbox{\CorCppFor:} +\vtop{\hbox{\CorCppFor:} \hbox{\hspace*{12mm} \const{SHMEM\_MAJOR\_VERSION}}} -& +& Integer representing the major version of \openshmem{} standard in use. \tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} +\vtop{\hbox{\CorCppFor:} \hbox{\hspace*{12mm} \const{SHMEM\_MINOR\_VERSION}}} -& +& Integer representing the minor version of \openshmem{} standard in use. \tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} +\vtop{\hbox{\CorCppFor:} \hbox{\hspace*{12mm} \const{SHMEM\_MAX\_NAME\_LEN}}} & Integer representing the length of vendor string. \tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\CorCppFor:} -\hbox{\hspace*{12mm} \const{SHMEM\_VENDOR\_STRING}}} +\vtop{\hbox{\CorCppFor:} +\hbox{\hspace*{12mm} \const{SHMEM\_VENDOR\_STRING}}} & String representing the vendor name of length less than -\const{SHMEM\_MAX\_NAME\_LEN}. In Fortran the string must be \const{SHMEM\_MAX\_NAME\_LEN} -and whitespace padded. It can also be equal in length to \const{SHMEM\_MAX\_NAME\_LEN} +\const{SHMEM\_MAX\_NAME\_LEN}. In Fortran the string must be \const{SHMEM\_MAX\_NAME\_LEN} +and whitespace padded. It can also be equal in length to \const{SHMEM\_MAX\_NAME\_LEN} since Fortran does not NULL terminate strings. \tabularnewline \hline \vspace{3mm} -\vtop{\hbox{\newtext{\CorCpp:}} +\vtop{\hbox{\newtext{\CorCpp:}} \hbox{\hspace*{12mm} \newtext{\const{SHMEM\_MAX\_PARTITIONS}}}} &\newtext{ Integer representing the implementations maximum for the number of partitions -that can be defined by the users.} %Implementations value must be at least 7 so -%that a portable program can assume at least that possible partitions} +that can be defined by the users. Implementations must support be at least +\textless X\textgreater so that a portable program can assume at least that +possible partitions} \tabularnewline \hline \vspace{3mm} @@ -145,7 +146,9 @@ \hbox{\hspace*{12mm} \newtext{\const{SHMEM\_MAX\_PARTITION\_ID}}}} &\newtext{ Integer representing the implementations maximum number that can be used to -represent the partition ID.} %The value should be at least 127} +represent the partition ID. Impementations must support at least \textless +X\textgreater so that a portable program can assume at least that possible +partition ID values} \tabularnewline \hline \end{tabular} diff --git a/content/mem_partition_env.tex b/content/mem_partition_env.tex index 0da453d2e..3a515561f 100644 --- a/content/mem_partition_env.tex +++ b/content/mem_partition_env.tex @@ -1,9 +1,8 @@ All OpenSHMEM implementations supporting the use of multiple Symmetric Memory Partitions provide support for a special environment variable called -SHMEM\_SYMMETRIC\_PARTITION to define the properties of the user-defined memory -locations which can be used to create symmetric heaps. The properties of the -memory partitions including the size of the symmetric heap to be created are -passed as specifiers to the environment variable. +SHMEM\_SYMMETRIC\_PARTITION. It defines the traits to establish the +characteristics of a memory partition. %These traits including the size of +%the symmetric heap are defined specifiers to the environment variable. \begin{envvardefinition} \begin{envvarname} @@ -11,67 +10,73 @@ \end{envvarname} \begin{envvararguments} - \envvarargument{REQ}{ID}{Integer to represent the partition number} + \envvarargument{REQ}{ID}{Integer used to identify the partition} \envvarargument{REQ}{SIZE}{Number of bytes to allocate for symmetric heap} - \envvarargument{OPT}{PGSIZE}{Number of bytes to specify the memory page size} - \envvarargument{OPT}{KIND}{String constants to specify the memory kind if - multiple memory kinds are supported by the system and identified by the + \envvarargument{OPT}{PGSIZE}{Number of bytes to specify the size of the + page used by the partition} + \envvarargument{OPT}{KIND}{String constants to specify the kind of memory + if multiple memory kinds are supported by the system and identified by the implementation} \envvarargument{OPT}{POLICY}{String constants to specify the memory - allocation policy} + allocation policy used by the partition} \end{envvararguments} One, two, or more partitions can be specified using this environment variable. Each partition is represented using a number called \emph{Partition ID}. \textless ID\textgreater is the user-specified part of the environment variable to represent the Partition ID. A maximum of SHMEM\_MAX\_PARTITIONS can be -defined and the Partition ID may be any non-zero positive integer between 1 and -SHMEM\_MAX\_PARTITION\_ID. +defined. These defined partitions can have any non-zero positive integer +between 1 and SHMEM\_MAX\_PARTITION\_ID as Partition ID. -Each partition can take a maximum of four property specifiers as input. The -\emph{SIZE} specifier is the only required value. It is used to specify the -required symmetric heap size on the partition. The \emph{PGSIZE} specifier is -used to specify the page size of the memory kind. +Each partition takes a maximum of four traits as input. The \emph{SIZE} is the +only required trait. It is used to specify the required symmetric heap size on +the partition. The \emph{PGSIZE} trait is used to specify the size of page used +by the partition. -On systems supporting multiple memory kinds, each memory kinds which are +The \emph{KIND} trait is used to specify the memory kind used by the partition. +On systems supporting multiple different kinds of memory, each memory kind identified by the implementation should be documented and made available as -string constants to the users. The \emph{KIND} specifier is used to specify the -memory kind to determine location of the memory partition. +case-sensitive string constants to the users. Along with other identified +memory kinds, every implementation should define the following two kinds. +\begin{itemize} +\item \emph{NORMALMEM} The primary memory kind of a node +\item \emph{SYSDEFAULT} System defined memory kind is used +\end{itemize} -The \emph{POLICY} is used to specify on how strictly the request optional -specifiers \emph{PGSIZE} and \emph{KIND} are to be honored. The following are -the accepted values for \emph{POLICY} specifier. +The \emph{POLICY} is used to specify on how strictly the user requested +optional traits (\emph{PGSIZE} and \emph{KIND}) are to be honored. The +following are the accepted values. \begin{itemize} \item \emph{MANDATORY} Any unavailability of the user requested optional -specifier would result in program termination with an informative error during -symmetric heap creation on the defined partition. -\item \emph{PREFERRED} Any unavailability of the user requested optional -specifier would allow the library to try to allocate the requested \emph{SIZE} -on the implementation defined location. -\item \emph{INTERLEAVED} Page allocations are interleaved across numa domains -associated with the memory kind. -\item \emph{SYSDEFAULT} System defined memory policy is used. +trait would result in program termination with an informative error message +during symmetric heap creation on the defined partition. +\item \emph{PREFERRED} Any unavailability of the user requested optional trait +would allow the library to try and allocate the requested \emph{SIZE} on an +implementation determined location. +\item \emph{INTERLEAVED} Page allocations are interleaved across NUMA domains +associated with the defined \emph{KIND}. +\item \emph{SYSDEFAULT} System defined default memory policy is used. \end{itemize} \envvarnotes{ + There are two types of memory allocations - generic (Refer + \ref{subsec:shfree}) and partition-specific allocations + (Refer \ref{subsec:shmem_kind}). Partition with ID 1 has a special usage. + When multiple partitions are created, any generic allocations of symmetric + data objects will use the symmetric heap created at partition with ID as 1. + SHMEM\_SYMMETRIC\_SIZE is supported with its current syntax and meaning. - But, it is an error to use both SHMEM\_SYMMETRIC\_SIZE and any form of - SHMEM\_SYMMETRIC\_PARTITION environment variables together. + But, it is an error to use both SHMEM\_SYMMETRIC\_SIZE and + SHMEM\_SYMMETRIC\_PARTITION1 environment variables together. - The total size of the symmetric heap is determined as the sum of sizes - of all \emph{SIZE} specifier in the defined memory partitions. + The total size of the symmetric heap is determined as the sum of all + \emph{SIZE} traits in the defined memory partitions. - All available page sizes and supported memory kinds be documented by the - implementation along with the \emph{SYSDEFAULT} values for the optional - properties. + All available page sizes and supported memory kinds should be documented + by the implementation along with the \emph{SYSDEFAULT} values for the + optional traits. +} - Partition ID 1 has a special usage. Symmetric data object allocations - identified using the partition ID (with \textit{shmem\_kind\_malloc} and - \textit{shmem\_kind\_align} routines) are allocated on the symmetric heap - created over the partition with requested ID. Any generic allocations - without the partition ID (using \textit{shmem\_malloc} and - \textit{shmem\_align}) would result in using the symmetric heap created over - partition with ID 1.} \end{envvardefinition} diff --git a/content/memory_model.tex b/content/memory_model.tex index 330221040..3dc15e04b 100644 --- a/content/memory_model.tex +++ b/content/memory_model.tex @@ -1,8 +1,8 @@ \begin{figure}[h] -\includegraphics[width=0.95\textwidth]{figures/mem_model} -\caption{\OSH Memory Model} -\label{fig:mem_model} -\end{figure} +\includegraphics[width=0.95\textwidth]{figures/mem_model} +\caption{\OSH Memory Model} +\label{fig:mem_model} +\end{figure} % An \openshmem program consists of data objects that are private to each \ac{PE} and data objects that are remotely accessible by all \ac{PE}s. Private data @@ -30,18 +30,18 @@ These data objects must not be defined in a dynamic shared object (DSO). \item Global and static \Clang{} and \Cpp variables. These data objects must not be defined in a DSO. - \item \Fortran{} arrays allocated with \textit{shpalloc} + \item \Fortran{} arrays allocated with \textit{shpalloc} \item \Clang{} and \Cpp data allocated by \textit{shmem\_malloc} -\end{itemize} +\end{itemize} \openshmem dynamic memory allocation routines (\textit{shpalloc} and \textit{shmem\_malloc}) allow collective allocation of \emph{Symmetric Data Objects} on a special memory region called the \emph{Symmetric Heap}. The Symmetric Heap is created during the execution of a program at a memory location -determined by the implementation \newtext{or on multiple user determined memory -locations called as the \emph{Symmetric Memory Partitions}}. The Symmetric -Heap may reside in different memory regions on different \acp{PE}. -Figure~\ref{fig:mem_model} shows how \openshmem implements a \ac{PGAS} model -using remotely accessible symmetric objects and private data objects when executing -an \openshmem program. Symmetric data objects are stored on the symmetric heap or -in the global/static memory section of each \ac{PE}. +determined by the implementation \newtext{or at multiple user determined memory +locations called as the \emph{Symmetric Memory Partitions}}. The Symmetric +Heap may reside in different memory regions on different \acp{PE}. +Figure~\ref{fig:mem_model} shows how \openshmem implements a \ac{PGAS} model +using remotely accessible symmetric objects and private data objects when executing +an \openshmem program. Symmetric data objects are stored on the symmetric heap or +in the global/static memory section of each \ac{PE}. diff --git a/content/memory_partitions.tex b/content/memory_partitions.tex index 47d446ccf..805de30a4 100644 --- a/content/memory_partitions.tex +++ b/content/memory_partitions.tex @@ -1,14 +1,14 @@ -\emph{Symmetric Heaps} can be created on a single memory location determined by -the implementation or on multiple memory locations determined by the users. The +\emph{Symmetric Heap} is created on a single memory location determined by the +implementation or at multiple memory locations determined by the users. The user-determined memory locations are called as the \emph{Symmetric Memory -Partitions}. Only a single symmetric heap can be created on each symmetric -memory partition. Each symmetric memory partitions is identified using its -\emph{Partition ID}, which is used on dynamic OpenSHMEM memory allocation -(\textit{shmem\_kind\_malloc} and \textit{shmem\_kind\_align}) routines. Apart -from the Partition ID, each symmetric memory partition have their own memory -properties. Refer to Section \ref{subsec:environment_variables} for the -complete list of properties which are available to characterize a memory -partition. Multiple symmetric heaps can be created by defining multiple -separate memory partitions. Apart from name, type and size attributes, +Partitions}. Only a single symmetric heap is created at each symmetric memory +partition. Each symmetric memory partition is identified using its +\emph{Partition ID}, which is used in dynamic OpenSHMEM memory allocation +(\textit{shmem\_partition\_malloc} and \textit{shmem\_partition\_align}) +routines. Apart from the Partition ID, each symmetric memory partition have +their own memory traits. Refer to Section \ref{subsec:environment_variables} +for a complete list of traits available to define the characteristics of a +memory partition. Multiple symmetric heaps can be created by defining multiple +separate memory partitions. Apart from name, type, and size attributes, symmetric data objects stored on symmetric heaps created over a memory partition have Partition ID as an extra attribute. \ No newline at end of file diff --git a/content/shmem_addr_accessible.tex b/content/shmem_addr_accessible.tex index 8383a4cb6..7aee5a24f 100644 --- a/content/shmem_addr_accessible.tex +++ b/content/shmem_addr_accessible.tex @@ -22,12 +22,12 @@ \apidescription{ \FUNC{shmem\_addr\_accessible} is a query routine that indicates whether a local - address is accessible via \openshmem routines from the specified remote \ac{PE}. - + address is accessible via \openshmem routines from the specified remote \ac{PE}. + This routine verifies that the data object is symmetric and accessible with respect to a remote \ac{PE} via \openshmem data transfer routines. The - specified address \VAR{addr} is a data object on the local \ac{PE}. - + specified address \VAR{addr} is a data object on the local \ac{PE}. + This routine may be particularly useful for hybrid programming with other communication libraries (such as \ac{MPI}) or parallel languages. For example, in SGI Altix series systems, for multiple executable MPI programs that @@ -35,20 +35,20 @@ \Fortran{} common block or \Clang{} global variable, is symmetric between processes running from the same executable file, but is not symmetric between processes running from different executable files. Data allocated from the - symmetric heap (\FUNC{shmem\_malloc}, \newtext{\FUNC{shmem\_kind\_malloc}} or - \FUNC{shpalloc}) is symmetric across the same or different executable files. + symmetric heap (\FUNC{shmem\_malloc}, \newtext{\FUNC{shmem\_partition\_malloc}} + or \FUNC{shpalloc}) is symmetric across the same or different executable files. } -\apireturnvalues{ +\apireturnvalues{ \CorCpp: The return value is \CONST{1} if \VAR{addr} is a symmetric data object and accessible via \openshmem routines from the specified remote \ac{PE}; otherwise, it is \CONST{0}. - + \Fortran: The return value is \CONST{.TRUE.} if \VAR{addr} is a symmetric data object and accessible via \openshmem routines from the specified remote \ac{PE}; otherwise, it is \CONST{.FALSE.}. } - + \apinotes{ None. } diff --git a/content/shmem_kind.tex b/content/shmem_kind.tex index 92284ec4d..83df68435 100644 --- a/content/shmem_kind.tex +++ b/content/shmem_kind.tex @@ -4,8 +4,8 @@ \begin{apidefinition} \begin{Csynopsis} -void *shmem_kind_malloc(size_t size, int partition_id); -void *shmem_kind_align(size_t alignment, size_t size, int partition_id); +void *shmem_partition_malloc(size_t size, int partition_id); +void *shmem_partition_align(size_t alignment, size_t size, int partition_id); \end{Csynopsis} \begin{apiarguments} @@ -15,38 +15,38 @@ \apiargument{IN}{alignment}{Byte alignment of the block allocated from the symmetric heap.} \apiargument{IN}{partition\_id}{Integer to identify the user-defined memory - partition on which the symemtric heap is created} + partition at which the symemtric heap is created} \end{apiarguments} \apidescription{ - The \FUNC{shmem\_kind\_malloc} routine returns a pointer to a block of at + The \FUNC{shmem\_partition\_malloc} routine returns a pointer to a block of at least \VAR{size} bytes suitably aligned for any use. This space is - allocated from the symmetric heap created on the user-defined memory - partition with characteristics associated with the partition\_id(in + allocated from the symmetric heap created within the user-defined memory + partition with characteristics associated with the \emph{partition\_id}(in contrast to \FUNC{malloc}, which allocates from the private heap). - - The \FUNC{shmem\_kind\_align} routine allocates a block in the symmetric - heap heap created on the user-defined memory partition with characteristics + + The \FUNC{shmem\_partition\_align} routine allocates a block in the symmetric + heap heap created within the user-defined memory partition with characteristics associated with the partition\_id that has a byte alignment specified by the alignment argument. } \apireturnvalues{ - The \FUNC{shmem\_kind\_malloc} routine returns a pointer to the allocated + The \FUNC{shmem\_partition\_malloc} routine returns a pointer to the allocated space; otherwise, it returns a \CONST{NULL} pointer. - - The \FUNC{shmem\_kind\_align} routine returns an aligned pointer to the + + The \FUNC{shmem\_partition\_align} routine returns an aligned pointer to the allocated space; otherwise, it returns a \CONST{NULL} pointer. } \apinotes{ Requirements for all PEs to use \FUNC{shmem\_malloc} and \FUNC{shmem\_align} memory management routines are the same for - \FUNC{shmem\_kind\_malloc} and \FUNC{shmem\_kind\_align} routines. + \FUNC{shmem\_partition\_malloc} and \FUNC{shmem\_partition\_align} routines. } \apiimpnotes{ In a quality implementation \FUNC{shmem\_realloc} and \FUNC{shmem\_free} - should also handle the symmetric memory partitions correctly. It is - assumed that the new memory address returned on \FUNC{shmem\_realloc} will - be in the same symmetric memory partition associated with the input + should handle the symmetric memory partitions correctly. It is assumed that + the new memory address returned on \FUNC{shmem\_realloc} will be in the + same symmetric memory partition associated with the input address. } diff --git a/content/shmem_malloc.tex b/content/shmem_malloc.tex index dbfdc68e5..b29dca774 100644 --- a/content/shmem_malloc.tex +++ b/content/shmem_malloc.tex @@ -25,14 +25,14 @@ \VAR{size} bytes suitably aligned for any use. This space is allocated from the symmetric heap (in contrast to \FUNC{malloc}, which allocates from the private heap). - + The \FUNC{shmem\_align} routine allocates a block in the symmetric heap that has a byte alignment specified by the alignment argument. - + The \FUNC{shmem\_free} routine causes the block to which \VAR{ptr} points to be deallocated, that is, made available for further allocation. If \VAR{ptr} is a - null pointer, no action occurs. - + null pointer, no action occurs. + The \FUNC{shmem\_realloc} routine changes the size of the block to which \VAR{ptr} points to the size (in bytes) specified by \VAR{size}. The contents of the block are unchanged up to the lesser of the new and old sizes. If the new @@ -42,7 +42,7 @@ the specified size. If \VAR{size} is \CONST{0} and \VAR{ptr} is not a \CONST{NULL} pointer, the block to which it points is freed. If the space cannot be allocated, the block to which \VAR{ptr} points is unchanged. - + The \FUNC{shmem\_malloc}, \FUNC{shmem\_free}, and \FUNC{shmem\_realloc} routines are provided so that multiple \ac{PE}s in a program can allocate symmetric, remotely accessible memory blocks. These memory blocks can then be used with @@ -58,36 +58,37 @@ \apireturnvalues{ The \FUNC{shmem\_malloc} routine returns a pointer to the allocated space; otherwise, it returns a \CONST{NULL} pointer. - + The \FUNC{shmem\_free} routine returns no value. - + The \FUNC{shmem\_realloc} routine returns a pointer to the allocated space (which may have moved); otherwise, it returns a null pointer. - + The \FUNC{shmem\_align} routine returns an aligned pointer to the allocated space; otherwise, it returns a \CONST{NULL} pointer. } -\apinotes{ +\apinotes{ As of Specification 1.2 the use of \FUNC{shmalloc}, \FUNC{shmemalign}, \FUNC{shfree}, and \FUNC{shrealloc} has been deprecated. Although OpenSHMEM libraries are required to support the calls, program users are encouraged to use \FUNC{shmem\_malloc}, \FUNC{shmem\_align}, \FUNC{shmem\_free}, and \FUNC{shmem\_realloc} instead. The behavior and signature of the routines remains unchanged from the deprecated versions. - + \newtext{Usage of \FUNC{shmem\_malloc} and \FUNC{shmem\_align} along with Symmetric Memory Partitions would result in the memory space to be - allocated on the symmetric heap created on partition with ID as 1.} + allocated on the symmetric heap created at partition with ID as 1.} The total size of the symmetric heap is determined at job startup. One can - adjust the size of the heap using the \CONST{SHMEM\_SYMMETRIC\_SIZE} or - \CONST{SHMEM\_SYMMETRIC\_PARTITION} environment variable (where available). - + adjust the size of the heap using the \CONST{SHMEM\_SYMMETRIC\_SIZE} + \newtext{or \CONST{SHMEM\_SYMMETRIC\_PARTITION}} environment variable + (where available). + The \FUNC{shmem\_malloc}, \FUNC{shmem\_free}, and \FUNC{shmem\_realloc} routines differ from the private heap allocation routines in that all \ac{PE}s in a program must call them (a barrier is used to ensure this). -} +} \apiimpnotes{ The symmetric heap allocation routines always return a pointer to corresponding diff --git a/main_spec.tex b/main_spec.tex index d44319fe5..ee1b72bd6 100644 --- a/main_spec.tex +++ b/main_spec.tex @@ -36,7 +36,7 @@ \section{Library Constants}\label{subsec:library_constants} \section{Environment Variables }\label{subsec:environment_variables} \input{content/environment_variables} \color{Green} -\subsection{User-defined Environment Variables}\label{subsubsec:usr_defn_env} +\subsection{Memory Partition Environment Variables}\label{subsubsec:usr_defn_env} \input{content/mem_partition_env} \color{Black} @@ -96,7 +96,7 @@ \subsubsection{\textbf{SHMEM\_MALLOC, SHMEM\_FREE, SHMEM\_REALLOC, SHMEM\_ALIGN} \input{content/shmem_malloc.tex} \color{Green} -\subsubsection{\textbf{SHMEM\_KIND\_MALLOC, SHMEM\_KIND\_ALIGN}}\label{subsec:shmem_kind} +\subsubsection{\textbf{SHMEM\_PARTITION\_MALLOC, SHMEM\_PARTITION\_ALIGN}}\label{subsec:shmem_kind} \input{content/shmem_kind.tex} \color{Black} diff --git a/utils/defs.tex b/utils/defs.tex index 722b2630e..a89500e07 100644 --- a/utils/defs.tex +++ b/utils/defs.tex @@ -3,8 +3,8 @@ \hyphenation{Open-SHMEM} -\renewcommand{\chaptername}{Chapter} -\renewcommand{\appendixname}{Annex} +\renewcommand{\chaptername}{Chapter} +\renewcommand{\appendixname}{Annex} % Place some penalty for doing the break % The penalty for a ``\gb'' should be greater than a \hyphenpenalty. @@ -13,15 +13,15 @@ \penalty10000} % This macro enables that all "_" (underscore) characters in the pfd -% file are searchable, and that cut&paste will copy the "_" as underscore. +% file are searchable, and that cut&paste will copy the "_" as underscore. % Without the following macro, the \_ is treated in searches and cut&paste -% as a " " (space character). -% This macro does not modify the behavior of _ in math or in verbatim +% as a " " (space character). +% This macro does not modify the behavior of _ in math or in verbatim % environments. In verbatim environments, the "_" is always treated % as a searchable character. % -\DeclareRobustCommand{\_}{\texttt{\char`\_}} -% +\DeclareRobustCommand{\_}{\texttt{\char`\_}} +% \def\colorswapnt{\colorlet{saved}{.}\color{ForestGreen}} \def\colorswapot{\colorlet{saved}{.}\color{red}} @@ -144,13 +144,13 @@ % Use Sans Serif font for sections, etc. % \makeatletter -\def\section{\@startsection {section}{1}{\z@}{-3.5ex plus -1ex minus +\def\section{\@startsection {section}{1}{\z@}{-3.5ex plus -1ex minus -.2ex}{2.3ex plus .2ex}{\Large\sf}} -\def\subsection{\@startsection{subsection}{2}{\z@}{-3.25ex plus -1ex minus +\def\subsection{\@startsection{subsection}{2}{\z@}{-3.25ex plus -1ex minus -.2ex}{1.5ex plus .2ex}{\large\sf}} -\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{-3.25ex plus +\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{-3.25ex plus -1ex minus -.2ex}{1.5ex plus .2ex}{\normalsize\sf\bf}} -\def\paragraph{\@startsection {paragraph}{4}{\z@}{3.25ex plus 1ex +\def\paragraph{\@startsection {paragraph}{4}{\z@}{3.25ex plus 1ex minus .2ex}{-1em}{\normalsize\sf}} \makeatother % @@ -168,12 +168,12 @@ basicstyle=\ttfamily\footnotesize, breaklines=true, % sets automatic line breaking escapeinside={|}{|}, % if you want to add LaTeX within your code - extendedchars=true, % lets you use non-ASCII characters; for 8-bits + extendedchars=true, % lets you use non-ASCII characters; for 8-bits % encodings only, does not work with UTF-8 - keepspaces=true, % keeps spaces in text, useful for keeping indentation of code + keepspaces=true, % keeps spaces in text, useful for keeping indentation of code % (possibly needs columns=flexible) morekeywords={*,...}, % if you want to add more keywords to the set - showspaces=false, % show spaces everywhere adding particular underscores; + showspaces=false, % show spaces everywhere adding particular underscores; % it overrides 'showstringspaces' showstringspaces=false, % underline spaces within strings only showtabs=false, % show tabs within strings adding particular underscores @@ -185,16 +185,16 @@ basicstyle=\ttfamily\footnotesize, breaklines=true, % sets automatic line breaking escapeinside={\%*}{*)}, % if you want to add LaTeX within your code - extendedchars=true, % lets you use non-ASCII characters; for 8-bits + extendedchars=true, % lets you use non-ASCII characters; for 8-bits % encodings only, does not work with UTF-8 keepspaces=true, % keeps spaces in text, useful for keeping % indentation of code (possibly needs columns=flexible) morekeywords={*,...}, % if you want to add more keywords to the set - showspaces=false, % show spaces everywhere adding particular underscores; + showspaces=false, % show spaces everywhere adding particular underscores; % it overrides 'showstringspaces' showstringspaces=false, % underline spaces within strings only showtabs=false, % show tabs within strings adding particular underscores - backgroundcolor=\color{gray}, + backgroundcolor=\color{gray}, } } @@ -316,7 +316,7 @@ \newenvironment{apidefinition}{ \begin{description} -\item[SYNOPSIS] \hfill \\ \\ +\item[SYNOPSIS] \hfill \\ \\ \vspace{-2em} } { @@ -330,34 +330,34 @@ morekeywords={size_t, TYPE, noreturn}, aboveskip=0pt, belowskip=0pt}}{} \lstnewenvironment{C11synopsis} -{ - \textbf{C11:} +{ + \textbf{C11:} \lstset{language={C}, backgroundcolor=\color{gray}, lineskip=2pt, morekeywords={size_t, TYPE, _Noreturn}, aboveskip=0pt, belowskip=0pt}}{} \lstnewenvironment{CsynopsisCol} -{ +{ \lstset{language={C}, backgroundcolor=\color{gray}, lineskip=2pt, morekeywords={size_t, TYPE, TYPENAME, SIZE}, aboveskip=0pt, belowskip=0pt}}{} \lstnewenvironment{Csynopsis} -{ - \textbf{C/C++:} +{ + \textbf{C/C++:} \lstset{language={C}, backgroundcolor=\color{gray}, lineskip=2pt, morekeywords={size_t, TYPE, TYPENAME, SIZE}, aboveskip=0pt, belowskip=0pt}}{} \lstnewenvironment{CsynopsisST} -{ - \textbf{C/C++:} - \color{red} +{ + \textbf{C/C++:} + \color{red} {\lstset{language={C}, backgroundcolor=\color{gray}, lineskip=2pt, morekeywords={size_t}, aboveskip=0pt, belowskip=0pt}} } {} - + \lstnewenvironment{Fsynopsis} -{ \textbf{FORTRAN:} +{ \textbf{FORTRAN:} \lstset{language={Fortran}, backgroundcolor=\color{gray}, lineskip=3pt, deletekeywords=[2]{STATUS}, deletekeywords=[3]{LOG}, aboveskip=0pt, @@ -366,7 +366,7 @@ \newenvironment{envvardefinition}{ \begin{description} -\item[SYNOPSIS] \hfill \\ \\ +\item[SYNOPSIS] \hfill \\ \\ } { \end{description} @@ -380,13 +380,13 @@ \newenvironment{envvararguments}{ \newcommand{\envvarargument}[3]{ \begin{tabular}{p{2cm} p{2cm} p{10cm}} -\textbf{##1} & \textit{##2} & {##3} \\ +\textbf{##1} & \textit{##2} & {##3} \\ \end{tabular} } \hfill \begin{description} -\item[Specifiers] \hfill \\ +\item[Memory Traits] \hfill \\ } { \hfill @@ -402,11 +402,11 @@ \newenvironment{apiarguments}{ \newcommand{\apiargument}[3]{ \begin{tabular}{p{2cm} p{2cm} p{10cm}} -\textbf{##1} & \textit{##2} & {##3} \\ +\textbf{##1} & \textit{##2} & {##3} \\ \end{tabular} } \hfill -\item[DESCRIPTION] \hfill +\item[DESCRIPTION] \hfill \begin{description} \item[Arguments] \hfill \\ @@ -419,7 +419,7 @@ \newcommand{\apidescription}[1]{ \begin{description} \vspace{-1em} -\item[API description] \hfill \\ +\item[API description] \hfill \\ #1 \hfill } @@ -431,10 +431,10 @@ \hline \tabularnewline \end{tabular}\\ #4 -} +} \newcommand{\apireturnvalues}[1]{ -\hfill +\hfill \item[Return Values] \hfill \\ #1 \\