lecture6.tex

%Template
%Copyright (C) 2019  Patrick Diehl
%
%This program is free software: you can redistribute it and/or modify
%it under the terms of the GNU General Public License as published by
%the Free Software Foundation, either version 3 of the License, or
%(at your option) any later version.

%This program is distributed in the hope that it will be useful,
%but WITHOUT ANY WARRANTY; without even the implied warranty of
%MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
%GNU General Public License for more details.

%You should have received a copy of the GNU General Public License
%along with this program.  If not, see <http://www.gnu.org/licenses/>.
\providecommand\classoption{12pt}
\documentclass[\classoption]{beamer}

\input{./template/packages}

\input{template/variables.tex}

% frame slide
\title{\coursename}
\subtitle{Lecture 6: Shared memory  parallelism}

\author{\tiny Patrick Diehl \orcid{0000-0003-3922-8419}}
%\institute {
%    \href{}{\tt \scriptsize \today}
%}
\date {
 \tiny \url{\courseurl}
\vspace{2cm}
\doclicenseThis  
  
}



\usepackage{ifxetex}

\ifxetex
\usepackage{fontspec}
\setmainfont{Raleway}
\fi

\ifluatex
\usepackage{fontspec}
\setmainfont{Raleway}
\fi

\begin{document} {
    \setbeamertemplate{footline}{}
    \frame {
        \titlepage
    }
}

\frame{

\tableofcontents

}

\AtBeginSection[]{
  \begin{frame}
  \vfill
  \centering
  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
    \usebeamerfont{title}\insertsectionhead\par%
  \end{beamercolorbox}
  \vfill
  \end{frame}
}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Reminder}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Lecture 5}
\begin{block}{What you should know from last lecture}
\begin{itemize}
\item Operator overloading
\item Header and class files
\item CMake
\end{itemize}
\end{block}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Shared memory parallelism}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Definition of parallelism}

\begin{itemize}
\item We need multiple resources which can operate at the same time
\item We have to have more than one task that can be performed at the same time
\item We have to do multiple tasks on multiple resources the same time
\end{itemize}

\end{frame}

\begin{frame}{Amdahl's Law (Strong scaling)~\cite{amdahl1967validity}}

\begin{align*}
S = \frac{1}{(1-P) + \frac{P}{N}}
\end{align*}
where $S$ is the speed up, $P$ the proportion of parallel code, and $N$ the numbers of threads.

\begin{block}{Example}
A program took 20 hours using a single thread and only the part took one hour can not be run in parallel, we will get $P=0.95$. So the theoretical speed up is $\frac{1}{(1-0.95)}=20$.
\end{block}
\begin{center}
\textcolor{blue}{Parallel computing with many threads is only beneficial for highly parallelizable programs.}
\end{center}

\end{frame}


\begin{frame}{}
\begin{figure}[tb]
\centering
\mode<beamer>{
 \begin{tikzpicture}
        \begin{axis}[
            no markers,
            samples=100,
            /pgf/declare function={
            f(\x,\a) = 1. / ((1.-\a)+(\a/x));
       		},
       		grid=both,
       		xlabel=$N$ number of threads,
       		ylabel=$S$ speedup, 	
       		legend pos=outer north east,
			legend style = {text=white,fill=background,draw=background}
        ]
            % ... and use it here
            \addplot+ [domain=1:2048,azure,thick] {f(x,0)};
            \addlegendentry{$P=0\%$}
            \addplot+ [domain=1:2048,cadetgrey,thick] {f(x,0.5)};
            \addlegendentry{$P=50\%$}
            \addplot+ [domain=1:2048,asparagus,thick] {f(x,.75)};
            \addlegendentry{$P=75\%$}
            \addplot+ [domain=1:2048,awesome,thick] {f(x,.90)};
            \addlegendentry{$P=90\%$}
            \addplot+ [domain=1:2048,black,atomictangerine,thick] {f(x,.95)};
            \addlegendentry{$P=95\%$}
       \end{axis}
    \end{tikzpicture}
    }
 \mode<handout>{
 \begin{tikzpicture}
        \begin{axis}[
            no markers,
            samples=100,
            /pgf/declare function={
            f(\x,\a) = 1. / ((1.-\a)+(\a/x));
       		},
       		grid=both,
       		xlabel=$N$ number of threads,
       		ylabel=$S$ speedup, 	
       		legend pos=outer north east
        ]
            % ... and use it here
            \addplot+ [domain=1:2048,azure,thick] {f(x,0)};
            \addlegendentry{$P=0\%$}
            \addplot+ [domain=1:2048,cadetgrey,thick] {f(x,0.5)};
            \addlegendentry{$P=50\%$}
            \addplot+ [domain=1:2048,asparagus,thick] {f(x,.75)};
            \addlegendentry{$P=75\%$}
            \addplot+ [domain=1:2048,awesome,thick] {f(x,.90)};
            \addlegendentry{$P=90\%$}
            \addplot+ [domain=1:2048,black,atomictangerine,thick] {f(x,.95)};
            \addlegendentry{$P=95\%$}
       \end{axis}
    \end{tikzpicture}
    }
\caption{Plot of Amdahl's law for different parallel portions of the code.}
\label{fig:amdals:law}
\end{figure}
\end{frame}


\begin{frame}{Example: Dot product}

\begin{center}
$S = \mathbf{X} \cdot \mathbf{V} = \sum\limits_i^N x_i y_i$
\end{center}
\vspace{-0.75cm}
\begin{align*}
\mathbf{X} = \lbrace x_1,x_2,\ldots,x_n \rbrace \\
\mathbf{Y} = \lbrace y_1,y_2,\ldots,y_n \rbrace \\
\end{align*}
\vspace{-1.5cm}
\begin{align*}
S = (x_1y_1) + (x_2y_2) + \ldots + (x_n y_n)
\end{align*}
\vspace{-1.cm}
\begin{block}{Flow chart: Sequential}
\begin{tikzpicture}
\draw (0,0) circle [radius=0.3] node {$\times$};
\draw (2,0) circle [radius=0.3] node {$\times$};
\draw (4,0) circle [radius=0.3] node {$\times$};
\draw (6,0) circle [radius=0.3] node {$\times$};
\draw (10,0) circle [radius=0.3] node {$\times$};
\node[above] at (8,0) {$\ldots$} ;

\draw (2,-1) circle [radius=0.3] node {$+$};
\draw (4,-1) circle [radius=0.3] node {$+$};
\draw (6,-1) circle [radius=0.3] node {$+$};
\draw (10,-1) circle [radius=0.3] node {$+$};
\node[above] at (8,-1) {$\ldots$} ;

\draw[->](0,-0.3) -- (1.7,-1);
\draw[->](2,-0.3) -- (2,-0.7);
\draw[->](4,-0.3) -- (4,-0.7);
\draw[->](6,-0.3) -- (6,-0.7);
\draw[->](10,-0.3) -- (10,-0.7);

\draw[->](2.3,-1) -- (3.7,-1.);
\draw[->](4.3,-1) -- (5.7,-1.);
\draw[->](6.3,-1) -- (7.7,-1.);
\draw[->](8.3,-1) -- (9.7,-1.);

\draw (-0.15,0.3) -- (-0.3,0.6);
\node[above] at (-0.3,0.6) {$x_1$} ;
\draw (2-0.15,0.3) -- (2-0.3,0.6);
\node[above] at (2-0.3,0.6) {$x_2$} ;
\draw (4-0.15,0.3) -- (4-0.3,0.6);
\node[above] at (4-0.3,0.6) {$x_3$} ;
\draw (6-0.15,0.3) -- (6-0.3,0.6);
\node[above] at (6-0.3,0.6) {$x_4$} ;
\draw (10-0.15,0.3) -- (10-0.3,0.6);
\node[above] at (10-0.3,0.6) {$x_n$} ;

\draw (0.15,0.3) -- (0.3,0.6);
\node[above] at (0.3,0.6) {$y_1$} ;
\draw (2+0.15,0.3) -- (2+0.3,0.6);
\node[above] at (2+0.3,0.6) {$y_2$} ;
\draw (4+0.15,0.3) -- (4+0.3,0.6);
\node[above] at (4+0.3,0.6) {$y_3$} ;
\draw (6+0.15,0.3) -- (6+0.3,0.6);
\node[above] at (6+0.3,0.6) {$y_4$} ;
\draw (10+0.15,0.3) -- (10+0.3,0.6);
\node[above] at (10+0.3,0.6) {$y_n$} ;

\draw[->](10,-1.3) -- (10,-1.6);
\node[below] at (10,-1.6) {$s$} ;
\end{tikzpicture}
\end{block}

\end{frame}



\begin{frame}{Parallelism approaches}

\begin{block}{Pipeline parallelism }
\begin{itemize}
\item Used in vector processors
\item Data passes between successive stages
\item Used in execution pipelines in all general microprocessors
\item Exploits
\begin{itemize}
\item Fine grain parallelism 
\item High clock speeds
\item Latency hiding
\end{itemize}
\end{itemize}
\end{block}

\begin{tikzpicture}
\draw (0,0) rectangle ++(0.75,0.75) node[pos=.5] {$+S$};
\draw (1.5,0) rectangle ++(0.75,0.75) node[pos=.5] {$xy$}; 
\draw (3.5,0) rectangle ++(2,0.75) node[pos=.5] {\text{get} $x_i$,$y_i$};

\node[above] at (8,0.5*0.75) {$\mathbf{X} = \lbrace x_1,x_2,\ldots,x_n \rbrace$} ;
\node[below] at (8,0.5*0.75) {$\mathbf{Y} = \lbrace y_1,y_2,\ldots,y_n \rbrace$} ;
\node at (-1,0.5*0.75) {$S$} ;

\draw[<-](-0.8,0.5*0.75) -- (0,0.5*0.75) ;
\draw[<-](0.75,0.5*0.75) -- (1.5,0.5*0.75) ;
\draw[<-](2.25,0.5*0.75) -- (3.5,0.5*0.75) ;
\draw[<-](5.75,0.8*0.75) -- (6,0.8*0.75) ;
\draw[<-](5.75,0.2*0.75) -- (6,0.2*0.75) ;

\end{tikzpicture}

More details~\cite{0072822562}
\end{frame}


\begin{frame}{Parallelism approaches}

\begin{block}{Single instructions and multiple data (SIMD)}
\begin{itemize}
\item All perform same operation at the same time
\item But may perform different operations at different times
\item Each operates on separate data
\item Used in accelerators on microprocessors
\item Scales as long as data scales
\end{itemize}
\end{block}

SIMD is part of Flynn's taxonomy, a classification of computer architectures, proposed by Michael J. Flynn in 1966~\cite{flynn1972some,duncan1990survey}.
\end{frame}


\begin{frame}{Flow chart: SIMD}

\begin{columns}[T]
\begin{column}{0.4\textwidth}
\begin{block}{Algorithm}
   \begin{enumerate}
   \item $S=0$
   \item Get $x_{i+1},y_{i+1}$
   \item Compute $xy$
   \item Add to $S$
   \item More data, go to 2
   \item Send $S$ to reduce
   \item Stop
   \end{enumerate}
   \end{block}
\end{column}
\begin{column}{0.6\textwidth}  %%<--- here
   \begin{tikzpicture}
   \draw (0,0) rectangle ++(0.75,0.75) node[pos=.5] {$P_1$}; 
   \draw (1.5,0) rectangle ++(0.75,0.75) node[pos=.5] {$P_2$}; 
   \draw (3,0) rectangle ++(0.75,0.75) node[pos=.5] {$P_3$}; 
   \draw (4.5,0) rectangle ++(0.75,0.75) node[pos=.5] {$P_4$}; 
   
   \draw (0.75,-2) rectangle ++(0.75,0.75) node[pos=.5] {$+$}; 
   \draw (3.75,-2) rectangle ++(0.75,0.75) node[pos=.5] {$+$}; 
   
   \draw (2.25,-4) rectangle ++(0.75,0.75) node[pos=.5] {$+$}; 
   
   \draw[->] (0.75+0.5*0.75,-2) -- (2.25,-4+0.75) ;
   \draw[->] (3.75+0.5*0.75,-2) -- (3,-4+0.75) ;
   
   \draw[->] (1.5+0.5*0.75,0) -- (1.5,-2+0.75) ;
   \draw[->] (0.5*0.75,0) -- (0.75,-2+0.75) ; 
   
   \draw[->] (3+0.5*0.75,0) -- (3.75,-2+0.75) ;
   \draw[->] (4.5+0.5*0.75,0) -- (4.5,-2+0.75) ; 
   
   \draw[->] (2.25+0.5*0.75,-4) -- (2.25+0.5*0.75,-4.5) ; 
   
   \draw[dashed] (0,-0.75) -- (5,-0.75) ;
   \node[below,rotate=90] at (-.75,-2) {\small Reduction tree}; 
   
   \node at (0.5*0.75,1.5) {\tiny $X=\lbrace x_1,x_2 \rbrace$};
   \node at (0.5*0.75,1.25) {\tiny $Y=\lbrace x_9,x_{10} \rbrace$};
   \node at (1.5+0.5*0.75,1.5) {\tiny $X=\lbrace x_3,x_4 \rbrace$};
   \node at (1.5+0.5*0.75,1.25) {\tiny $Y=\lbrace x_{11},x_{12} \rbrace$};
   \node at (3.+0.5*0.75,1.5) {\tiny $X=\lbrace x_5,x_6 \rbrace$};
   \node at (3.+0.5*0.75,1.25) {\tiny $Y=\lbrace x_{13},x_{14} \rbrace$}; 
   \node at (4.5+0.5*0.75,1.5) {\tiny $X=\lbrace x_7,x_8 \rbrace$};
   \node at (4.5+0.5*0.75,1.25) {\tiny $Y=\lbrace x_{15},x_{16} \rbrace$};
   
   \draw[->] (0.5*0.75,1.125) -- (0.5*0.75,0.75) ;
   \draw[->] (1.5+0.5*0.75,1.125) -- (1.5+0.5*0.75,0.75) ;
   \draw[->] (3.+0.5*0.75,1.125) -- (3.+0.5*0.75,0.75) ;
   \draw[->] (4.5+0.5*0.75,1.125) -- (4.5+0.5*0.75,0.75) ;
   \end{tikzpicture}
\end{column}
\end{columns}
\vspace{0.25cm}
Reduction tree: Exploits fine grain functions and need global communications
\end{frame}

\begin{frame}{Uniform memory access (UMA)}

\begin{center}
\begin{tikzpicture}
%Threads
\draw (0,3) rectangle (.5,3.5) node[pos=.5] {1};
\draw (0.5,3) rectangle (1.0,3.5) node[pos=.5] {..};
\draw (1.,3) rectangle (1.5,3.5) node[pos=.5] {n};

\draw (2.5,3) rectangle (3.,3.5) node[pos=.5] {1};
\draw (3.,3) rectangle (3.5,3.5) node[pos=.5] {..};
\draw (3.5,3) rectangle (4,3.5) node[pos=.5] {n};

\draw (0.75,2.5) -- (0.75,3.);
\draw (3.25,2.5) -- (3.25,3.);

%BUS
\draw (0,1) rectangle (4,1.5) node[pos=.5] {Bus};


%CPU
\draw (0,2) rectangle (1.5,2.5) node[pos=.5] {CPU 1};
\draw (2.5,2) rectangle (4,2.5) node[pos=.5] {CPU 2};
\draw (0.75,2) -- (0.75,1.5);
\draw (3.25,2) -- (3.25,1.5);

%Memory
\draw (0,0) rectangle (4,0.5) node[pos=.5] {Memory};
\draw (0.75,.5) -- (0.75,1.);
\draw (3.25,.5) -- (3.25,1.);
\end{tikzpicture}
\end{center}

\begin{block}{Access times}
\begin{itemize}
\item Memory access times are the same
\end{itemize}
\end{block}
More details~\cite{el2005advanced,hager2010introduction}.
\end{frame}

\begin{frame}{Non-uniform memory access (NUMA)}


\begin{center}
\begin{tikzpicture}
%Threads
\draw (0,3) rectangle (.5,3.5) node[pos=.5] {1};
\draw (0.5,3) rectangle (1.0,3.5) node[pos=.5] {..};
\draw (1.,3) rectangle (1.5,3.5) node[pos=.5] {n};

\draw (2.5,3) rectangle (3.,3.5) node[pos=.5] {1};
\draw (3.,3) rectangle (3.5,3.5) node[pos=.5] {..};
\draw (3.5,3) rectangle (4,3.5) node[pos=.5] {n};

\draw (0.75,2.5) -- (0.75,3.);
\draw (3.25,2.5) -- (3.25,3.);

%BUS
\draw (0,1) rectangle (1.5,1.5) node[pos=.5] {Bus};
\draw (2.5,1) rectangle (4,1.5) node[pos=.5] {Bus};

%CPU
\draw (0,2) rectangle (1.5,2.5) node[pos=.5] {CPU 1};
\draw (2.5,2) rectangle (4,2.5) node[pos=.5] {CPU 2};
\draw (0.75,2) -- (0.75,1.5);
\draw (3.25,2) -- (3.25,1.5);

%Memory
\draw (0,0) rectangle (1.5,0.5) node[pos=.5] {Memory};
\draw (2.5,0) rectangle (4,0.5) node[pos=.5] {Memory};
\draw (0.75,.5) -- (0.75,1.);
\draw (3.25,.5) -- (3.25,1.);
\end{tikzpicture}
\end{center}

Access time to the memory depends on the memory location relative to the CPU.

\begin{block}{Access times}
\begin{itemize}
\item Local memory access is fast
\item Non-local memory access has some overhead
\end{itemize}
\end{block}
\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Parallel algorithms}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[fragile]{Parallel algorithms in C++ 17\footnote{\tiny\url{https://en.cppreference.com/w/cpp/experimental/parallelism}}}
\begin{itemize}
\item C++17 added support for parallel algorithms to the standard library, to help programs take advantage of parallel execution for improved performance.
\item Parallelized versions of 69 algorithms from \lstinline|<algorithm>|, \lstinline|<numeric>| and \lstinline|<memory>| are available. 
\end{itemize}

\begin{block}{Recently new feature!}
Only recently released compilers (gcc 9 and MSVC 19.14)\footnote{\tiny\url{https://en.cppreference.com/w/cpp/compiler_support}} implement these new features and some of them are still experimental.\\
\vspace{0.5cm}
Some special compiler flags are needed to use these features:

\begin{lstlisting}[language=bash]
g++ -std=c++1z -ltbb lecture6-loops.cpp 
\end{lstlisting}
\end{block}
\end{frame}

\begin{frame}[fragile]{Example: Accumulate}

\begin{lstlisting}
std::vector<int> nums(1000000,1);
\end{lstlisting}

\begin{block}{Sequential\footnote{\tiny\url{https://en.cppreference.com/w/cpp/algorithm/accumulate}}}
\begin{lstlisting}
auto result = std::accumulate(nums.begin(), 
                              nums.end(),
                              0.0);
\end{lstlisting}
\end{block}

\begin{block}{Parallel\footnote{\tiny\url{https://en.cppreference.com/w/cpp/experimental/reduce}}}
\begin{lstlisting}
auto result = std::reduce(
              std::execution::par,
              nums.begin(), nums.end());
\end{lstlisting}
\end{block}

Important: \lstinline|std::execution::par| from \lstinline|#include<execution>|\footnote{\tiny\url{https://en.cppreference.com/w/cpp/experimental/execution_policy_tag}}
\end{frame}


\begin{frame}[fragile]{Execution time}

\begin{block}{Time measurements}
\begin{lstlisting}[language=bash]
g++ -std=c++1z -ltbb lecture6-loops.cpp 
./a.out
std::accumulate result 9e+08 took 10370.689498 ms
std::reduce result 9.000000e+08 took 612.173647 ms
\end{lstlisting}


\end{block}


\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Execution policies}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\begin{frame}{Execution policies}

\begin{itemize}
\item \lstinline|std::execution::seq| \\
The algorithm is executed sequential, like \lstinline|std::accumulate| in the previous example and using only once thread.
\item \lstinline|std::execution::par| \\
The algorithm is executed in parallel and used multiple threads.
\item \lstinline| std::execution::par_unseq| \\
The algorithm is executed in parallel and vectorization is used.
\end{itemize}
Note we will not cover vectorization in this course. \\
\vspace{0.5cm}
Fore more details: CppCon 2016: Bryce Adelstein Lelbach “The C++17 Parallel Algorithms Library and Beyond"\footnote{\tiny{\url{https://www.youtube.com/watch?v=Vck6kzWjY88}}}
\end{frame}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Be aware of: Data races and Deadlocks}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}{Be aware of}

\begin{center}
With great power comes great responsibility!

\end{center}


\begin{block}{You are responsible}
When using parallel execution policy, it is the programmer's responsibility to avoid
\begin{itemize}
\item data races
\item race conditions
\item deadlocks
\end{itemize} 
\end{block}

\end{frame}

\begin{frame}[fragile]{Data race}

\begin{lstlisting}
//Compute the sum of the array a in parallel
int a[] = {0,1};
int sum = 0;
std::for_each(std::execution::par, 
              std::begin(a), 
              std::end(a), [&](int i) {
  sum += a[i]; // Error: Data race
});
\end{lstlisting}

\begin{block}{Data race:}
A data race exists when multithreaded (or otherwise parallel) code that would access a shared resource could do so in such a way as to cause unexpected results.
\end{block}

\end{frame}

\begin{frame}[fragile]{Solution I: data races}

\begin{block}{\lstinline|std::atomic|\footnote{\tiny\url{https://en.cppreference.com/w/cpp/atomic/atomic}}}
\begin{lstlisting}
//Compute the sum of the array a in parallel
int a[] = {0,1};
std::atomic<int> sum{0};
std::for_each(std::execution::par, 
              std::begin(a), 
              std::end(a), [&](int i) {
  sum += a[i]; 
});
\end{lstlisting}
\end{block}

The atomic library\footnote{\tiny\url{https://en.cppreference.com/w/cpp/atomic}} provides components for fine-grained atomic operations allowing for lockless concurrent programming. Each atomic operation is indivisible with regards to any other atomic operation that involves the same object. Atomic objects are free of data races. 
\end{frame}


\begin{frame}[fragile]{Solution 2: data races}
\begin{block}{\lstinline|std::mutex|\footnote{\tiny\url{https://en.cppreference.com/w/cpp/thread/mutex}}}
\begin{lstlisting}
//Compute the sum of the array a in parallel
int a[] = {0,1};
int sum = 0;
std::mutex m;
std::for_each(std::execution::par, 
              std::begin(a), 
              std::end(a), [&](int i) {
  m.lock();
  sum += a[i];
  m.unlock(); 
});
\end{lstlisting}
\end{block}
The mutex class is a synchronization primitive that can be used to protect shared data from being simultaneously accessed by multiple threads. 
\end{frame}


\begin{frame}[fragile]{Race condition}

\begin{lstlisting}

if (x == 5)  // Checking x
{
   // Different thread could change x 
      
   y = x * 2; // Using x
}
// It is not sure if y is 10 or any other value.
\end{lstlisting}

\begin{block}{Race condition}
A check of a shared variable within a parallel execution and another thread could change this variable before it is used.
\end{block}

\end{frame}

\begin{frame}[fragile]{Solution: Race condition}


\begin{lstlisting}
std::mutex m;

m.lock();
if (x == 5)  // Checking x
{
   // Different thread could change x 
      
   y = x * 2; // Using x
}
m.unlock();
// Now it is sure that y will be 10
\end{lstlisting}

\begin{block}{Race condition}
A check of a shared variable within a parallel execution and another thread could change this variable before it is used.
\end{block}


\end{frame}

\begin{frame}{Deadlocks}
Deadlock describes a situation where two or more threads are blocked forever, waiting for each other.

\begin{block}{Example (Taken from\footnote{\tiny{\url{https://docs.oracle.com/javase/tutorial/essential/concurrency/deadlock.html}}})}
Alphonse and Gaston are friends, and great believers in courtesy. A strict rule of courtesy is that when you bow to a friend, you must remain bowed until your friend has a chance to return the bow. Unfortunately, this rule does not account for the possibility that two friends might bow to each other at the same time.
\end{block}
\vspace{1cm}
Example: lecture7-deadlocks.cpp

\end{frame}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Summary}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{frame}{Summary}
\begin{block}{After this lecture, you should know}
\begin{itemize}
\item Shared memory parallelism
\item Parallel algorithms
\item Execution policies
\item Race condition, data race, and deadlocks
\end{itemize}
\end{block}

\begin{block}{Further reading:}
C++ Lecture 3 - Modern Paralization Techniques\footnote{\tiny\url{https://www.youtube.com/watch?v=1DUW5Qw3eck}}: OpenMP for shared memory parallelism and the Message Passing Interface for distributed memory parallelism. Note that HPX which will we cover after the midterm is introduced there.
\end{block}

\end{frame}




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{References}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\begin{frame}[t, allowframebreaks]
\frametitle{References}
\bibliographystyle{plain}
\bibliography{bib}
\end{frame}


\end{document}