\documentclass[a4paper,11pt]{article}
%\documentclass[a4paper,11pt]{scrartcl}

\input{../preambles/preamble}
\input{../preambles/unicode}

\setmainlanguage{english}
\setotherlanguages{german,greek,russian}

\input{../preambles/math-single}
\input{../preambles/math-brac}
\input{../preambles/math-thm}
\input{../preambles/phys-chem}

%\setromanfont[Mapping=tex-text]{Linux Libertine O}
% \setsansfont[Mapping=tex-text]{DejaVu Sans}
% \setmonofont[Mapping=tex-text]{DejaVu Sans Mono}

\usepackage[%style=authoryear-icomp,
			backend=biber]{biblatex}
\addbibresource{./maxwell-proca.bib}

\title{Hamiltonian Dynamics of Maxwell--Proca Theory}
\author{Yi-Fan Wang (王\ 一帆)}
%\date{}

\begin{document}
\maketitle

\section{Maxwell--Proca theory in flat space-time}

Consider a Maxwell--Proca theory in Minkowski space-time with source
\begin{equation}
\Ld = -\frac{1}{4} F_{\mu\nu} F^{\mu\nu} - \frac{1}{2} m^2 A_\mu A^\mu
+ A_\mu J^\mu,
\end{equation}
where $m > 0$ corresponds to the Proca theory \cite[sec.\ 2.3]{Gitman1990}, and
$m = 0$ the Maxwell theory \cite[sec.\ 3.3.3]{Rothe2010}, \cite[sec.\
2.4]{Gitman1990}.

Since the source is external and not dynamical, it seems necessary to impose
$\partial_\mu J^\mu = 0$ by hand.

The action with velocity is
\begin{align}
\sfun{S^\text{v}}{A, \Pi, V} \coloneqq \int\dd t\int\dd^d x\,
\cbr{\Ld^\text{v} + \Pi^\mu\rbr{\dot A_\mu-V_\mu}},
\end{align}
where the Lagrangian density with velocity reads
\begin{align}
\Ld^\text{v} = \frac{1}{2} \rbr{V_i - \partial_i A_0}^2 - \frac{1}{4} F_{ij}^2
+ \frac{m^2}{2} \rbr{A_0^2 - A_i^2} + A_0 J^0 + A_i J^i.
\label{eq:lagrangian-density-with-velocity}
\end{align}
On the velocity shell, the canonical momenta densities are
\begin{equation}
\Pi^0 \coloneqq \frpa{\Ld^\text{v}}{V_0} = 0,\qquad
\Pi^i \coloneqq \frpa{\Ld^\text{v}}{V_i} = V^i - \partial^i A_0.
\end{equation}
The fundamental Poisson brackets are
\begin{equation}
\sbr{\rfun{A_\mu}{\vec x_1}, \rfun{\Pi^\nu}{\vec x_2}}_\text{P} =
\delta_\mu{}^\nu\rfun{\delta^d}{\vec x_1 - \vec x_2}.
\end{equation}

Brining the $V_i$'s on shell, the primary action reads
\begin{align}
\sfun{S^\text{p}}{A, \Pi, V_0} = \int \dd t\int\dd^d x
\rbr{\mscrH^\text{p}+\Pi^\mu \dot A_\mu + \partial_i\rbr{\Pi^i A_0}},
\end{align}
in which the primary Hamiltonian is
\begin{align}
\mscrH^\text{p} &= \frac{1}{2} \rbr{\Pi^i}^2 + \frac{1}{4} F_{ij}^2 +
\frac{m^2}{2} \rbr{-A_0^2 + A_i^2} - A_i J^i \nonumber \\
&\quad + V_0 \Pi^0 - A_0 \rbr{\partial_i \Pi^i + J^0},
\label{eq:primary-Hamil-general-MP}
\end{align}
and
\begin{equation}
\Phi_1 \coloneqq \Pi^0
\end{equation}
is the only primary constraint.

In $\rbr{3+1}$-dimensions, the electromagnetic potentials, fields and current
are
\begin{align}
\Phi &= -A_0,\qquad \vec A^i = A^i;
\\
E_i &= -F_{0i} = F_{i0} =
\partial_i A_0 - \partial_0 A_i = \partial_i A_0 - V_i = -\Pi_i;
\\
B^i = \frac{1}{2} \epsilon^{ijk} F_{ij},\qquad
F_{ij} &= \epsilon_{ijk}B^k;
\\
\rho &= J^0, \qquad \vec J^i = J^i.
\end{align}
\Cref{eq:lagrangian-density-with-velocity,eq:primary-Hamil-general-MP} can also
be written as
\begin{align}
\Ld^\text{v} &= \frac{1}{2} \rbr{\vec E^2 - \vec B^2}
+ \frac{m^2}{2} \rbr{\Phi^2 - \vec A^2} - \rho\Phi + \vec A\cdot\vec J,
\\
\mscrH^\text{p} &= \frac{1}{2}\rbr{\vec E^2+\vec B^2}
- \frac{m^2}{2} \rbr{\Phi^2-\vec A^2} - \vec A\cdot J
+ V_0 \Pi^0 + \Phi\rbr{-\nabla\cdot E + \rho}
\end{align}

\subsection{Constraint algebra}
The Poisson bracket of $\Phi_1$ and $\mscrH^\text{p}$ is
\begin{align}
\sbr{\rfun{\Phi_1}{\vec{x}_1},\rfun{\mscrH^\text{p}}{\vec{x}_2}}_\text{P} &=
\sbr{\rbr{\Pi^0}_1,
-\frac{m^2}{2}A_0^2-A_0\rbr{\partial_i \Pi^i + J^0}}_\text{P}
\nonumber \\
&= \rbr{-m^2 A^0 + \partial_i \Pi^i + J^0}_2
\rfun{\delta}{\vec{x}_1-\vec{x}_2}.
\end{align}
Integration with $\dd^d x_2$ yields the secondary constraint
\begin{equation}
\sbr{\Phi_1, H^\text{p}}_\text{P} = -m^2 A^0 + \partial_i \Pi^i + J^0
\eqqcolon \Phi_2,
\label{eq:secondary-general-MP}
\end{equation}
so that
\begin{equation}
\sbr{\rfun{\Phi_1}{\vec{x}_1},\rfun{\Phi_2}{\vec{x}_2}}_\text{P} =
-m^2\rfun{\delta}{\vec{x}_1-\vec{x}_2}.
\label{eq:Maxwell-Proca-Q}
\end{equation}
One may further compute
\begin{align}
&\quad\sbr{\rfun{\Phi_2}{\vec{x}_1},
\rfun{\mscrH^\text{p}}{\vec{x}_2}}_\text{P} \nonumber \\
&= \sbr{\rbr{\partial_i \Pi^i}_1,
\rbr{\frac{1}{4} F_{jk}^2 + \frac{m^2}{2} A_j^2 - A_j J^j}_2}_\text{P}
+ \sbr{-m^2 \rbr{A^0}_1, \rbr{V_0 \Pi^0}_2}_\text{P},
\end{align}
in which
\begin{align}
\sbr{\rbr{\partial_i \Pi^i}_1, \rbr{\frac{1}{4}
F_{jk}^2}_2}_\text{P} &=
\rbr{\partial_j A_k - \partial_k A_j}_2 \rbr{\partial_i}_1
\sbr{\rbr{\Pi^i}_1, \rbr{\partial^j A^k}_2}_\text{P} \nonumber \\
&= -\rbr{F^{ij} \partial_j}_2 \rbr{\partial_i}_1
\rfun{\delta}{\vec{x}_1-\vec{x}_2}.
\end{align}
The Poisson bracket can be evaluated as
\begin{align}
&\quad \sbr{\rfun{\Phi_2}{\vec{x}_1},
\rfun{\mscrH^\text{p}}{\vec{x}_2}}_\text{P} \\
&= \rbr{-\rbr{F^{ij} \partial_j + m^2 A^i + J^i}_2 \rbr{\partial_i}_1 + m^2
\rbr{V_0}_2}
\rfun{\delta}{\vec{x}_1-\vec{x}_2}.
\end{align}
Integration with $\dd^d x_2$ yields
\begin{equation}
\sbr{\Phi_2, H^\text{p}}_\text{P} = -\partial_i\rbr{m^2 A^i + J^i} + m^2 V_0.
\label{eq:persistence-Phi2}
\end{equation}

\subsection{Proca theory}

For Proca theory $m > 0$ , then the algorithm terminates, and one obtains a
pure second-class system.
\begin{equation}
\mbfQ = \begin{pmatrix}0 & -m^2 \\ +m^2 & 0\end{pmatrix},
\qquad
\mbfQ^{-1} = \begin{pmatrix}0 & +m^{-2} \\ -m^{-2} & 0\end{pmatrix}.
\end{equation}
Dirac bracket
\begin{equation}
\begin{split}
&\phantom{}
\sbr{\rfun{f}{\vec{x}_1},\rfun{g}{\vec{x}_2}}_\text{D} =
\sbr{\rbr{f}_1, \rbr{g}_2}_\text{P} \\
+ \int \dd^d x_3
\Bigl(&- \sbr{\rbr{f}_1, \rbr{\Pi^0}_3}_\text{P}
\sbr{\rbr{m^{-2}\partial_i \Pi^i + A_0}_3, \rbr{g}_2}_\text{P} \\
&+ \sbr{\rbr{f}_1, \rbr{m^{-2}\partial_i \Pi^i + A_0}_3}_\text{P}
\sbr{\rbr{\Pi^0}_3, \rbr{g}_2}_\text{P} \Bigr).
\end{split}
\end{equation}
The fundamental Dirac brackets, which are different from Poisson brackets, are
\begin{equation}
\begin{split}
\sbr{\rfun{A_0}{\vec{x}_1},\rfun{A_i}{\vec{x}_2}}_\text{D} &=
m^{-2} \rbr{\partial_i}_1 \rfun{\delta}{\vec{x}_1 - \vec{x}_2}, \\
\sbr{\rfun{A_0}{\vec{x}_1},\rfun{\Pi^0}{\vec{x}_2}}_\text{D} &= 0.
\end{split}
\end{equation}

\subsubsection{Physical coordinates}

Introducing the regularising coordinates
\begin{alignat}{3}
\alpha_i &= A_i + m^{-2}\rbr{\partial_i \Pi^0 - J_i},&\qquad \beta^i &= \Pi^i;
\\
\alpha_0 &= A_0 + m^{-2}\rbr{\partial_i \Pi^i - J_0},&\qquad \beta^0 &= \Pi^0.
\end{alignat}
It is easy\footnote{Really? Have I done it?} to show that
\begin{align}
\sbr{\rfun{\alpha_i}{\vec{x}_1},\rfun{\beta^j}{\vec{x}_2}}_\text{D} &=
\delta^i{}_j\rfun{\delta}{\vec{x}_1,\vec{x}_2},\\
\sbr{\rfun{\alpha_i}{\vec{x}_1},\rfun{\alpha_j}{\vec{x}_2}}_\text{D} &= 0 =
\sbr{\rfun{\beta^i}{\vec{x}_1},\rfun{\beta^j}{\vec{x}_2}}_\text{D}.
\end{align}
Furthermore, one has
\begin{equation}
\mscrH^\text{p} = \mscrH^\text{phy}+\mscrH^\text{con}+\mscrH^\text{irr},
\end{equation}
where\footnote{This is to be re-calculated, since a boundary term has been
split at the beginning.}
\begin{align}
\begin{split}
\mscrH^\text{phy} &= \frac{1}{2}\rbr{\beta^i}^2 + \frac{m^2}{2}\alpha_i^2 +
\frac{1}{4}\rbr{\partial_i \alpha_j - \partial_j \alpha_i}^2 +
\frac{1}{2m^2}\rbr{\partial_i \beta^i}^2 \\
&\quad +\frac{1}{m^2} J^0 \partial_i \beta^i,
\end{split}\\
\mscrH^\text{con} &= -\frac{m^2}{2}\alpha_0^2 - \frac{1}{2m^2}
\rbr{\partial_i \beta^0}^2,\\
\begin{split}
\mscrH^\text{irr} &= \partial_i\rbr{\alpha_0\beta^i - \beta^0\alpha_i +
\frac{1}{m^2}\rbr{\beta^0\partial_i\beta^0 -
\beta^i\partial_j\beta^j - J^0 \beta^i}} \\
&\quad + \frac{1}{2m^2}\rbr{\rbr{J^0}^2 - \rbr{J^i}^2}.
\end{split}
\end{align}
Further more,
\begin{equation}
\Phi_1 = \beta^0,\qquad\Phi_2 = m^2\alpha_0 \propto \alpha_0.
\end{equation}
Thus the $\rbr{\alpha_i, \beta^i}$ are regular pairs of canonical variables,
whereas $\rbr{\alpha_0, \beta^0}$ are the singular variables as constraints.
The canonical dynamics of the physical $\rbr{\alpha_i, \beta^i}$'s are
determined by $\mscrH^\text{phy}$ as a regular system.

\subsection{Free Maxwell theory}

For Maxwell theory $m = 0$. The primary Hamiltonian in
\cref{eq:primary-Hamil-general-MP} takes the form
\begin{equation}
\mscrH^\text{p} = \frac{1}{2} \rbr{\Pi^i}^2 + \frac{1}{4} F_{ij}^2 - A_i J^i
+V_0 \Pi^0 - A_0 \rbr{\partial_i \Pi^i + J^0},
\label{eq:primary-Hamil-Maxwell}
\end{equation}
the secondary constraint $\Phi_2$ in \cref{eq:secondary-general-MP} now
reads
\begin{equation}
\Phi_2 = \partial_i \Pi^i + J^0.
\end{equation}
In $\rbr{3+1}$ dimensions, the first two terms in
\cref{eq:primary-Hamil-Maxwell} reads
\begin{equation}
\frac{1}{2}\rbr{\Pi^i}^2 + \frac{1}{4}F_{ij}^2 =
\frac{1}{2}\rbr{\vec E^2 + \vec B^2}.
\end{equation}

Since the Poisson bracket of $\Phi_2$ and $H^\text{p}$
\begin{equation}
\sbr{\Phi_2, H^\text{p}}_\text{P} = -\partial_i J^i
\end{equation}
contains now no canonical variable, the algorithm terminates. Furthermore,
the constraint algebra is commutative, hence the system is a purely
first-class one.

Persistence condition on $\Phi_2$ requires
\begin{align}
\partial_i J^i = 0,
\end{align}
which is confusing.

\subsubsection{Gauge transformation}

\subsubsection{Physical coordinates}

\printbibliography

\end{document}


%=============================================================================
%=============================================================================

\chapter{Structured-Grid System Interface (Struct)}
\label{ch-Struct}

In order to get access to the most efficient and scalable solvers for
scalar structured-grid applications, users should use the
\code{Struct} interface described in this chapter.  This interface
will also provide access (this is not yet supported) to solvers in
\hypre{} that were designed for unstructured-grid applications and
sparse linear systems in general.  These additional solvers are
usually provided via the unstructured-grid interface (\code{FEI}) or
the linear-algebraic interface (\code{IJ}) described in Chapters
\ref{ch-FEI} and \ref{ch-IJ}.

Figure~\ref{fig-struct-example} gives an example of the type of grid
currently supported by the \code{Struct} interface.  The interface
uses a finite-difference or finite-volume style, and currently
supports only scalar PDEs (i.e., one unknown per gridpoint).
\begin{figure}
\centering
\includegraphics[width=.5\textwidth]{figStructExample1}
\caption{%
An example 2D structured grid, distributed accross two processors.}
\label{fig-struct-example}
\end{figure}
There are four basic steps involved in setting up the linear system
to be solved:
%\begin{enumerate}
\begin{list}{\arabic{enumi}.}{\usecounter{enumi}\setlength{\itemsep}{0in}}
\item set up the grid,
\item set up the stencil,
\item set up the matrix,
\item set up the right-hand-side vector.
\end{list}
%\end{enumerate}
To describe each of these steps in more detail, consider solving the
2D Laplacian problem
\begin{equation}\label{eqn-laplacian}
\left \{
\begin{array}{ll}
\nabla^2 u = f , & \mbox{in the domain}, \\
u = 0,           & \mbox{on the boundary}.
\end{array}
\right .
\end{equation}
Assume (\ref{eqn-laplacian}) is discretized using standard 5-pt finite-volumes
on the uniform grid pictured in \ref{fig-struct-example}, and assume that the
problem data is distributed across two processes as depicted.

%-----------------------------------------------------------------------------

\section{Setting Up the Struct Grid}
\label{sec-Struct-Grid}

The grid is described via a global {\em index space}, i.e., via integer singles
in 1D, tuples in 2D, or triples in 3D (see Figure~\ref{fig-struct-boxes}).
\begin{figure}
\centering
\includegraphics[width=.5\textwidth]{figStructGridBoxes}
\caption{%
A box is a collection of abstract cell-centered indices, described by its
minimum and maximum indices.  Here, two boxes are illustrated.}
\label{fig-struct-boxes}
\end{figure}
The integers may have any value, negative or positive.  The global indexes
allow \hypre{} to discern how data is related spatially, and how it is
distributed across the parallel machine.  The basic component of the grid is a
{\em box}: a collection of abstract cell-centered indices in index space,
described by its ``lower'' and ``upper'' corner indices.  The scalar grid data
is always associated with cell centers, unlike the more general \code{SStruct}
interface which allows data to be associated with box indices in several
different ways.

Each process describes that portion of the grid that it ``owns'', one box at a
time.  For example, the global grid in Figure~\ref{fig-struct-example} can be
described in terms of three boxes, two owned by process 0, and one owned by
process 1.  Figure~\ref{fig-struct-grid} shows the code for setting up the grid
on process 0 (the code for process 1 is similar).
\begin{figure}
\centering
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructGrid1}\vspace{-.5em} \\ 1
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructGrid2}\vspace{-.5em} \\ 2
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructGrid3}\vspace{-.5em} \\ 3
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructGrid4}\vspace{-.5em} \\ 4
\end{tabular}
\vspace{2em} \\
\begin{minipage}{0.7\textwidth}
\begin{verbatim}

    HYPRE_StructGrid grid;
    int ndim        = 2;
    int ilower[][2] = {{-3,1}, {0,1}};
    int iupper[][2] = {{-1,2}, {2,4}};

    /* Create the grid object */
1:  HYPRE_StructGridCreate(MPI_COMM_WORLD, ndim, &grid);

    /* Set grid extents for the first box */
2:  HYPRE_StructGridSetExtents(grid, ilower[0], iupper[0]);

    /* Set grid extents for the second box */
3:  HYPRE_StructGridSetExtents(grid, ilower[1], iupper[1]);

    /* Assemble the grid */
4:  HYPRE_StructGridAssemble(grid);

\end{verbatim}
\end{minipage}
\caption{%
Code on process 0 for setting up the grid in Figure~\ref{fig-struct-example}.}
\label{fig-struct-grid}
\end{figure}
The ``icons'' at the top of the figure illustrate the result of the numbered
lines of code.  The \code{Create()} routine creates an empty 2D grid object
that lives on the \code{MPI_COMM_WORLD} communicator.  The \code{SetExtents()}
routine adds a new box to the grid.  The \code{Assemble()} routine is a
collective call (i.e., must be called on all processes from a common
synchronization point), and finalizes the grid assembly, making the grid
``ready to use''.

%-----------------------------------------------------------------------------

\section{Setting Up the Struct Stencil}
\label{sec-Struct-Stencil}

The geometry of the discretization stencil is described by an array of indexes,
each representing a relative offset from any given gridpoint on the grid.  For
example, the geometry of the 5-pt stencil for the example problem being
considered can be represented by the list of index offsets shown in
Figure~\ref{fig-struct-stencil-a}.
\begin{figure}
\centering
\mbox{}\hfill
\includegraphics[width=.3\textwidth]{figStructStenc0}
\hfill
\includegraphics[width=.25\textwidth]{figStructStenc7}
\hfill\mbox{}
\caption{%
Representation of the 5-point discretization stencil for the example problem.}
\label{fig-struct-stencil-a}
\end{figure}
Here, the $(0,0)$ entry represents the ``center'' coefficient, and is the 0th
stencil entry.  The $(0,-1)$ entry represents the ``south'' coefficient, and is
the 3rd stencil entry.  And so on.

On process 0 or 1, the code in Figure~\ref{fig-struct-stencil-b} will set up
the stencil in Figure~\ref{fig-struct-stencil-a}.  The stencil must be the same
on all processes.
\begin{figure}
\centering
\mbox{}\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructStenc1} \\ 1
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructStenc2} \\ 2
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructStenc3} \\ 3
\end{tabular}
\hfill\mbox{}
\vspace{1em} \\
\mbox{}\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructStenc4} \\ 4
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructStenc5} \\ 5
\end{tabular}
\hfill
\begin{tabular}{@{}c@{}}
\includegraphics[width=.22\textwidth]{figStructStenc6} \\ 6
\end{tabular}
\hfill\mbox{}
\vspace{2em} \\
\begin{minipage}{0.85\textwidth}
\begin{verbatim}

      HYPRE_StructStencil stencil;
      int ndim         = 2;
      int size         = 5;
      int entry;
      int offsets[][2] = {{0,0}, {-1,0}, {1,0}, {0,-1}, {0,1}};

      /* Create the stencil object */
  1:  HYPRE_StructStencilCreate(ndim, size, &stencil);

      /* Set stencil entries */
      for (entry = 0; entry < size; entry++)
      {
2-6:     HYPRE_StructStencilSetElement(stencil, entry, offsets[entry]);
      }

      /* Thats it!  There is no assemble routine */

\end{verbatim}
\end{minipage}
\caption{%
Code for setting up the stencil in Figure~\ref{fig-struct-stencil-a}.}
\label{fig-struct-stencil-b}
\end{figure}
The \code{Create()} routine creates an empty 2D, 5-pt stencil object.  The
\code{SetElement()} routine defines the geometry of the stencil and assigns the
stencil numbers for each of the stencil entries.  None of the calls are
collective calls.

%-----------------------------------------------------------------------------

\section{Setting Up the Struct Matrix}
\label{sec-Struct-Matrix}

The matrix is set up in terms of the grid and stencil objects described in
Sections \ref{sec-Struct-Grid} and \ref{sec-Struct-Stencil}.  The coefficients
associated with each stencil entry will typically vary from gridpoint to
gridpoint, but in the example problem being considered, they are as follows
over the entire grid (except at boundaries; see below):
\begin{equation}\label{eqn-stencil-laplacian}
\left [
\begin{array}{ccc}
    & -1 &    \\
 -1 &  4 & -1 \\
    & -1 &
\end{array}
\right ] .
\end{equation}

On process 0, the code in Figure~\ref{fig-struct-matrix} will set up matrix
values associated with the center (entry 0) and south (entry 3) stencil entries
as given by \ref{eqn-stencil-laplacian} and Figure~\ref{fig-struct-matrix}
(boundaries are ignored here temporarily).
\begin{figure}
\centering
\begin{minipage}{0.75\textwidth}
\begin{verbatim}

HYPRE_StructMatrix  A;
double              values[36];
int                 stencil_indices[2] = {0,3};
int                 i;

HYPRE_StructMatrixCreate(MPI_COMM_WORLD, grid, stencil, &A);
HYPRE_StructMatrixInitialize(A);

for (i = 0; i < 36; i += 2)
{
   values[i]   =  4.0;
   values[i+1] = -1.0;
}

HYPRE_StructMatrixSetBoxValues(A, ilower[0], iupper[0], 2,
                               stencil_indices, values);
HYPRE_StructMatrixSetBoxValues(A, ilower[1], iupper[1], 2,
                               stencil_indices, values);

/* set boundary conditions */
...

HYPRE_StructMatrixAssemble(A);

\end{verbatim}
\end{minipage}
\caption{%
Code for setting up matrix values associated with stencil entries 0 and 3 as
given by \ref{eqn-stencil-laplacian} and Figure~\ref{fig-struct-stencil-a}.}
\label{fig-struct-matrix}
\end{figure}
The \code{Create()} routine creates an empty matrix object.  The
\code{Initialize()} routine indicates that the matrix coefficients (or values)
are ready to be set.  This routine may or may not involve the allocation of
memory for the coefficient data, depending on the implementation.  The optional
\code{Set} routines mentioned later in this chapter and in the Reference
Manual, should be called before this step.  The \code{SetBoxValues()} routine
sets the matrix coefficients for some set of stencil entries over the
gridpoints in some box.  Note that the box need not correspond to any of the
boxes used to create the grid, but values should be set for all gridpoints that
this process ``owns''.  The \code{Assemble()} routine is a collective call, and
finalizes the matrix assembly, making the matrix ``ready to use''.

Matrix coefficients that reach outside of the boundary should be set to zero.
For efficiency reasons, \hypre{} does not do this automatically.  The most
natural time to insure this is when the boundary conditions are being set, and
this is most naturally done after the coefficients on the grid's interior have
been set.  For example, during the implementation of the Dirichlet boundary
condition on the lower boundary of the grid in Figure~\ref{fig-struct-example},
the ``south'' coefficient must be set to zero.  To do this on process 0, the
code in Figure~\ref{fig-struct-matrix-boundary} could be used:
\begin{figure}
\centering
\begin{minipage}{0.8\textwidth}
\begin{verbatim}

int  ilower[2] = {-3, 1};
int  iupper[2] = { 2, 1};

/* create matrix and set interior coefficients */
...

/* implement boundary conditions */
...

for (i = 0; i < 12; i++)
{
   values[i] =  0.0;
}

i = 3;
HYPRE_StructMatrixSetBoxValues(A, ilower, iupper, 1, &i, values);

/* complete implementation of boundary conditions */
...

\end{verbatim}
\end{minipage}
\caption{%
Code for adjusting boundary conditions along the lower grid boundary in
Figure~\ref{fig-struct-example}.}
\label{fig-struct-matrix-boundary}
\end{figure}

%-----------------------------------------------------------------------------

\section{Setting Up the Struct Right-Hand-Side Vector}
\label{sec-Struct-RHS}

The right-hand-side vector is set up similarly to the matrix set up described
in Section \ref{sec-Struct-Matrix} above.  The main difference is that there is
no stencil (note that a stencil currently does appear in the interface, but
this will eventually be removed).

On process 0, the code in Figure~\ref{fig-struct-rhs} will set up the
right-hand-side vector values.
\begin{figure}
\centering
\begin{minipage}{0.8\textwidth}
\begin{verbatim}

HYPRE_StructVector  b;
double              values[18];
int                 i;

HYPRE_StructVectorCreate(MPI_COMM_WORLD, grid, &b);
HYPRE_StructVectorInitialize(b);

for (i = 0; i < 18; i++)
{
   values[i]   =  0.0;
}

HYPRE_StructVectorSetBoxValues(b, ilower[0], iupper[0], values);
HYPRE_StructVectorSetBoxValues(b, ilower[1], iupper[1], values);

HYPRE_StructVectorAssemble(b);

\end{verbatim}
\end{minipage}
\caption{%
Code for setting up right-hand-side vector values.}
\label{fig-struct-rhs}
\end{figure}
The \code{Create()} routine creates an empty vector object.  The
\code{Initialize()} routine indicates that the vector coefficients (or values)
are ready to be set.  This routine follows the same rules as its corresponding
\code{Matrix} routine.  The \code{SetBoxValues()} routine sets the vector
coefficients over the gridpoints in some box, and again, follows the same rules
as its corresponding \code{Matrix} routine.  The \code{Assemble()} routine is a
collective call, and finalizes the vector assembly, making the vector ``ready
to use''.

%-----------------------------------------------------------------------------

\section{Symmetric Matrices}
\label{sec-Symmetric-Matrices}

Some solvers and matrix storage schemes provide capabilities for significantly
reducing memory usage when the coefficient matrix is symmetric.  In this
situation, each off-diagonal coefficient appears twice in the matrix, but only
one copy needs to be stored.  The \code{Struct} interface provides support for
matrix and solver implementations that use symmetric storage via the
\code{SetSymmetric()} routine.

To describe this in more detail, consider again the 5-pt finite-volume
discretization of (\ref{eqn-laplacian}) on the grid pictured in
Figure~\ref{fig-struct-example}.  Because the discretization is symmetric, only
half of the off-diagonal coefficients need to be stored.  To turn symmetric
storage on, the following line of code needs to be inserted somewhere between
the \code{Create()} and \code{Initialize()} calls.
\begin{display}
\begin{verbatim}

HYPRE_StructMatrixSetSymmetric(A, 1);

\end{verbatim}
\end{display}
The coefficients for the entire stencil can be passed in as before.  Note that
symmetric storage may or may not actually be used, depending on the underlying
storage scheme.  Currently in \hypre{}, the \code{Struct} interface always uses
symmetric storage.

To most efficiently utilize the \code{Struct} interface for symmetric matrices,
notice that only half of the off-diagonal coefficients need to be set.  To do
this for the example being considered, we simply need to redefine the 5-pt
stencil of Section \ref{sec-Struct-Stencil} to an ``appropriate'' 3-pt stencil,
then set matrix coefficients (as in Section \ref{sec-Struct-Matrix}) for these
three stencil elements {\em only}.  For example, we could use the following
stencil
\begin{equation}\label{eqn-symmetric-stencil}
\left [
\begin{array}{ccc}
~~~~~~ & ( 0, 1) &         \\
~~~~~~ & ( 0, 0) & ( 1, 0) \\
~~~~~~ &         &
\end{array}
\right ] .
\end{equation}
This 3-pt stencil provides enough information to recover the full 5-pt stencil
geometry and associated matrix coefficients.


\subsection{Synthetic controls}


\section{Suffix Automaton}

\inputminted{cpp}{code/maximalist_tomato.cpp}

\section{Palindromic Tree}

\inputminted{cpp}{code/palindromic_tree.cpp}


\documentclass[journal]{IEEEtran}

\input{../../utils/header_final.tex}

\begin{document}

\title{Facial Keypoints Detection}
\author{Will Clark and Matthew DeLio\\
University of Chicago Booth School of Business\\
\textsf{\{will.clark,mdelio\}@chicagobooth.edu}}

% The paper headers
\markboth{Machine Learning (BUS 41201) Final Project}
{Machine Learning (BUS 41204) Final Project}

\maketitle

\section{Introduction}\label{intro}

\subsection{Problem Summary}

The goal of this paper is to predict the $(x,y)$ coordinates of 15 facial keypoints for a given set of 96-by-96 grayscale images. The keypoints are:
\begin{itemize}
\item Left and right eye center (2)
\item Left and right eye inner and outer corners (4)
\item Left and right eyebrow inner and outer ends (4)
\item Nose tip (1)
\item Mouth left and right corner (2)
\item Mouth top and bottom lip center (2)
\end{itemize}
A random sample of images and the associated keypoints are shown for reference in \cref{fig:random_faces}. Note that of the displayed faces, only two have the full set of keypoints plotted. This is a feature of the data, namely, that most faces are missing at least multiple keypoints. Accordingly, we predict two values for each keypoint: (1) whether it is present in an image or not, and (2) what its $(x,y)$ coordinates would be if it were present.raining

There are 7049 images in the training set provided by Kaggle. We randomly subset the data so that 70 percent of images are for training a model and the other 30 percent are for validation. There is also a true test set of 1783 images for which we only have the image data but not the keypoint data. This is the set on which the Kaggle contest is judged.

The criterion for model accuracy (and evaluation in the Kaggle contest) is the root means square error of all keypoint coordinates:
\[ \text{RMSE}=\sqrt{\frac{1}{n} \sum_{i=1}^{n} \left( y_i - \hat{y}_i\right)^2} \]

\begin{figure}[!ht]
  \centering
  \caption{Random Sub-Sample of Faces and Keypoints}
  \includegraphics[scale=.49]{random_faces.pdf}
  \label{fig:random_faces}
\end{figure}

\subsection{Data}

As discussed in \cref{intro}, most images do not contain all keypoints, which presents a problem when we are trying to train our model. One naive solution would be to throw out all faces without all keypoints, but doing so would remove 4909 images from our training data set (out of 7049 total images)--about 70 percent of our data.

An important observation is that keypoints tend to be missing in groups, so that (as an example) if an image does not have one of the eyebrow keypoints, it tends to also not have all other eyebrow keypoints. Accordingly, we bundle the keypoints into the following groups (the number of images containing all keypoints in the group are reported in parentheses):
\begin{itemize}
\item eye center: left eye center, right eye center (7033)
\item eye corner: left eye inner corner, left eye outer corner, right eye inner corner, right eye outer corner (2247)
\item eyebrow: left eyebrow inner end, left eyebrow outer end, right eyebrow inner end, right eyebrow outer end (2190)
\item mouth (bottom): mouth center bottom lip (7016)
\item mouth (excl. bottom): mouth left corner, mouth right corner, mouth center top lip (2260)
\item nose: nose tip (7049)
\end{itemize}
For each feature, then, we can group it with other features that tend to be present in the same image. Within a group, we throw out images in which all keypoints \textit{in that group} are not present, but in 22 out of 30 cases this requires us to remove less than 1 percent of the available images (in the other 8 cases, we throw out between 1 and 4 percent of available images).

Splitting the features into groups like this requires that we train six models, but each model will have more available training data than a single model required to predict all features.

A related point is that we will also need to predict whether or not a keypoint is contained in a feature, so that we can predict an empty value for keypoints that are likely to be missing in the test set. We discuss this model more fully in \cref{missing}.

\subsection{Tools and Software}

The main tool we use in this paper is a multilayer convolutional neural network. We built and trained the model in Python using the Theano and Lasagne libraries.\footnote{Lasagne is "a lightweight library to build and train neural networks in Theano." See \url{https://github.com/Lasagne/Lasagne} for details.} One advantage of this technology stack is that computation can be carried out on a graphics card (GPU) which can significantly improve calculation speed.

To take advantage of Theano's GPU capability, we set up our computing environment using a g2.2xlarge EC2 instance from Amazon Web Services. Doing so gave us access to an NVIDIA GPUs with 1536 CUDA cores that performed the bulk of the calculation. In our case, the performance improvement from utilizing the g2.2xlarge instance (compared to running locally) was approximately 40x.

\section{Missing Feature Model}\label{missing}

Since the data contain many missing features caused in some cases by partially obscured images and just poor labelling, our resulting model must also predict the presence of features.  To predict this we turn to a slightly modified version of the neural network described in \cref{neural} with a sigmoid non-linearity on the output layer and the binary cross-entropy loss function defined below:

\[\label{eq:bin_cross}
 L = -target \log(p) - (1 - target) \log(1 - p)
\]
where \texttt{target} indicates whether the feature is actually missing ($\in \{0,1\}$) and \texttt{p} is the predicted probability of the feature being missing

For many of the features (see \cref{fig:logistic_boxplots_eye,fig:logistic_boxplots_eyebrow,fig:logistic_boxplots_mouth}, the neural network does a good job separating the two classes and producing probabilities that make logical sense.  The ones that it doesn't separate well, like \texttt{Left Eye Center}, \texttt{Right Eye Center} and \texttt{Mouth Center Bottom Lip}, have few missing data-point exemplars in the training data (9, 9, and 23 respectively out of 4934 samples).

\begin{figure}[!ht]
  \centering
  \caption{Boxplots for Missing Eye Features}
  \includegraphics[scale=.49]{logistic_boxplots_eye.pdf}
  \label{fig:logistic_boxplots_eye}
\end{figure}

% We can probably drop these next two, they don't add much to the discussion...maybe we can just mention they they look identical (throw them in an appendix)
\begin{figure}[!ht]
  \centering
  \caption{Boxplots for Missing Eyebrow Features}
  \includegraphics[scale=.49]{logistic_boxplots_eyebrow.pdf}
  \label{fig:logistic_boxplots_eyebrow}
\end{figure}

\begin{figure}[!ht]
  \centering
  \caption{Boxplots for Missing Mouth Features}
  \includegraphics[scale=.49]{logistic_boxplots_mouth.pdf}
  \label{fig:logistic_boxplots_mouth}
\end{figure}

% \begin{figure}[!ht]
%   \centering
%   \caption{Boxplots for Missing Nose Feature}
%   \includegraphics[scale=.49]{logistic_boxplots_nose.pdf}
%   \label{fig:logistic_boxplots_nose}
% \end{figure}

With the output trained, we next turn to determining the optimal cutoff.  Using the R package \texttt{OptimalCutpoints} we choose to maximize the accuracy area \cite{lewis2008use,greiner1995two,greiner1996two} which is defined in \cref{eq:roc_aa}.

\[\label{eq:roc_aa}
AA(c)=\frac{TP(c)TN(c)}{(TP(c)+FN(c))(FP(c)+TN(c))}
\]
where $TP$ = True Positives, $TN$ = True Negatives, $FN$ = False Negatives, \& $FP$ = False Positives

Looking over our boxplots, we see there are generally two categories of features: those that are missing often and those that are not.  For the ones that are missing often, we have a relatively good separability between the predicted probabilities.  For those that are not missing often, our predictive power diminishes.  Turning to the ROC plot for a feature that is missing quite often, \texttt{Left Eye Inner Corner}, (see \cref{fig:roc_left_eye_inner_corner}) we see there is a lot of area under the ROC curve and the optimal cutoff point of 0.5 seems to put us right on the upper-left edge of the curve.  Next, looking at \texttt{Left Eye Center}, a feature with few missing exemplars, (see \cref{fig:roc_left_eye_center}) we see less area and a clear trade-off between specificity and sensitivity.  Again, a value of 0.5 seems to put us on the upper-left edge of the curve.

It turns out that, given the optimization condition, the best cutoff for all of these features is 0.5 (see \cref{tab:logistic_cutoff_table}\footnote{Note: that the \texttt{Nose Tip} feature is present in all training and validation images.}).  In each of the cases, we end up with quite a large sensitivity and a reasonable specificity.

\begin{figure}[!htb]
  \centering
  \caption{ROC Curve for Left Eye Inner Corner}
  \includegraphics[scale=.49]{roc_left_eye_inner_corner.pdf}
  \label{fig:roc_left_eye_inner_corner}
\end{figure}

\begin{figure}[!htb]
  \centering
  \caption{ROC Curve For Left Eye Center Feature}
  \includegraphics[scale=.49]{roc_left_eye_center.pdf}
  \label{fig:roc_left_eye_center}
\end{figure}

\input{logistic_cutoff_table.tex}

\section{Individual Keypoint Models}\label{neural}

In this section we will present a subset of the models we experimented with and report performance statistics for each. We will also discuss some auxiliary modeling techniques that we investigated. For each of the models we discuss below, we use a rectifier nonlinear activation and adjust the learning rate and momentum dynamically (discussed in detail below).

\subsection{Baseline Model}

Our baseline model is a neural network with six layers: 3 convolutional/pooling layers, and 3 hidden layers. We began our modeling with only three layers, but following \cite{benlecun2007}, our idea was that by increasing the depth of the network, we would increase its predictive accuracy as well. Additional layers should, in theory, allow the network to discover or learn more than just the low-level features of the images and hopefully increase its ability to identify specific keypoints.

The hope for the convolutional layer is that it would allow the network to automatically train 2-dimensional filters.  The hope is that the neural network will automatically adapt the filter taps to detect low-level features like edges.  It's possible also that higher-level features (like eyes, noses, mouths) might be trained as well.

The loss function on the validation data set for this set of models is displayed in \cref{fig:val_loss_default}. The RMSE on the validation set is 3.85 (see \cref{tab:model_rmse}).

\begin{figure}[!htb]
  \centering
  \caption{Validation Loss for Feature Detection Models (Baseline)}
  \includegraphics[scale=.49]{val_loss_default.pdf}
  \label{fig:val_loss_default}
\end{figure}

\subsection{Baseline Model with L1 and L2 Regularization}

We augmented our baseline model (described above) with L1 and L2 regularization. Compared to the model without regularization, we find that the loss function falls more gradually and more smoothly. This is because the cost function now has a second term that penalizes model complexity, so that early on in the training cycle the model is less able to fit the data. By not over-fitting the data, the hope is that the model is better at understanding images that it has not seen before. In other words, we are hoping to move the model more towards the optimal trade-off between bias and variance.

The loss function on the validation data set for this set of models is displayed in \cref{fig:val_loss_default_w_l1_and_l2}. The RMSE on the validation set is in excess of 35 (see \cref{tab:model_rmse}), making it far and away our worst performing model. One explanation is that the model has been over-regularized; we have put too much of a premium on building a simple model, so that the result is very bad predictive capability. We can see this a bit in \cref{fig:val_loss_default_w_l1_and_l2}, as the loss function for three of the feature models are not able to stabilize even after 100 epochs.\footnote{We show the results here for 100 epochs; we trained some models for up to 1000 epochs but saw no performance improvements.}

\begin{figure}[!htb]
  \centering
  \caption{Validation Loss for Feature Detection Models (Baseline with Regularization)}
  \includegraphics[scale=.49]{val_loss_default_w_l1_and_l2.pdf}
  \label{fig:val_loss_default_w_l1_and_l2}
\end{figure}

\subsection{Inverted Hourglass}

We coined the term inverted hourglass as we explored different neural network structures.  In this model we decided to place more convolutional filters near the input to see if it would improve over our existing model.  Therefore, we essentially flipped the convolutional part of our initial model over (hence the inverted part of the name) to start with 32 filters at the beginning and whittling this down to eight just before the densely connected layer.  Having a much smaller set of filters at the output of the convolutional layer meant that we would use a smaller initial dense layer, which we then doubled and halved again before the output layer (hence the hourglass part).

The loss function on the validation data set for this set of models is displayed in \cref{fig:val_loss_6_level_inverted_hourglass}. The RMSE on the validation set is 3.70, making it our best performing model (see \cref{tab:model_rmse}).

\begin{figure}[!htb]
  \centering
  \caption{Validation Loss for Feature Detection Models (Inverted Hourglass)}
  \includegraphics[scale=.49]{val_loss_6_level_inverted_hourglass.pdf}
  \label{fig:val_loss_6_level_inverted_hourglass}
\end{figure}

\subsection{Inverted Hourglass with Regularization}

We augmented our six layer model described above to include only the L1 regularization described above. Once again, the goal here is to prevent the model from over-fitting to the images that it trains on, so penalizing model complexity should increase the out-of-sample accuracy. Following \cite{ng2004feature}, we chose L1 regularization because we have a dimensionally large input data set (each image has 96x96 pixels, with each pixel being an input feature).

The loss function on the validation data set for this set of models is displayed in \cref{fig:val_loss_6_level_inverted_hourglass_reg}. The RMSE on the validation set is 3.98; predictive performance has declined by adding regularization.

\begin{figure}[!htb]
  \centering
  \caption{Validation Loss for Feature Detection Models (Inverted Hourglass with Regularization)}
  \includegraphics[scale=.49]{val_loss_6_level_inverted_hourglass_reg.pdf}
  \label{fig:val_loss_6_level_inverted_hourglass_reg}
\end{figure}

\subsection{Inverted Hourglass with Regularization and Dropout}

Finally, we further augmented our six layer model with L1 regularization to also include dropout. Similar to regularization, drop helps us ensure that we're not focusing too much on the training set, trying to find the real underlying structure.  Again, the goal is to choose a simpler model, but instead of adjusting the cost function we instead alter the structure of the network by deleting nodes in the intermediate layers during training.

The loss function on the validation data set for this set of models is displayed in \cref{fig:val_loss_6_level_inverted_hourglass_reg_do}. The RMSE on the validation set is 4.40; predictive performance has declined again by combining dropout with regularization.

\begin{figure}[!htb]
  \centering
  \caption{Validation Loss for Feature Detection Models (Six Layers)}
  \includegraphics[scale=.49]{val_loss_6_level_inverted_hourglass_reg_do.pdf}
  \label{fig:val_loss_6_level_inverted_hourglass_reg_do}
\end{figure}

\input{model_rmse.tex}

\subsection{Other Modeling Choices}

After being somewhat displeased with the results, we tried many other methods to see if we could train the model a bit better.  These experiments included and discussed in the following subsections.

\subsubsection{Batch-Size}
While batch-size shouldn't really matter, we did want to see if it had any effect on the training.  As it turns out, we did notices that larger batch-sizes did help lower the resulting RMSE slightly.  This was likely because the nesterov momentum update algorithm could incorporate a larger amount of the data-set in each update.

\subsubsection{Learning-Rate}
When we initially set the learning rate too high, we quickly saturated our 32-bit floating-point accumulators (GPUs can only handle single-precision floating point operations). Additionally, we knew that if we set it too low, we would wait forever to converge to an answer and, worse-yet, risked ending up in a local minima.

We had to balance setting it too high and too low.  After some consideration we saw that dynamic learning-rates were common-place \cite{gorr,dnouri} and we opted to implement and experiment with both a log and a linear learning-rate backoff based, for simplicity, solely on the number of epochs trained.

\subsubsection{Momentum}
Our update algorithm (nesterov updates) included both a learning-rate and momentum parameter.  After reading a bit in \cite{gorr} about the advantages of momentum in the overall amount of time needed to train a network we set about determining a reasonable value.  Further reading in the lasagne manual \cite{lasagnenesterov} led us to experiment not only with static momentum parameters but also with dynamic ones that increased momentum as learning-rate decreased.

\subsubsection{Dropout}
We played a bit with varying the amount of dropout present in the network.  We found that too little and too much dropout had, unsurprisingly, negative effects on the training.  In the former case, we likely were over-fitting our data, in the latter one, we introduced too much noise to train on.  We found that 10-20\% dropout was appropriate for our network.

\subsubsection{Scaling Output}
Finally, in an effort to see if we could get the neural network to train a little better, we tried scaling the output to lie between -1 and 1 in one experiment.  In another we tried normalizing it to so that coordinate positions were measured in standard deviations away from the mean.  Unfortunately we did not see improvements in the RMSEs produced by either of these experiments (reported training RMSEs were higher since the scale was smaller, but the resulting de-scaled RMSEs were worse).

\subsection{Image Pre-processing}

One key observation of our models outputs is that most of the predicted keypoints are very close to the average location of each keypoint. That is, to a first approximation our models just predict a keypoint will be at its average location; the variance is possibly due to the model learning to understand localized variation in the images.

One idea to help the model learn the ``true'' features of each face would be to pre-process the images by rotating them. Our goal would be that the model could not simply predict average locations for each keypoint, but would have to truly learn what signals the location of an eye, or a nose, etc.

To implement this, we augmented the training data set with 3 copies of each image, rotated 90, 180, or 270 degrees. Unfortunately, the results did not meet our expectations, as the RMSE of the model trained on the augmented data set exceeded the RMSE of the baseline model by at least 2 times. It appeared as though the model was still, to a first approximation, predicting the average location of each keypoint, although because the images were rotated, the average location was nowhere close to the actual location for any of the images.

\subsection{Learning Representation}

We also tried using the penultimate layer of our baseline neural network as the input into a different type of model (random forest or penalized regression, in our case). The idea is to use the neural network to learn and identify important features of the data, and then to use these important features in a final predictive model. Unfortunately, this methodology did not produce successful results in our case. The RMSE produced by this additional layer of modeling was worse than the traditional baseline model, so we did not investigate it any further.

\subsection{Combined Model}

In the combined model, we attempted to train both the missing data probability and the feature coordinate locations simultaneously.  This involved a fair amount of additional work to better understand the inner-workings of Theano and Lasagne, but the idea was that if we could train both models at once, we wouldn't have to train 7 models in total (6 feature-sets + the missing probability).  Additionally, because the neural network had to detect the presence of the missing feature, it was hoped it could use this to predict location better. Finally because the neural network could see all images in the training dataset, it could theoretically do a better job learning.  To do this, we had to create a loss-function that combined RMSEs from the keypoint coordinates and Binary-Cross-Entropy values from the logistic output.  To make matters more complicated, we had to write this loss-function to not penalize the network for a coordinate given on a missing feature.  Unfortunately, when the model was built and run, it fared worse than the model trained on the 7 separate features. It is likely that a better loss-function could have helped this, but due to time-constraints we were forced to move-on.

\section{Discussion}

\subsection{Best and Worst Performances}

In \cref{fig:avg_face_rmse}, we show an ``average'' face, i.e., one that is a composite of all images in our data set. On the image is the average location of each keypoint (marked with a ``+'' sign), and the shaded circles show the average deviation of the predicted keypoints from the actual keypoints. Colors correspond to the model used for each set of keypoints (i.e. eye corners were all predicted with one model, so they are all depicted in yellow).

We can consider the accuracy of our model in two different ways: the RMSE of each keypoint coordinate (shown in \cref{tab:rmse}), or the Euclidean distance of the predicted keypoint $(x,y)$ coordinate from the actual $(x,y)$ coordinate (shown in \cref{tab:rmse}; these are represented by the circles in \cref{fig:avg_face_rmse}).

From the distances between predicted in actual keypoints in \cref{tab:radius}, we can see that the model is best at predicting eye features. The top of the list (ordered from most to least accurate) is comprised entirely of the eye-related features (eye centers, eye corners, and eyebrows). We can see in \cref{fig:avg_face_rmse} that the radii around the eye features are smaller than those of the nose and mouth features. The most accurate keypoint is the inner corner of the right eye, which has a distance between the predicted and actual keypoints of just over three pixels.

Conversely, the model is not very good at predicting where the tip of a nose is. It is the least accurate keypoint, with a mean distance between the predicted and actual value of over six pixels.

\input{rmse.tex}
\input{radius.tex}

\subsection{Model Evaluation}

One baseline to judge model performance is how well we predict relative to naive averaging of keypoints. That is, one simple algorithm would be to predict--for all faces in the data set--the average of each keypoint across all faces. Doing so yields an average RMSE (across all keypoints) of 3.6. This actually compares favorably to the predictive capability of our best model, which produces an RMSE of 3.7. It appears that the model configurations we experimented with were not able to truly learn the features of facial keypoints.

One feature that stands out when examining the badly mis-predicted faces in \cref{fig:worst_faces} is that the model tends to predict keypoints close to their average values regardless of how the face in a given image is oriented. We can see this playing out in the plots above showing the validation losses for each model; in most cases, the validation loss relatively quickly drops then stays constant for the remainder of the training run. One interpretation of this is that the model quickly learns where the average keypoints are on an image, then fails to learn any other features of the data sets. It is possible that there are too few exemplars with tilted faces for the model to train against and thus it does a relatively poor job with these features.

\begin{figure}[!htb]
  \centering
  \caption{Average Face with Estimated Keypoints}
  \includegraphics[scale=.49]{avg_face_rmse.pdf}
  \label{fig:avg_face_rmse}
\end{figure}

\begin{figure}[!htb]
  \centering
  \caption{Most Accurate Predictions (Average RMSE = 1.768764)}
  \includegraphics[scale=.49]{best_faces.pdf}
  \label{fig:best_faces}
\end{figure}

\begin{figure}[!htb]
  \centering
  \caption{Least Accurate Predictions (Average RMSE = 7.874467)}
  \includegraphics[scale=.49]{worst_faces.pdf}
  \label{fig:worst_faces}
\end{figure}

\section{Kaggle Submission}

As a final and true out-of-sample test, we used the model with the best RMSE on the validation data set (the Inverted Hourglass model) to predict on the Kaggle test set. This is a set of 1783 images for which we know the pixel values but not the true keypoint values. The only way to judge the accuracy of the model on this set is to submit results to Kaggle. Our submission was graded to have an RMSE of 4.16987, which at the time of this writing puts us in 105th place. The RMSE of this submission is significantly higher than our validation RMSE. One possibility is that the training set is different than the testing set, as the training set is composed of images without all keypoints while the testing set is composed of images with all keypoints labeled, suggesting they are perhaps from different sources. Another possibility is that our model has over-trained on the training set, but given the relatively high validation RMSE we suspect this is not likely.

\bibliographystyle{IEEEtran}
\bibliography{citations}

\end{document}

% \input{.tex}

% \begin{figure}[!htb]
%   \centering
%   \begin{subfigure}[b]{0.49\textwidth}
%     \caption{}
%     \includegraphics[width=\textwidth]{.pdf}
%     \label{fig:}
%   \end{subfigure}
%   \hfill
%   \begin{subfigure}[b]{0.49\textwidth}
%     \caption{}
%     \includegraphics[width=\textwidth]{.pdf}
%     \label{fig:}
%   \end{subfigure}
%   \caption{}
% \end{figure}

% \begin{figure}[!htb]
%   \centering
%   \caption{}
%   \includegraphics[scale=.49]{.pdf}
%   \label{fig:}
% \end{figure}


\documentclass[11pt,twocolumn]{article}
\usepackage{enumerate}
\usepackage[hidelinks]{hyperref}
\usepackage{amsmath, amssymb, textcomp}
\usepackage{graphicx}
\usepackage[section]{placeins}
\usepackage{cite}
\usepackage{color}

\begin{document}

\title{The Ising model\\ {\large Computational Physics}}
\author{Tobias de Jong\\ s0881260}
\date{\today} % Voeg automatisch de datum in

\maketitle
\newcommand{\Part}[3][ ]{\ensuremath{\frac{\partial^{#1} #2}{{\partial #3}^{#1}}}}
\newcommand{\Dif}[3][ ]{\ensuremath{\frac{d^{#1} #2}{{d #3}^{#1}}}}
\renewcommand{\O}[1]{\ensuremath{\mathcal{O}\left(#1\right)}}
\renewcommand{\vec}{\bold}
\section{Introduction}
In this paper we will investigate the different properties of the Ising model in 2D and 3D using Monte Carlo simulation methods. The canonical Metropolis single spin flip method as well as the Wolff cluster flip algorithm are used to obtain values for the critical exponents $\chi$, $\gamma$ and $\nu$ in both 2D and 3D. %for different lattices
\section{Theory}
\subsection{Ising model}
\subsection{Monte Carlo approximation}
\section{Algorithms}
\subsection{Metropolis}
The Metropolis algorithm~\cite{Metropolis} utilizes single spin flips to transition between states. By selecting a random spin at each step we make sure that the transition chance  between any two neighbouring states $\mu$, $\nu$ is equal:
\[g(\mu,\nu) = g(\mu,\nu) = \frac1N\]
\subsection{Wolff}
The Metropolis algorithm has as major disadvantage that the thermalization slows down exponentially around the critical point. As this is the area in parameter space we are most interested in, another algorithm is needed. We use an algorithm proposed by Wolff in 1989~\cite{Wolff}. In essence, this algorithm stochastically builds up a cluster which it will flip at once, thus negating the critical slowdown.
\section{Implementation}
Both the Metropolis and the Wolff algorithm were implemented using Python and NumPy. While this choice of language will not result in the fastest implementation it yields easily readable and adaptable code.
Moreover, simple libraries exist to parallelise the different runs to keep the total wall clock computation time reasonable.
The other major benefit of using python is of course easy access to extensive plotting libraries.
\section{Results}
\subsection{Thermalization \& Auto-correlation}
To visualize the thermalization we do several runs for each value of $\beta$, calculate the magnetisation for each and take the mean across different runs. This way, the plot will be a flat line at the equilibrium value when thermalized. Results for a $40\times40$ grid and 100 runs per value of beta is given in figure \ref{therm}. The obvious conclusion here is that the thermalisation time depends quite strongly on $\beta$, especially around the critical point, which constitutes the interesting part of the data. An algorithm which would bypass this slow-down behaviour would thus be beneficial. As mentioned, the Wolff algorithm is such an algorithm, as although it still needs to thermalize, the thermalization time does not diverge around the critical point.
\begin{figure}[!ht]
\centering \includegraphics[width=\columnwidth]{figs/therm.pdf}
\caption{Thermalization}
\label{therm}
\end{figure}
The normalized autocorrelation of the Magnetisation and the energy for diverse values of $\beta J$ is given in figures \ref{magcorrel} and \ref{Ecorrel}.
\subsection{Finite size scaling}
\section{Data collapse}
\subsection{Rectangular 2D lattice}
To obtain a data collapse we use the values as found by Newman and Barkema \cite{Thebook}: $\gamma = 1.76$, $\nu = 1.00$ and $T_c=2.269J$. For the exponent $\beta$ we used the exact value $\frac18$. The result is shown in figure \ref{2Dcollapse}. Shuffling with the zoomed dataset gives the best collapse for $T_c = 2.271$, $\beta =0.139$, $\nu =0.997$, $\gamma =1.76$.
\subsection{Hexagonal 2d lattice}
To check the universality of the critical exponents we also ran the simulations for a hexagonal lattice. All values are kept the same as compared to the previous section, except for $T_c$, which we starting with the critical exponents found for the rectangular lattice. This yields $T_c=3.6378$. The result can be seen in figure \ref{Hexcollapse}.
\subsection{3 dimensions}
To estimate the values of the critical exponents for 3 dimensions, we use Wolff's algorithm to run for lattice sizes of $10^3$, $20^3$ and $30^3$ spins, both in rectangular and hexagonal close packing lattices. We use the first $10^3$ run of both lattices to estimate the critical temperature and adjust the range of simulated temperatures closer around the critical point, to keep computation time wihin reasonable boundaries.\\
An attempt at a data collapse is then visually attempted, by varying the critical exponents and -temperature until the curves best overlap. The resulting plot for a rectangular lattice is shown in \ref{collapse3Drect}, the resulting one for the hcp-lattice in figure \ref{collapse3Dhcp}. Note that due to the higher coordination number the critical temperature is shifted, but the critical exponents are the same.
\bibliography{verslag}{}
\bibliographystyle{plain}

\end{document}


I have written unit tests without thinking about how I chose
what to test. Foremost in my mind is risk, which I estimate
from how likely something is to go wrong, how bad it will be,
and how hard it is to mitigate. That's the first two steps
of choosing unit tests.

Often, we're given code that has some functions that do important
work but have too little documentation and no tests. It's easy
to add tests, in this situation, in order to understand what
those functions do. The tests can then tell the story of how
to set up data for those functions and work with their output.

Methods to test parameter logic all have the same underpinning,
that they will try to test at least one set of parameters from
each of the equivalence classes of inputs. Deciding what is an
equivalence class can be difficult. Sometimes two parameters
sets are equivalent in terms of the mathematical statement of
the model, but their representation in code makes one set
have a different risk from another. For instance, a function of small
values would be well-defined as an equation, but it could cause
denormalization when written in code.
It can also be the other way around, that
the mathematical statement of a model makes clear what parameter
sets may be equivalent. Information from both statements of
a function will be helpful.

Not every parallel implementation is a brute-force retelling
of a model. Most mathematical functions provide finger-holds
support of their correctness.

The simplest of these are parameters for which a function
takes a known value. The function could be a mess of
polynomials of trigonometric quantities,
\begin{equation}
  y = (\sin x)^3 - 6 (\cos x)^2 + 2,
\end{equation}
but these simplify considerably at $x=0, \pi/4, \pi/2, \pi,$
and so on. If the function should divide by zero for some
number, then that, too, is a parameter for which the function
has a known value, an exception.

\subsubsection{Checkout testing}

Let's say you have a cluster of computers. They are all Intel machines,
but they come from different years. They could be different architectures
or just different steppings of the \cpu, which is the least difference
one can have between two chipsets. You compile code on one machine,
or you install some \rlang package that compiles code underneath. When
you move to another computer on another day, that same code fails
to run with the message, ``invalid opcode.''

An opcode is a machine-language instruction. The computer you used
yesterday knew a command that's slightly too fancy for the computer
you're using today, and it failed. You have to clean out the code
and recompile today. It's not that the new computer is just older.
It's still possible that, were you to switch back, there might be
the same message again. The history of \cpu opcodes isn't a forward
march into a larger, better set.

That scenario would be annoying, and it's a headache for people
saddled with heterogenous clusters. They solve it with, for instance,
creating architecture-specific shared drives so that they can
install a different version of the software for each version of
system architecture on the cluster. That scenario is, however, a
best case scenario.

The worse scenario, and I wish this were rarer, is that the
software runs on the other computer but gives the wrong error.
It can fail silently. This seems like an abrogation of a compiler's
contract with its faithful coder, but it isn't. That compiler
ensured that the code ran well enough on one target architecture.
The problem is that compiler optimizations at level \textsc{o3}, and above,
are willing to make slightly riskier choices that yield great
performance benefits. These choices are tuned to the architecture
at hand, or as requested by compiler options, and, while they may
run on similar architectures, carry more risk of error on them.
It solves the problem of having languages
that don't clearly represent time-saving invariants and of coders
who can't afford motorcycles.

There's a solution for this, too. Turn all optimizations down
to \textsc{o2} or lower. Or claim to the compiler that the \cpu
understands no later than the \textsc{sse2} instruction set from 2000.
Unless you're intentionally commemorating the year Phil Katz died, you'll see
this as a severe loss of performance.

The last option to deal with this problem is called checkout,
which is ``testing conducted in the environment to ensure that a software product performs as required after installation. (IEEE 829-2008)'' It means that you create a test to run before you
run code on a new machine. It won't do the work of creating
a separate version of the code, if needed, but it tells you if
you need it. You don't generally care about integer performance
here. You care about vectorized code and floating-point code.
That's where the problems will be, so you can subset the test
suite for these acceptance tests.

Designating a subset of all tests as acceptance tests is an
example of test case prioritization~\cite{rothermel1999test}.
We prioritize tests by hand, by marking tests to be run under
different conditions, but tests for scientific code can be lengthy,
which makes it worth considering automatically reordering tests
so that those that are most sensitive to correctness run by
themselves, or run first~\cite{yoo2012regression}.

\subsubsection{Stratification and filtering}\label{sec:parameters-stratification}
The sections above all imply that there could be some way
to emphasize which test are chosen, based on an assessment
of which parameters carry the most risk. For instance, if
a parameter is an option to make a copy of the input data
before further work, it would be enough to test that exactly
once, and not in all other parameter combinations. On
the other hand, there may
be certain parameter values that should be thoroughly-tested
because they challenge the numerical precision of the algorithm.

The ability to prefer tests of certain sets of parameter values
is called \emph{stratification.} That means you take any
of the test generation methods above and give the test author
the ability to weight which tests it should generate.

The throw out some of the generated tests is called
\emph{filtering.} Sometimes it's easier to generate tests
and toss useless ones than it is to generate the most-desired
tests in the first place.

Some of the more professional testing software includes
these capabilities, such as \textsc{aetg}~\citep{cohen1997aetg},
\textsc{cts}~\citep{hartman2004problems},
and the \textsc{act} library~\citep{kuhn2008automated}.

\subsubsection{All other methods}
The orthogonal arrays, mentioned in Sec.~\ref{sec:parameter-decision},
are yet another way to generate tests that span the space of
all possible parameters~\citep{Owen1992}.
Whichever method you choose, it's
good to have one of these methods available when it's time to
do user-level testing on scientific code.
\citet{petke2015practical} cover these strategies in their book,
and there are a few survey papers on the topic~\citep{grindal2005,nie2011survey,khalsa2014orchestrated}.

Our attention shifts to asking, which parameter sets should I test
for this function? Which parameter sets are meaningfully different
test? How do I know when I've tested enough of them?

\section{disaggregating function}

We could be testing a function where we know what answer it should
give. For instance, take a function which disaggregates its input
vector into finer bins.
\begin{lstlisting}
function disaggregate(coarse, coarse_bins, fine_bins)
\end{lstlisting}
For the unit test, we can create a smooth-enough function on
the fine bins, aggregate it to the coarse bins, and use
that as our known input.

\section{Faults in Mathematical Code}\label{sec:faults-and-failures}

For each section, ask what faults does this technique limit?
For instance, using a limit test for constant-mortality mean age
ensures you don't have a numerical fault.

\subsection{Fault versus Failure}
Jorgensen's \emph{Software Testing: A Craftsman's Approach} distinguishes
faults from failures~\citep{jorgensen2013}.
A fault is the characters that were typed
incorrectly. A failure is observation that something went wrong.
For instance, there may be a fault that 0.3 is assigned to an integer
instead of assigning it to a double. The failure will happen later,
in the code that divides $1 / 0$.

Any failure is a problem. We don't want to throw exceptions or segfault.
Worse for mathematical code is when an error in the code isn't visible
but does give a wrong answer.
The worst problem is faults that aren't failures.

\subsection{Classes of Faults}
Jorgensen gives a set of fault categories and examples, elided here:
\begin{itemize}
    \item Input/Output - incorrect input accepted, incomplete result
    \item Logic - missing cases, test of wrong variable
    \item Computation - incorrect algorithm, parenthesis error
    \item Interface - parameter mismatch, call to wrong procedure
    \item Data - Incorrect type, wrong data, off by one
\end{itemize}
These all apply and, yes, read Jorgensen, but most of this article
expands upon a single category of fault: incorrect algorithm.

This document covers two different kinds of failure-free faults.
\begin{itemize}
\item Simple data manipulation that doesn't do what the user
   expected it to do.

\item Mathematical functions that return a result that looks
   reasonable but isn't correct.
\end{itemize}
While these faults are within the taxonomy of faults
in any book on testing, we can discuss some of the particular
challenges that complicated mathematical functions introduce.

\section{Statistics}

Our tools to
understand and interpret this experimental data fall into a
few categories of \emph{statistical} tests.
\begin{itemize}
	\item A statistical test that a measure of the data matches
	      a known value. We could ask whether the mean of the durations
	      is what we expect from the construction of the function.

	\item A statistical test of differences between two experiments.
	      This could be a comparison of outputs from parturition
	      for two different cattle breeds.

	\item A statistical test to characterize the shape of an
	      association. For instance, does the duration between
	      births increase as weight increases?
\end{itemize}
For each of these kinds of statistical tests, there are a range of
standard methods. We choose among those methods according to what
our goals are for particular software tests.

\begin{itemize}
	\item Develop new code.
	\item Unit test that code to ensure it isn't broken by a change.
	\item Regression test when writing a new version of that code,
	      either for refactoring or for a modification in function.
	\item Acceptance test, or checkout test that code.
\end{itemize}