%% ------------------------------------------------------------------------- %%
\chapter{Theoretical Background}
\label{cap:conceitos}
In this chapter, the theoretical concepts that apply to this research are stated. First, we define and explain the concept of a digital image in Section~\ref{sec:digital_image} as well as basic relationship between pixels in Section~\ref{sec:image_components_relation}. In Section~\ref{sec:image_histogram} we will see how image histograms are fundamental for the methods described in Chapter~\ref{cap:proposed-solution} of the proposed solution. Also, the background of image segmentation, mainly for the thresholding technique is shown in Section~\ref{sec:image_segmentation}. Thereafter, a brief introduction to color models is provided in Section~\ref{sec:color_models} in order to give an overview of the main characteristics of some of the most used in the computer vision and image processing area, on which this research is based.

%% ------------------------------------------------------------------------- %%
\section{Digital image}
\label{sec:digital_image}
By definition, an image is a two-dimensional function where its amplitude, at any pair of coordinates, is called the \textit{intensity} or \textit{gray level} of the image at that point. The image is said digital when this function is converted to a discrete form, $f(x, y)$ let's say. This is made by a process called \textit{digitalization}, which consists of two steps: \textit{sampling} and \textit{quantization}~\citep{gonzalez:02}.

Each element of the discrete function $f(x, y)$ is called \textit{pixel} (picture element), where $0 \leq x \leq H - 1$ and $0 \leq y \leq W - 1$. This means that the image can be represented in a matrix form (see Eq.~\ref{eq:image_function}), where $H$ is the number of rows and $W$ the number of columns of the matrix. Therefore, $H$ and $W$ define the size or resolution of the image~\citep{pedrini:08}.

\begin{equation}
f(x, y) =
 \begin{bmatrix}
  f(0, 0)     & f(0, 1)     & \cdots & f(0, W - 1) \\
  f(1, 0)     & f(1, 1)     & \cdots & f(1, W - 1) \\
  \vdots      & \vdots      & \ddots & \vdots  \\
  f(H - 1, 0) & f(H - 1, 1) & \cdots & f(H - 1, W - 1)
 \end{bmatrix}
\label{eq:image_function}
\end{equation}

Usually, pixels are stored, and therefore read, in this matrix in an order known as \textit{raster} (see Fig.~\ref{fig:raster}). This information is important so that capture and display devices may be able to establish a common interface, and make necessary transformations in the coordinates, when needed.

\begin{figure}[ht]
    \centering
    \begin{tikzpicture}
    \draw[thick,->] (0,4.5) -- (4.5,4.5) node[anchor=south west] {W};
    \draw[thick,->] (0,4.5) -- (0,0) node[anchor=south east] {H};
    \end{tikzpicture}

    \caption[Representation of the raster order of an image]{Representation of the raster order of an image. The origin coordinates $(0, 0)$ starts in the top left corner, where both axes rise. Source: proposed by the author.}
    \label{fig:raster}
\end{figure}

In a monochromatic digital image, the value of a pixel is a scalar in the range $[L_{min}, L_{max}]$, where $L_{min}$ and $L_{max}$ are, in general, positive integers\footnote{In some rare occasions, $L_{min}$ can be a negative integer.}, respectively, the minimum and maximum gray level values~\citep{pedrini:08}.

In a multispectral image, each pixel has a vector value such that $f(x, y) = (L_1, L_2, \ldots L_n)$ where $L^i_{min} \leq L_i \leq L^i_{max}$ and $i = 1, 2, 3, \ldots, n$. In general, $L_i$ can represent different measures for each of $(x, y)$ coordinate as well as different intervals~\citep{pedrini:08}.

A colored image is a multispectral image, where the color in each $(x, y)$ point is given by three variables: brightness, hue and saturation~\citep{pedrini:08}. The brightness gives the notion of chromatic intensity. Hue represents the dominant color perceived by an observer. Saturation refers to the relative purity or amount of white light applied to the hue. Combined, hue and saturation are known as chromaticity and, therefore, a color must be characterized by its brightness and chromaticity \citep{gonzalez:02}.

%% ------------------------------------------------------------------------- %%
\section{Basic relationship between pixels}
\label{sec:image_components_relation}
There is a number of applications in image processing and computer vision that uses information of relationship among pixels to create knowledge. Some of these important relationships will be described in the following sections once we will apply it further in Chapter~\ref{cap:proposed-solution} further. It is worth mentioning that we defined an image as a function $f(x, y)$. In this section, when referring to a particular pixel, we will denote it in lowercase letters such as $p$.

%% ------------------------------------------------------------------------- %%
\subsection{Neighborhood}
\label{sec:neighborhood}
A pixel $p$ with coordinates $(x, y)$ can have up to four\footnote{Respecting the image domain limits.} horizontal and vertical neighbors whose coordinates are given by:

\begin{equation*}
    (x + 1, y), (x - 1, y), (x, y + 1), (x, y - 1)
    \label{eq:n4_neighbors}
\end{equation*}

This set of pixels, called the 4-\textit{neighbors} of $p$, is denoted by $N_4(p)$~\citep{gonzalez:02}. See Figure~\ref{fig:n4-neighbors} for a reference on how this neighborhood looks like.

\begin{figure}[ht]
    \centering
    \begin{tikzpicture}
    \draw[step=0.75cm,black,thin] (0,0) grid (0.75,2.25);
    \draw[black,thin] (0.75,0.75) rectangle (1.5,1.5);
    \draw[black,thin] (-0.75,0.75) rectangle (0,1.5);
    \fill[gray!40, draw=black] (0,0.75) rectangle (0.75,1.5);

    \end{tikzpicture}

    \caption[The 4-\textit{neighbors} representation of a pixel $p$]{The 4-neighbors representation of a pixel $p$. The pixel $p$ is centered on the grid with gray background. Source: adapted from~\citet{pedrini:08}.}
    \label{fig:n4-neighbors}
\end{figure}

Some neighbors of $p$ might lie outside of the image boundaries if $(x, y)$ is on the border of the image~\citep{gonzalez:02}. Those who will use neighborhood operations in the image might take this in consideration to avoid index out of range in those regions.

The four coordinates of the diagonals of $p$, if they exist in the domain of the image, are given by:
\begin{equation*}
    (x + 1, y + 1), (x + 1, y - 1), (x - 1, y + 1), (x - 1, y - 1)
    \label{eq:nd_neighbors}
\end{equation*}

This set of pixels are denoted by $N_D(p)$. When combined, the 4-\textit{neighbors} and $N_D(p)$ will generate the 8-\textit{neighbors} of $p$, known as $N_8(p)$~\citep{gonzalez:02}. Formally, we have:
\begin{equation*}
    N_8(p) = N_4(p) \cup N_D(p)
    \label{eq:n8_neighbors}
\end{equation*}

See Figure~\ref{fig:n8-neighbors} for a reference on how the $N_8(p)$ neighborhood looks like.

\begin{figure}[ht]
    \centering
    \begin{tikzpicture}
    \draw[step=0.75cm,black,thin] (0,0) grid (2.25,2.25);
    \fill[gray!40, draw=black] (0.75,0.75) rectangle (1.5,1.5);
    \end{tikzpicture}

    \caption[The 8-\textit{neighbors} representation of a pixel $p$]{The 8-neighbors representation of a pixel $p$. The pixel $p$ is centered on the grid with gray background. Source: adapted from~\citet{pedrini:08}.}
    \label{fig:n8-neighbors}
\end{figure}

Despite these are the most common neighbors used in applications, other different distances from $p$ as well as connectivity can be applied. The idea of neighborhood can also be extended to 3-dimensional images where, instead of pixels, the voxels are the coordinates considered~\citep{pedrini:08}.

%% ------------------------------------------------------------------------- %%
\subsection{Connectivity}
\label{sec:connectivity}
The connectivity between pixels is a very important concept used to establish the boundaries of objects and regions in an image. To figure out if two pixels are connected, it must be determined if they are neighbors and if their gray levels satisfy some similarity criteria, such as gray levels, color or texture equality. For instance, in a binary image, where the pixels values vary in the range $[0, 1]$, two pixels may be 4-\textit{neighbors}, but they are connected if and only if they have the same value~\citep{gonzalez:02}.

%% ------------------------------------------------------------------------- %%
\subsection{Image boundaries}
\label{sec:image_boundaries}

When working with neighborhood operations, the mask being used, independently of its size, can fall beyond the image boundaries. In other words, this means that, for an image $f$, with size $W \times H$, some parts of the mask operator will be outside of the domain of the image (i.e. out of the range given by $W \times H$ matrix)~\citep{pedrini:08}. Figure~\ref{fig:img_boundaries} shows an example of this phenomenon in a $10 \times 8$ image being used as input for an 4-\textit{neighbors} operator.

\begin{figure}[ht]
    \centering
    \begin{tikzpicture}
    \draw[step=0.75cm,black,thin] (0,0) grid (7.5,6.0);
    \fill[pattern=north east lines, pattern color=blue, draw=black] (4.5,5.25) rectangle (5.25,6.0);
    \fill[pattern=north east lines, pattern color=blue, draw=black] (5.25,4.5) rectangle (6.0,5.25);
    \fill[pattern=north east lines, pattern color=blue, draw=black] (5.25,5.25) rectangle (6.0,6.0);
    \fill[pattern=north east lines, pattern color=blue, draw=black] (5.25,6.0) rectangle (6.0,6.75);
    \fill[pattern=north east lines, pattern color=blue, draw=black] (6.0,5.25) rectangle (6.75,6.0);
    \end{tikzpicture}

    \caption[Representation of a 4-\textit{neighbors} window mask going beyond image borders]{Representation of a 4-\textit{neighbors} window mask going beyond image borders. Source: adapted from~\citet{pedrini:08}.}
    \label{fig:img_boundaries}
\end{figure}

There are several ways to work around this problem. One simple mechanism is to simply ignore the pixels on the border where the mask go beyond -- this is the approach we used in this work. Despite this will avoid index out of bound errors, border pixels of the image will not be looked at. Another approach is to copy the corresponding pixels from the input image. Once again, the resulting image will have some not processed pixels in the border. Another strategy is to apply a different mask for the borders, which may consider the difference in the corners as well, to perform the operation, which can turn the operation more complex and computational costly~\citep{pedrini:08}.

%% ------------------------------------------------------------------------- %%
\section{Image histogram}
\label{sec:image_histogram}
In particular for image processing, when we are referring to an image histogram, we are talking about to count the frequency of the pixels for each intensity level in separate bins, in a multi-valued (intensity levels) sequence~\citep{fisher:03}. The most common visual representation is a graph or plot, which gives an overall idea about the intensity distribution of an image. The foundation of a histogram can be seen as a set of bins, where each bin is representing a certain intensity value from a given range. A simple algorithm to compute a histogram of an image may examines all pixels in the image and assigns each to a bin depending on the pixel intensity. In the end, each bin will have the number of pixels of its own intensity value~\citep{gonzalez:02}.

For instance, in an 8-bit grayscale image there are 256 different possible intensities, and so, the histogram will graphically display 256 integer numbers showing the counting of occurrences of those grayscale values. It is also possible to compute histograms of color images. In this particular case, the channels of the color space in turn are split individually from where a separate histogram is calculated~\citep{fisher:03}. We are using color histograms of the YCbCr channels image in this work.

\begin{figure}[H]
    \centering
    \begin{subfigure}[t]{0.4\textwidth}
        \includegraphics[width=\textwidth]{obama_gs}
    \end{subfigure}
    ~
    \begin{subfigure}[t]{0.54\textwidth}
        \includegraphics[width=\textwidth]{obama_histogram}
    \end{subfigure}
    \caption[Grayscale image with its respective histogram]{Grayscale image with its respective histogram. Left image is a sample from Pratheepan dataset, transformed to grayscale. On the right we have its histogram of pixel intensity values in the range [0, 255]. Source: \citet{tan:12} and the author.}
    \label{fig:obama_hist}
\end{figure}

Several statistical measures can also be obtained from a histogram such that minimum and maximum values, mean, median, standard deviation, and percentiles~\citep{pedrini:08}. Those measures, specifically percentiles, as well as the histogram itself are fundamental for the methods described in Chapter~\ref{cap:proposed-solution} of the proposed solution.

%% ------------------------------------------------------------------------- %%
\section{Image segmentation}
\label{sec:image_segmentation}
Image segmentation is the task of partition the image domain so each part is semantically different from each of its neighbors. These regions which, in general, are the representations of specific objects, are the fundamental parts of an image. This partitioning process is also used by humans, through their visual perception, into surrounding environment to help themselves to recognize them, guide their movements, and for almost every other day by day tasks in their lives~\citep{konstantinos:00}.

Segmentation of complex images is one of the most difficult tasks in the field of image processing, where the accuracy determines whether a successful operation or not. It is a heavy process that consists of many steps such as the analysis of color, shape, motion, geometry, and texture of objects in images~\citep{gonzalez:02}.

Usually, the first task of any image analysis process is image segmentation. All subsequent tasks, such as feature extraction and object recognition rely heavily on the quality of the segmentation~\citep{konstantinos:00}. The algorithms created for image segmentation generally are based on one of two basic properties of intensity values: \emph{discontinuity}, where abrupt changes in intensity, such as points, lines, and edges are detected, and \emph{similarity}, whose approaches are based on regions partitioning, according to a set of predefined criteria~\citep{gonzalez:02}.

One of the \emph{similarity} methods it is the thresholding: a popular and simple method used to segment regions of an image, especially in applications where speed is an important factor -- that is the case for skin detection, once it is used, in general, for face detection, gesture analysis, face tracking, video surveillance systems, medical image analysis, and other human-related image processing applications. Due to the simplicity of implementation, several authors apply this technique for skin detection task~\citep{kovac:03, chai:99, basilio:11, kaur:12, shaik:15, kumar:15}. Others extend this application by using adaptive thresholding~\citep{yogarajah:11, tan:12}.

%% ------------------------------------------------------------------------- %%
\subsection{Thresholding}
\label{sec:thresholding}
Histogram thresholding is one of the simplest pixel-based techniques for image segmentation due to its intuitive properties and easy implementation~\citep{gonzalez:02}. Roughly speaking, if an image is composed of distinct regions -- that means, each region in the histogram is represented by a peak --, adjacent regions could be split into separated groups, once adjacent peaks are likely separated by a valley~\citep{konstantinos:00}. Those valleys can be determined by means of one or more threshold values. However, to find out the threshold value that is the lower bound of a valley is definitely not a trivial task~\citep{konstantinos:00}.

\begin{figure}[h]
\centering

\begin{tikzpicture}
    \begin{axis}[width=0.45\textwidth,
       height=0.3\textwidth,ybar,enlargelimits=0.02,yticklabels=\empty,xticklabels=\empty,bar width=0.2,axis x line*=bottom,axis y line*=left,xlabel={(a)}]
    \addplot [draw=blue,pattern=horizontal lines light blue] coordinates {(1,1) (2,2) (3,3) (4,4) (5,5) (6,7) (7
    ,8) (8,9) (9,9) (10,8) (11,6) (12,6) (13,5) (14,4) (15,3) (16,2) (17,1) (18,0.5) (19,1) (20,2) (21,3) (22,4) (23,5) (24,6) (25,6) (26,5) (27,4) (28,3) (29,2) (30,1)};
    \addplot[mark=none] coordinates {(17.5,0.5)} node[pin=90:{$T$}]{} ;
    \end{axis}
\end{tikzpicture}
~
\begin{tikzpicture}
    \begin{axis}[width=0.65\textwidth,
       height=0.3\textwidth,ybar,enlargelimits=0.02,yticklabels=\empty,xticklabels=\empty,bar width=0.2,axis x line*=bottom,axis y line*=left,xlabel={(b)}]
    \addplot [draw=blue,pattern=horizontal lines light blue] coordinates {(1,1) (2,2) (3,3) (4,4) (5,5) (6,7) (7
    ,8) (8,9) (9,9) (10,8) (11,6) (12,6) (13,5) (14,4) (15,3) (16,2) (17,1) (18,0.5) (19,1) (20,2) (21,3) (22,4) (23,5) (24,6) (25,6) (26,5) (27,4) (28,3) (29,2) (30,1) (31,0.5) (32,1) (33,2) (34,3) (35,4) (36,4.5) (37,5) (38,5.5) (39,6) (40,5) (41,3) (42,2) (43,1)};
    \addplot[mark=none] coordinates {(17.5,0.5)} node[pin=90:{$T_1$}]{} ;
    \addplot[mark=none] coordinates {(30.5,0.5)} node[pin=90:{$T_2$}]{} ;
    \end{axis}
\end{tikzpicture}
\caption[Gray level histograms that can be partitioned by (a) a single threshold, and (b) multiple thresholds]{Gray level histograms that can be partitioned by (a) a single threshold, and (b) multiple thresholds. Source: adapted from~\citet{gonzalez:02}.}
\label{fig:histogram_thresholding}
\end{figure}

Let Figure~\ref{fig:histogram_thresholding}(a) be the gray levels histogram of an image $f(x, y)$. We can clearly see two dominant groups of pixels. One obvious way to separate the objects in this image is to select a threshold $T$ that splits these groups. In other words, for every single point $(x, y)$, if $f(x, y) \leq T$, then $(x, y)$ belongs to the first object (or group), otherwise it belongs to the second object. In general, one of these objects is seen as the background of the image and the other, the portion of interest in the image~\citep{gonzalez:02}.

The resulting image $g(x, y)$ after the thresholding is given by~\citep{gonzalez:02}:
\begin{equation}
g(x, y) =   \begin{cases}
                0, \quad &\text{if}\ f(x, y) \leq T\\[0.7em]
                1, \quad &\text{otherwise}
            \end{cases}
\end{equation}

Therefore, pixels labeled 0 corresponds to the object left of $T$ in the histogram, and pixels labeled 1 corresponds to the object right of $T$ in histogram -- or background as commonly used in literature. The \emph{single-level thresholding} procedure that produces $g(x, y)$ image is known as \emph{binarization}~\citep{gonzalez:02}.

Figure~\ref{fig:histogram_thresholding}(b) shows a slightly more general case of thresholding, where three distinct groups of the histogram are split by $T_1$ and $T_2$ threshold values. Here, \emph{multi-level thresholding} classifies a point $(x, y)$ as belonging to a specific object class as follows~\citep{gonzalez:02}:

\begin{equation}
g(x, y) =   \begin{cases}
                l_1, \quad &\text{if}\ f(x, y) \leq T_1\\[0.7em]
                l_2, \quad &\text{if}\ T_1 < f(x, y) \leq T_2\\[0.7em]
                l_3, \quad &\text{if}\ f(x, y) > T_2
            \end{cases}
\end{equation}
where $l_1, l_2, l_3$ are different gray levels used to represent each interval given by the thresholding values.

When $T$ depends only on $f(x, y)$, that is, only on gray level values, the threshold is called \emph{global}. When another property -- e.g. the average gray level of a neighborhood centered on $(x, y)$ -- is used to define $T$, the threshold is called \emph{local}. If, in addition, $T$ depends on spatial coordinates $x$ and $y$, the threshold is called \emph{dynamic} or \emph{adaptive}~\citep{gonzalez:02}.

%% ------------------------------------------------------------------------- %%
\section{Color models}
\label{sec:color_models}

In general, the colors are represented by their brightness, hue, and saturation, which are usually the features used to distinguish one color from another. The brightness gives the notion of chromatic intensity. Hue represents the dominant color perceived by an observer. Saturation refers to the relative purity or amount of white light applied to the hue. Combined, hue and saturation are known as chromaticity and, therefore, a color must be characterized by its brightness and chromaticity~\citep{gonzalez:02}.

Colors can be specified by mathematical models in tuples of numbers in a coordinate system and a subspace within that system where each color is represented by a single point. Such models are known as the color models~\citep{gonzalez:02}.

These models can be classified as of two types: the additive models in which the primary color intensities are added to produce other colors and subtractive, where colors are generated by subtracting the length of the dominant wave from the white light.

The following sections briefly describe some color models, as well as their variants and main areas of application.

%% ------------------------------------------------------------------------- %%
\subsection{Munsell color model}
\label{sec:modelo_cores_munsell}

Pioneer in an attempt to organize the perception of the colors in a color space, Albert Henry Munsell was able to combine the art and science of colors in a single theory~\citep{konstantinos:00}.

The principle of equality of perceptual separation between the components of the model is the essential idea of the Munsell color model. These components are hue, value, corresponding to luminance, and chroma, corresponding to saturation~\citep{konstantinos:00}.

The model is represented by a cylindrical coordinate system and it can be seen in the Figure~\ref{fig:munsell-system}. The hue is arranged in the circular axis consisting of five bases as well as five secondary colors, the saturation in the radial axis and the luminance in the vertical axis in a range varying from 0 to 10.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=.6\textwidth]{munsell-system}
  \caption[Munsell color model.]{Munsell color model represented by a cylindrical coordinate system. The hue is arranged on the circular axis consisting of five bases and five secondary colors, the chroma (saturation) on the radial axis and the luminance (value) on the vertical axis in a range varying from 0 to 10. Source: \citet{rus:07}.}
  \label{fig:munsell-system}
\end{figure}

%% ------------------------------------------------------------------------- %%
\subsection{CIE color model}
\label{sec:modelo_cores_cie}

In 1931, the CIE established the first mathematical model of a color numerical specification, whose objective was to analyze the relationship between the physical aspects of colors in the electromagnetic spectrum and their perception by the human visual system to determine how an ordinary person perceives the color. A review of this specification was published in 1964~\citep{gonzalez:02}.

The experiment that originated the standard consisted of detecting the colors perceived by an observer from a mixture of three primary colors X, Y and Z, called tristimulus values. These coordinates gave rise to the CIE XYZ color space which encompasses all the colors that can be perceived by an ordinary human being. For this reason, it is considered a device independent representation~\citep{konstantinos:00}.

The system proposed by the CIE XYZ to describe a color is based on a luminance component Y, and two additional components X and Z, that bring the chromaticity information. This system is formed by imaginary colors that can be expressed as combinations of the normalized measures shown in Equations~\ref{eq:cie_x}, \ref{eq:cie_y} and \ref{eq:cie_z}~\citep{konstantinos:00}.

\begin{equation}
  x = \frac{X}{X + Y + Z}
\label{eq:cie_x}
\end{equation}

\begin{equation}
  y = \frac{Y}{X + Y + Z}
\label{eq:cie_y}
\end{equation}

\begin{equation}
  z = \frac{Z}{X + Y + Z}
\label{eq:cie_z}
\end{equation}

where $x + y+ z = 1$.

Combinations of negative values and other problems related to selecting a set of real primaries are eliminated. The chromaticity coordinates $x$ and $y$ allow to represent all colors in a two-dimensional plane, also known as a chromaticity diagram, which can be seen in the Figure~\ref{fig:cie-cromaticity-diagram}.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=.55\textwidth]{cie-cromaticity-diagram}
  \caption[CIE 1931 chromaticity diagram]{CIE 1931 chromaticity diagram. The points representing pure colors in the electromagnetic spectrum are labeled according to their wavelengths and are located along the curve from the right end of the x-axis, corresponding to the red color, to the left end of the same axis, corresponding to the violet color, forming a polygon similar to a horseshoe. The internal points correspond to all possible combinations of visible colors. Source: \citet{ben:09}.}
  \label{fig:cie-cromaticity-diagram}
\end{figure}

The coordinates $(x = 1/3, y = 1/3)$ correspond to the location of the white light, also known as white point, and serve as a reference in the process of image capture, coding, or reproduction.

CIE also derived and standardized two other color models based on CIE XYZ specification and, likewise, are device independent. Both are perceptually uniform, which means that equal perceptual distances separate all colors in the system~\citep{vezhnevets:03}. As an example, the gray scale of the space should allow for a smooth transition between black and white.

The first one was designed to reduce the problem of perceptual non-uniformity. Some Uniform Chromaticity Scale (UCS) diagrams were proposed based on mathematical equations to transform the values XYZ or the coordinates $x, y$ into a new set of values $(u, v)$, which gave rise to the 1960 CIE $uv$ chromaticity diagram~\citep{gevers:12}.

With unsatisfactory results yet, the CIE made a new change by multiplying the $v$ component by a factor of 1.5. In addition, the brightness scale given by the Y component has been replaced by $L^* = [0, 100]$ to better represent the differences in luminosity that are equivalent. This revision originated the CIE 1976 $L^*u^*v^*$ color model, commonly known by the acronym CIELuv~\citep{gevers:12}.

In 1976 the CIE adopted a new color model, based on the $L, a, b$ model, proposed by Richard Hunter in 1948, which best represented the uniform spacing of colors. Named CIE $L^*a^*b^*$ and known by the acronym CIELab, it is a space based on opponent colors \footnote{Theory started around 1500 when Leonardo da Vinci concluded that colors are produced by mixing yellow and blue, green and red, and white and black. In 1950, this theory was confirmed when optically-colored signals were detected at the optical connection between the retina and the brain~\citep{gevers:12}.} in which the color stimuli of retina is converted to distinctions between light and dark, red and green, and blue and yellow, represented by the axes $L^*$, $a^*$, and $b^*$, respectively~\citep{gevers:12}.

%% ------------------------------------------------------------------------- %%
\subsection{RGB color model}
\label{sec:modelo_cores_rgb}

The RGB model, an acronym for Red, Green, and Blue, is an additive color model in which the three primary colors red, green and blue are added to produce the others~\citep{gonzalez:02}.

This system was based on the trichromatic theory of Thomas Young and Hermann Helmholtz in the mid-19th century and can be represented graphically (see Fig.~\ref{fig:rgb-cube}) through the unit cube defined on the axes R, G, and B~\citep{konstantinos:00}.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=.35\textwidth]{rgb-cube}
  \caption[Unit cube representing the colors of the RGB model]{Unit cube representing the colors of the RGB model. The origin, given by the vertex $(0, 0, 0)$, represents the black color. The vertex $(1, 1, 1)$, opposite to the origin, represents the white color. The highlighted vertices on the axes represent the primary colors and the others are the complement of each. Each point inside the cube corresponds to a color that can be represented by the triple $(r, g, b)$, where $r, g, b \in [0, 1]$. The shades of gray are represented along the main diagonal of the cube, with each point along this diagonal being formed by equal contributions of each primary color. Source: adapted from~\citet{gonzalez:02}.}
  \label{fig:rgb-cube}
\end{figure}

It is noteworthy that there are two ways to represent the RGB space: linear and non-linear. The aforementioned system shows the non-linear model, whose abbreviation is $R'G'B'$ and is most used by devices and applications because of their similarity to the human visual system. In the literature, this system is frequently cited with the acronym RGB, which makes the nomenclature dubious, since the linear model is also called RGB and, therefore, the conversion between color spaces must be done with some caution. It is also important to note that linear RGB values are rarely used to represent an image since they are perceptually highly non-uniform~\citep{konstantinos:00}.

%% ------------------------------------------------------------------------- %%
\subsection{CMY color model}
\label{sec:modelo_cores_cmy}

The CMY model is based on the complementary primary colors of the R, G, and B coordinates, which are given by Cyan, Magenta, and Yellow. Unlike RGB, it is a subtractive color model, and so, non-additive, in which colors are generated by subtracting the length of the dominant wave from the white light and, therefore, the resulting color corresponds to the light that is reflected~\citep{gonzalez:02}.

One way to get the CMY coordinates from the RGB system is:\\
\begin{equation}
  \begin{bmatrix}
    C \\ M \\ Y
  \end{bmatrix} =
  \begin{bmatrix}
    B \\ R \\ R
  \end{bmatrix} +
  \begin{bmatrix}
    G \\ B \\ G
  \end{bmatrix}
\end{equation}

\noindent or by making a change of coordinates by subtracting the primary colors R, G and B of the white color $W = (1, 1, 1)$~\citep{gonzalez:02}:
\begin{equation}
  \begin{bmatrix}
    C \\ M \\ Y
  \end{bmatrix} =
  \begin{bmatrix}
    1 \\ 1 \\ 1
  \end{bmatrix} -
  \begin{bmatrix}
    R \\ G \\ B
  \end{bmatrix}
\end{equation}

Similarly to RGB, CMY is device dependent. The model is widely used in equipment that deposits colored pigments on paper, such as color printers or photocopiers. The Figure~\ref{fig:cmy-model} shows how the model components are combined to generate the other colors.

\begin{figure}[!ht]
  \centering
  \includegraphics[width=.25\textwidth]{cmy-model}
  \caption[CMY subtractive color model]{CMY subtractive color model. It is noteworthy that the intersection of yellow with magenta generates the red color, magenta with cyan generates the blue color and cyan with yellow generates the green color. Source: \citet{rus:08}.}
  \label{fig:cmy-model}
\end{figure}

Overlapping the CMY primary colors in equal amounts to generate the black color typically creates a tint that is close to the brown or dark green. To avoid this undesired effect, the black component is usually added to the system, represented by the letter K. This operation gives rise to a new model known as \textbf{CMYK}~\citep{gonzalez:02}.

%% ------------------------------------------------------------------------- %%
\subsection{Color models of the YUV family}
\label{sec:modelo_cores_yuv}

Color models of the YUV family are also known as orthogonal color spaces. In addition, one of their key features is to reduce the redundancy present in RGB color channels~\citep{kakumanu:07}.

The acronym YUV stands to a set of color spaces of which the luminance information, represented by the Y component, is coded separately from the chrominance, given by the components U and V. The components U and V are representations of signals of the difference of the blue subtracted from luminance ($B-Y$) and red subtracted from luminance ($R-Y$). It is used to represent colors in analog television transmission systems in the Phase Alternating Line (PAL) and Sequential Color with Memory (SECAM)~\citep{pedrini:08}.

The transformation of the RGB space to YUV is given by~\citep{pedrini:08}:\\
\begin{equation}
  \begin{bmatrix}
    Y \\ U \\ V
  \end{bmatrix} =
  \begin{bmatrix}
     0.299 &  0.587 &  0.114 \\
    -0.147 & -0.289 &  0.436 \\
     0.615 & -0.515 & -0.100 \\
  \end{bmatrix}
  \begin{bmatrix}
    R \\ G \\ B
  \end{bmatrix}
\end{equation}
where $0 \leq R, G, B \leq 1$.

Analogous to the YUV, the YIQ model was adopted in 1950 by the National Television System Committee (NTSC), an American standard for color television signal transmission. In this model, the Y component corresponds to luminance and the components I and Q encode the chrominance information~\citep{pedrini:08}.

The transformation of the RGB space to YIQ is given by~\citep{pedrini:08}:\\
\begin{equation}
  \begin{bmatrix}
    Y \\ I \\ Q
  \end{bmatrix} =
  \begin{bmatrix}
    0.299  &  0.587 &  0.114 \\
    0.596  & -0.275 & -0.321 \\
    0.212  & -0.523 & -0.311 \\
  \end{bmatrix}
  \begin{bmatrix}
    R \\ G \\ B
  \end{bmatrix}
\end{equation}
where $0 \leq R, G, B \leq 1$.

Another color model of the YUV family is the YCbCr, mathematically defined by a coordinate transformation with respect to some RGB space~\citep{pedrini:08}. YCbCr color model is often used in image and video compression standards, because its particular representation that it is easy to get rid of some redundant color information. The Figure~\ref{fig:ycbcr-model} shows the CbCr plane of the YCbCr color model for a specific Y value.

\begin{figure}[!htb]
  \centering
  \includegraphics[width=.7\textwidth]{ycbcr_plane}
  \caption[CbCr plane of the YCbCr color model]{CbCr plane of the YCbCr color model. Y value is set to 0.5, Cb on the horizontal axis in the range [-1, 1], Cr on the vertical axis in the range [-1, 1]. Source: \citet{eugster:10}.}
  \label{fig:ycbcr-model}
\end{figure}

In this system, the Y component represents luminance, computed as a weighted sum of RGB values. Cb component gives the measurement of the difference between the blue color and a reference value, similar to the Cr component which is the measurement of the difference between the red color and a reference value~\citep{pedrini:08}.

The transformation of the RGB space to YCbCr is given by~\citep{pedrini:08}:\\
\begin{equation}
  \begin{bmatrix}
    Y \\ Cb \\ Cr
  \end{bmatrix} =
  \begin{bmatrix}
     0.299 &  0.587 &  0.114 \\
    -0.169 & -0.331 &  0.5   \\
     0.5   & -0.419 & -0.081 \\
  \end{bmatrix}
  \begin{bmatrix}
    R \\ G \\ B
  \end{bmatrix}
\end{equation}

The choice of YCbCr color model for skin segmentation is, in general, twofold: first, YCbCr is one of the color models where it is possible to split the chrominance (Cb, Cr) components from the luminance (Y), and second, it is commonly used in the digital video domain. Furthermore, chrominance of different skin colors (e.g. white, black, yellow and brown) shares a marked similarity, which can be used to distinguish between skin and non-skin colors~\citep{phung:02}.

%% ------------------------------------------------------------------------- %%
\subsection{Color models of the HSI family}
\label{sec:modelo_cores_hsi}

Hue, Saturation, and Intensity (HSI) models are best suited for image processing applications from the user point of view, due to the correlation with human perception of the color \citep{konstantinos:00}.

In this model, as well in YIQ color space, the intensity given by I component is decomposed from the chrominance information, represented by the hue (H) and saturation (S). The combination of these components results in a pyramidal structure which can be seen in Figure~\ref{fig:hsi-model}~\citep{konstantinos:00}.

\begin{figure}[!hbt]
  \centering
  \includegraphics[width=.75\textwidth]{hsi-model}
  \caption[Graphical representation of the HSI model]{Graphical representation of the HSI model. The hue describes the color itself, in the form of an angle $\theta$, where $\theta \in [0, 360]$. Red is at 0 degrees, yellow at 60, green at 120, and so on. The saturation component, which varies between 0 and 1, indicates how much color is polluted with white color. The intensity scale is between $[0, 1]$, where 0 means black and 1, white. Source: \citet{blackice:16}.}
  \label{fig:hsi-model}
\end{figure}

The transformation of the components of the RGB space to HSI is given by the equations \citep{pedrini:08}:
\begin{align}
\label{eq:rgb_para_hsi}
\begin{split}
  \theta &= cos^{-1} \bigg( \frac{(R - G) + (R - B)}{2 \sqrt{(R - G)^2 + (R - B)(G - B)}} \bigg)
  \\[0.5em]
  H &= \begin{cases}
            \theta,       & \text{if}\ B \leq G\\
            360 - \theta, & \text{otherwise}\\
       \end{cases}
  \\[0.5em]
  S &= 1 - \frac{3 min(R, G, B)}{R + G + B}
  \\[0.5em]
  I &= \frac{R + G + B}{3}
\end{split}
\end{align}

It is important to note that the values R, G, and B must be normalized in the interval between 0 and 1. The intensity $I$ and the saturation $S$ are also normalized between 0 and 1.

Another model of this family is formed by the components Hue, Saturation and Value (HSV) and its three-dimensional graphical representation is a hexagonal pyramid derived from the RGB cube~\citep{pedrini:08}. Value, in this context, is the luminance component.

The various hue shades are represented at the top of the pyramid, the saturation is measured along the horizontal axis and value is measured along the vertical axis, which passes through the center of the pyramid. The hue, which corresponds to the edges around the vertical axis, varies from 0 (red) to 360 degrees and the angle between the vertices is 60 degrees. The saturation varies from 0 to 1 and it is represented as the ratio of the purity of a given hue to its maximum purity, that is when $S = 1$. Value varies from 0, at the peak of the pyramid representing the black color, to 1 at the base, where the intensities of the colors are maximum~\citep{pedrini:08}.

The transformation of the components of the RGB space to HSV is given by the equations \citep{pedrini:08}:
\begin{align}
\label{eq:rgb_para_hsv}
\begin{split}
  H &=  \begin{cases}
            60\ffrac{(G - B)}{M - m}, & \text{if}\ M = R\\[0.7em]
            60\ffrac{(B - R)}{M - m} + 120, & \text{if}\ M = G\\[0.7em]
            60\ffrac{(R - G)}{M - m} + 240, & \text{if}\ M = B
       \end{cases}
  \\[0.5em]
  S &=  \begin{cases}
            \ffrac{(M - m)}{M}, \quad &\text{if}\ M \neq 0\\[0.7em]
            0, \quad &\text{otherwise}\\
       \end{cases}
  \\[0.5em]
  V &= M
\end{split}
\end{align}

\noindent where $m = min(R, G ,B)$ and $M = max(R, G ,B)$. The luminance $V$ and saturation $S$ are normalized between 0 and 1. The $H$ hue ranges from 0 to 360 degrees.

Similarly to HSV, the Hue, Saturation, and Lightness (HSL) model is a three-dimensional representation (cylindrical geometry) and is formed by two cones of height 1, whose bases are coincident~\citep{pedrini:08}.

The hue is determined by the points in the circle of the common bases to the cones. The saturation varies from 0 to 1, depending on the distance to the axis of the cone. The lightness is along the vertical axis common to the two cones and varies in the range $[0, 1]$, where 0 means black and 1, white~\citep{pedrini:08}.

The conversion of the RGB space to HSL is given by the equations \citep{pedrini:08}:
\begin{align}
\label{eq:rgb_para_hsl}
\begin{split}
  H &=  \begin{cases}
            60\ffrac{(G - B)}{M - m}, & \text{if}\ M = R\\[0.7em]
            60\ffrac{(B - R)}{M - m} + 120, & \text{if}\ M = G\\[0.7em]
            60\ffrac{(R - G)}{M - m} + 240, & \text{if}\ M = B
       \end{cases}
  \\[0.5em]
  S &=  \begin{cases}
            \ffrac{(M - m)}{M + m}, & \text{if}\ 0 < L \leq 0,5\\[0.7em]
            \ffrac{(M - m)}{2 - (M + m)}, & \text{if}\ L > 0,5\\[0.5em]
            0, & \text{if}\ M = m\\
       \end{cases}
  \\[0.5em]
  L &= \frac{M + m}{2}
\end{split}
\end{align}

\noindent where $m = min(R, G, B)$ and $M = max(R, G, B)$. The lightness $L$ and saturation $S$ are normalized between 0 and 1. Note that the transformation of the $H$ component is the same as that used in the conversion of the RGB to HSV space in Eq.~\ref{eq:rgb_para_hsv} and varies between 0 and 360 degrees.

All the color models of this family have the property of thinking of lighter colors, obtained by increasing the brightness or lightness, and darker colors, by the diminution of the same values. The intermediate colors are produced by decreasing the saturation~\citep{pedrini:08}.


\documentclass[10pt]{beamer}

\usepackage[spanish,activeacute]{babel}
\usepackage[utf8x]{inputenc}
\usepackage{eurosym}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{graphicx}
\usepackage{lmodern}
\usepackage{enumerate}

\def\X{\ve{X}}
\def\x{\ve{x}}
\def\z{\ve{z}}
\def\y{\ve{y}}

\newcommand{\p}[1]{p_{_{#1}}} %pdf or pmf

\definecolor{darkblue}{rgb}{0.0, 0.0, 0.55}
\setbeamercolor{title}{fg=darkblue}
\setbeamercolor{frametitle}{fg=darkblue}
\newcommand{\myitem}{\item[$\bullet$]}
\definecolor{darkgreen}{rgb}{0, 0.55, 0}
\definecolor{darkred}{rgb}{0.55, 0,0}

\usepackage{tikz}
\usetikzlibrary{fit,positioning}
\usetikzlibrary{shapes,matrix,decorations.markings,arrows}
\usetikzlibrary{graphs}
\newcommand{\ve}[1]{\boldsymbol{#1}}
\newcommand{\set}[1]{\mathcal{#1}}
    \usetikzlibrary{chains,positioning}
    \usepackage{mathtools}

\newcommand{\noteB}[1]{\textbf{\textcolor{darkblue}{#1}}}

\newcommand{\noteR}[1]{\textbf{\textcolor{darkred}{#1}}}

\newcommand{\noteG}[1]{\textbf{\textcolor{darkgreen}{#1}}}

\newcommand{\mes}[2]{m_{#1\rightarrow#2}}
\newcommand{\logmes}[2]{l_{#1\rightarrow#2}}
\newcommand{\Iset}[1]{\mathtt{#1}} %Index set

\usepackage{bbm}

\setbeamertemplate{navigation symbols}{\usebeamerfont{footline} \insertframenumber/\inserttotalframenumber}
\logo{\includegraphics[scale=0.04]{logouc3m.eps}}

\title{Parameter learning with EM for discrete BNs}
\author{Pablo M. Olmos, olmos@tsc.uc3m.es}
\date{Course on Bayesian Networks,  November 2016}

%\AtBeginSection{\frame{\tableofcontents[currentsection]}}
\AtBeginSection{\frame{\sectionpage}}
%\AtBeginSubsection{\frame{\subsectionpage}}
\AtBeginSubsection{\frame{\frametitle{Bayessian Networks}\tableofcontents[currentsection,currentsubsection]}}

\begin{document}

\frame{\titlepage}

\begin{frame}{Index}
\tableofcontents
\end{frame}

\section{Learning parameters with full observations}

\begin{frame}{Motivation}

\begin{itemize}
\item Let $X_j\in\set{X}$, $j=1,\ldots,5$, be discrete R.V., where $K\doteq|\set{X}|$
\item Consider the following BN:
\end{itemize}
\begin{align*}
p(\x)=p(x_1)p(x_2|x_1)p(x_4)p(x_3|x_2,x_4)p(x_5|x_4)p(x_6|x_2)
\end{align*}
\begin{figure}
\centering
\begin{tikzpicture}[scale=0.8]
\tikzstyle{factor}=[rectangle,minimum size = 3mm, thick, draw =black,fill=black]
\tikzstyle{var}=[circle,minimum size = 3mm, thick, draw =black,fill=black]
\tikzstyle{var2}=[circle, thick, draw =black]
\tikzstyle{second}=[circle, minimum size = 10mm, thick]
\tikzstyle{box}=[rectangle, draw=black!100]
\tikzstyle{connect}=[-latex, thick]
\tikzstyle{arrow}=[draw, -latex]

\draw[step=5cm];
	\node[var2] (x_1) at (-1,0) []{$x_1$};
	\node[var2] (x_2) at (1,0) []{$x_2$};
	\node[var2] (x_3) at (3,0) []{$x_3$};
	\node[var2] (x_4) at (3,-2) []{$x_4$};
	\node[var2] (x_5) at (5,-2) []{$x_5$};
	\node[var2] (x_6) at (1,-2) []{$x_6$};

	\path [arrow]
		(x_1) edge []  (x_2)
		(x_2) edge []  (x_3)
		(x_2) edge []  (x_6)
		(x_4) edge []  (x_3)
		(x_4) edge []  (x_5);

\end{tikzpicture}
\end{figure}

\begin{itemize}
\item \noteG{BN structure is known}, \noteR{CPD tables are unknown}.
\item Our goal is to \noteB{estimate the CPD tables from $N$ independent samples} $\x_1,\x_2,\ldots,\x_N$,  drawn from $p(\x)$.
\end{itemize}

\end{frame}

\begin{frame}{Log-Likelihood of data (I)}
\begin{itemize}
\item Let $\x\in\mathcal{X}^V$ be a discrete R.V. such that $p(\x)=\prod_{t=1}^{V}p(x_t|\x_{\text{pa}(t)})$.
\item Consider $N$ independent samples $\mathcal{D}=\{\x_i\}_{i=1}^{N}$. For each sample, we can write each
$p(x_t|\x_{\text{pa}(t)})$ term, $t=1,\ldots,V$ as follows:
\begin{align*}
p(x_{it}|\x_{i,\text{pa}(t)})=\color{darkgreen}{}\prod_{c=1}^{K_{\text{pa}(t)}}\color{darkblue}{}\prod_{k=1}^{K_t}\color{darkred}{}\theta_{tck}^{\mathbbm{1}[x_{it}=k,\x_{i,\text{pa}(t)}=c]}
\end{align*}
\item \noteG{$\prod_{c=1}^{K_{\text{pa}(t)}}$} $\rightarrow$ product over all possible values of $\x_{i,\text{pa}(t)}$.
\item \noteB{$\prod_{k=1}^{K}$} $\rightarrow$ product over all possible values of $x_{i,t}$.
\item \noteR{$\theta_{tck}^{\mathbbm{1}[x_{it}=k,\x_{i,\text{pa}(t)}=c]}$} $\rightarrow$ equal to $\theta_{tck}$ if $x_{it}=k$, and $\x_{i,\text{pa}(t)}=c$. Thus,
$$ p(x_t=k|\x_{\text{pa}(t)}=c)\doteq \theta_{tck}$$
\item Note that $\sum_{k=1}^{K_t}\theta_{tck}=1$.
\end{itemize}

\end{frame}

\begin{frame}{Log-Likelihood of data (II)}
\begin{itemize}
\item The log-likelihood of the complete data is given by
\begin{align*}
\log p(\mathcal{D}|\ve{\theta})=\sum_{t=1}^{V}\sum_{c=1}^{K_{\text{pa}(t)}}\sum_{k=1}^{K_t} \color{darkgreen} N_{tck} \color{black}{}\log \theta_{tck}
\end{align*}
where
\begin{align*}
\color{darkgreen} N_{tck} =\sum_{i=1}^{N}\mathbbm{1}[x_{it}=k,\x_{i,\text{pa}(t)}=c]
\end{align*}
are the empirical counts.
\item Log-likelihood is maximized if we take
\begin{align*}
\hat{\theta}_{tck}=\frac{N_{tck}}{\sum_{k'=1}^{K_t} N_{tck'}}
\end{align*}
\item \noteR{ML solution is very simple! We calculate frequencies!}
\end{itemize}

\end{frame}

\begin{frame}{MAP solution}
\begin{itemize}
\item Dirichlet prior over $\ve{\theta}_{tc}=[\theta_{tc1}, \theta_{tc2}, \ldots, \theta_{tcK_t}]$:
\begin{align*}
\ve{\theta}_{tc} \sim \text{Dir}\left[\alpha_{tc1}, \alpha_{tc2}, \ldots, \alpha_{tcK_t}\right]
\end{align*}
\item It is easy to show that $p(\ve{\theta}_{tc}|\mathcal{D})$ is another Dirichlet distribution with parameters
\begin{align*}
\ve{\theta}_{tc} \sim \text{Dir}\left[\alpha_{tc1}+N_{tc1}, \alpha_{tc2}+N_{tc2}, \ldots, \alpha_{tcK_t}+N_{tcK_t}\right]
\end{align*}
\item The mean of $\theta$ w.r.t. the posterior distribution is
\begin{align*}
\mathbb{E}_{p(\ve{\theta}_{tc}|\mathcal{D})}[\theta_{tck}]=\frac{\alpha_{tck}+N_{tck}}{\sum_{k'=1}^{K_t} \alpha_{tck'}+N_{tck'}}
\end{align*}
\item $\theta_{tck}$ act as \emph{pseudocounts}, avoiding to assign zero probability to unobserved outcomes in our data
\end{itemize}

\end{frame}

\section{Learning with partial observations}

\begin{frame}{Motivation}

\begin{itemize}
\item Let $X_j\in\set{X}$, $j=1,\ldots,5$, be discrete R.V., where $K\doteq|\set{X}|$
\item Consider the following BN:
\end{itemize}
\begin{align*}
p(\x)=p(x_1)p(x_2|x_1)p(x_4)p(x_3|x_2,x_4)p(x_5|x_4)p(x_6|x_2)
\end{align*}
\begin{figure}
\centering
\begin{tikzpicture}[scale=0.8]
\tikzstyle{factor}=[rectangle,minimum size = 3mm, thick, draw =black,fill=black]
\tikzstyle{var}=[circle,minimum size = 3mm, thick, draw =black,fill=black]
\tikzstyle{var2}=[circle, thick, draw =black]
\tikzstyle{var3}=[circle, thick, draw =black,fill=gray]
\tikzstyle{second}=[circle, minimum size = 10mm, thick]
\tikzstyle{box}=[rectangle, draw=black!100]
\tikzstyle{connect}=[-latex, thick]
\tikzstyle{arrow}=[draw, -latex]

\draw[step=5cm];
	\node[var3] (x_1) at (-1,0) []{$x_1$};
	\node[var2] (x_2) at (1,0) []{$x_2$};
	\node[var2] (x_3) at (3,0) []{$x_3$};
	\node[var3] (x_4) at (3,-2) []{$x_4$};
	\node[var3] (x_5) at (5,-2) []{$x_5$};
	\node[var2] (x_6) at (1,-2) []{$x_6$};

	\path [arrow]
		(x_1) edge []  (x_2)
		(x_2) edge []  (x_3)
		(x_2) edge []  (x_6)
		(x_4) edge []  (x_3)
		(x_4) edge []  (x_5);

\end{tikzpicture}
\end{figure}

\begin{itemize}
\item \noteG{BN structure is known}, \noteR{CPD tables are unknown}.
\item Our goal is to \noteB{estimate the CPD tables from $N$ independent samples} $\x_1,\x_2,\ldots,\x_N$,  drawn from $p(\x)$.
\item Only a few elements of $\x$ are observed!
\end{itemize}

\end{frame}

\frame{
\frametitle{EM in a nutshell}

\begin{itemize}
\item $p(\y_i,\x_i|\ve{\theta})$, where $\y_i$ is observed and $\x_i$ is hidden, $i=1,\ldots,N$.
\item Our goal is to estimate $\ve{\theta}$ to maximize $p(\mathcal{D}|\ve{\theta})$ (ML) or as the mode of the posterior distribution $p(\ve{\theta}|\mathcal{D})$. \noteR{Complex! We have to marginalize first over $\x_i$, $i=1,\ldots,N$.}
\item EM algorithm. Initialize $\ve{\theta}$ to $\ve{\theta}^0$. For $\ell=1,2, \ldots$
\begin{enumerate}
\item \noteG{E-step}:
\begin{align*}
Q(\ve{\theta},\ve{\theta}^{\ell-1})&=\sum_{i=1}^{N}\int_{\x_i} p(\x_i|\y_i,\ve{\theta}^{\ell-1}) \log \left(p(\x_i,\y_i|\ve{\theta})\right) \text{d}\ve{x}_i\\&=\sum_{i=1}^{N}\mathbb{E}_{p(\x_i|\y_i,\ve{\theta}^{\ell-1})}[\log \left(p(\x_i,\y_i|\ve{\theta}\right)]
\end{align*}
\item \noteB{M-step}:
\begin{align*}
\ve{\theta}^{\ell}&=\arg\text{max}_{\ve{\theta}} ~Q(\ve{\theta},\ve{\theta}^{\ell-1}) \qquad (\text{ML estimation})\\\\
\ve{\theta}^{\ell}&=\arg\text{max}_{\ve{\theta}} ~Q(\ve{\theta},\ve{\theta}^{\ell-1}) +\log p(\ve{\theta}) \qquad (\text{MAP estimation})
\end{align*}
\end{enumerate}
\end{itemize}

}

\frame{
\frametitle{EM for discrete BNs (I)}
\begin{itemize}
\item $\y_i\rightarrow$ Set of observed variables for $i$-th data, $i=1, \ldots, N$.
\item The log-likelihood of the complete data is given by
\begin{align*}
\log p(\{\y_i,\x_i\}_{i=1}^N\|\ve{\theta})=\sum_{t=1}^{V}\sum_{c=1}^{K_{\text{pa}(t)}}\sum_{k=1}^{K_t} \color{darkgreen} N_{tck} \color{black}{}\log \theta_{tck}
\end{align*}
where
\begin{align*}
\color{darkgreen} N_{tck} =\sum_{i=1}^{N}\mathbbm{1}[x_{it}=k,\x_{i,\text{pa}(t)}=c]
\end{align*}
\end{itemize}
}

\frame{
\frametitle{EM for discrete BNs (II)}
\begin{itemize}
\item \noteG{E-step}:
\begin{align*}
Q(\ve{\theta},\ve{\theta}^{\ell-1})&=\sum_{i=1}^{N}\sum_{c=1}^{K_{\text{pa}(t)}}\sum_{k=1}^{K_t} \widetilde{N_{tck}} \log \theta_{tck}
\end{align*}
where
\begin{align*}
\widetilde{N_{tck}}&=\sum_{i=1}^{N}\mathbb{E}\left[\mathbbm{1}[x_{it}=k,\x_{i,\text{pa}(t)}=c]\right]\\
&=\sum_{i=1}^{N} p(x_{it}=k,\x_{i,\text{pa}(t)}=c|\y_i,\boldsymbol{\theta}^{\ell-1})
\end{align*}
\item Thus, given each pair $(\x_i,\y_i)$, we simply have to compute the marginal joint probabilities $p(x_{it}=k,\x_{i,\text{pa}(t)}=c|\y_i)$, $t=1,\ldots,V$.
\item We will use \noteB{Belief Propagation for this task!}
\end{itemize}
}

\frame{
\frametitle{EM for discrete BNs (III)}
\begin{itemize}
\item \noteG{M-step}:
\begin{align*}
\hat{\theta}_{tck}&=\frac{\widetilde{N_{tck}}}{\sum_{k'=1}^{K_t} \widetilde{N_{tck'}}} \qquad (\text{ML estimation})\\\\
\hat{\theta}_{tck}&=\frac{\alpha_{tck}+\widetilde{N_{tck}}}{\sum_{k'=1}^{K_t} \alpha_{tck'}+\widetilde{N_{tck'}}} \qquad (\text{MAP estimation})
\end{align*}
\end{itemize}

}

\end{document}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: t
%%% End:


\chapter{Generic Band interpolation}\label{ch:geninterp}

By setting $\verb#geninterp#=\verb#TRUE#$, \postw\ will calculate the
band energies (and possibly the band derivatives, if also
\verb#geninterp_alsofirstder# is set to \verb#TRUE#) on a generic list
of $k$ points provided by the user.

The list of parameters of the Generic Band Interpolation module are
summarized in Table~\ref{parameter_keywords_geninterp}.
The list of input $k$ points for which the band have to be calculated
is read from the file named {\tt seedname\_geninterp.kpt}. The format
of this file is
described below.

Note also that you might want to activate the flag
{\tt use\_ws\_distance=.true.}
to improve the interpolation results. For an explanation, see
Chapter~\ref{chap:interpolation}.

\section{Files}
\subsection{{\tt seedname\_geninterp.kpt}}
INPUT. Read by \postw\ if {\tt geninterp} is \verb#true#.

The first line is a comment (its maximum allowed length is 500
characters).

The second line must contain \verb#crystal# (or \verb#frac#) if the
$k$-point coordinates are given in crystallographic units,
i.e., in fractional units with respect to the primitive reciprocal
lattice vectors.
Otherwise, it must contain \verb#cart# (or \verb#abs#) if instead the
$k-$point coordinates are given in absolute
coordinates (in units of 1/\AA) along the $k_x$, $k_y$ and $k_z$
axes.

\emph{Note on units}: In the case of absolute coordinates,
if $a_{lat}$ is the lattice constant expressed in angstrom,
and you want to represent for instance the point
$X=\frac {2\pi}{a_{lat}} [0.5, 0, 0]$, then you have to input for its $x$ coordinate
$k_x = 0.5 * 2 * \pi / a_{lat}$. As a practical example, if
$a_{lat}=4$\AA, then $k_x = 0.78539816339745$ in
absolute coordinates in units of 1/\AA.

The third line must contain the number $n$ of following $k$ points.

The following $n$ lines must contain the list of $k$ points in the
format
\begin{verbatim}
kpointidx k1 k2 k3
\end{verbatim}
where \verb#kpointidx# is an integer identifying the given $k$ point,
and \verb#k1#, \verb#k2# and \verb#k3# are the three coordinates of the
$k$ points in the chosen units.

\subsection{{\tt seedname\_geninterp.dat} or {\tt
    seedname\_geninterp\_NNNNN.dat}}
\label{sec:seedname.geninterp.dat}
OUTPUT. This file/these files contain the interpolated band energies (and also the band
velocities if the input flag \verb#geninterp_alsofirstder# is \verb#true#).

If the flag \verb|geninterp_single_file| is \verb|true|, then a single
file {\tt seedname\_geninterp.dat} is written by the code at the end
of the calculation. If instead one sets \verb|geninterp_single_file|
to \verb|false|, each process writes its own output file, named
{\tt seedname\_geninterp\_00000.dat}, {\tt
  seedname\_geninterp\_00001.dat}, \ldots

This flag is useful when one wants to parallelize the calculation on
many nodes, and it should be used especially for systems with a small
number of Wannier functions, when one wants to compute the bands on a
large number of $k$ points (if the flag \verb|geninterp_single_file|
is \verb|true|, instead, all the I/O is made by the root node, which
is a significant bottleneck).

{\bf Important!} The files are not deleted before the start of a
calculation, but only the relevant files are overwritten. Therefore,
if one first performs a calculation and then a second one with a smaller
number of processors, care is needed to avoid to mix the results of
the older calculations with those of the new one. In case of doubt,
either check the date stamp in the first line of the {\tt
    seedname\_geninterp\_*.dat} files, or simply
delete the  {\tt
    seedname\_geninterp\_*.dat} files before starting the new
  calculation.

To join the files, on can simply use the following command:
\begin{verbatim}
cat seedname_geninterp_*.dat > seedname_geninterp.dat
\end{verbatim}
or, if one wants to remove the comment lines:
\begin{verbatim}
rm seedname_geninterp.dat
for i in seedname_geninterp_*.dat ; do grep -v \# "$i" >> \
seedname_geninterp.dat ; done
\end{verbatim}

The first few lines of each files are comments (starting with \#),
containing a datestamp, the
comment line as it is read from the input file, and a header.
The following lines contain the band energies (and
derivatives) for each band and $k$ point (the energy
index runs faster than the $k$-point index).
For each of these lines, the first four columns contain the $k$-point index as provided in the
input, and the $k$ coordinates (always in absolute coordinates, in
units of 1/\AA).
The fifth column contains the band energy.

If \verb#geninterp_alsofirstder# is \verb#true#, three further columns
are printed, containing the three first derivatives of the bands along the $k_x$, $k_y$
and $k_z$ directions.

The $k$ point coordinates are in units of 1/\AA, the band energy is in eV.


\documentclass{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{stmaryrd}
\usepackage{listings}
\usepackage{proof}
\usepackage{supertabular}
\usepackage{geometry}
\usepackage{color}
\usepackage{proof}
\usepackage{theorem}

\newtheorem{lem}{Lemma}
\newtheorem{thm}{Theorem}
\newtheorem{defn}{Definition}
\newtheorem{cor}{Corollary}

\newcommand{\Tr}{\mathit{Tr}}
\newcommand{\secref}[1]{Section \ref{#1}}
\newcommand{\figref}[1]{Figure \ref{#1}}
\newcommand{\lstref}[1]{Listing \ref{#1}}
\newcommand{\appref}[1]{Appendix \ref{#1}}
\newcommand{\lemref}[1]{Lemma \ref{#1}}
\newcommand{\corref}[1]{Corollary \ref{#1}}

\newcommand{\rlett}{\textsc{r\_let\_term} }
\newcommand{\rletv}{\textsc{r\_let\_value} }
\newcommand{\rseqlt}{\textsc{r\_seq\_left\_term} }
\newcommand{\rseqlv}{\textsc{r\_seq\_left\_value} }
\newcommand{\rfunc}{\textsc{r\_fun\_call} }
\newcommand{\rmethc}{\textsc{r\_meth\_call} }
\newcommand{\rifexpr}{\textsc{r\_if\_term} }
\newcommand{\riftrue}{\textsc{r\_if\_true} }
\newcommand{\riffalse}{\textsc{r\_if\_false} }

\newcommand{\tunit}{\textsc{t\_unit} }
\newcommand{\ttrue}{\textsc{t\_true} }
\newcommand{\tfalse}{\textsc{t\_false} }
\newcommand{\tobj}{\textsc{t\_object} }
\newcommand{\tfundef}{\textsc{t\_fun\_def} }
\newcommand{\tlet}{\textsc{t\_let} }
\newcommand{\tif}{\textsc{t\_if} }
\newcommand{\tseq}{\textsc{t\_seq} }
\newcommand{\tfunc}{\textsc{t\_fun\_call} }
\newcommand{\tmethc}{\textsc{t\_meth\_call} }
\newcommand{\tdrop}{\textsc{t\_drop} }
\newcommand{\tsub}{\textsc{t\_sub} }
\newcommand{\tnarrow}{\textsc{t\_narrow} }
\newcommand{\twiden}{\textsc{t\_widen} }

\newcommand{\subrefl}{\textsc{sub\_refl}}
\newcommand{\subtrans}{\textsc{sub\_trans}}
\newcommand{\subfn}{\textsc{sub\_fn}}
\newcommand{\subobj}{\textsc{sub\_obj}}

\newcommand{\remapunit}{\textsc{remap\_unit}}
\newcommand{\remapbool}{\textsc{remap\_bool}}
\newcommand{\remapfunc}{\textsc{remap\_func}}
\newcommand{\remapobj}{\textsc{remap\_obj}}

\newcommand{\remapapp}[3]{\ensuremath{\mathbf{remap}(#1, #2 \gg #3)}}

\newcommand{\typerule}[4]{#1 \triangleright #2 : #3 \triangleleft #4}
\newcommand{\oprule}[4]{#1 \mid #2\;\longrightarrow\;#3 \mid #4}
\newcommand{\inferrule}[4]{\left\llbracket #1 \triangleright #2 : #3 \triangleleft #4 \right\rrbracket}
\newcommand{\inferlhs}[1]{\left\llbracket \gamma \triangleright #1 : \alpha \triangleleft \gamma' \right\rrbracket}
\newcommand{\subst}[3]{#3 \{\overline{^{#1}/_{#2}}\}}
\newcommand{\substxy}[1]{\subst{x_i}{y_i}{#1}}
\newcommand{\unitv}{\mathtt{unit}}
\newcommand{\unitt}{\mathbf{Unit}}
\newcommand{\truev}{\mathtt{true}}
\newcommand{\falsev}{\mathtt{false}}
\newcommand{\boolt}{\mathbf{Bool}}
\newcommand{\funv}[4]{\lambda(\overline{#1 : #2 \gg #3}).#4}
\newcommand{\funt}[3]{(\overline{#1 \gg #2}) \rightarrow #3}
\newcommand{\lett}[3]{\mathtt{let}\:#1\:\mathtt{=}\:#2\:\mathtt{in}\:#3}
\newcommand{\ift}[3]{\mathtt{if} \; #1 \; \mathtt{then} \; #2 \; \mathtt{else} \; #3}
\newcommand{\cand}{\:\wedge\:}
\newcommand{\free}[1]{\mathbf{free}\:#1}
\newcommand{\fresh}[1]{\mathbf{fresh}\:#1}

\newcommand{\qed}{$\blacksquare$}

\newenvironment{proof}{\vspace{1ex}\noindent{\bf Proof}\hspace{0.5em}}
  {\hfill\qed\vspace{1ex}}

\input{ts_defns}

\begin{document}

\title{TS1 - A simple linear calculus for typestate inference}
\author{Iain McGinniss}

\section{Grammar}

\ottgrammartabular{
\ottt\ottinterrule
\ottv\ottinterrule
\otto\ottinterrule
\ottT\ottinterrule
\ottO\ottinterrule
}

\section{Operational Semantics}

\ottgrammartabular{
\ottmu\ottinterrule
}

%\ottdefnvalidstore

\ottdefnreduce

\section{Type rules}
\label{sec:typerules}

\ottgrammartabular{
\ottG\ottinterrule
\ottT\ottinterrule
}

%\ottdefnvalidgamma

\ottdefnstoretype

\ottdefnmethtype

\ottdefnsubtype

\ottdefnobjeffextract

\ottdefntype

\section{Soundness proofs}

\subsection{Supporting lemmas}

\begin{lem}
\label{lem:valuectx}
If $v$ is a value and $\typerule{\Gamma}{v}{T}{\Gamma'}$, then $\Gamma = \Gamma'$.
\end{lem}
\begin{proof}
Directly from the typing judgements. $v$ is either {\tt unit}, {\tt true},
{\tt false}, a function literal or an object literal.
Correspondingly, by rules \tunit,
\ttrue, \tfalse,
\tfundef and \tobj, $\Gamma = \Gamma'$.
\end{proof}

\begin{lem}[Object state sets]
Given an object state set $\overline{S}$ and a state $S'$, if
$S' \in \overline{S}$ then $O@S' <: O@\overline{S}$.
\end{lem}
\begin{proof}
$O@\overline{S}$ is defined as equivalent to
$\bigsqcup_{S \in \overline{S}}{O@S}$. Given that $S \in \overline{S}$,
we must have that $\overline{S} = \overline{S'} \cup \{ S \}$ for
some $\overline{S'}$.
it follows that $O@\overline{S} = O@S' \sqcup \left(
\bigsqcup_{S \in \overline{S'}}{O@S} \right)$.
By the definition of join,
$O@S' <: O@S' \sqcup \bigsqcup_{S \in \overline{S}}{O@S}$.
\end{proof}

\begin{lem}
\label{lem:ssetinc}
Given an object protocol $O = \left\{ \overline{ S_i \{ ... \} } \right\}$ and two state sets
$\overline{S} \subseteq \overline{S'} \subseteq \overline{S_i}$, it follows
that $O@\overline{S} <: O@\overline{S'}$.
\end{lem}
\begin{proof}
From the definition of state sets, we have that
$O@\overline{S} = \bigsqcup_{S \in \overline{S}}{O@S}$ and that
$O@\overline{S'} = \bigsqcup_{S \in \overline{S'}}{O@S}$.
As $\overline{S} \subseteq \overline{S'}$ there exists an
$\overline{S''}$ such that $\overline{S'} = \overline{S} \cup \overline{S''}$
where $\overline{S} \cap \overline{S''} = \emptyset$. Therefore
$O@\overline{S'} = \left(\bigsqcup_{S \in \overline{S}}{O@S}\right) \sqcup \left(\bigsqcup_{S' \in \overline{S'} - \overline{S}}{O@S'}\right)$.

As $A <: A \sqcup B$ for all $A$ and $B$ where $A \sqcup B$ is defined,
it follows that $O@\overline{S} <: O@\overline{S'}$.
\end{proof}

\begin{lem}
\label{lem:remap}
Given types $T$, $U$, and $V$, such that
$\remapapp{T}{U}{V}$ is defined, it follows that $\remapapp{T}{U}{V} <: V$.
\end{lem}
\begin{proof}
By case analysis of V.

\begin{itemize}
\item $V = \unitt$ or $V = \boolt$. By either \remapunit or \remapbool
it must be the case that $U = V$. It must also be the case that
$T = U$, as $\unitt$ and $\boolt$ do not have any subtypes other than
themselves. Therefore $V = \remapapp{T}{U}{V} <: V$ by \subrefl.

\item $V = \funv{V_i}{V_i'}{V'}$ for some $\overline{V_i}$, $\overline{V_i'}$
and $V'$. By \remapfunc, $T <: U$ and $T = \remapapp{T}{U}{V}$. Therefore,
$T <: V$.

\item $V = O@\overline{S'}$ for some $O$ and $\overline{S'}$. By
\remapobj, $O'@\overline{S'''} = \remapapp{T}{U}{V}$ where
$T = O'@\overline{S''}$ and $U = O@\overline{S}$ and
$\overline{S'''} = \{ S | O'@S \sim O@\overline{S'}, S \in O' \}$
for some $O'$, $\overline{S}$ and $\overline{S''}$. By \subobj,
$T = O'@\overline{S'''} <: O@\overline{S'} = V$ as for each $S \in \overline{S'''}$,
$O'@S <: O@\overline{S'}$.
\end{itemize}
\end{proof}

\begin{lem}[Remap is covariant]
\label{lem:remapcv}
Given types $T, T', U$ and $V$ such that $T <: T'$,
$\remapapp{T}{U}{V} <: \remapapp{T'}{U}{V}$.
\end{lem}

\begin{lem}
\label{lem:remapeff}
Given types $T$, $U$ and $V$ such that $U <: V$ and $T = \remapapp{U}{V}{V}$,
it follows that $U <: T$.
\end{lem}
\begin{proof}
By case analysis of the type $V$.

\begin{itemize}
\item $V = \unitt$ or $V = \boolt$.
It must be the case that $V = U$ also, as $\unitt$ and $\boolt$
have no subtypes other than themselves by rule \subrefl.
Therefore, by rule \remapunit or \remapbool, $T = V$,
and $U <: T$ by rule \subrefl.

\item $V = \funv{T_i}{T_i'}{T_r}$ for some $\overline{T_i}$, $\overline{T_i'}$
and $T_r$. As $U <: V$, it follows by rule \remapfunc that
$T = U$, and $U <: T$ by \subrefl.

\item $V = O@\overline{S}$ for some $O$ and $\overline{S}$.
It follows that $U = O'@\overline{S'}$
for some $O'$ and $\overline{S'}$
such that $O'@\overline{S'} \sim O@\overline{S}$
by rule \subobj.
By rule \remapobj, it follows that $T = O'@\overline{S''}$
such that $\overline{S''} = \{ S | O'@S \sim O@\overline{S}, S \in O' \}$.
By \lemref{lem:ssetinc} it follows that
$O'@\overline{S'} <: O'@\overline{S''}$.

\end{itemize}
\end{proof}

\begin{cor}
\label{lem:remapsame}
Given a type $T$, $T <: \remapapp{T}{T}{T}$.
\end{cor}

\begin{lem}[remap]
\label{lem:upgrade}
If a term $t$ can be typed such that
$\typerule{\Gamma, x : T}{t}{U}{\Gamma', x : V}$, and if
$T' <: T$, then there exists a $V' <: \remapapp{T'}{T}{V}$
such that $\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.
\end{lem}
\begin{proof}
by induction on the structure of $t$.
The term $t$ is of one of the following forms:

\begin{itemize}
\item $t$ is a value, so by \lemref{lem:valuectx} it follows that
$\Gamma = \Gamma'$ and $T = V$.
The environment can be arbitrary in the typing rules for values,
therefore we can type $t$ such that
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : T'}$.
Let $V' = T'$.
By \lemref{lem:remapeff}, we have that $V' <: \remapapp{T'}{T}{V}$.
Therefore, $\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.

\item $t = \lett{y}{t'}{t''}$. It follows from the rule \tlet that
$x \neq y$ (as variables cannot be rebound) and that there exists
$\Gamma_1, \Gamma_2, T_2, T_y, T_y'$ such that
$\typerule{\Gamma, x : T}{t'}{T_y}{\Gamma_1, x : T_2}$ and that
$\typerule{\Gamma_1, x : T_2, y : T_y}{t''}{U}{\Gamma', x : V, y : T_y'}$.
By induction, it follows that there exists a type $V_2 <: \remapapp{T'}{T}{T_2}$ such that
$\typerule{\Gamma, x : T'}{t'}{T_y}{\Gamma_1, x : V_2}$.
By \lemref{lem:remap} we have that $V_2 <: T_2$.
Therefore it also follows by induction that
there exists a type $V_3 <: \remapapp{V_2}{T_2}{V}$
such that
$\typerule{\Gamma_1, x : V_2, y : T_y}{t''}{U}{\Gamma', x : T_3, y : V_3}$.
Let $V' = V_3$. The rules of \tlet are satisfied such that
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.

\item $t = y ( \overline{z_i} )$.
It follows from the rule \tfunc that there exists
$\Gamma_1, \overline{T_i}, \overline{U_i}, \overline{V_i}, \overline{T_i'}$
such that
$\Gamma, x : T = \Gamma_1, y : \funt{U_i}{V_i}{U}, \overline{z_i : T_i}$
and that
$\Gamma', x : V = \Gamma_1, y : \funt{U_i}{V_i}{U}, \overline{z_i : T_i'}$
such that for each $i$, $T_i <: U_i$, $T_i' <: V_i$
and $U <: U'$.

There are three cases to consider:
\begin{itemize}
\item $x \neq y$ and $x \notin \overline{x_i}$, meaning that
$\Gamma_1 = \Gamma_2, x : T$ for some $\Gamma_2$, that
$\Gamma = \Gamma_2, y : \funt{U_i}{V_i}{U}, \overline{z_i : T_i}$
and that $\Gamma' = \Gamma_2, y : \funt{U_i}{V_i}{U}, \overline{z_i : T_i'}$.
Additionally, $T = V$.
It follows that $t$ can be typed without $x$
such that $\typerule{\Gamma}{t}{U}{\Gamma'}$. Let $V' = T'$.
By the weakening lemma, $\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.
By \lemref{lem:remapeff}, $V' <: \remapapp{T'}{T}{V}$.

\item $x = y$, meaning $\Gamma = \Gamma_1, \overline{z_i : T_i}$ and
$\Gamma' = \Gamma_1, \overline{z_i : T_i'}$.
Additionally, $T = V = \funt{U_i}{V_i}{U}$.
$T'$ must therefore be of form $\funt{U_i'}{V_i'}{U'}$
such that for each $i$, $U_i <: U_i'$, $V_i' <: V_i$, $U' <: U$ by the
definition of subtyping. Let $V' = T'$.
By \lemref{lem:remapeff}, $V' <: \remapapp{T'}{T}{V}$.
It follows by rule \tfunc, and \tsub to replace $U'$ with $U$, that
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.

\item $x \in \overline{z_i}$, meaning that $T = T_i$ and
$V = T_i' = \remapapp{T}{U_i}{V_i}$ for some $i$.
Let $V' = \remapapp{T'}{U_i}{V_i}$.
By \lemref{lem:remapcv}, $V' <: V$.
As $T' <: T$, we can type $t$ by rule \tfunc such that
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma, x : V'}$.

\end{itemize}

\item $t = y.m$. By the rule \tmethc it follows that
$\Gamma, x : T = \Gamma_1, y : O@\overline{S}$
and that
$\Gamma', x : V = \Gamma_1, y : O@\overline{S'}$
for some $\Gamma_1, O, \overline{S}$ and $\overline{S'}$
where
$\overline{S'} = \{ S_n | m : T_r \Rightarrow S_n \in O@S, S \in \overline{S} \}$.
Additionally,
$U = \bigsqcup{ \{ T_r | m : T_r \Rightarrow S_n \in O@S, S \in \overline{S} \} }$.
There are two cases to consider:

\begin{itemize}
\item $x = y$. It follows that $\Gamma = \Gamma_1 = \Gamma'$ and
that $T = O@\overline{S}, V = O@\overline{S'}$.
$T'$ must be of form $O'@\overline{S}''$ such that
$O'@\overline{S}'' \sim O@\overline{S}$ by rule \subobj.

Let $V' = O'@\overline{S}''' = \remapapp{T'}{T}{V}$
where
$S''' = \{ S_n' | m : T_r' \Rightarrow S_n' \in O'@S'', S'' \in \overline{S}''\}$.
Let $U' = \bigsqcup{ \{ T_r' | m : T_r' \Rightarrow S_n' \in O'@S'', S'' \in \overline{S}'' \} }$.
As $O'@\overline{S}'' \sim O@\overline{S}$, it follows that
$O'@\overline{S}''' \sim O@\overline{S}'$.
Therefore, $U' <: U$ and $V' <: V$. By the rule \tmethc, and \tsub
to replace $U'$ with $U$, it follows that
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.

\item $x \neq y$. It follows that $\Gamma = \Gamma'', y : O@\overline{S}$
and that $\Gamma' = \Gamma'', y : O@\overline{S'}$
for some $\Gamma''$, $O$ and $\overline{S}$, and that
$T = V$.
By rule \tmethc we have that $\typerule{\Gamma}{t}{U}{\Gamma'}$.
Let $V' = T'$. By \lemref{lem:remapeff}, $V' <: \remapapp{T'}{T}{V}$.
By the weakening lemma,
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.

\end{itemize}

\item $t = t' ; t''$. It follows by rule \tlet that there exists
$\Gamma'', V_{mid}, U'$
such that $\typerule{\Gamma, x : T}{t'}{U'}{\Gamma'', x : V_{mid}}$
and that $\typerule{\Gamma'', x : V_{mid}}{t''}{U}{\Gamma', x : V}$.
By induction, it follows that there exists $V_{mid}' <: \remapapp{T'}{T}{V_{mid}}$
such that $\typerule{\Gamma, x : T'}{t'}{U'}{\Gamma'', x : V_{mid}'}$.
By \lemref{lem:remap} we have that $V_{mid}' <: V_{mid}$.
There it also follows by induction that
there exists $V_{end} <: \remapapp{V_{mid}'}{V_{mid}}{V}$ such that
$\typerule{\Gamma'', x : V_{mid}'}{t''}{U}{\Gamma'', x : V_{end}}$.
Let $V' = V_{end}$. By rule \tlet it follows that
$\typerule{\Gamma, x : T'}{t' ; t''}{U}{\Gamma'', x : V'}$.

\item $t = \ift{t_c}{t_t}{t_f}$. It follows from rule \tif that there
exists $\Gamma_1$, $\Gamma_2$, $\Gamma_3$, $U_t$, $U_f$, $V_c$, $V_t$ and
$V_f$
such that
$\typerule{\Gamma, x : T}{t_c}{\boolt}{\Gamma_1, x : V_c}$ and that
$\typerule{\Gamma_1, x : V_c}{t_t}{U_t}{\Gamma_2, x : V_t}$ and that
$\typerule{\Gamma_1, x : V_c}{t_f}{U_f}{\Gamma_3, x : V_f}$ with
$\Gamma' = \Gamma_2 \sqcup \Gamma_3$, $U = U_t \sqcup U_f$ and
$V = V_t \sqcup V_f$.

By induction, there exists a $V_c' <: \remapapp{T'}{T}{V_c}$
such that $\typerule{\Gamma, x : T'}{t_c}{\boolt}{\Gamma_1, x : V_c'}$.
By \lemref{lem:remap} we also have that $V_c' <: V_c$.
Therefore by induction there also exists a
$V_t' <: \remapapp{V_c'}{V_c}{V_t}$ and $V_f' <: \remapapp{V_c'}{V_c}{V_f}$
such that $\typerule{\Gamma_1, x : V_c'}{t_t}{T_t}{\Gamma_2, x : V_t'}$
and that $\typerule{\Gamma_1, x : V_c'}{t_f}{T_f}{\Gamma_3, x : V_f'}$.
By \lemref{lem:remap} we also have that $V_t' <: V_t$ and that
$V_f' <: V_f$.
Let $V' = V_t' \sqcup V_f'$. By the definition of join, $V' <: V$.
It follows by rule \tif that
$\typerule{\Gamma, x : T'}{t}{U}{\Gamma', x : V'}$.

\end{itemize}

\end{proof}

\begin{lem}[Weakening]
If $\typerule{\Gamma}{t}{T}{\Gamma'}$,
then for all $\Gamma''$ such that $dom(\Gamma) \cap dom(\Gamma'') = \emptyset$,
$t$ can also be typed such that
$\typerule{\Gamma, \Gamma''}{t}{T}{\Gamma', \Gamma''}$.
\end{lem}
\begin{proof}
by induction on the structure of $t$.

If $t$ is some value $v$, it follows by \lemref{lem:valuectx} that
$\Gamma = \Gamma'$ and that the typing derivation will still be valid as long
as the input and output contexts are identical. Therefore,
$\typerule{\Gamma , \Gamma''}{v}{T}{\Gamma', \Gamma''}$.

If $t$ is not a value, then it is of one of the following forms:

\begin{itemize}
\item $t = \lett{x}{t'}{t''}$. It follows from rule \tlet that
there exists $\Gamma_1, \Gamma_2, T', T''$ such that
$\typerule{\Gamma}{t'}{T'}{\Gamma_1}$ and
$\typerule{\Gamma_1, x : T'}{t''}{T}{\Gamma', x : T''}$.
By \tlet we have that $dom(\Gamma') = dom(\Gamma)$.
By induction, $t'$ can be typed such
that $\typerule{\Gamma,\Gamma''}{t'}{T'}{\Gamma_1,\Gamma''}$,
and $t''$ can be typed such that
$\typerule{\Gamma_1, \Gamma'', x : T'}{t''}{T}{\Gamma', \Gamma'', x : T''}$,
as we can assume by Barendregt's convention that $x$ can be made distinct from
any variable names in $\Gamma''$ by relabeling $x$.
Therefore,
$\typerule{\Gamma, \Gamma''}{\lett{x}{t'}{t''}}{T}{\Gamma', \Gamma''}$.

\item $t = x ( \overline{x_i} )$. It follows from the rule \tfunc that
there exists a $\Gamma_1, \overline{T_i}, \overline{U_i}, \overline{V_i}, \overline{T_i'}$
such that
$\Gamma = \Gamma_1, x : (\overline{U_i \gg V_i}) \rightarrow T, \overline{x_i : T_i}$
and that
$\Gamma' = \Gamma_1, x : (\overline{U_i \gg V_i}) \rightarrow T, \overline{x_i : T_i'}$.
As $\Gamma_1$ can be arbitrary in the rule \tfunc, it follows that we
can extend the input and output contexts such that
$\typerule{\Gamma, \Gamma''}{x ( \overline{x_i} )}{T}{\Gamma', \Gamma''}$.

\item $t = x.m$. It follows from the rule \tmethc that there exists
a $\Gamma_1$ such that $\Gamma = \Gamma_1, x : O@\overline{S}$ and that
$\Gamma' = \Gamma_1, x : O@\overline{S'}$.
As $\Gamma_1$ can be arbitrary in rule \tmethc, it follows that we can
extend the input and out contexts such that
$\typerule{\Gamma, \Gamma''}{x.m}{T}{\Gamma', \Gamma''}$.

\item $t = t' ; t''$. It follows from the rule \tseq that there exists
$\Gamma_1$ and $T'$ such that $\typerule{\Gamma}{t'}{T'}{\Gamma_1}$
and $\typerule{\Gamma_1}{t''}{T}{\Gamma'}$. By induction
$\typerule{\Gamma, \Gamma''}{t'}{T'}{\Gamma_1, \Gamma''}$
as $dom(\Gamma_1) \subseteq dom(\Gamma)$ and therefore
$\Gamma_1$ and $\Gamma''$ are disjoint.
Similarly, as $dom(\Gamma_1) = dom(\Gamma')$, by induction
$\typerule{\Gamma_1, \Gamma''}{t''}{T}{\Gamma', \Gamma''}$.
Therefore $\typerule{\Gamma, \Gamma''}{t'}{T'}{\Gamma_1, \Gamma''}$.

\item $t = \ift{t_c}{t_t}{t_f}$. It follows from rule \tif that there
exists $\Gamma_1$, $\Gamma_2$, $\Gamma_3$, $T_t$ and $T_f$ such that
$\typerule{\Gamma}{t_c}{\boolt}{\Gamma_1}$ and that
$\typerule{\Gamma_1}{t_t}{T_t}{\Gamma_2}$ and that
$\typerule{\Gamma_1}{t_f}{T_f}{\Gamma_3}$ with
$\Gamma' = \Gamma_2 \sqcup \Gamma_3$ and $T = T_t \sqcup T_f$.
By induction each of $\Gamma_1$, $\Gamma_2$ and $\Gamma_3$ can
be extended by $\Gamma''$ in the typing judgements for $t_c$,
$t_t$ and $t_f$. As $\Gamma'' \sqcup \Gamma'' = \Gamma''$, it
follows that $(\Gamma_2, \Gamma'') \sqcup (\Gamma_3, \Gamma'') =
\Gamma_2 \sqcup \Gamma_3, \Gamma''$ and therefore $t$ can be
typed such that $\typerule{\Gamma, \Gamma''}{t}{T}{\Gamma', \Gamma''}$.

\end{itemize}

\end{proof}

\begin{lem}[Substitution]
If $\typerule{\Gamma}{t}{T}{\Gamma'}$, then
$\typerule
{\subst{x_i}{y_i}{\Gamma}}
{\subst{x_i}{y_i}{t}}
{T}
{\subst{x_i}{y_i}{\Gamma'}}$
where each $x_i$ and $y_i$ is distinct, and $\overline{x_i} \cap \overline{y_i} = \emptyset$.
\end{lem}
\begin{proof}
by induction on the structure of $t$.

If $t$ is some value $v$, then $\Gamma = \Gamma'$, therefore
$\subst{x_i}{y_i}{\Gamma} = \subst{x_i}{y_i}{\Gamma'}$
and substitution has no effect on the value, which has no specific requirements
of either $\Gamma$ or $\Gamma'$ other than that they are equal.
Therefore, $\typerule{\subst{x_i}{y_i}{\Gamma}}{\subst{x_i}{y_i}{v}}{T}{\subst{x_i}{y_i}{\Gamma'}}$.

If $t$ is a term, then it is of one of the following forms:

\begin{itemize}
\item $t = \lett{x}{t''}{t'}$.
Substitution is defined on this term such that \\
$\subst{x_i}{y_i}{t} \equiv
\lett{x}{( \subst{x_i}{y_i}{t''} )}{(\subst{x_j}{y_j}{t'})}$
, where $\overline{y_j} = \overline{y_i} - \{ x \}$.

By rule \tlet, it follows
that there exists $T', T'', \Gamma_1$ such that
$\typerule{\Gamma}{t''}{T''}{\Gamma_1}$ and that
$\typerule{\Gamma_1, x : T''}{t''}{T}{\Gamma', x : T'}$.
Additionally, $x \notin dom(\Gamma)$.

By induction, $t''$ can be substituted such that
$\typerule
{\subst{x_i}{y_i}{\Gamma}}
{\subst{x_i}{y_i}{t''}}
{\subst{x_i}{y_i}{\Gamma_1}}
$.
Additionally, $t'$ can be substituted such that
$(\subst{x_j}{y_j}{\Gamma_1}) , x : T'' \triangleright \subst{x_j}{y_j}{t''} \triangleleft \{ (\overline{x_j / y_j} \} \Gamma') , x : T'$ as it is guaranteed that $x \notin \overline{y_j}$.
As $dom(\Gamma') = dom(\Gamma)$, and that $x \notin dom(\Gamma)$, it follows that $\subst{x_i}{y_i}{\Gamma_1} = (\subst{x_j}{y_j}{\Gamma_1})$ and
that $\overline{x_i / y_i} \} \Gamma' = \overline{x_j / y_j} \} \Gamma'$.

Therefore, the requirements of \tlet are satisfied such that
$\typerule{\subst{x_i}{y_i}{\Gamma}}{\subst{x_i}{y_i}{t}}{T}{\subst{x_i}{y_i}{\Gamma'}}$.

\item $t = t_l ; t_r$.
Substitution is defined on this term such that
$\{ \overline{x_i / y_i} \} ( t_l ; t_r ) \equiv \subst{x_i}{y_i}{t_l} ; \subst{x_i}{y_i}{t_r}$.

By rule \tseq, there exists $\Gamma'', T'$ such that
$\typerule{\Gamma}{t_l}{T'}{\Gamma''}$ and
$\typerule{\Gamma''}{t_r}{T}{\Gamma'}$. By induction,
$\typerule{\subst{x_i}{y_i}{\Gamma}}{\subst{x_i}{y_i}{t_l}}{T'}{\subst{x_i}{y_i}{\Gamma''}}$ and
$\typerule{\subst{x_i}{y_i}{\Gamma''}}{\subst{x_i}{y_i}{t_r}}{T}{\subst{x_i}{y_i}{\Gamma'}}$.
Therefore, $t$ can by typed \tseq after the substitution.

\item $t = x_0 ( \overline{x_k} )$.
Substition is defined on this term such that
$\subst{x_i}{y_i}{x_0} ( \overline{x_k} ) \equiv y_0 ( \overline{y_k} )$
where for each $x \in \{ x_0, \overline{x_k} \}$,
$x = y_j \implies y = x_j$, otherwise $y$ is the original value.

For each substitution that occurs in the term, the variable must exist in
both the input and output contexts by rule \tfunc, and will be substituted
for the same name. Therefore, $t$ can be typed by \tfunc after the
substitution.

\item $t = x.m$.
Substition is defined on this term such that
$\subst{x_i}{y_i}{x.m} \equiv y.m$
where $y = x_j$ if $x = y_j$ for some $y_j \in \overline{y_i}$.
Otherwise, $y = x$.

Rule \tmethc requires that $x$ exist in both $\Gamma$ and $\Gamma'$ and
therefore any substitution of x in the term will be matched with the
same substitution in the contexts. Therefore, $t$ can be typed by
\tmethc after the substitution.

\item $t = \ift{t_c}{t_t}{t_f}$. Substitution is defined on this term such
that \\
$\subst{x_i}{y_i}{(\ift{t_c}{t_t}{t_f})} \equiv
\ift{\substxy{t_c}}{\substxy{t_t}}{\substxy{t_f}}$.
By rule \tif there exists $\Gamma_1$, $\Gamma_2$, $\Gamma_3$, $T_t$, $T_f$
such that
$\typerule{\Gamma}{t_c}{\boolt}{\Gamma_1}$ and that
$\typerule{\Gamma_1}{t_t}{T_t}{\Gamma_2}$ and that
$\typerule{\Gamma_1}{t_f}{T_f}{\Gamma_3}$ where
$\Gamma' = \Gamma_2 \sqcup \Gamma_3$ and $T = T_t \sqcup T_f$. By induction
substitution can be performed on $t_c$, $t_t$ and $t_f$ such that
$\typerule{\substxy{\Gamma}}{\substxy{t_c}}{\boolt}{\substxy{\Gamma_1}}$ and that
$\typerule{\substxy{\Gamma_1}}{\substxy{t_t}}{T_t}{\substxy{\Gamma_2}}$ and
$\typerule{\substxy{\Gamma_1}}{\substxy{t_f}}{T_f}{\substxy{\Gamma_3}}$.
By the definition of substitution on contexts,
$\substxy{\Gamma_2} \sqcup \substxy{\Gamma_3} = \substxy{\Gamma'}$. Therefore,
by rule \tif,
$\typerule{\substxy{\Gamma}}{\substxy{\ift{t_c}{t_t}{t_f}}}{T}{\substxy{\Gamma'}}$.
\end{itemize}
\end{proof}

\subsection{Progress and Preservation}

\begin{thm}[Progress and Preservation]
Given a term $t$ such that $\typerule{\Gamma}{t}{T}{\Gamma'}$
with a store $\mu$ such that $\Gamma \vdash \mu$, either $t$ is a value or
there exist $t'$ and $\mu'$ such that
$\oprule{t}{\mu}{t'}{\mu'}$, with some $\Gamma''$ such that
$\Gamma'' \vdash \mu'$ and $\typerule{\Gamma''}{t'}{T}{\Gamma'}$.
\end{thm}

\begin{proof}
by induction on the structure of $t$. Assume $t$ is not a value. It is
therefore of one of the following forms:

\begin{itemize}
\item $t = \lett{x}{t''}{t'}$.
It follows by rule \tlet that there exist $\Gamma_1, T', T''$
such that
$\typerule{\Gamma}{t''}{T'}{\Gamma_1}$ and that
$\typerule{\Gamma_1, x : T'}{t'}{T}{\Gamma', x : T''}$. There are two
possibilities:

	\begin{itemize}
	\item If $t''$ is some value $v$, then by \lemref{lem:valuectx}
	we have that $\Gamma = \Gamma_1$ and reduction can occur
	by \rletv, such that
	$\oprule{\lett{x}{v}{t'}}{\mu}{t'}{\mu [ x \mapsto v ]}$.
	Let $\mu' = \mu [ x \mapsto v ]$. Let $\Gamma'' = \Gamma, x : T'$.
	It follows that that $\Gamma'' \vdash \mu'$, with
	$\typerule{\Gamma''}{t'}{T}{\Gamma', x : T'}$.
	The requirements of rule \tdrop are satisfied by this, therefore we can
	type $t'$ such that $\typerule{\Gamma''}{t'}{T}{\Gamma'}$.

	\item If $t''$ is a term,
	by induction we have that $\oprule{t''}{\mu}{t'''}{\mu'}$, with
	some $\Gamma'' \vdash \mu'$ and $\typerule{\Gamma''}{t'''}{T'}{\Gamma_1}$.
	This satisfies the requirements of \rlett, meaning $t$ itself can reduce
	such that $\oprule{\lett{x}{t''}{t'}}{\mu}{\lett{x}{t'''}{t'}}{\mu'}$.
	Additionally, the conditions of \tlet are satisfied such that
	$\typerule{\Gamma''}{\lett{x}{t'''}{t'}}{T}{\Gamma'}$.

	\end{itemize}

\item $t = t_l ; t_r$. It follows by rule
\tseq that $\typerule{\Gamma}{t_l}{T_l}{\Gamma_{mid}}$ and
$\typerule{\Gamma_{mid}}{t_r}{T}{\Gamma'}$.

	\begin{itemize}
	\item If $t_l$ is some value $v$, then
	reduction can occur by \\
	\rseqlv such that
	$t_l; t_r \mid \mu \longrightarrow t_r \mu$. By \lemref{lem:valuectx},
	$\Gamma'' = \Gamma = \Gamma_{mid}$
	and $\mu' = \mu$, therefore $\Gamma'' \vdash \mu'$ and
	$\typerule{\Gamma''}{t_r}{T}{\Gamma'}$.

	\item If $t_l$ is a term, by induction $t_l \mid \mu \longrightarrow t_l' \mid \mu'$
	with some $\Gamma'' \vdash \mu'$ and
	$\typerule{\Gamma''}{t_l'}{T_l}{\Gamma_{mid}}$.
	This satisfies the requirements of \rseqlt
	such that $t_l; t_r \mid \mu \longrightarrow t_l'; t_r \mid \mu'$.
	Additionally, the requirements of \tseq are satisfied such that
	$\typerule{\Gamma''}{t_l'; t_r}{T}{\Gamma'}$.

	\end{itemize}

\item $t = x ( \overline{x_i} )$. It follows by rule \tfunc that
$\Gamma = \Gamma_1, \overline{x_i : T_i}$
with $\Gamma_1(x) = \funt{U_i}{V_i}{T}$
such that $\overline{T_i <: U_i}$
and that $\Gamma' = \Gamma_1, \overline{x_i : T_i'}$
where $T_i' = \mathbf{remap} ( T_i, U_i >> V_i )$.

As $\Gamma \vdash \mu$,
it follows that $\mu(x) = \funv{y_i}{U_i'}{V_i'}{t'}$
where $\typerule{\overline{y_i : U_i'}}{t'}{T'}{\overline{y_i : V_i'}}$,
with $\funt{U_i'}{V_i'}{T'} <: \funt{U_i}{V_i}{T}$. It
follows by \subfn that for each $i$, $U_i <: U_i'$, $V_i' <: V_i$
and $T' <: T$.
Additionally for each $x_i$, $\mu(x_i) = v_i$ such that
$\typerule{\emptyset}{v_i}{T_i''}{\emptyset}$
where $T_i'' <: T_i$.

Let $t_{sub} = \subst{x_i}{y_i}{t'}$.
The term $t$ can be reduced by \rfunc such that
$x ( \overline{x_i} ) \mid \mu \longrightarrow t_{sub} \mid \mu$.
By the substitution lemma,
$\typerule{\overline{x_i : U_i'}}{t_{sub}}{T'}{\overline{x_i : V_i'}}$.
By lemma \lemref{lem:upgrading}, it follows that
for each $i$, there exists a $V_i'' = \remapapp{T_i''}{U_i'}{V_i'}$
such that
$\typerule{\overline{x_i : U_i}}{T_{sub}}{T'}{\overline{x_i : V_i''}}$.
By \lemref{lem:remap} for each $i$ that $V_i'' <: V_i'$.

Therefore, by given that $T_i <: U_i$ and that $V_i'' <: V_i' <: V_i$
and that $T' <: T$ we can apply \tnarrow, \twiden and \tsub
such that $\typerule{\overline{x_i : T_i}}{t'}{T}{\overline{x_i : V_i}}$.

Let $\mu' = \mu$. Let $\Gamma'' = \Gamma$. It follows that
$\Gamma'' \vdash \mu'$. By applying the weakening lemma, we can type
$t_{sub}$ such that $\typerule{\Gamma''}{t''}{T}{\Gamma'}$.

\item $t = x.m$. It follows by rule \tmethc that
$\Gamma = \Gamma_1, x : O@\overline{S}$ and that
$\Gamma' = \Gamma_1, x : O@\overline{S'}$, with
$\overline{S'} = \{ S_n | m : T_r \Rightarrow S_n \in O@S, S \in \overline{S} \}$.
As
$\Gamma \vdash \mu$, we must have $\mu = \mu_1, x \mapsto o'@S_c$ with
$\Gamma_1 \vdash \mu_1$ and where
$\typerule{\emptyset}{o'@S_c}{O'@S_c}{\emptyset}$ such that
$O'@S_c <: O@\overline{S}$.

By the definition of subtyping for object types,
we must have that
$o' = [ S' \{ m = (v,S_n) ; ... \} ... ]$, where $O'@S_n <: O@\overline{S'}$.
Where the value $v$ is concerned, $\typerule{\emptyset}{v}{T'}{\emptyset}$
for some $T'$ such that $T' <: T = \bigsqcup{ \{ T_r | m : T_r \Rightarrow S_n \in O@S, S \in \overline{S} \} }$.

Therefore reduction can occur by \rmethc such that
$x.m \mid \mu_1, x \mapsto o@S \longrightarrow v \mid \mu_1, x \mapsto o@S_n$.
Let $\mu' = \mu_1, x \mapsto o@S'$. Let $\Gamma'' = \Gamma'$. It follows that
$\Gamma'' \vdash \mu'$, as $O'@S_n <: O@\overline{S'}$.
By application of the weakening lemma and rule \tsub, it follows that
$\typerule{\Gamma''}{v}{T}{\Gamma'}$.

\item $t = \ift{t_c}{t_t}{t_f}$. It follows by rule \tif that there exists
$\Gamma_1$, $\Gamma_2$, $\Gamma_3$, $T_t$ and $T_f$ such that
$\typerule{\Gamma}{t_c}{\boolt}{\Gamma_1}$ and that
$\typerule{\Gamma_1}{t_t}{T_t}{\Gamma_2}$ and that
$\typerule{\Gamma_2}{t_f}{T_f}{\Gamma_3}$ with
$\Gamma' = \Gamma_2 \sqcup \Gamma_3$ and $T = T_t \sqcup T_f$. There are
three possibilities for reduction:

	\begin{itemize}
	\item $t_c = \truev$. By \lemref{lem:valuectx} we have that
	$\Gamma = \Gamma_1$. Reduction can occur by \riftrue such that
	$\oprule{t}{\mu}{t_t}{\mu}$. By \tsub we have that
	$\typerule{\Gamma_1}{t_t}{T}{\Gamma_2}$ as $T_t <: T$. Additionally, for
	all $x \in \Gamma_2$, $\Gamma_2(x) <: \Gamma'(x)$. Therefore by
	repeatedly applying \twiden for all $x$, we have that
	$\typerule{\Gamma_1}{t_t}{T}{\Gamma'}$.
	Let $\Gamma'' = \Gamma$, $\mu' = \mu$ and
	$t' = t_t$. Therefore, $\typerule{\Gamma''}{t'}{T}{\Gamma'}$ with
	$\Gamma'' \vdash \mu'$.

	\item $t_c = \falsev$. By \lemref{lem:valuectx} we have that
	$\Gamma = \Gamma_1$. Reduction can occur by \riffalse such that
	$\oprule{t}{\mu}{t_f}{\mu}$. By \tsub we hhave that
	$\typerule{\Gamma_1}{t_f}{T}{\Gamma_3}$ as $T_f <: T$. Additionally, for
	all $x \in \Gamma_3$, $\Gamma_3(x) <: \Gamma'(x)$. Therefore by
	repeatedly applying \twiden for all $x$, we have that
	$\typerule{\Gamma_1}{t_f}{T}{\Gamma'}$. Let $\Gamma'' = \Gamma$,
	$\mu' = \mu$ and $t' = t_f$. Therefore,
	$\typerule{\Gamma''}{t'}{T}{\Gamma'}$ with $\Gamma'' \vdash \mu'$.

	\item $t_c$ is not a value. By induction, there exists $t_c'$,
	$\mu'$, $\Gamma''$ such that
	$\oprule{t_c}{\mu}{t_c'}{\mu'}$ and
	$\typerule{\Gamma''}{t_c'}{\boolt}{\Gamma_1}$ with $\Gamma'' \vdash \mu'$.
	Reduction of $t$ can therefore occur by \rifexpr such that
	$\oprule{t}{\mu}{\ift{t_c'}{t_t}{t_f}}{\mu'}$. It follows
	by \tif that $\typerule{\Gamma''}{\ift{t_c'}{t_t}{t_f}}{T}{\Gamma'}$.
	\end{itemize}

\end{itemize}
\end{proof}

\section{Type inference}

Type inference in the style of Pottier, i.e. separate constraint generation
and constraint solving phases to determine the principal typing of a term,
can be defined by observing the requirements imposed by the typing judgements
defined in Section \ref{sec:typerules}.

\ottgrammartabular{
\ottC\ottinterrule
\ottctxc\ottinterrule
\otttyc\ottinterrule
\ottprotoc\ottinterrule
\otttinf\ottinterrule
}

\[
\begin{array}{l}

% unit value
\inferlhs{\unitv} \triangleq \alpha = \unitt \cand \gamma = \gamma'
\\\\

% true value
\inferlhs{\truev} \triangleq \alpha = \boolt \cand \gamma = \gamma'
\\\\

% false value
\inferlhs{\falsev} \triangleq \alpha = \boolt \cand \gamma = \gamma'
\\\\

% object value
\inferlhs{\left[ \overline{S_i \left\{ \overline{ m_{ij} = (v_{ij}, S_{ij}) } \right\} } \right]@S}
\triangleq
\exists \alpha_o, \alpha_s, \overline{\alpha_{ij}}, \overline{\gamma_{ij}} . \\
\alpha = \alpha_o@\alpha_s
\cand
\left\{ \overline{ S_i \left\{ \overline{ m_{ij} : \alpha_{ij} >> S_{ij} } \right\} }\right\}@S <: \alpha_o@\alpha_s
\cand
\gamma = \gamma'
\cand
\\
\overline{\gamma_{ij} = \emptyset}
\cand
\overline{\inferrule{\gamma_{ij}}{v_{ij}}{\alpha_{ij}}{\gamma_{ij}}}
\\\\

% function value
\inferlhs{\lambda ( \overline{x_i} ) . t} \triangleq
\exists \gamma_r, \gamma_a, \gamma_b, \overline{\alpha_{ai}}, \overline{\alpha_{bi}}, \alpha_r . \\
\alpha = (\overline{\alpha_{ai} \gg \alpha_{bi}}) \rightarrow \alpha_r
\cand
\gamma = \gamma'
\cand
\gamma_r = \emptyset
\cand \\
\gamma_a = \gamma_r, \overline{x_i : \alpha_{ai}}
\cand
\overline{x_i : \alpha_{bi} \in \gamma_b}
\cand
\inferrule{\gamma_a}{t}{\alpha_r}{\gamma_b}
\\\\

% let binding
\inferlhs{\lett{x}{t'}{t}} \triangleq \exists \gamma_v, \gamma_{b1}, \gamma_{b2}, \alpha' . \\
\gamma_{b1} = \gamma_v, x : \alpha'
\cand
\gamma' = \gamma_{b2} - \{ x \}
\cand \\
\inferrule{\gamma}{t'}{\alpha'}{\gamma_v}
\cand
\inferrule{\gamma_{b1}}{t}{\alpha}{\gamma_{b2}}
\\\\

% sequence
\inferlhs{t ; t'} \triangleq
\exists \gamma'', \alpha' . \\
\inferrule{\gamma}{t}{\alpha'}{\gamma''}
\cand
\inferrule{\gamma''}{t'}{\alpha}{\gamma'}
\\\\

% function call
\inferlhs{x ( \overline{x_i} )} \triangleq
\exists \overline{\alpha_i}, \overline{\alpha_i'},\overline{\alpha_i''} . \\
x : ( \overline{\alpha_i \gg \alpha_i'} ) \rightarrow \alpha \in \gamma
\cand
\overline{x_i : \alpha_i'' \in \gamma}
\cand
\\
\overline{\alpha_i'' <: \alpha_i}
\cand
\gamma' = \gamma [ \overline{x_i \mapsto \alpha_i'} ]
\\\\

% method call
\inferlhs{x.m} \triangleq
\exists \alpha_o, \alpha_s, \alpha_s' . \\
x : \alpha_o @ \alpha_s \in \gamma
\cand
m : \alpha \gg \alpha_s' \in \alpha_o @ \alpha_s
\cand  \\
\gamma' = \gamma[ x \mapsto \alpha_o @ \alpha_s' ]
\\\\

% if-then-else
\inferlhs{\ift{t_c}{t_t}{t_f}} \triangleq
\\
\exists \alpha_c, \alpha_t, \alpha_f, \gamma_c, \gamma_t, \gamma_f . \\
\inferrule{\gamma}{t_c}{\alpha_c}{\gamma_c}
\cand
\inferrule{\gamma_c}{t_t}{\alpha_t}{\gamma_t}
\cand
\\
\inferrule{\gamma_c}{t_f}{\alpha_f}{\gamma_f}
\cand
\alpha_c = \boolt
\cand
\\
\alpha_t <: \alpha
\cand
\alpha_f <: \alpha
\cand
\gamma' = \gamma_t \sqcup \gamma_f
\\\\

\end{array}
\]

\subsection{Context constraint solving}

Constraint solving is performed in phases. In the first phase, the context
constraints
are normalised using the rules in \figref{fig:contextsolve} --- where the set of
constraints in the premise exists, they are replaced with those in the
conclusion. Normalisation means that every context variable will have a single
defining constraint of form $\gamma = \emptyset$, $\free{\gamma}$ or
$\gamma = \gamma', \overline{x_i : \mathbb{T}_i}$, where $\gamma'$ is
either empty or free. A context $\Gamma$ for type checking can be easily
constructed from constraints in this form, as once the type variable
constraints are solved we can say $\Gamma = \overline{x_i : T_i}$ for
every $\gamma = \gamma', \overline{x_i : \mathbb{T}_i}$ where $T_i$ is the
solved form of $\mathbb{T}_i$.

The context normalisation rules do the following:

\begin{itemize}
\item \textsc{ceq} --- Where $\gamma$ is directly equivalent
to $\gamma'$, which is defined in terms of extension of
$\gamma''$, rewrite the constraints such that
$\gamma$ directly copies the extension of $\gamma''$.
\item \textsc{crem} --- Where $\gamma$ removes a variable $x$ from $\gamma'$,
which is defined in terms of extension of $\gamma''$, rewrite the
constraints such that $\gamma$ copies the extension of $\gamma''$ with
the variable $x$ removed from the extension. There is no need for an
error-producing variant of \textsc{crem}, where the variable $x$ is missing,
as context removal constraints are always generated in a pair with a context
extension constraint for let binding.
\item \textsc{cext} --- Where $\gamma$ extends $\gamma'$ with
a variable set $\overline{y_j}$, and $\gamma'$ is defined in terms of extension of
$\gamma''$ with a disjoint set of variables in its extension set, rewrite the
constraints such that $\gamma$'s extension set includes both the $y_j$
definitions and $x_j$ definitions.
\item $\textsc{cext}_2$ --- Where $\gamma$ extends $\gamma'$ with
a new variable $x$, and $\gamma'$ is defined in terms of extension of
some $\gamma''$ and the extension set already contains $x$, this
represents an attempt to redeclare $x$. As such, generate an error
$\mathbf{bound}\:x$, terminating the constraint solving process.
\item \textsc{crep} --- where $\gamma$ replaces the type of some
variable $x$ in $\gamma'$, where $\gamma$ is defined in terms of extension
of $\gamma''$ and $x$ is in the extension set, rewrite the constraints
such that $\gamma$ copies the extension set of $\gamma'$ with $x$
replaced as specified.
\item $\textsc{crep}_2$ --- where $\gamma$ replaces variable $x$ in
$\gamma'$ which is defined in terms of extension of $\gamma''$
which is free and $x$ is not in the extension set, $x$ is a
free variable in the term being processed. To introduce $x$ in such a way that
it will be included in all contexts, $\gamma''$ is redefined in
terms of extension of a new context $\gamma_r$ with variable $x$
in the extension set mapped to a new type variable $\alpha$. Following
this, \textsc{cext} can be applied to $\gamma'$, then \textsc{crep} can
be applied to $\gamma$.
\item $\textsc{crep}_3$ --- where $\gamma$ replaces variable $x$ in
$\gamma'$ which is defined in terms of an empty root $\gamma''$,
and $x$ is not in $\gamma$'s extension set, this indicates the usage
of an unbound variable within a function body. As such, generate an
error $\mathbf{unbound}\:x$ indicating that x is unbound within this
context, terminating the contraint solving process.
\item \textsc{cvar} --- where a variable constraint for $x$ exists
for context variable $\gamma$, which is defined in terms of
extension of $\gamma_r$ which is free, and $x$ is not in the extension
set of $\gamma$, $x$ is a free variable in the term being processed.
Similar to $\textsc{crep}_2$, redefine $\gamma_r$
in terms of extension of a new root $\gamma_r'$, with $x$ in
the extension set with the observed type $\mathbb{T}$.
\item $\textsc{cvar}_2$ --- where a variable constraint for $x$ exists
for context variable $\gamma$, which is defined in terms of $\gamma'$
which is empty, and $x$ is not in the extension set of $\gamma$,
$x$ is an unbound variable within a function body.
\end{itemize}

The rules are repeatedly applied to any matching sets of constraints
until no further rewriting is possible, producing either an error
or a set of context constraints which are fully normalised, and
can be converted into contexts for type checking easily.

The way in which the context constraints are generated guarantees
that the dependencies between contexts form a directed acyclic graph. For
this reason, we believe that that the first phase will always terminate,
but this has not yet been proven.

\begin{center}
\begin{tabular}{c}
	\infer[\textsc{ceq}]
	{\gamma = \gamma'', \overline{x_i : \mathbb{T}_i} \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}}
	{\gamma = \gamma' \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}}
\\\\
	\infer[\textsc{crem}]
	{
	\gamma = \gamma'', \overline{x_i : \mathbb{T}_i} \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}, x : \mathbb{T}
	}
	{
	\gamma = \gamma' - \{x\} \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}, x : \mathbb{T}
	}
\\\\
	\infer[\textsc{cext}]
	{\gamma = \gamma'', \overline{x_i : \mathbb{T}_i}, \overline{y_j : \mathbb{T}_j} \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}}
	{
		\gamma = \gamma', \overline{y_j : \mathbb{T}_j} \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}
		& \overline{y_j} \cap \overline{x_i} = \emptyset
	}
\\\\
	\infer[\textsc{cext\_2}]
	{\mathbf{bound}\:x}
	{
		\gamma = \gamma', \overline{y_j : \mathbb{T}_j}, x : \mathbb{T} \cand \gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}
		& x \in \overline{x_i}
	}
\\\\
	\infer[\textsc{crep}]
	{\gamma = \gamma'', \overline{x_i : \mathbb{T}_i}, x : \mathbb{T} \cand
	\gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}, x : \mathbb{T}'}
	{\gamma = \gamma'[x \mapsto \mathbb{T}] \cand
	\gamma' = \gamma'', \overline{x_i : \mathbb{T}_i}, x : \mathbb{T}'}
\\\\
	\infer[\textsc{crep\_2}]
	{
	\begin{array}{c}
	\exists \alpha, \gamma_r .
	\gamma = \gamma'[x \mapsto \mathbb{T}] \cand
	\gamma' = \gamma'', \overline{x_i : \mathbb{T}_i} \cand
	\\
	\gamma'' = \gamma_r, x : \alpha \cand
	\free{\gamma_r}
	\end{array}
	}
	{
	\begin{array}{c}
	\begin{array}{ccc}
	x \notin \overline{x_i}
	&
	\fresh{\gamma_r}
	&
	\fresh{\alpha}
	\end{array}
	\\
	\gamma = \gamma'[x \mapsto \mathbb{T}] \cand
	\gamma' = \gamma'', \overline{x_i : \mathbb{T}_i} \cand
	\free{\gamma''}
	\end{array}
	}
\\\\
	\infer[\textsc{crep\_3}]
	{\mathbf{unbound}\:x}
	{
	\begin{array}{c}
	x \notin \overline{x_i}
	\\
	\gamma = \gamma'[x \mapsto \mathbb{T}] \cand
	\gamma' = \gamma'', \overline{x_i : \mathbb{T}_i} \cand
	\gamma'' = \emptyset
	\end{array}
	}
\\\\
	\infer[\textsc{cvar}]
	{\exists \gamma_r' .
	\gamma = \gamma_r, \overline{x_i : \mathbb{T}_i} \cand
	\gamma_r = \gamma_r', x : \mathbb{T} \cand
	\free{\gamma_r'}
	}
	{
	\begin{array}{c}
	\begin{array}{cc}
	x \notin \overline{x_i}
	&
	\fresh{\gamma_r'}
	\end{array}
	\\
	\gamma = \gamma_r, \overline{x_i : \mathbb{T}_i} \cand
	x : \mathbb{T} \in \gamma \cand
	\free{\gamma_r}
	\end{array}
	}
\\\\
	\infer[\textsc{cvar\_2}]
	{\mathbf{unbound}\:x}
	{x : \mathbb{T} \in \gamma \cand \gamma = \gamma', \overline{x_i : \mathbb{T}_i} \cand \gamma' = \emptyset
	&
	x \notin \overline{x_i}}
\\\\
	\infer[\textsc{cjoin}]
	{
	\begin{array}{c}
	\exists \overline{\alpha_i} .
	\gamma = \gamma_r, \overline{x_i : \alpha_i}
	\cand
	\overline{\alpha_i <: \mathbb{T}_i}
	\cand
	\overline{\alpha_i <: \mathbb{U}_i}
	\cand
	\\
	\gamma' = \gamma_r, \overline{x_i : \mathbb{T}_i}
	\cand
	\gamma'' = \gamma_r, \overline{x_i : \mathbb{U}_i}
	\end{array}
	}
	{
	\gamma = \gamma' \sqcup \gamma''
	\cand
	\gamma' = \gamma_r, \overline{x_i : \mathbb{T}_i}
	\cand
	\gamma'' = \gamma_r, \overline{x_i : \mathbb{U}_i}
	}

\end{tabular}
\end{center}

\end{document}