\documentclass[a3paper]{article}
\usepackage[hyphens]{url}
\usepackage[utf8]{inputenc}
\usepackage[pdftex]{graphicx}
\usepackage{epstopdf}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{mathrsfs}
\usepackage{booktabs}
\usepackage[ruled,linesnumbered]{algorithm2e}
\usepackage{cleveref}
\usepackage[misc]{ifsym}
\usepackage{longtable}
\urlstyle{same}
\usepackage[a4paper, total={7in, 10in}]{geometry}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{cleveref}
\usepackage{booktabs}
\usepackage[switch]{lineno}
\usepackage{ntheorem}

\usepackage{float}
\restylefloat{table}

\newtheorem{theorem}{Theorem}
\newtheorem{lemma}{Lemma}
\newtheorem*{proof}{\it{Proof.}\rm}

\newcommand{\IE}{\textit{i.e.}, }
\newcommand{\EG}{\textit{e.g.}, }
\newcommand{\ET}{\textit{et al.}}
\newcommand{\ST}{\textit{s.t.}}

\newcommand{\dd}{\mathrm{d}}
\newcommand{\vv}[1]{\bm{\mathrm{{#1}}}}
\newcommand{\E}{\operatorname{\mathbb{E}}}
\newcommand{\EE}[1]{\operatorname{\mathbb{E}}\left[{#1}\right]}
\newcommand{\EEE}[2]{\operatorname{\mathbb{E}}_{{#1}}\left[{#2}\right]}
\newcommand{\KLD}{\operatorname{KL}}
\newcommand{\KLDD}[2]{\operatorname{KL}\left[{#1}\,\big\|\,{#2}\right]}
\newcommand{\abs}[1]{\left|#1\right|}
\newcommand{\Entropy}{\operatorname{H}}
\newcommand{\Entropyy}[1]{\operatorname{H}\left[#1\right]}
\newcommand{\pin}{p_{in}}
\newcommand{\pout}{p_{out}}
\newcommand{\pmix}{p_{mix}}
\newcommand*{\QED}{\hfill\ensuremath{\square}}  %自定义，空心
% Following comment is from ijcai97-submit.tex:
% The preparation of these files was supported by Schlumberger Palo Alto
% Research, AT\&T Bell Laboratories, and Morgan Kaufmann Publishers.
% Shirley Jowell, of Morgan Kaufmann Publishers, and Peter F.
% Patel-Schneider, of AT\&T Bell Laboratories collaborated on their
% preparation.

% These instructions can be modified and used in other conferences as long
% as credit to the authors and supporting agencies is retained, this notice
% is not changed, and further modification or reuse is not restricted.
% Neither Shirley Jowell nor Peter F. Patel-Schneider can be listed as
% contacts for providing assistance without their prior permission.

% To use for other conferences, change references to files and the
% conference appropriate and use other authors, contacts, publishers, and
% organizations.
% Also change the deadline and address for returning papers and the length and
% page charge instructions.
% Put where the files are available in the appropriate places.

\title{Technique Appendix}
\begin{document}

\pagestyle{plain}

\newpage
\appendix

\section{Counter-examples}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/log_prob_histogram}
\includegraphics[width=0.5\columnwidth]{figures/log_prob_histogram-1}
\caption{Counter example of $\log p_\theta(x)$ with AUROC 0.0609 and 0.1104}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/log_prob_histogram-3}
\includegraphics[width=0.5\columnwidth]{figures/log_prob_histogram-4}
\caption{Counter example of $T_{perm}(x)$ with AUROC 0.3933 and 0.4692}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/grad_norm_histogram}
\includegraphics[width=0.5\columnwidth]{figures/grad_norm_histogram-1}
\caption{Counter example of $\|\nabla_x \log p_\theta(x)\|$ with AUROC 0.0196 and 0.0023}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/ll_with_complexity_histogram}
\includegraphics[width=0.5\columnwidth]{figures/ll_with_complexity_histogram-1}
\caption{Counter example of $S(x)$ with AUROC 0.2636 and 0.0062}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/kl_histogram}
\includegraphics[width=0.5\columnwidth]{figures/kl_histogram-1}
\caption{Counter example of $LLR(x)$ with AUROC 0.1054 and 0.5143}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/ll_waic_histogram}
\includegraphics[width=0.5\columnwidth]{figures/ll_waic_histogram-1}
\caption{Counter example of $WAIC(x)$ with AUROC 0.1041 and 0.2135}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/var_log_prob_histogram}
\includegraphics[width=0.5\columnwidth]{figures/var_log_prob_histogram-1}
\caption{Counter example of $\mathop{VAR}_\theta \log p_\theta(x)$ with AUROC 0.5358 and 0.3756}
\end{figure}

\begin{figure}[H]
\includegraphics[width=0.5\columnwidth]{figures/log_prob_histogram-5}
\includegraphics[width=0.5\columnwidth]{figures/log_prob_histogram-6}
\caption{Counter example of $\log p_\theta(x|y)$ with AUROC 0.3822 and 0.1150}
\end{figure}

\section{Experiments}~\label{app:b}

\subsection{Datasets}
%All the datasets considered are listed below:
%\textbf{CIFAR-10}~\cite{krizhevsky2009learning} is a natural image datasets with 10 classes including animal, ship, airplane and etc.
%\textbf{CIFAR-100}~\cite{krizhevsky2009learning} is just like CIFAR-10, including 100 classes.
%\textbf{SVHN}~\cite{netzer2011reading} includes house numbers from 0 to 9.
%\textbf{CelebA}~\cite{liu2015deep} is a large-scale face attributes dataset.
%\textbf{TinyImageNet} consists of a subset of ImageNet images. It contains 200 different classes.
%\textbf{LSUN} has a testing set of 10 different scenes.
%\textbf{iSUN} is subset of SUN, including 8925 scene images in 899 different scenes.
%\textbf{MNIST} consists of handwritten digits from 0 to 9.
%\textbf{Fashion MNIST} includes 10 kinds of clothes and shoes.
%\textbf{Not MNIST} includes letters from A to J on various typefaces.
%\textbf{KMNIST} includes 10 kinds of Kanji characters.
%\textbf{Omniglot} contains 1623 different handwritten characters from 50 different alphabets.
%\textbf{Noise} is created by uniformly randomly sampling.
%\textbf{Constant} includes images whose pixels have same color.
The following datasets are considered: MNIST~\cite{lecun1998gradient}, FashionMNIST~\cite{xiao2017/online}, KMNIST~\cite{clanuwat2018deep}, NOT-MNIST, Omniglot~\cite{lake2015human}, CIFAR-10~\cite{krizhevsky2009learning}, CIFAR-100~\cite{krizhevsky2009learning}, TinyImagenet~\cite{deng2009imagenet}, SVHN~\cite{netzer2011reading}, iSUN~\cite{xu2015turkergaze}, CelebA~\cite{liu2015deep}, LSUN~\cite{yu2015lsun}, Noise and Constant.

Natural images are resized to 32x32x3 and grey images are 28x28x1. All pair in natural image datasets and all pair in grey image datasets are considered in our experiments. Only CIFAR-10, CIFAR-100 and TinyImageNet are not simply-classified and intuitively they have similar classes. LSUN and iSUN are only used as out-of-distribution. CelebA, Noise and Constant have no labels, and we set random labels from 0 to 9 on them. We end up with 92 dataset pairs.

\subsection{Metrics}
Following standard metrics are adopted to measure the effectiveness of a method in out-of-distribution detection:

\noindent \textbf{AUROC} is the Area Under the Receiver Operating Characteristic curve, a threshold-independent metric~\cite{davis2006relationship} and widely used in OoD domain.
%AUROC can be interpreted as the probability that a sample from in-distribution is assigned a lower detection score than a sample from out-of-distribution~\cite{fawcett2006introduction}. It is widely applied in OoD domain. We select AUROC as our major metrics.

\noindent \textbf{AP} is the Average Precision, summarizing the precision-recall curve as the weighted mean of precisions.

\noindent \textbf{FPR@TPR95} is the False Positive Rate when True Positive Rate is over 95\%., which means the probability that an out-of-distribution example is misclassified as in-distribution when over 95\% in-distribution is detected accurately.

\noindent \textbf{AUPR} is the Area under the Precision-Recall curve, which is also threshold independent~\cite{saito2015precision}. % AUPR-In and AUPR-Out denotes the AUPR where in-distribution and out-of-distribution are positive, respectively.

\subsection{Setups}
For fair comparison, all indicators are based on common models with standard training as their proposers suggest, including ResNet34, VAE, PixelCNN, RealNVP. ResNet34 validates whether two datasets are simply-classified and serves for the OoD indicators based on classifier.

In our experiments, there is no validation set. For indicators depending on hyper-parameters, we try grid-searching as their proposers suggest and report the performance with all hyper-parameters considered. To ensure the generality on all datasets, it is forbidden to specify the hyper-parameters or architectures on special dataset.

Our experiments are running on 18 Nvidia GTX 2080Ti of 3 GPU servers. The detailed setup is described in `supplemental/code/README'.
\subsection{Architecture}
The architecture of VAE is:

\begin{table}[H]
\centering
\begin{tabular}{lllllll}
Layer     & Channel & Stride & Kernel & Activation  \\
\toprule
Resnet & 16 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 32 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 64 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet & 128 & 2x2 & 3x3 & Leaky ReLU \\
Flatten & \\
Dense & 256 &  &  & None \\
\bottomrule
\end{tabular}
\caption{Encoder architecture of VAE}
\end{table}

Before the decoder, we will use dense layer to map $z$ to a tensor with shape $8hw$ and reshape $(h / 4, w / 4, 128)$ where $h, w$ is the height and width of data.

\begin{table}[H]
\centering
\begin{tabular}{lllllll}
Layer     & Channel & Stride & Kernel & Activation  \\
\toprule
Resnet Deconv & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet Deconv & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet Deconv & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet Deconv & 64 & 1x1 & 3x3 & Leaky ReLU \\
Resnet Deconv & 32 & 1x1 & 3x3 & Leaky ReLU \\
Resnet Deconv & 16 & 1x1 & 3x3 & Leaky ReLU \\
Conv2d & 3 or 1 & 1x1 & 1x1 & None \\
\bottomrule
\end{tabular}
\caption{Decoder architecture of VAE}
\end{table}
We apply a Discretized Logistic Distribution as $p_\theta(x|z)$ and gaussian as $q_\phi(z|x)$.

The architecture of PixelCNN is:

\begin{table}[H]
\centering
\begin{tabular}{lllllll}
Layer     & Channel & Vertical Kernel & Horizontal Kernel& Activation & Dropout  \\
\toprule
Resnet & 64 & 2x3 & 2x2 & Leaky ReLU & 0.2\\
Resnet & 64 & 2x3 & 2x2 & Leaky ReLU & 0.2\\
Resnet & 64 & 2x3 & 2x2 & Leaky ReLU & 0.2\\
Resnet & 64 & 2x3 & 2x2 & Leaky ReLU & 0.2\\
Resnet & 64 & 2x3 & 2x2 & Leaky ReLU & 0.2\\
Resnet & 256 & 2x3 & 2x2 & Leaky ReLU & 0.2\\
Flatten & \\
Dense & 256 &  &  & None \\
\bottomrule
\end{tabular}
\caption{Architecture of PixelCNN}
\end{table}

The architecture of Wasserstein is:

\begin{table}[H]
\centering
\begin{tabular}{lllllll}
Layer     & Channel & Stride & Kernel & Activation  \\
\toprule
Resnet Deconv & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet Deconv & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet Deconv & 128 & 1x1 & 3x3 & Leaky ReLU \\
Resnet Deconv & 64 & 1x1 & 3x3 & Leaky ReLU \\
Resnet Deconv & 32 & 1x1 & 3x3 & Leaky ReLU \\
Resnet Deconv & 16 & 1x1 & 3x3 & Leaky ReLU \\
Conv2d & 1 & 1x1 & 1x1 & None \\
\bottomrule
\end{tabular}
\caption{Generator architecture of WGAN}
\end{table}

\begin{table}[H]
\centering
\begin{tabular}{lllllll}
Layer     & Channel & Stride & Kernel & Activation  \\
\toprule
Resnet & 16 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 32 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 64 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 128 & 1x1 & 3x3 & Leaky ReLU \\
Resnet & 128 & 2x2 & 3x3 & Leaky ReLU \\
Resnet & 128 & 2x2 & 3x3 & Leaky ReLU \\
\bottomrule
\end{tabular}
\caption{Discriminator architecture of WGAN}
\end{table}

RealNVP has 3 level and 6 blocks at each level. Channel is 128 and activation is ReLU in each block.
The average runtime memory of VAE, PixelCNN, WGAN and Glow are 2.949GB, 2.851GB, 2.826GB and 2.917GB.

\subsection{Data-augmentation and steps}
In single-shot fine-tune, the image is randomly shifted in -10\% to 10\% vertically and horizontally, and rotated in -60 to 60 degree, as basic data-augmentation.

The other data-augmentations, containing shifting, rotation, blur, noise, scale, cropping, flipping and modifying contrast and lightness are shown in the following. As can be seen, their performance has no significant difference ($\leq 0.3$\%) to the basic data-augmentation containing rotation and shift. If do not use data-augmentation, fine-tuning is easy to fail (\EG some parameters becomes NaN during fine-tuning).

\begin{table}[t]
	\centering
	\begin{tabular}{lll}
		\toprule
		Data-augmentation & AUROC \\
		Basic & 95.2470\\
		+Flip & 95.3480\\
		+Crop & 95.1940\\
		+Blur & 95.2464\\
		+Noise & 95.4416\\
		+Contrast & 95.2464\\
		+Light & 95.2680\\
		+Scale & 95.3076\\
		\bottomrule
	\end{tabular}
	\caption{Ablation study about data-augmentations. }
\end{table}

In single-shot fine-tune, the number of steps influence the performance, as shown in the following.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{auroc_steps}
\caption{The AUROC when step is from 1 to 256. When step is less than 64, step significant influence the AUROC, and when step is larger than 64, AUROC is nearly same. Therefore, we set step = 64 as the default parameter of single-shot fine-tune. Additional, when step =64, the time-cost is only 7s per testing sample.}
\end{figure}

%The hyper-parameters for VAE, PixelCNN, WGAN and Glow are shown in the scripts, shown in code appendix.
%The detailed experiments are shown in appendix D.

\section{Theorem}~\label{app:c}

We will derive the theorems in our papers by the assumptions:

\noindent \textbf{1.} $D^{train}_{in}, D^{test}_{in}$ are i.i.d. from distribution $\pin$ and $D^{test}_{out}, D^{train}_{out}$ are i.i.d. from distribution $\pout$.

\noindent \textbf{2.}  $div$ is a divergence , satisfying that $div(\pin, \pout) \gg 0$ and $div(\pout, \pin) \gg 0$.

\noindent \textbf{3.}  If $div(\pin, \pout)$ and $div(\pout, \pin)$ can be represented by sampling formula, \IE $div(\pin, \pout) \approx \sum_{i} f(x_i)$, then $f(x_i) \gg 0$ for most $x_i$ in $\pin$.

\noindent \textbf{4.} $f: \mathcal{X} \rightarrow \mathcal{R}$ maps the distribution $\pin, \pout$ into two Gaussian distribution with constant variance.

\noindent \textbf{5.} Assumption 3 and 4 hold for any $\hat{f}$ approximating $f$.

\textbf{Lemma 1.}\textit{
	Indicators $f_0$ and $f_1$ satisfying that if $f_0(x_1) < f_0(x_2)$ then $f_1(x_1) < f_1(x_2)$ and if $f_0(x_1) > f_0(x_2)$ then $f_1(x_1) > f_1(x_2)$ for $x_1, x_2$ in mixture distribution, have same performance for OoD. We call $f_0 \triangleq f_1$.
}

\begin{proof}\rm
	Assume we get data $x_0, x_1, \ldots, x_n$ from mixture distribution. The metrics AUROC, AUPR, FPR@TPR95 and AP for indicator $f$ are all dependent on the order of $f(x_0), f(x_1), \ldots, f(x_n)$. The array $f_0(x_0), f_0(x_1), \ldots, f_0(x_n)$ and $f_1(x_0), f_1(x_1), \ldots, f_1(x_n)$ have the same order and then they have same performance for all metrics considered in OoD.
\end{proof}

\begin{theorem}~\label{thm1}
	$\log \pin(x) - \log \pout(x)$ and $\log p_\theta(x) - \log p_\omega(x)$ and are effective symmetric indicators, \IE it reaches same performance in experiment A vs B and B vs A, with threshold 0, where $p_\theta \rightarrow \pin$ and $p_\omega \rightarrow \pout$. $\log p_\theta(x)$ maps $\pin$ into a gaussian distribution.
\end{theorem}

\begin{proof}\rm
	We select $KL$ as $div$. By assumption 3 and 4, we have
	\begin{equation*}
		\log \pin(x_1) - \log \pout(x_1) \gg 0 \quad and \quad \log \pin(x_2) - \log \pout(x_2) \ll 0
	\end{equation*}
	where $x_1 \sim \pin$ and $x_2 \sim \pout$. By assumption 5, above equation holds for $\log p_\theta(x) - \log p_\omega(x)$. Therefore, $\log \pin(x) - \log \pout(x)$ and $\log p_\theta(x) - \log p_\omega(x)$ can detect most of OoD, with threshold zero.

	In experiment A vs B and B vs A, the indicator is $\log p_A(x) - \log p_B(x)$ and $\log p_B(x) - \log p_A(x)$. These two experiments have same mixture distribution. For any testing data $x_1, x_2$ in mixture distribution, if $\log p_A(x_1) - \log p_B(x_1) > \log p_A(x_2) - \log p_B(x_2)$, then $\log p_B(x_1) - \log p_A(x_1) < \log p_B(x_2) - \log p_A(x_2)$. Conversely, if $\log p_A(x_1) - \log p_B(x_1) < \log p_A(x_2) - \log p_B(x_2)$, then $\log p_B(x_1) - \log p_A(x_1) > \log p_B(x_2) - \log p_A(x_2)$. Thus, they have inverse order.
	Noting that the positive and negative labels in experiment A vs B and B vs A are also inverse, $\log \pin(x) - \log \pout(x)$ reaches same performance in experiment A vs B and B vs A. The proof for $\log p_\theta(x) - \log p_\omega$ is the same as above.

	Let $f(x) = \log \pin(x)$ and $\hat{f}(x) = \log p_\theta(x)$. Since $p_\theta \rightarrow \pin$, $\hat{f} \rightarrow f$. By assumption 4,5, we know $\hat{f} = \log p_\theta(x)$ maps $\pin$ into a gaussian distribution.
\end{proof}

\begin{theorem}~\label{thm2}
	For any mixture distribution $\pmix = \alpha \pin + \beta \pout$ where $\alpha + \beta = 1$ and $\alpha, \beta > 0$, the performance of indicator $\log \pin(x) - \log \pmix(x)$ and indicator $\log \pin(x) - \log \pout(x)$ is equal for OoD detection.
\end{theorem}

\begin{proof}\rm
	For any $x1,x2$ satisfying $\log \pin(x_1) - \log \pout(x_1) < \log \pin(x_2) - \log \pout(x_2)$, we have $\log \pin(x_1) - \log \pmix(x_1) < \log \pin(x_2) - \log \pmix(x_2)$ by
	\begin{align*}
		&\log \pin(x) - \log \pmix(x) = -\log \frac{\alpha \pin(x) + \beta \pout(x)}{\pin(x)} = -\log
		\Big(\alpha + \beta \frac{\pout(x)}{\pin(x)}\Big) \\
		&\log \frac{\pin(x_1)}{\pout(x_1)} < \log \frac{\pin(x_2)}{\pout(x_2)} \Rightarrow \frac{\pin(x_1)}{\pout(x_1)} < \frac{\pin(x_2)}{\pout(x_2)} \Rightarrow \frac{\pout(x_1)}{\pin(x_1)} > \frac{\pout(x_2)}{\pin(x_2)} \\
	 &\Rightarrow \log \Big(\alpha + \beta \frac{\pout(x_1)}{\pin(x_1)}\Big) > \log
		\Big(\alpha + \beta \frac{\pout(x_2)}{\pin(x_2)}\Big) \\
		&\Rightarrow -(\log \pin(x_1) - \log \pmix(x_1)) > -(\log \pin(x_2) - \log \pmix(x_2)) \\
		&\Rightarrow \log \frac{\pin(x_1)}{\pmix(x_1)} < \log \frac{\pin(x_2)}{\pmix(x_2)}
	\end{align*}

	By the same way, for any $x1,x2$ satisfying $\log \pin(x_1) - \log \pout(x_1) > \log \pin(x_2) - \log \pout(x_2)$, we have $\log \pin(x_1) - \log \pmix(x_1) > \log \pin(x_2) - \log \pmix(x_2)$.  By \cref{lemma1}, performance of indicator $\log \pin(x) - \log \pmix(x)$ and indicator $\log \pin(x) - \log \pout(x)$ is equal for OoD detection.

	\noindent \textbf{Remark.} In the proof, we do not use the condition that $\alpha > 0$ but only $\beta > 0$ and $\pmix(x) > 0$. It means that \cref{thm2} holds even when $\alpha < 0, \beta > 0$ and $\pmix(x) > 0$.
\end{proof}

\begin{theorem}~\label{thm3}On any dataset pair that log-likelihood works well, \IE $\log \pin(x_1) > \log \pin(x_2)$ for most $x_1 \sim \pin, x_2 \sim \pout$, KL-based indicator can reach better performance.
\end{theorem}

\begin{proof}\rm
	Noting that the AUROC is related to the Mann–Whitney U, which tests whether positives are ranked higher than negatives, whose number called distortion.
	For each $x_1 \sim \pin, x_2 \sim \pout$,
	assume $\log \pout(x_1) < \log \pout(x_2)$, then $\log \pin(x_1) - \log \pout(x_1) > \log \pin(x_2) - \log \pout(x_2)$.
	KL-based indicator will get same performance than log-likelihood indicator.

	Assume $\log \pout(x_1) > \log \pout(x_2)$, then likelihood indicator will make mistake in experiment B vs A (if current experiment is A vs B). Meanwhile KL-based indicator might detect OoD in both A vs B and B vs A by \cref{thm1}. In conclusion, likelihood leads to more distortion than KL-based indicator, in experiment B vs A and experiment A vs B.

	It means KL-based indicator is always better than log-likelihood indicator.
\end{proof}

For the proofing of following theorems, we need an additional mathematical simplification: by assumption 4 and 5, the indicators for OoD maps the distribution $\pin, \pout$ into two Gaussian distribution with constant variance, therefore, the AUROC between these two Gaussian distribution is dependent on the difference of their mean, \IE $\E_{\pin(x)} f(x) - \E_{\pout(x)} f(x)$ is our simplified metric for mathematical inference.

\begin{theorem}~\label{thm4}
For any likelihood-ratio indicator $\log \pin(x) - \log g(x)$ where $g$ is a continuous differentiable probability distribution, KL-based indicator outperforms them.
\end{theorem}
\begin{proof}\rm
	Consider the following optimization with subsidiary conditions for searching the optimal $g$ for likelihood ratio:
	\begin{align*}
	\max_g \Big\{\E_{\pin(x)} \big[\log \pin(x) - \log g(x)\big] &- \E_{\pout(x)} \big[\log \pin(x) - \log g(x)\big]\Big\} \\
	\textbf{s.t.} KL(\pin, g) = L \text{ and } &\int g(x) \dd x = 1
	\end{align*}
	where $L$ is a constraint to the optimization domain of $g$. It could be solved by Lagrange multiplier method introduced by calculus of variation.
	Let $\lambda, \psi > 0$ be Lagrange Multiplier for the two constraint. The Lagrange function is
	\begin{align*}
		J[g] &= \E_{\pin(x)} \big[\log \pin(x) - \log g(x)\big] - \E_{\pout(x)} \big[\log \pin(x) - \log g(x)\big] \\
		&-\lambda KL(\pin, g) - \psi \int g(x) \dd x \\
		&= \int \pin(x) \log \pin(x) - \pin(x) \log g(x) - \pout(x) \log \pin(x) \\
		&+ \pout(x) \log g(x) - \lambda \pin(x) \log \frac{\pin(x)}{g} - \psi g(x) \dd x = \int F(g, x) \dd x
	\end{align*}
	By calculus of variation, the extremal point for $J[g]$ satisfies that $\delta J = 0$, which is equal to
	\begin{align*}
		\frac{\partial F(g, x)}{\partial g} = 0 \Rightarrow -\frac{\pin(x)}{g(x)} + \frac{\pout(x)}{g(x)} + \frac{\lambda \pin(x)}{g(x)} - \psi = 0 \\
		\Rightarrow g(x) = \frac{1}{\psi} ((\lambda - 1)\pin(x) + \pout(x))
	\end{align*}
	Considering the subsidiary condition $\int g(x) \dd x = 1$, we have
	\begin{equation*}
		\int g(x) \dd x = \frac{1}{\psi} \int (\lambda - 1) \pin(x) + \pout(x) \dd x = \frac{1}{\psi} ((\lambda - 1) + 1) = \frac{\lambda}{\psi} = 1
	\end{equation*}
	Therefore, $\lambda = \psi$ and $g^*(x) = \frac{\lambda - 1}{\lambda} \pin(x) + \frac{1}{\lambda} \pout(x)$ is the extremal point.
	By the remark of \cref{thm1}, $\log \pin(x) - \log g^*(x)$ gets same performance as $\log \pin(x) - \log \pout(x)$ in OoD detection.
	For any $L$, we can get $g^*(x)$, which has same performance as $\log \pin(x) - \log \pout(x)$. It means that KL-based indicator is the optimal indicator among all likelihood-ratio indicators.
\end{proof}

\begin{theorem}~\label{thm5}
	$\log \frac{p_\theta(x)}{p_\gamma(x)}$ can reach better AUROC than KL-based indicator, when $p_\gamma$ is well-trained, \IE $p_\gamma$ reaches better likelihood on $\pmix$ than $p_{\hat{\gamma}} = \alpha p_{\theta} + \beta p_{\omega}$.
\end{theorem}
\begin{proof}\rm
	Let $p_{\hat{\gamma}}(x) = \alpha p_\theta(x) + \beta p_\omega(x)$. $\hat{\gamma}$ is the parameters that can be obtained by optimizer.
	Considering the indicator $\log p_\theta(x) - \log p_{\hat{\gamma}}(x) = -\log (\alpha + \beta \frac{p_\omega(x)}{p_\theta(x)})$, for $x_1 \sim \pin$ and $x_2 \sim \pout$, we have
	\begin{align*}
		\log \frac{p_\theta(x_1)}{p_{\hat{\gamma}}(x_1)} - \log \frac{p_\theta(x_2)}{p_{\hat{\gamma}}(x_2)} = \log (\alpha + \beta \frac{p_\omega(x_2)}{p_\theta(x_2)}) - \log (\alpha + \beta \frac{p_\omega(x_1)}{p_\theta(x_1)})
	\end{align*}
	We have known that $\log \frac{p_\theta(x_1)}{p_\omega(x_1)} \gg 0$ and $\log \frac{p_\theta(x_2)}{p_\omega(x_2)} \ll 0$ by \cref{thm1}. Then, we have $\log \frac{p_\theta(x_1)}{p_{\hat{\gamma}}(x_1)} - \log \frac{p_\theta(x_2)}{p_{\hat{\gamma}}(x_2)} > 0$. It means that $\log \frac{p_\theta(x)}{p_{\hat{\gamma}}(x)}$ can be used for detecting OoD.

	Next, consider the $\gamma$ is optimized better than $\hat{\gamma}$, \IE $\E_{\pmix} \log p_\gamma(x) \geq \E_{\pmix} \log p_{\hat{\gamma}}(x)$.

	\begin{align*}
		\E_{\pmix} \log p_\gamma(x) = \alpha \E_{\pin} \log p_\gamma(x) + \beta \E_{\pout} \log p_\gamma(x)
	\end{align*}
	By $\E_{\pmix} \log p_\gamma(x) \geq \E_{\pmix} \log p_{\hat{\gamma}}(x)$, there are 3 case:

	\textbf{1.} $\E_{\pin} \log p_\gamma(x) \leq \E_{\pin} \log p_{\hat\gamma}(x)$ and $\E_{\pout} \log p_\gamma(x) \geq \E_{\pout} \log p_{\hat\gamma}(x)$.

	\textbf{2.} $\E_{\pin} \log p_\gamma(x) \geq \E_{\pin} \log p_{\hat\gamma}(x)$ and $\E_{\pout} \log p_\gamma(x) \geq \E_{\pout} \log p_{\hat\gamma}(x)$.

	\textbf{3.} $\E_{\pin} \log p_\gamma(x) \geq \E_{\pin} \log p_{\hat\gamma}(x)$ and $\E_{\pout} \log p_\gamma(x) \leq \E_{\pout} \log p_{\hat\gamma}(x)$.

	By assumption 4, 5 and simplified metric, we know that if  $\E_{\pin} \log p_\gamma(x)$ increase, the KL-based indicator $\log \frac{p_\theta(x)}{p_{{\gamma}}(x)}$ will be worse and if  $\E_{\pout} \log p_\gamma(x)$ increase, the KL-based indicator $\log \frac{p_\theta(x)}{p_{{\gamma}}(x)}$ will be better.

	Therefore, in case 1, the performance of $\log \frac{p_\theta(x_1)}{p_{\gamma}(x_1)}$ will be better than $\log \frac{p_\theta(x_1)}{p_{\hat\gamma}(x_1)}$. In case 2 and 3, we know $\E_{\pin} \log p_{\hat\gamma}(x) \leq \E_{\pin} \log p_\theta(x)$ and $\E_{\pin} \log p_\gamma(x) \leq \E_{\pin} \log p_\theta(x)$ since $\theta$ is well-trained for such loss function.
	\begin{align*}
		\beta \E_{\pout} \log p_\gamma(x) = \E_{\pmix} \log p_\gamma(x) - \alpha \E_{\pin} \log p_\gamma(x) \\
		\geq \E_{\pmix} \log p_{\hat\gamma}(x) - \alpha \E_{\pin} \log p_\theta(x) \\
		= \alpha \E_{\pin} \log \frac{p_{\hat \gamma}(x)}{p_\theta(x)} + \beta \E_{\pout} \log p_{\hat \gamma}(x) \\
	\Rightarrow \E_{\pout} \log \frac{p_\gamma(x)}{p_{\hat\gamma}(x)} \geq \frac{\alpha}{\beta} \E_{\pin} \log \frac{p_{\hat \gamma}(x)}{p_\theta(x)}
	\end{align*}
	And we have
	\begin{equation*}
		\E_{\pin} \log \frac{p_\gamma(x)}{p_{\hat\gamma}(x)} \leq \E_{\pin} \log \frac{p_\theta(x)}{p_{\hat\gamma}(x)}
	\end{equation*}
	Therefore, by our simplified metric, we have
	\begin{align*}
		(\E_{\pin(x)} \log \frac{p_\theta(x)}{p_{{\gamma}}(x)} - \E_{\pout(x)} \log \frac{p_\theta(x)}{p_{{\gamma}}(x)}) - (\E_{\pin(x)} \log \frac{p_\theta(x)}{p_{{\hat\gamma}}(x)} - \E_{\pout(x)} \log \frac{p_\theta(x)}{p_{\hat\gamma}(x)}) \\
		=  \E_{\pout} \log \frac{p_\gamma(x)}{p_{\hat\gamma}(x)} -\E_{\pin} \log \frac{p_{\gamma}(x)}{p_{\hat\gamma}(x)}\geq \frac{\alpha}{\beta} \E_{\pin} \log \frac{p_{\hat \gamma}(x)}{p_\theta(x)} - \E_{\pin} \log \frac{p_\theta(x)}{p_{\hat\gamma}(x)} \\
		= \frac{\alpha + \beta}{\beta} \E_{\pin} \log \frac{p_\theta(x)}{p_{\hat\gamma}(x)} = \frac{1}{\beta} \E_{\pin} \log \frac{p_\theta(x)}{p_{\hat\gamma}(x)} \geq 0
	\end{align*}
	It means that $\frac{p_\theta(x)}{p_{{\gamma}}(x)}$ is a better OoD indicator than $\frac{p_\theta(x)}{p_{{\hat\gamma}}(x)}$ in case 2 and 3. In conclusion,  $\frac{p_\theta(x)}{p_{{\gamma}}(x)}$ will be a better OoD indicator and thus they can be used to detect OoD if $\gamma$ is trained better than $\hat{\gamma}$.
%	Additional, by the above proof, since $p_\gamma$ is optimized from $p_\theta$, the sign of $\log \frac{p_\theta(x)}{p_\gamma(x)}$ represents whether the likelihood of $x$ in mixture distribution is improved during the mixture training.

\end{proof}
%
%\begin{theorem}\label{thm6}
%	Assumption 2 is a corollary of definition of OoD problem when $div$ is Wasserstein distance.
%\end{theorem}
%\begin{proof}
%	By the definition of OoD, exist a $L > 0$, for any $x_1 \sim \pin, x_2 \sim \pout$, their distance $\|x_1 - x_2\|$ is at least $L$.
%	\begin{equation*}
%		 W^1(\pin, \pout) = \min_{f \in \Gamma(\pin, \pout)} \iint f(x_1, x_2) \|x_1 - x_2\| \dd x_1 \dd x_2 \geq L
%	\end{equation*}
%\end{proof}
%
%\begin{theorem}\label{thm7}
%	$D(x)$ is a symmetric indicator.
%\end{theorem}
%\begin{proof}
%	Let $D$ is the optimal discriminator in $W^1(\pin, \pout)$, for any $D'$ satisfying $Lip(D') \leq 1$, we have
%	\begin{align*}
%		\E_{\pout} D'(x) - \E_{\pin} D'(x) = \E_{\pin} [-D'(x)] - \E_{\pout} [-D'(x)] \\
%		\leq  \E_{\pin} D(x) - \E_{\pout} D(x) =  \E_{\pout} [-D(x)] - \E_{\pin} [-D(x)]
%	\end{align*}
%	Therefore, $-D(x)$ is the optimal discriminator in $W^1(\pout, \pin)$. By the same way in \cref{thm1},
%	$D(x)$ is a symmetric indicator.
%\end{proof}
%
%\begin{theorem}\label{thm8}
%	$\hat{D}$ that is optimal solution in $W^1(\pin, \pmix)$ is same to the optimal solution $D$ in $W^1(\pin, \pout)$. Moreover, the neural networks trained by $W^1(\pin, \pmix)$ and $W^1(\pin, \pout)$ will share the same optimization process.
%\end{theorem}
%
%\begin{proof}
%	For any $D$ satisfying $Lip(D) \leq 1$,
%	\begin{align*}
%		\E_{\pin} D(x) - \E_{\pmix} D(x) = \E_{\pin} D(x) - \alpha\E_{\pin} D(x) - \beta\E_{\pout} D(x) \\
%		= (1 - \alpha)\E_{\pin} D(x) - \beta\E_{\pout} D(x)
%		= \beta \E_{\pin} D(x) - \beta \E_{\pout} D(x) \\
%		\leq \beta \E_{\pin} \hat{D}(x) - \beta \E_{\pout} \hat{D}(x) = (1 - \alpha)\E_{\pin} \hat{D}(x) - \beta\E_{\pout} \hat{D}(x) \\
%		= \E_{\pin} \hat{D}(x) - \alpha\E_{\pin} \hat{D}(x) - \beta\E_{\pout} \hat{D}(x) = \E_{\pin} \hat{D}(x) - \E_{\pmix} \hat{D}(x)
%	\end{align*}
%	Therefore, $\hat{D}$ is also the optimal discriminator in $W^1(\pin, \pmix)$. Additional, in above proof, the loss functions for $D$ in $W^1(\pin, \pmix)$ and $W^1(\pin, \pout)$ are
%	\begin{equation*}
%		\E_{\pin} D(x) - \E_{\pmix} D(x) = \beta (\E_{\pin} D(x) - \E_{\pout} D(x))
%	\end{equation*}
%	The two loss functions have same gradient direction for neural network optimization, thus the training optimization processes are same.
%\end{proof}
%
%\begin{theorem}\label{thm9}
%	The discriminator in $W^1(\pin, \pout)$ is the best indicator among all indicators that is 1-Lipschitz. Moreover, it is the best indicator who has limited gradient.
%\end{theorem}
%\begin{proof}
%	By our simplified metric, for any indicator $f$,
%	\begin{equation*}
%		\E_{\pin} f(x) - \E_{\pout} f(x)
%	\end{equation*}
%	measures the performance of OoD detection. Thus
%	\begin{equation*}
%		\max_{Lip(f) \leq 1} \E_{\pin} f(x) - \E_{\pout} f(x) = W^1(\pin, \pout)
%	\end{equation*}
%	will search the best indicator among all indicators that is 1-Lipschitz. Especially, for a indicator $f$ has limited gradient $K$, function $g(x) = f(x) / K$ satisfies that $Lip(g) \leq 1$. By \cref{thm1}, $f \triangleq g$, and $g$ is worse than the optimal discriminator in $W^1(\pin, \pout)$.
%
%\end{proof}

\section{Detailed Experiments}
See the `detailed\_experiment' in supplemental file.

%% The file named.bst is a bibliography style file for BibTeX 0.99c

%% The file named.bst is a bibliography style file for BibTeX 0.99c
\bibliographystyle{splncs04}
\bibliography{reference}

\end{document}


\documentclass{article} % For LaTeX2e
%\usepackage{format/nips13submit_e}
%\nipsfinalcopy % Uncomment for camera-ready version
\usepackage{times}
\usepackage{hyperref}
\usepackage{url}
\usepackage{color}
\definecolor{mydarkblue}{rgb}{0,0.08,0.45}
\hypersetup{
    pdfpagemode=UseNone,
    colorlinks=true,
    linkcolor=mydarkblue,
    citecolor=mydarkblue,
    filecolor=mydarkblue,
    urlcolor=mydarkblue,
    pdfview=FitH}

\usepackage{graphicx, amsmath, amsfonts, bm, lipsum, capt-of}

\usepackage{natbib, xcolor, wrapfig, booktabs, multirow, caption}

\usepackage{float}

\def\ie{i.e.\ }
\def\eg{e.g.\ }

\title{An automatic report for the dataset : stocks-0-amorepacific}

\author{
The Automatic Statistician
}

\newcommand{\fix}{\marginpar{FIX}}
\newcommand{\new}{\marginpar{NEW}}

\setlength{\marginparwidth}{0.9in}
%\input{include/commenting.tex}

%% For submission, make all render blank.
%\renewcommand{\LATER}[1]{}
%\renewcommand{\fLATER}[1]{}
%\renewcommand{\TBD}[1]{}
%\renewcommand{\fTBD}[1]{}
%\renewcommand{\PROBLEM}[1]{}
%\renewcommand{\fPROBLEM}[1]{}
%\renewcommand{\NA}[1]{#1}  % Note, NA's pass through!

\begin{document}

\allowdisplaybreaks

\maketitle

\begin{abstract}
This report was produced by the Automatic Bayesian Covariance Discovery (ABCD) algorithm.
%See \url{http://arxiv.org/abs/1302.4922} and \url{http://www-kd.iai.uni-bonn.de/cml/proceedings/papers/2.pdf} for preliminary papers.
\end{abstract}

\section{Executive summary}

The raw data and full model posterior with extrapolations are shown in figure~\ref{fig:rawandfit}.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_raw_data} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_all}
\end{tabular}
\caption{Raw data (left) and model posterior with extrapolation (right)}
\label{fig:rawandfit}
\end{figure}

The structure search algorithm has identified six additive components in the data.
The  first 3 additive components explain 97.2\% of the variation in the data as shown by the coefficient of determination ($R^2$) values in table~\ref{table:stats}.
The  first 4 additive components explain 99.6\% of the variation in the data.
After the first 4 components the cross validated mean absolute error (MAE) does not decrease by more than 0.1\%.
This suggests that subsequent terms are modelling very short term trends, uncorrelated noise or are artefacts of the model or search procedure.
Short summaries of the additive components are as follows:
\begin{itemize}

  \item \input{stocks-0-amorepacific/stocks-0-amorepacific_1_short_description.tex}

  \item \input{stocks-0-amorepacific/stocks-0-amorepacific_2_short_description.tex}

  \item \input{stocks-0-amorepacific/stocks-0-amorepacific_3_short_description.tex}

  \item \input{stocks-0-amorepacific/stocks-0-amorepacific_4_short_description.tex}

  \item \input{stocks-0-amorepacific/stocks-0-amorepacific_5_short_description.tex}

  \item \input{stocks-0-amorepacific/stocks-0-amorepacific_6_short_description.tex}

\end{itemize}

\begin{table}[htb]
\begin{center}
{\small
\begin{tabular}{|r|rrrrr|}
\hline
\bf{\#} & {$R^2$ (\%)} & {$\Delta R^2$ (\%)} & {Residual $R^2$ (\%)} & {Cross validated MAE} & Reduction in MAE (\%)\\
\hline
- & - & - & - & 204356.84 & -\\

1 & 52.0 & 52.0 & 52.0 & 20006.35 & 90.2\\

2 & 83.1 & 31.1 & 64.8 & 12663.35 & 36.7\\

3 & 97.2 & 14.1 & 83.3 & 8612.25 & 32.0\\

4 & 99.6 & 2.4 & 86.2 & 8266.45 & 4.0\\

5 & 99.6 & -0.0 & -0.0 & 8266.45 & 0.0\\

6 & 100.0 & 0.4 & 100.0 & 8266.45 & 0.0\\

\hline
\end{tabular}
\caption{
Summary statistics for cumulative additive fits to the data.
The residual coefficient of determination ($R^2$) values are computed using the residuals from the previous fit as the target values; this measures how much of the residual variance is explained by each new component.
The mean absolute error (MAE) is calculated using 10 fold cross validation with a contiguous block design; this measures the ability of the model to interpolate and extrapolate over moderate distances.
The model is fit using the full data and the MAE values are calculated using this model; this double use of data means that the MAE values cannot be used reliably as an estimate of out-of-sample predictive performance.
}
\label{table:stats}
}
\end{center}
\end{table}

Model checking statistics are summarised in table~\ref{table:check} in section~\ref{sec:check}.
These statistics have not revealed any inconsistencies between the model and observed data.

The rest of the document is structured as follows.
In section~\ref{sec:discussion} the forms of the additive components are described and their posterior distributions are displayed.
In section~\ref{sec:extrap} the modelling assumptions of each component are discussed with reference to how this affects the extrapolations made by the model.
Section~\ref{sec:check} discusses model checking statistics, with plots showing the form of any detected discrepancies between the model and observed data.

\section{Detailed discussion of additive components}
\label{sec:discussion}

\subsection{Component 1 : A very smooth function. This function applies until 21 Nov 2018 and from 21 Nov 2018 onwards}

\input{stocks-0-amorepacific/stocks-0-amorepacific_1_description.tex}

This component explains 52.0\% of the total variance.
The addition of this component reduces the cross validated MAE by 90.2\% from 204356.8 to 20006.4.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1_cum}
\end{tabular}
\caption{Pointwise posterior of component 1 (left) and the posterior of the cumulative sum of components with data (right)}
\label{fig:comp1}
\end{figure}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1_anti_cum}
\end{tabular}
\caption{Pointwise posterior of residuals after adding component 1}
\label{fig:comp1}
\end{figure}

\subsection{Component 2 : A smooth function. This function applies from 20 Sep 2018 until 21 Nov 2018 and from 21 Nov 2018 until 29 Nov 2018}

\input{stocks-0-amorepacific/stocks-0-amorepacific_2_description.tex}

This component explains 64.8\% of the residual variance; this increases the total variance explained from 52.0\% to 83.1\%.
The addition of this component reduces the cross validated MAE by 36.70\% from 20006.35 to 12663.35.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2_cum}
\end{tabular}
\caption{Pointwise posterior of component 2 (left) and the posterior of the cumulative sum of components with data (right)}
\label{fig:comp2}
\end{figure}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2_anti_cum}
\end{tabular}
\caption{Pointwise posterior of residuals after adding component 2}
\label{fig:comp2}
\end{figure}

\subsection{Component 3 : A smooth function. This function applies until 20 Sep 2018 and from 29 Nov 2018 onwards}

\input{stocks-0-amorepacific/stocks-0-amorepacific_3_description.tex}

This component explains 83.3\% of the residual variance; this increases the total variance explained from 83.1\% to 97.2\%.
The addition of this component reduces the cross validated MAE by 31.99\% from 12663.35 to 8612.25.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3_cum}
\end{tabular}
\caption{Pointwise posterior of component 3 (left) and the posterior of the cumulative sum of components with data (right)}
\label{fig:comp3}
\end{figure}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3_anti_cum}
\end{tabular}
\caption{Pointwise posterior of residuals after adding component 3}
\label{fig:comp3}
\end{figure}

\subsection{Component 4 : A smooth function. This function applies from 21 Nov 2018 until 21 Nov 2018}

\input{stocks-0-amorepacific/stocks-0-amorepacific_4_description.tex}

This component explains 86.2\% of the residual variance; this increases the total variance explained from 97.2\% to 99.6\%.
The addition of this component reduces the cross validated MAE by 4.02\% from 8612.25 to 8266.45.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4_cum}
\end{tabular}
\caption{Pointwise posterior of component 4 (left) and the posterior of the cumulative sum of components with data (right)}
\label{fig:comp4}
\end{figure}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4_anti_cum}
\end{tabular}
\caption{Pointwise posterior of residuals after adding component 4}
\label{fig:comp4}
\end{figure}

\subsection{Component 5 : A constant}

\input{stocks-0-amorepacific/stocks-0-amorepacific_5_description.tex}

This component explains -0.0\% of the residual variance; this reduces the total variance explained from 99.6\% to 99.6\%.
The addition of this component reduces the cross validated MAE by 0.00\% from 8266.45 to 8266.45.
This component neither explains residual variance nor improves MAE and therefore is likely to be an artefact of the model or search procedure.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5_cum}
\end{tabular}
\caption{Pointwise posterior of component 5 (left) and the posterior of the cumulative sum of components with data (right)}
\label{fig:comp5}
\end{figure}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5_anti_cum}
\end{tabular}
\caption{Pointwise posterior of residuals after adding component 5}
\label{fig:comp5}
\end{figure}

\subsection{Component 6 : Uncorrelated noise}

\input{stocks-0-amorepacific/stocks-0-amorepacific_6_description.tex}

This component explains 100.0\% of the residual variance; this increases the total variance explained from 99.6\% to 100.0\%.
The addition of this component reduces the cross validated MAE by 0.00\% from 8266.45 to 8266.45.
This component explains residual variance but does not improve MAE which suggests that this component describes very short term patterns, uncorrelated noise or is an artefact of the model or search procedure.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_6} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_6_cum}
\end{tabular}
\caption{Pointwise posterior of component 6 (left) and the posterior of the cumulative sum of components with data (right)}
\label{fig:comp6}
\end{figure}

\section{Extrapolation}
\label{sec:extrap}

Summaries of the posterior distribution of the full model are shown in figure~\ref{fig:extrap}.
The plot on the left displays the mean of the posterior together with pointwise variance.
The plot on the right displays three random samples from the posterior.

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_all} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_all_sample}
\end{tabular}
\caption{Full model posterior with extrapolation. Mean and pointwise variance (left) and three random samples (right)}
\label{fig:extrap}
\end{figure}

Below are descriptions of the modelling assumptions associated with each additive component and how they affect the predictive posterior.
Plots of the pointwise posterior and samples from the posterior are also presented, showing extrapolations from each component and the cuulative sum of components.

\subsection{Component 1 : A very smooth function. This function applies until 21 Nov 2018 and from 21 Nov 2018 onwards}

\input{stocks-0-amorepacific/stocks-0-amorepacific_1_extrap_description.tex}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1_sample} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1_cum_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_1_cum_sample}
\end{tabular}
\caption{Posterior of component 1 (top) and cumulative sum of components (bottom) with extrapolation. Mean and pointwise variance (left) and three random samples from the posterior distribution (right).}
\label{fig:extrap1}
\end{figure}

\subsection{Component 2 : A smooth function. This function applies from 20 Sep 2018 until 21 Nov 2018 and from 21 Nov 2018 until 29 Nov 2018}

\input{stocks-0-amorepacific/stocks-0-amorepacific_2_extrap_description.tex}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2_sample} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2_cum_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_2_cum_sample}
\end{tabular}
\caption{Posterior of component 2 (top) and cumulative sum of components (bottom) with extrapolation. Mean and pointwise variance (left) and three random samples from the posterior distribution (right).}
\label{fig:extrap2}
\end{figure}

\subsection{Component 3 : A smooth function. This function applies until 20 Sep 2018 and from 29 Nov 2018 onwards}

\input{stocks-0-amorepacific/stocks-0-amorepacific_3_extrap_description.tex}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3_sample} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3_cum_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_3_cum_sample}
\end{tabular}
\caption{Posterior of component 3 (top) and cumulative sum of components (bottom) with extrapolation. Mean and pointwise variance (left) and three random samples from the posterior distribution (right).}
\label{fig:extrap3}
\end{figure}

\subsection{Component 4 : A smooth function. This function applies from 21 Nov 2018 until 21 Nov 2018}

\input{stocks-0-amorepacific/stocks-0-amorepacific_4_extrap_description.tex}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4_sample} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4_cum_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_4_cum_sample}
\end{tabular}
\caption{Posterior of component 4 (top) and cumulative sum of components (bottom) with extrapolation. Mean and pointwise variance (left) and three random samples from the posterior distribution (right).}
\label{fig:extrap4}
\end{figure}

\subsection{Component 5 : A constant}

\input{stocks-0-amorepacific/stocks-0-amorepacific_5_extrap_description.tex}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5_sample} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5_cum_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_5_cum_sample}
\end{tabular}
\caption{Posterior of component 5 (top) and cumulative sum of components (bottom) with extrapolation. Mean and pointwise variance (left) and three random samples from the posterior distribution (right).}
\label{fig:extrap5}
\end{figure}

\subsection{Component 6 : Uncorrelated noise}

\input{stocks-0-amorepacific/stocks-0-amorepacific_6_extrap_description.tex}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_6_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_6_sample} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_6_cum_extrap} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_6_cum_sample}
\end{tabular}
\caption{Posterior of component 6 (top) and cumulative sum of components (bottom) with extrapolation. Mean and pointwise variance (left) and three random samples from the posterior distribution (right).}
\label{fig:extrap6}
\end{figure}

\section{Model checking}
\label{sec:check}

Several posterior predictive checks have been performed to assess how well the model describes the observed data.
These tests take the form of comparing statistics evaluated on samples from the prior and posterior distributions for each additive component.
The statistics are derived from autocorrelation function (ACF) estimates, periodograms and quantile-quantile (qq) plots.

Table~\ref{table:check} displays cumulative probability and $p$-value estimates for these quantities.
Cumulative probabilities near 0/1 indicate that the test statistic was lower/higher under the posterior compared to the prior unexpectedly often \ie they contain the same information as a $p$-value for a two-tailed test and they also express if the test statistic was higher or lower than expected.
$p$-values near 0 indicate that the test statistic was larger in magnitude under the posterior compared to the prior unexpectedly often.

\begin{table}[htb]
\begin{center}
{\small
\begin{tabular}{|r|rr|rr|rr|}
\hline
 & \multicolumn{2}{|c|}{ACF} & \multicolumn{2}{|c|}{Periodogram} & \multicolumn{2}{|c|}{QQ} \\
\bf{\#} & {min} & {min loc} & {max} & {max loc} & {max} & {min}\\
\hline

1 & \textcolor{gray}{0.813} & \textcolor{gray}{0.836} & \textcolor{gray}{0.674} & \textcolor{gray}{0.434} & \textcolor{gray}{0.169} & \textcolor{gray}{0.857}\\

2 & \textcolor{gray}{0.510} & \textcolor{gray}{0.770} & \textcolor{gray}{0.919} & \textcolor{gray}{0.340} & \textcolor{gray}{0.929} & \textcolor{gray}{0.108}\\

3 & \textcolor{gray}{0.518} & \textcolor{gray}{0.426} & \textcolor{gray}{0.675} & \textcolor{gray}{0.478} & \textcolor{gray}{0.423} & \textcolor{gray}{0.790}\\

4 & \textcolor{gray}{0.490} & \textcolor{gray}{0.572} & \textcolor{gray}{0.775} & \textcolor{gray}{0.611} & \textcolor{gray}{0.664} & \textcolor{gray}{0.442}\\

5 & \textcolor{gray}{0.495} & \textcolor{gray}{0.519} & \textcolor{gray}{0.494} & \textcolor{gray}{0.482} & \textcolor{gray}{0.507} & \textcolor{gray}{0.493}\\

6 & \textcolor{gray}{0.531} & \textcolor{gray}{0.506} & \textcolor{gray}{0.552} & \textcolor{gray}{0.511} & \textcolor{gray}{0.650} & \textcolor{gray}{0.442}\\

\hline
\end{tabular}
\caption{
Model checking statistics for each component.
Cumulative probabilities for minimum of autocorrelation function (ACF) and its location.
Cumulative probabilities for maximum of periodogram and its location.
$p$-values for maximum and minimum deviations of QQ-plot from straight line.
}
\label{table:check}
}
\end{center}
\end{table}

No statistically significant discrepancies between the data and model have been detected but model checking plots for each component are presented below.

\subsection{Model checking plots for components without statistically significant discrepancies}

\subsubsection{Component 1 : A very smooth function. This function applies until 21 Nov 2018 and from 21 Nov 2018 onwards}

No discrepancies between the prior and posterior of this component have been detected

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_acf_bands_1} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_pxx_bands_1} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_qq_bands_1}
\end{tabular}
\caption{
ACF (top left), periodogram (top right) and quantile-quantile (bottom left) uncertainty plots.
The blue line and shading are the pointwise mean and 90\% confidence interval of the plots under the prior distribution for component 1.
The green line and green dashed lines are the corresponding quantities under the posterior.}
\label{fig:check1}
\end{figure}

\subsubsection{Component 2 : A smooth function. This function applies from 20 Sep 2018 until 21 Nov 2018 and from 21 Nov 2018 until 29 Nov 2018}

No discrepancies between the prior and posterior of this component have been detected

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_acf_bands_2} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_pxx_bands_2} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_qq_bands_2}
\end{tabular}
\caption{
ACF (top left), periodogram (top right) and quantile-quantile (bottom left) uncertainty plots.
The blue line and shading are the pointwise mean and 90\% confidence interval of the plots under the prior distribution for component 2.
The green line and green dashed lines are the corresponding quantities under the posterior.}
\label{fig:check2}
\end{figure}

\subsubsection{Component 3 : A smooth function. This function applies until 20 Sep 2018 and from 29 Nov 2018 onwards}

No discrepancies between the prior and posterior of this component have been detected

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_acf_bands_3} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_pxx_bands_3} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_qq_bands_3}
\end{tabular}
\caption{
ACF (top left), periodogram (top right) and quantile-quantile (bottom left) uncertainty plots.
The blue line and shading are the pointwise mean and 90\% confidence interval of the plots under the prior distribution for component 3.
The green line and green dashed lines are the corresponding quantities under the posterior.}
\label{fig:check3}
\end{figure}

\subsubsection{Component 4 : A smooth function. This function applies from 21 Nov 2018 until 21 Nov 2018}

No discrepancies between the prior and posterior of this component have been detected

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_acf_bands_4} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_pxx_bands_4} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_qq_bands_4}
\end{tabular}
\caption{
ACF (top left), periodogram (top right) and quantile-quantile (bottom left) uncertainty plots.
The blue line and shading are the pointwise mean and 90\% confidence interval of the plots under the prior distribution for component 4.
The green line and green dashed lines are the corresponding quantities under the posterior.}
\label{fig:check4}
\end{figure}

\subsubsection{Component 5 : A constant}

No discrepancies between the prior and posterior of this component have been detected

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_acf_bands_5} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_pxx_bands_5} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_qq_bands_5}
\end{tabular}
\caption{
ACF (top left), periodogram (top right) and quantile-quantile (bottom left) uncertainty plots.
The blue line and shading are the pointwise mean and 90\% confidence interval of the plots under the prior distribution for component 5.
The green line and green dashed lines are the corresponding quantities under the posterior.}
\label{fig:check5}
\end{figure}

\subsubsection{Component 6 : Uncorrelated noise}

No discrepancies between the prior and posterior of this component have been detected

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\begin{tabular}{cc}
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_acf_bands_6} & \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_pxx_bands_6} \\
\mbm \includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_qq_bands_6}
\end{tabular}
\caption{
ACF (top left), periodogram (top right) and quantile-quantile (bottom left) uncertainty plots.
The blue line and shading are the pointwise mean and 90\% confidence interval of the plots under the prior distribution for component 6.
The green line and green dashed lines are the corresponding quantities under the posterior.}
\label{fig:check6}
\end{figure}

\section{MMD - experimental section}
\label{sec:mmd}

\begin{table}[htb]
\begin{center}
{\small
\begin{tabular}{|r|r|}
\hline
\bf{\#} & {mmd}\\
\hline

1 & \textbf{0.000}\\

2 & \textbf{0.009}\\

3 & \textbf{0.000}\\

4 & \textbf{0.000}\\

5 & \textcolor{gray}{0.963}\\

6 & 0.032\\

\hline
\end{tabular}
\caption{
MMD $p$-values
}
\label{table:mmd}
}
\end{center}
\end{table}

\subsubsection{Component 1 : A very smooth function. This function applies until 21 Nov 2018 and from 21 Nov 2018 onwards}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_mmd_1}
\caption{
MMD plot}
\label{fig:mmd1}
\end{figure}

\subsubsection{Component 2 : A smooth function. This function applies from 20 Sep 2018 until 21 Nov 2018 and from 21 Nov 2018 until 29 Nov 2018}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_mmd_2}
\caption{
MMD plot}
\label{fig:mmd2}
\end{figure}

\subsubsection{Component 3 : A smooth function. This function applies until 20 Sep 2018 and from 29 Nov 2018 onwards}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_mmd_3}
\caption{
MMD plot}
\label{fig:mmd3}
\end{figure}

\subsubsection{Component 4 : A smooth function. This function applies from 21 Nov 2018 until 21 Nov 2018}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_mmd_4}
\caption{
MMD plot}
\label{fig:mmd4}
\end{figure}

\subsubsection{Component 5 : A constant}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_mmd_5}
\caption{
MMD plot}
\label{fig:mmd5}
\end{figure}

\subsubsection{Component 6 : Uncorrelated noise}

\begin{figure}[H]
\newcommand{\wmgd}{0.5\columnwidth}
\newcommand{\hmgd}{3.0cm}
\newcommand{\mdrd}{stocks-0-amorepacific}
\newcommand{\mbm}{\hspace{-0.3cm}}
\includegraphics[width=\wmgd,height=\hmgd]{\mdrd/stocks-0-amorepacific_mmd_6}
\caption{
MMD plot}
\label{fig:mmd6}
\end{figure}

\end{document}


\section{Related Work}\label{sec:related}

\paragraph{SMT-Based Verification}
%
SMT solvers have been extensively used to automate
reasoning on verification languages like
Dafny~\cite{dafny}, Fstar~\cite{fstar} and Why3~\cite{why3}.
%
These languages are designed for verification,
thus have limited support for commonly used language
features like parallelism and optimized libraries
that we use in our verified implementation.
%
Refinement Types~\cite{ConstableS87,FreemanPfenningDONTCITE91,Rushby98}
on the other hand, target existing general purpose languages,
such as
ML~\cite{pfenningxi98,GordonRefinement09,LiquidPLDI08},
C~\cite{deputy,LiquidPOPL10},
Haskell~\cite{Vazou14},
Racket~\cite{RefinedRacket}
and Scala~\cite{refinedscala}.
However, before Refinement Reflection~\cite{reflection} was introduced,
they only allowed ``shallow'' program specifications,
that is, properties that only talk about behaviors of program functions
but not functions themselves.
%

\paragraph{Dependent Types}
Unlike Refinement Types, dependent type systems,
like Coq~\cite{coq-book}, Adga~\cite{agda} and Isabelle/HOL~\cite{isabelle} allow for ``deep'' specifications
which talk about program functions,
such as the program equivalence reasoning we presented.
%
Compared to (Liquid) Haskell,
these systems allow for tactics and heuristics
that automate proof term generation
but lack SMT automations and
general-purpose language features,
like non-termination, exceptions and IO.
%
Zombie~\cite{Zombie,Sjoberg2015} and Fstar~\cite{fstar} allow dependent types to
coexist with divergent and effectful programs,
but still lack the optimized libraries,
like @ByteSting@, which come
with a general purpose language
with long history, like Haskell.

\paragraph{Parallel Code Verification}
Dependent type theorem provers have been used before to
verify parallel code.
%
BSP-Why~\cite{bspwhy} is an extension to Why2 that is using both Coq and SMTs
to discharge user specified verification conditions.
%
Daum~\cite{daum07} used Isabelle to formalize the semantics
of a type-safe subset of C,
by extending Schirmer's~\cite{schirmer06}
formalization of sequential imperative languages.
%
Finally, Swierstra~\cite{wouter10} formalized mutable arrays in Agda
and used them to reason about distributed maps and sums.

One work  closely related to ours is
SyDPaCC~\cite{SyDPaCC}, a Coq library that
automatically parallelizes list homomorphisms
by extracting parallel Ocaml versions of user provided Coq functions.
%
Unlike SyDPaCC, we are not automatically generating the parallel
function version, because of engineering limitations
(\S~\ref{sec:evaluation}).  Once these are addressed, code extraction
can be naturally implemented by turning
Theorem~\ref{theorem:two-level} into a Haskell type class with a
default parallelization method.
%
SyDPaCC used maximum prefix sum as a case study,
whose morphism verification is
much simpler than our string matching case study.
%
Finally, our implementation consists of
2K lines of Liquid Haskell, which we consider verbose and aim to
use tactics to simplify.
On the contrary, the SyDPaCC implementation
requires three different languages:
2K lines of Coq with tactics, 600 lines of Ocaml and 120 lines of C,
and is considered ``very concise''.


\documentclass[main.tex]{subfiles}
\begin{document}

\subsection{IN-IN formalism}

\marginpar{Monday\\ 2021-1-11, \\ compiled \\ \today}

In cosmology we directly observe correlation functions, as opposed to particle accelerators.
In particle physics speak, they are VEVs of products of fields: here we will be interested in
%
\begin{align}
Q(t) = \delta
\phi _{\vec{k}_1}(t)
\phi _{\vec{k}_2}(t)
\phi _{\vec{k}_3}(t)
\,,
\end{align}
%
where \(\delta \phi \) is the inflaton perturbation. This is the \textbf{bispectrum}.

The IN-IN formalism gives us a formula like
%
\begin{align}
\ev{Q(t)}{\Omega }
= \bra{0}
\qty[\overline{T} \exp(i \int_{t_0 }^{t} \dd{t'} H' _{\text{int}} (t'))]
Q'(t)
\qty[\overline{T} \exp(-i \int_{t_0 }^{t} \dd{t'} H' _{\text{int}} (t'))]
\ket{0}
\,.
\end{align}

Short description of the interaction picture for a field described by  Hamiltonian \(H = H_0 + H _{\text{int}}\).
Fields evolve according to the free time-evolution operator only.
Then, we get
%
\begin{align}
\expval{Q(t)}
&= \ev{Q(t)}{\Omega } = \ev{U^{-1}(t, t_0 ) Q (t_0 ) U(t, t_0 )}{\Omega } \\
&= \ev{U^{-1} (t, t_0 ) U _{\text{free}} (t, t_0 ) Q' (t) U^{-1} _{\text{free}} (t, t_0 ) U(t, t_0 )}{\Omega }  \\
&= \ev{F^{-1}(t, t_0 ) Q'(t) F (t, t_0 )}{\Omega }
\,,
\end{align}
%
where
%
\begin{align}
\dv{F(t, t_0 )}{t} = -i H _{\text{int}} F(t, t_0 )
\,.
\end{align}

We can get a formal solution as the exponential of a time-ordered integral;
using Wick's theorem we can expand the exponential.

How do we have \(\ket{0}\), the vacuum of the free theory, instead of \(\ket{\Omega }\)?

In QFT we usually have the \(S\)-matrix formalism, which is an ``in-out'' formalism; here instead we use an ``in-in'' one.
We turn off the interaction theory in the far past; we can then replace \(\ket{\Omega }\) with \(\ket{0}\).
We use the Bunch-Davies vacuum as \(\ket{0}\): we say that the quantum fluctuation on small scales are approximately plane waves.

As an example, we will compute the bispectrum for a model
%
\begin{align}
S = \int \dd[4]{x} \sqrt{-g}
\qty[ \frac{M _{\text{Pl}}^2}{2} R
- \frac{1}{2} g^{\mu \nu } \partial_{\mu } \phi (t, \vec{x}) \partial_{\nu } (t, \vec{x})
- V[\phi (t, \vec{x})]]
\,,
\end{align}
%
where \(M _{\text{Pl}}\) is the reduced Planck mass.
For simplicity, let us take \(g_{\mu \nu } = a^2(\tau) \eta _{\mu \nu }\).

We expand
%
\begin{align}
\phi (t, \vec{x}) &= \phi_0 (t) + \delta \phi (t, \vec{x})  \\
V(\phi ) &= V(\phi_0 )
+ \eval{\dv{V}{\phi }}_{\phi_0 } \delta \phi
+ \frac{1}{2} \eval{\dv[2]{V}{\phi }}_{\phi_0 } \delta \phi^2
+ \frac{1}{3!} \eval{\dv[3]{V}{\phi }}_{\phi_0 } \delta \phi^3
\,.
\end{align}

We substitute these into the action (see slides) and the first term yields the unperturbed EOM:
%
\begin{align}
\ddot{\phi}_0 + 3 H \dot{\phi}_0 + \dv{V}{\phi_0 } = 0
\,.
\end{align}

The second term vanishes.
We define the parameters
%
\begin{align}
\eval{\dv[2]{V}{\phi}}_{\phi_0 } &= m_{ \delta \phi}^2 &
\eval{\dv[3]{V}{\phi}}_{\phi_0 } &= - \lambda
\,.
\end{align}

We impose \(m_{ \delta \phi } = 0\) to simplify our discussion.
The Lagrangian density then becomes
%
\begin{align}
\mathcal{L} = \underbrace{\frac{a}{2} \dot{\delta}_\chi^2
- \dot{a} \dot{\delta}_\chi \delta_\chi
+ \frac{1}{2} \frac{\dot{a}^2}{a} \delta_{ \chi  }^2
- \frac{\delta^{ij}}{2a} \partial_{i} \delta _\chi \partial_{j} \delta _\chi }_{\mathcal{L} _{\text{free}}}
+ \underbrace{\frac{\lambda}{3!} \delta_\chi^3}_{\mathcal{L} _{\text{int}}}
\,,
\end{align}
%
in terms of \(\delta _\chi = a \delta \phi \).

The EOM read
%
\begin{align}
\dots
\,.
\end{align}

The bispectrum is connected to the three-point function since we lose one independent momentum because of the requirement of momentum conservation.

What follows is a long discussion on how to perform the integrals involved in the computation of the bispectrum.

\end{document}


\documentclass[a4paper]{article}

%% Language and font encodings
\usepackage[english]{babel}
\usepackage[utf8x]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{caption}

%% Sets page size and margins
\usepackage[a4paper,top=3cm,bottom=2cm,left=3cm,right=3cm,marginparwidth=1.75cm]{geometry}

%% Useful packages
\usepackage{amsmath}
\usepackage{amsfonts}

\usepackage{graphicx}
\usepackage{tikz}
\usetikzlibrary{arrows.meta}

\usepackage[colorinlistoftodos]{todonotes}
\usepackage[colorlinks=true, allcolors=blue]{hyperref}

\usepackage{color}
\usepackage{url}

%% display solutions or not
\newif\ifsol
\soltrue % comment out to hide solutions

%% todo tracker -- overleaf v2 has better one it uses but will default to this if compiled on something that doesn't have built-in todo
%\newcommand{\todo}[1]{\textbf{\textcolor{red}{#1}}}

\title{Section 6: Reinforcement Learning}
\author{CS 182 - Artificial Intelligence}
\date{}

\begin{document}
\maketitle

\noindent We use \textbf{Reinforcement Learning (RL)} when we have a Markov Decision Process (MDP) for which we know:
\begin{itemize}
    \item $S$: set of states
    \item $A$: set of actions (for any given state)
    \item $s_0$: start state
    \item $s_g$: goal state (or set of goal states)
\end{itemize}
but do not know:
\begin{itemize}
    \item $T : S \times A \to S$: transition map (or model)
    \item $R : S \times A \to \mathbb{R}$: reward function
\end{itemize}
The goal of reinforcement learning is to learn a \textbf{policy} $\pi: S \to A$ which returns the best action for any given state in the problem. In the process of looking for the policy, RL algorithms often calculate:
\begin{itemize}
    \item $V : S \to \mathbb{R}$: \textbf{value function}, which stores the utility of every state
    \item $Q : S \times A \to \mathbb{R}$: \textbf{Q-value function}, which stores the value of every (state, action) pair \\
\end{itemize}

\noindent In \textbf{Passive Reinforcement Learning}, an agent has a fixed policy, and learns about the environment while executing that policy. In lecture, we discussed two algorithms:
\begin{enumerate}
    \item \textbf{Direct evaluation} runs many "experiments", or "simulations", resulting from following a given policy $\pi$, and keeps track of the total discounted rewards of the visited states. Afterwards, the reward instances for each state are averaged to determine the state's value. This could be written as
    \[
    V(s) = \frac{1}{n} \sum_i (R(s_i) + \gamma R(s_i') + \gamma^2 R(s_i'') + ...)
    \]
    where $s_i$, $s_i'$, $s_i''$, $\dots$, are the states visited in the $i$th simulation. Direct evaluation can also be referred to as a \textbf{Monte Carlo} approach, since it depends on averaging over many randomized simulations or samples.
    \item \textbf{Temporal difference learning} iteratively runs experiments following a given policy. With each transition, a sample-based Bellman update is made to the value of the visited state (both equations are computing the same thing):
    \[
    V^{\pi}(s) \leftarrow V^{\pi}(s) + \alpha [R(s, \pi(s), s') + \gamma V^{\pi}(s') - V^{\pi}(s)]
    \]
    \[
    V^{\pi}(s) \leftarrow (1- \alpha) V^{\pi}(s) + \alpha [R(s, \pi(s), s') + \gamma V^{\pi}(s')]
    \]
    We can think of this algorithm can be thought of as sample-based policy evaluation. Remember policy evaluation was:
    $$V^{\pi}(s) \leftarrow \sum_{s'} T(s, \pi_k(s), s') [R(s,\pi(s),s') + \gamma V^{\pi}(s')]$$
    In Q-learning we are still executing some policy $\pi$ we just don't know what the transition model is and thus can't compute all of the possible transitions to sum over. Instead we are simply computing one of the values in the sum and making a weighted update of our current value to incorporate the new information.
\end{enumerate}

\noindent In \textbf{Active Reinforcement Learning}, and agent chooses actions and tries to find an optimal policy while learning about the environment. The primary algorithm used in active RL is \textbf{Q-learning}. Similarly to TD learning, Q-learning makes sample-based Bellman updates, but to Q-values instead of values:
\[
Q(s, a) \leftarrow Q(s, a) + \alpha [R(s, a, s') + \gamma \max_{a'} Q(s', a') - Q(s, a)]
\]
\[
Q(s, a) \leftarrow (1 - \alpha) Q(s, a) + \alpha [R(s, a, s') + \gamma \max_{a'} Q(s', a')]
\]
Since we are learning Q-values and not values we can extract the current estimate of the optimal policy by doing a one-step argmaximization over the Q-value immediately at any time: \\
\[
\pi(s) = arg\max_{a} Q(s,a)
\]

\noindent Some algorithmic details and variations:
\begin{enumerate}
    \item In active algorithms, we must choose actions to take from states. One approach is \textbf{$\epsilon$-greedy action selection}: with (small) probability $\epsilon$, actions are chosen randomly; with (large) probability $1 - \epsilon$, we choose the current optimal action. This encourages exploration of the state space and is generally helpful in practice. Note that this is called off-policy learning as we learn the exact Q-values but execute with a different policy, the $\epsilon$-greedy policy.
    \item How can we evaluate the choice of parameters or exploration functions in Q-learning? One approach is to measure \textbf{regret} -- the difference between expected rewards and final optimal rewards.
    \item In linear \textbf{approximate} (or \textbf{feature based}) \textbf{Q-learning} we approximate the Q-function with a set of functions which are intended to represent higher-level features of the state space:
    \[
    Q(s,a) = w_1 f_1(s,a) + w_2 f_2(s,a) + \dots + w_m f_m(s,a)
    \]
    For example, for PACMAN they could be the number of ghosts, the distance to the nearest ghost, the distance to the nearest food, etc. The advantage here is that we simply need to learn the correct weights, and thus we have fewer parameters to learn than in standard Q-learning, where we have to learn a table of Q-values for every single state-action pair. The disadvantage is that the quality of the Q-values is highly dependent on the quality of the functions. The update of the weights based on new information is:
    \[
    w_i \leftarrow w_i + \alpha f_i(s,a) [R(s, a, s') + \gamma \max_{a'} Q(s', a') - Q(s, a)]
    \]

\end{enumerate}

\noindent Algorithmic parameters:
\begin{itemize}
    \item $\alpha$: learning rate (determines the integration of new information into current estimates)
    \item $\gamma$: discount rate (determines how quickly rewards decay with time)
\end{itemize}

\newpage
\section*{Practice Problems}
\begin{enumerate}
    \item Suppose we are Q-learning with $\epsilon$-greedy action selection, starting with $\epsilon = 0.2$. When is it a good idea to...
    \begin{itemize}
        \item keep $\epsilon$ constant over time?
        \item decrease $\epsilon$ to 0 over time?
        \item increase $\epsilon$ over time?
    \end{itemize}

    \ifsol
        \textcolor{blue}{It is generally good to decrease $\epsilon$ to 0 with time, especially when using on-policy learning methods. The reason is that as the agent learns the actual optimal policy for the world, it should switch from a mix of exploration and exploitation to mostly exploitation. However, if the world is changing, we may leave $\epsilon$ high such that the agent would continue to explore. It never makes sense to continuously increase $\epsilon$, as this counteract convergence.}
    \else
        \vspace{7em}
    \fi

    \item Suppose we are Q-learning. When is it a good idea to...
    \begin{itemize}
        \item set $\alpha = 0$?
        \item set $\alpha = 1$?
        \item set $\alpha = 0.2$ and decrease it to 0 over time?
    \end{itemize}

    \ifsol
        \textcolor{blue}{If the learning rate $\alpha$ is set to 0, we will not make any updates to the value function, so this is never a good idea. When $\alpha$ is set to 1, we will be fully overriding prior value estimates with the new update. While this is generally undesirable, it is actually optimal for a fully deterministic MDP. The third option -- setting alpha to a low value and decreasing it over time -- is the most reasonable approach.}
    \else
        \vspace{7em}
    \fi

    \item What would the returned value assignments be for Direct Evaluation over a deterministic MDP? Can you find a simple upper bound on the number of experiments that need to be run by the algorithm for the value function to converge? What if the MDP is not deterministic?

    \ifsol
        \textcolor{blue}{If the MDP is deterministic, Direct Evaluation would assign the exact value of every visited state under the given policy, so the number of necessary experiments would be no larger than the total number of start states. If the MDP is not deterministic, we do not have this guarantee, since separate experiments would have different returns.}
    \else
        \vspace{7em}
    \fi

    \item When using features to represent the $Q$-function, is it guaranteed that the feature-based $Q$-learning finds the same optimal $Q^∗$ as would be found when using a tabular representation for the $Q$-function?

    \ifsol
        \textcolor{blue}{No -- the optimal Q-function $Q∗$ would depend on the choice of features. Even if the choice of features was fully informed, it may not be possible to represent the optimal $Q^*$ with a (linear) weighted combination of features.}
    \else
        \vspace{7em}
    \fi

    \item Why is temporal difference (TD) learning of Q-values (Q-learning) superior to TD learning of values?

    \ifsol
        \textcolor{blue}{If only values are found, it is difficult to extract a policy -- this would require knowing or calculating the transition function. If Q-values are found, the policy can be easily computed by $\pi(s) = \text{argmax}_a \; Q(s, a)$.}
    \else
        \vspace{7em}
    \fi

    \newpage
    \item Recall a problem from last week's section: Consider an MDP having three states, (1, 2, 3), where State 3 is a terminal state. In States 1 and 2, there are two possible actions: $A$ and $B$. Unlike last week, we no longer know the rewards or transition model ahead of time. \\

    Suppose we decide to run Q-learning on this problem with $\alpha = 0.2$ and a 0.9 discount ($\gamma = 0.9$). All Q-values are initialized to 0. While running one episode/simulation within the algorithm, we get the following $(s, a, s', r)$ samples (in order): \\

    (State 1, Action B, State 1, reward = -1) \\
    (State 1, Action A, State 1, reward = -1) \\
    (State 1, Action A, State 2, reward = -2) \\
    (State 2, Action B, State 2, reward = -2) \\
    (State 2, Action B, State 3, reward = 0) \\

    Run Q-learning based on these samples. How are the Q-values updated? \\
    \ifsol
        \textcolor{blue}{
    Initialization:
    \begin{center}
    \begin{tabular}{ |c|c|c| }
     \hline
             & Action A & Action B \\
     State 1 & 0 & 0 \\
     State 2 & 0 & 0 \\
     State 3 & 0 & 0 \\
     \hline
    \end{tabular}
    \end{center}
    (State 1, Action B, State 1, reward = -1)  $\rightarrow$  (0.8 * 0 + 0.2 (-1 + 0.9 max(0, 0)) = -0.2)
    \begin{center}
    \begin{tabular}{ |c|c|c| }
     \hline
             & Action A & Action B \\
     State 1 & 0 & -0.2 \\
     State 2 & 0 & 0 \\
     State 3 & 0 & 0 \\
     \hline
    \end{tabular}
    \end{center}
    (State 1, Action A, State 1, reward = -1)  $\rightarrow$  (0.8 * 0 + 0.2 (-1 + 0.9 max(0, -0.2)) = -0.2)
    \begin{center}
    \begin{tabular}{ |c|c|c| }
     \hline
             & Action A & Action B \\
     State 1 & -0.2 & -0.2 \\
     State 2 & 0 & 0 \\
     State 3 & 0 & 0 \\
     \hline
    \end{tabular}
    \end{center}
    (State 1, Action A, State 2, reward = -2)  $\rightarrow$  (0.8 * (-0.2) + 0.2 (-2 + 0.9 max(0, 0)) = -0.56)
    \begin{center}
    \begin{tabular}{ |c|c|c| }
     \hline
             & Action A & Action B \\
     State 1 & -0.56 & -0.2 \\
     State 2 & 0 & 0 \\
     State 3 & 0 & 0 \\
     \hline
    \end{tabular}
    \end{center}
    (State 2, Action B, State 2, reward = -2)  $\rightarrow$  (0.8 * 0 + 0.2 (-2 + 0.9 max(0, 0)) = -0.4)
    \begin{center}
    \begin{tabular}{ |c|c|c| }
     \hline
             & Action A & Action B \\
     State 1 & -0.56 & -0.2 \\
     State 2 & 0 & -0.4 \\
     State 3 & 0 & 0 \\
     \hline
    \end{tabular}
    \end{center}
    (State 2, Action B, State 3, reward = 0)  $\rightarrow$  (0.8 * (-0.4) + 0.2 (0 + 0.9 max(0, 0)) = -0.32)
    \begin{center}
    \begin{tabular}{ |c|c|c| }
     \hline
             & Action A & Action B \\
     State 1 & -0.56 & -0.2 \\
     State 2 & 0 & -0.32 \\
     State 3 & 0 & 0 \\
     \hline
    \end{tabular}
    \end{center}
    }
    \else
        \vspace{28em}
    \fi

    How would the solution be different if we chose all actions following the optimal policy? \\
    \ifsol
        \textcolor{blue}{At every step, the next chosen action (from a given state) will be the one corresponding to the largest current Q-values of that state. In the samples given above, all actions follow the current optimal policy except the last sample, where action A would have been taken instead of Action B, since $Q(\text{State 2}, \text{Action A}) > Q(\text{State 2}, \text{Action B})$.}
    \else
        \vspace{7em}
    \fi
    \newpage
    \item Consider the Pacman-inspired game board below. Suppose that every action has a 80\% chance of moving in the desired direction, and a 10\% chance of moving in each of two perpendicular directions (though we only learn this transition function by executing actions in the world). The lower left-hand square is numbered (0, 0), the start square is (3, 0), and the goal square is (0, 6). The goal of this game is for Pacman to navigate to the exit (reward of 100) without touching the dangerous explosion square at (1, 4), which terminates the game with a reward of $-100$. Pacman cannot cross the thick wall in the middle of the board; an action causing a wall collision would make Pacman stay in place.

    \begin{center}
    \includegraphics[scale=0.25]{grid.png}
    \end{center}

    Let's say we choose to solve this problem with feature-based Q-learning, with the features: \\
    $f_1 =$ the Manhattan distance to the goal \\
    $f_2 =$ 0 or 1 indicating whether the explosion square is in 1-square radius of Pacman.
    \begin{enumerate}
        \item Which gameboard squares will be equivalent to the state defined by ($f_1 = 4$, $f_2 = 1$)? \\
        \ifsol
            \textcolor{blue}{Squares (1, 3) and (2, 4) both have a Manhattan distance of $4$ to the goal, and are one square away from the square $(1, 4)$.}
        \else
            \vspace{5em}
        \fi

        \item What are the advantages and disadvantages of using this feature representation? \\
        \ifsol
            \textcolor{blue}{The advantage of a feature representation is that it decreases the number of states needed for the problem, so each transition informs a greater number of (grid) state values. The downside is that features may not capture important information. In this problem, if the explosion square is nearby but behind a wall, it does not pose a threat; but the feature representation equates this situation to one where there is a dangerous square and no wall.}
        \else
            \vspace{5em}
        \fi
    \end{enumerate}
\end{enumerate}

\end{document}


\section{Data description, visualization and statistical analysis} \label{data-visualization}

We used a dataset given by the teacher containing two distinct folders: one with face image examples and the other with non-face image examples. The images are in gray scale and weren't normalized as can be seen in figures \ref{fig:faces-overview} and \ref{fig:non-faces-overview}.

\begin{figure}[htbp]
\centerline{\includegraphics[scale=0.35]{images/faces_overview.png}}
\caption{Overview of the images with faces in the dataset.}
\label{fig:faces-overview}
\end{figure}

\begin{figure}[htbp]
\centerline{\includegraphics[scale=0.35]{images/non_faces_overview.png}}
\caption{Overview of the images without faces in the dataset.}
\label{fig:non-faces-overview}
\end{figure}

This dataset contains 124 images, 69 of which are face images and 55 are non-face images, with a window of pixels of \(27 \times 18\). We can conclude that the dataset is a little skewed, having more face images than non-face images as it is represented in figure \ref{fig:dataset-distribution}.

\begin{figure}[htbp]
\centerline{\includegraphics[scale=0.35]{images/data_distribution.png}}
\caption{Analysis of the number of examples for each type of image.}
\label{fig:dataset-distribution}
\end{figure}


\subsection{Methods}
\label{sec:methods}

The hadronic and leptonic decay mode combinations in di-$\tau$ system are listed in Table.~\ref{tab:taudecaymodes}. In \tauhh (hh) channel, according to the number of charged and neutral pions, decay mode ApBn denotes the decay mode with A pions and B neutrons in decay products. For example, 1pXn denotes $\tau$ decaying to one pion and multiple neutrons. While in \taulh (lh) channel, decay mode 'l' indicates $\tau$ decay to leptons, i.e. $e$ and $\mu$. The method of constructing the \phistarCP observable in di-$\tau$ decays depends mainly on the hadronic $\tau$ decay modes.
%
\begin{table}[htbp]
	\centering
	\caption{Di-$\tau$ decay mode combinations and corresponding method and branching ratio to construct \phistar observable used in this analysis.
	}
	\begin{tabular}{cccc}
		\toprule
		Decay channel & Decay mode combination & Method & Branching Ratio in all di-$\tau$ decays\\
		\midrule
		\multirow{3}{*}{\taulh} & l-1p0n & IP-IP & 7.6\% \\
		& l-1p1n & IP-$\rho$ & 17.8\%\\
		& l-1pXn & IP-$\rho$ & 7.3\%\\
		\midrule
		\multirow{6}{*}{\tauhh} & 1p0n-1p0n & IP-IP & 1.2\%\\
		& 1p0n-1p1n & IP-$\rho$ & 5.5\%\\
		& 1p1n-1p1n & $\rho$-$\rho$ & 6.4\%\\
		& 1p0n-1pXn & IP-$\rho$ & 2.2\%\\
		& 1p1n-1pXn & $\rho$-$\rho$ & 5.3\%\\
		& 1p1n-3p0n & $\rho$-$a_{1}$ & 4.7\%\\
		\bottomrule
	\end{tabular}
	\label{tab:taudecaymodes}
\end{table}
%

Fig.~\ref{fig:observables:methods} illustrates the methods of building \phistarCP corresponding to $\tau$ decay modes. Impact Parameter (IP) method is applied when there is only one visible and charged particle in $\tau$ decay, i.e. 1p0n and leptonic decay channels. \phistarCP defined by IP method is the acoplanarity angle between the $\tau$ decay planes spanned by the transverse component of impact parameter vectors $\hat{\textbf{n}}^{*\pm}_\perp$ boosted into visible di-$\tau$ ZMF.

For $\rho$ method, \phistarCP is defined as the angle between the tau decay plane formed from the spatial vectors of the charged and neutral components (charged: $\hat{\textbf{q}}^{*\pm}$, neutral:  $\hat{\textbf{q}}^{*0\pm}$) in visible di-$\tau$ ZMF. This method is used in 1p1n and 1pXn $\tau$ decay channels.

For IP-$\rho$ mixed method, the tau decay plane is spanned by the normalized spatial vector $\hat{\textbf{q}}^{*\pm}$, $\hat{\textbf{q}}^{*0\pm}$, and the transverse component of the impact parameter vector, $\hat{\textbf{n}}^{*\pm}_\perp$ in visible di-$\tau$ ZMF. IP-$\rho$ method is applied in 1p0n-1p1n, 1p0n-1pXn, l-1p1n, and l-1pXn di-$\tau$ decay channels.
%
\begin{figure}[h!]
	\begin{center}
		\begin{tabular}{ccc}
			\subfigure[]{\includegraphics[width=.2\textwidth]{figures/observables/IP-IP.jpg}}
			\subfigure[]{\includegraphics[width=.21\textwidth]{figures/observables/rho-rho.jpg}}
			\subfigure[]{\includegraphics[width=.2\textwidth]{figures/observables/IP-rho.jpg}}
		\end{tabular}
	\end{center}
	\caption{Illustration of the \phistarCP constructed by (a) IP method, (b) $\rho$ method, (c) IP-$\rho$ method. }
	\label{fig:observables:methods}
\end{figure}
%


\section*{Private Property Model}
Our study is carried out for the prisoner's dilemma game (PD), which is often used to model selfish behavior of individuals when cooperation is risky and defection is tempting, but where the outcome of mutual defection is inferior to cooperation on both sides \cite{axelrod1981evolution,nowak2006five}.
Formally, the so-called reward $R$ represents the payoff for mutual cooperation, while the payoff for defection of both sides is the {\it punishment} $P$. $T$ represents the {\it temptation} to unilaterally defect, which results in the {\it sucker's payoff} $S$ for the cooperating individual. The inequalities $T > R > P > S$ and $2R > T + S$ define the classical prisoner's dilemma, in which it is more profitable to defect, no matter what strategy the other individual selects. Therefore, rationally behaving individuals would be expected to defect when they meet once. However, defection by everyone is implied as well by the game-dynamical replicator equation \cite{epstein1998zones}, which takes into account imitation of superior strategies, or payoff-driven birth-and-death processes. In contrast, a coexistence of cooperators and defectors is predicted for the snowdrift game (SD). Although it is also used to study social cooperation, its payoffs are characterized by $T > R > S > P$ (i.e., $S > P$ rather than $P > S$) \cite{}.\\

As is well-known \cite{nowak2006five}, cooperation can, for example, be supported by repeated interactions \cite{axelrod1981evolution}, by intergroup competition with or without altruistic punishment\cite{traulsen2006evolution,fehr2002altruistic,boyd2003evolution}, and by network reciprocity based on the clustering of cooperators \cite{nowak1992evolutionary,szabo2002phase,hauert2004spatial}. In the latter case, the level of cooperation in 2-dimensional spatial games is further enhanced by ``disordered environments" (10\% inaccessible empty locations) \cite{vainstein2001disordered}, and by
diffusive mobility, provided that the mobility parameter is in a suitable range \cite{vainstein2007does}. Usually, strategy mutations, random relocations, and other sources of stochasticity can significantly challenge the formation and survival of cooperative clusters \cite{}. {\it Success-driven} migration, in contrast, is a robust mechanism: By leaving unfavorable neighborhoods, seeking more favorable
ones, and remaining in cooperative neighborhoods, it supports cooperative clusters very efficiently against the destructive effects of noise, thus preventing defector invasion in a large area of payoff parameters \cite{helbing2009outbreak}.\\

The {\it private property} game implies the possibility to expel players from locations with higher pay-off, and as such, it extends the success-driven migration game by allowing migrations to any location within the migration range. We assume $N$ individuals on a two dimension square with periodic boundary conditions and $L\times L$ sites, which are either empty or occupied by one individual. Population density $d$ is a parameter with values between $0.2$ and $0.8$.\footnote{below and beyond these values, a migration game makes little sense as interactions in low density worlds may only be achievedby large migration ranges, and densities $\rightarrow 1$ almost completely remove migration opportunities.} Individuals are updated asynchronously, in a random sequential order, and each individual gets updated on average $N$ times (i.e., the number of Monte Carlo Steps $MCS = N \times L^2$). At each step, the randomly selected individual performs simultaneous interactions with the $4$ direct neighbors and compares the overall payoff with that of these neighbors. If one neighbor has a better payoff, the individual updates her strategy with the one of her best performing neighbor.\footnote{Performing the same simulations with 8 neighbors does not change the picture overall.} In absence of noise ($r=0$, equiv Fermi temperature equals to 1), this update is sure, and the individual cannot spontaneously start to cooperate ($q=0$).\\

The migration step is performed before the imitation step\cite{helbing2009outbreak}: An individual explores the expected payoffs for {\it all} sites in the Moore neighborhood $(2M + 1) \times (2M + 1)$ of range $M$. If the fictitious payoff is higher than in the current location, the individual is assumed to move to the site with the highest payoff with probability $m$ (in absence of migration noise, $m=1$). If the target site is empty, then {\it success-driven} migration occurs. On the contrary, if the site with highest payoff is already occupied by another individual, the {\it property-violation} migration occurs. With probability $s$, the focal player takes the site of the incumbent individual, who in turn is expelled to the best empty site in her own Moore neighborhood (see Figure \ref{fig:migration_diagram} for a graphical representation). With probability $1-s$ however, property violation is aborted and the individual migrates to the best empty site within the migration range. The probability $s$ encompasses the unsure nature of the property violation endeavor such as the possibility to get caught by law enforcement, or limited by some physical or technical constraints. \\

While small quantities of property violation introduce some amount of randomness that enhances cooperation (see Figure \ref{fig:phase_transition}a), our model is aimed at uncovering the limit conditions and the mechanisms, which help maintain or restore cooperative societies in presence of property violation. Indeed, it makes particularly sense to study limit conditions in the case of private property violation, because society want to optimize their law enforcement, hence invest the minimum for the maximum effects.\footnote{Here, we can enrich the paragraph by pulling out complex system research on transition from order to disorder (and vice-versa) when a system is optimized.}

%The resulting strategy mutations reflect deficient imitation or trial-and-error behavior. As a side effect, such noise leads to an independence of the final cooperation level from the initial one (at $t=0$), and a {\it qualitatively different} pattern formation dynamics for the same payoff values, update rules, and initial conditions ( c.f. SI Fig.1). Using the alternative Fermi update rule \cite{} would have been possible as well. However, resetting strategies rather than inverting them, combined with values $q$ much smaller than $0.5$, creates particularly adverse conditions for cooperation.\\