\input{boilerplate}
\newcommand{\beginsupplement}{%
        \setcounter{table}{0}
        \renewcommand{\thetable}{S\arabic{table}}%
        \setcounter{figure}{0}
        \renewcommand{\thefigure}{S\arabic{figure}}%
     }
\newcommand{\norm}[1]{\left\lVert #1 \right\rVert}
\usepackage{amsmath}

\usepackage[mathlines]{lineno}
\linenumbers
\usepackage{adjustbox}
\usepackage[natbibapa]{apacite}
\bibliographystyle{apacite}
\begin{document}
%\title{Exploring the role of auxiliary parameters in evaluating voxel-wise encoding models}
\title{The role of auxiliary parameters in evaluating voxel-wise encoding models for 3T and 7T BOLD fMRI data}

\author[1]{Moritz~Boos}
\author[2]{J.~Swaroop~Guntupalli}
\author[1]{Jochem~W.~Rieger}
\author[3,4]{Michael~Hanke}

\affil[1]{Applied Neurocognitive Psychology Lab and Cluster of Excellence "Hearing4all", University of Oldenburg, Oldenburg, Germany}
\affil[2]{Vicarious AI, Union City CA, USA}
\affil[3]{Institute of Neuroscience and Medicine, Brain \& Behaviour (INM-7), Research Centre Jülich, Jülich, Germany}
\affil[4]{Institute of Systems Neuroscience, Medical Faculty, Heinrich Heine University Düsseldorf, Düsseldorf, Germany}
\maketitle

\begin{abstract}
In neuroimaging, voxel-wise encoding models are a popular tool to predict brain activity elicited by a stimulus.
To evaluate the accuracy of these predictions across multiple voxels, one can choose between multiple quality metrics.
However, each quality metric requires specifying auxiliary parameters such as the number and selection criteria of voxels, whose influence on model validation is unknown.
In this study, we systematically vary these parameters and observe their effects on three common quality metrics of voxel-wise encoding models in two open datasets of 3- and 7-Tesla BOLD fMRI activity elicited by musical stimuli.
We show that such auxiliary parameters not only exert substantial influence on model validation, but also differ in how they affect each quality metric.
Finally, we give several recommendations for validating voxel-wise encoding models that may limit variability in validation due to different numbers of voxels, voxel selection criteria, and magnetic field strengths.
\end{abstract}

\clearpage

\section*{Introduction}

In functional magnetic resonance imaging (f{MRI}) research, voxel-wise encoding models are an increasingly popular tool to characterize the relationship between a real world stimulus and BOLD activity patterns \citep{TD+06,KG+08,NG11,huth2012continuous,holdgraf2016rapid}. However, researchers face many degrees of freedom in constructing voxel-wise encoding models, such as how to represent the stimulus or how to estimate an accurate model. To evaluate the fidelity of these choices and the resulting voxel-wise encoding models, multiple quality metrics exist that assess how well these models approximate spatial patterns of brain activity across multiple voxels. \citet{ML08} first introduced binary retrieval accuracy --- a measure that assesses whether an encoding model's predicted spatial activation patterns can, on average, identify the correct stimulus against a decoy --- to evaluate the predicted f{MRI} images for the meaning of nouns. \citet{KG+08} used stimulus identification accuracy, and \citet{NG09} used stimulus reconstruction quality as a metric for encoding performance. In auditory neuroscience, only binary retrieval accuracy \citep{CTK+2012,hoefle2018identifying} and stimulus identification \citep{SF14,allen2018encoding} have been used.
However, while providing novel insights into the cortical processing of sensory stimuli, it is unknown how additional parameters of the data analysis that are unrelated to the encoding models themselves affect these quality metrics. Hence, it remains difficult to compare the quality of voxel-wise encoding models when they differ in such auxiliary parameters.
In this study, we aim to tackle this problem by documenting the effect of two common auxiliary parameters in the validation of voxel-wise encoding models --- the number of voxels used and how they were selected --- on different quality metrics for low and high field strength (3 and 7-Tesla f{MRI}). To this end, we utilize a 3T f{MRI} study on the perception of musical genres \citep{CTK+2012} which has recently been replicated in 7T \citep{HDH+2015}. Using these datasets, we apply three approaches to encoding model validation, --- binary retrieval accuracy \citep{ML08}, stimulus identification \citep{KG+08,SF14} and decoding accuracy of the stimulus \citep{NG09} ---, and compare them in a 3T and 7T dataset with identical stimuli and comparable design, for different choices in data analysis parameters.
%make claims more conservative

\section*{Methods}

\subsection*{Stimuli}

Stimuli were five natural, stereo, high-quality music stimuli (\unit[6]{s} duration; \unit[44.1]{kHz} sampling rate) for each of five different musical genres: 1) Ambient, 2) Roots Country 3) Heavy Metal, 4) 50s Rock'n'Roll, and 5) Symphonic. Previously, all 25 stimuli have been made publicly available \citep{HDH+2015}.

\subsection*{f{MRI} data}

The analyses presented here were performed on two independently recorded, and previously published datasets \citep{CTK+2012,HDH+2015}. While these datasets have been acquired using identical stimuli, with the same number of acquisition runs and number of stimulation trials, they nevertheless differ in their precise stimulation timing, stimulation setup and equipment, as well as other acquisition details. A brief description of both datasets is provided below. For more information the reader is referred to the respective publications.

\paragraph{\unit[3]{Tesla}}
%
Nineteen participants (11 male, age 19-37 years, mean age 24.7 years) were scanned in a Philips Intera Achieva scanner with 32 channel SENSE head coil at the Center for Cognitive Neuroscience at Dartmouth College. Functional scans were acquired with an echo planar imaging sequence (\unit[2]{s} TR; \unit[35]{ms} TR, \unit[90]{\textdegree} flip angle) with \unit[3]{mm} isotropic voxels.
% MIH: as it was published before we probably don't have to include this
%All participants consented in accordance with the procedures set by the Committee
%for the Protection of Human Subjects at the Dartmouth College.
Each participant participated in eight functional runs. Stimuli were presented in an event-related design with a variable trial duration. Each run consisted of a total of 29 trials corresponding to 25 music clips and 4 catch trials presented randomly during each run. Each trial started with a \unit[6]{s} music clip followed by \unit[4-8]{s} of fixation. For catch trials, a question appeared after the audio presentation asking whether an acoustic feature is present in the music clip such as vocals, guitar, etc. Subjects responded “Yes” or “No” with a button box. Catch trials were supposed to help keep the participants’ attention to the music and were discarded from the analyses. Each run had \unit[4]{s} of fixation at the beginning and \unit[10]{s} of fixation at the end.

For further details, see \citet{CTK+2012}

\paragraph{\sevenT}
%
The procedures for the \unit[7]{Tesla} acquisition were highly similar and only critical differences are reported here.
The dataset consists of twenty participants (12 male, age 21-38 years, mean age 26.6 years).
Echo-planar BOLD images (gradient-echo, \unit[2]{s} repetition time (TR), \unit[22]{ms} echo time, \unit[0.78]{ms} echo spacing, GRAPPA acceleration factor 3) were acquired using a whole-body \sevenT\ Siemens MAGNETOM magnetic resonance scanner equipped with a 32 channel brain receive coil. 36 axial slices (thickness \unit[1.4]{mm}, \unit[1.4 $\times$ 1.4]{mm} in-plane resolution) with a 10\% inter-slice gap were recorded in ascending order. Slices were oriented to include the ventral portions of frontal and occipital cortex while minimizing intersection with the eyeballs. The field-of-view was centered on the approximate location of Heschl's gyrus along the rostral-caudal axis.

Instead of dedicated catch trials, similar catch questions as for the 3T acquisition were presented \unit[4]{s} after the end of the stimulus in trials with an \unit[8]{s} inter-stimulus delay. Consequently, each run consisted of 25 trials, and no trials were discarded from the analysis.  There was no additional fixation at the start of a run. For further details, see \citet{HDH+2015}, and \citet{HBI+14} for details on MRI acquisition methods.

\subsection*{Preprocessing}

Approximate temporal lobe masks for each participant were extracted from Montreal Neurological Institute coordinate space using FSL \citep{SJB+04,JBB+12}, and projected into the subject-specific coordinate system.  Each voxel inside the temporal lobe mask was run-wise $Z$-scored and linearly de-trended using PyMVPA \citep{HHS09b}. After preprocessing, 3T f{MRI} data consisted of 16999 voxels and 7T f{MRI} data of 115439 voxels.

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{pics/encoding_scheme}

  \caption{A schematic overview of the encoding process. The spectrogram for
    each stimulus is transformed into its low-quefrency mel-frequency spectrum
    (LQ-MFS). Then, the encoding features are extracted by a sliding window
    from the LQ-MFS. Using these features, encoding model is trained on all
    runs in the training set, and used to predict the BOLD activity of the
  left-out run.  These predictions are subsequently used for validation.}

 \label{fig:encoding_scheme}
\end{figure}

\subsection*{Encoding model}

To build an encoding model with high predictive power, one needs to find an appropriate feature representation of the music stimuli \citep{holdgraf2017encoding}. \citet{CTK+2012} compared different feature representations of the same stimuli in the 3T dataset. They found features corresponding to the timbre of the stimulus offer the best discriminative power. We chose a similar feature set, related to timbre, that was made available by \citet{HDH+2015}, the low-quefrency mel-frequency spectrum (LQ-MFS) of the stimulus. This stimulus representation consists of 48 LQ-MFS coefficients for each 100 ms segment of a stimulus, thus each stimulus overall comprises 2880 LQ-MFS coefficients.

For each f{MRI} sample $y_{vt}$ (where $t=1,2,,\ldots,T$ denotes the time-points, $T$ the number of samples in each run, and $v=1,2,\ldots,V$ denotes the voxels) the LQ-MFS features $x_{t}$ $[1\times\widetilde{M}]$ (where $\widetilde{M}$ is the number of LQ-MFS coefficients) of the corresponding two second part of the stimulus -- immediately prior to the acquisition time point -- were computed. In case there was no stimulus presented at time-point $t$, a zero vector $[1\times\widetilde{M}]$ was used.

As the BOLD response is delayed,  the most recent feature vector was removed for each f{MRI} sample (this corresponds to an assumed \unit[2]{s} stimulus-response delay), and the new feature vector at time-point $t$ was created by concatenating the prior feature vectors $x_{t-1}$,$x_{t-2}$ and $x_{t-3}$ (see Figure \ref{fig:encoding_scheme}). From now on, we denote this stacked feature vector as $x_{t}$.  Feature vectors (and the corresponding f{MRI} sample) were removed from the analysis, if two-thirds or more of the concatenated feature vectors were zero-vectors.

The BOLD activity time-series, as well as the feature time-series, were vertically stacked, resulting in a matrix of features $X$ $[N\times M]$ (where $N$ is the number of f{MRI} samples, and $M$ is number of LQ-MFS coefficients, with $M=3\widetilde{M}$) and a matrix of BOLD activity $Y$ $[N\times V]$ (where $V$ is the number of voxels and $N$ as above). This lagging of the stimulus allows us to train the encoding model to predict the f{MRI} time-series without explicitly modeling the BOLD response. The encoding model could then be expressed as the probability to observe the BOLD activity at time-point $t$ and voxel $v$:
\begin{equation}
  \label{eq:encmo}
  p(y_{vt}|x_{t}) = N(y_{vt};x_{t}\beta_{v},\sigma)
\end{equation}
where $N(y;\mu,\sigma)$ denotes the probability density at $y$ for a
Gaussian with mean $\mu$ and standard deviation $\sigma$, and $\beta_{v}$ is a
$[M\times1]$ vector of regression coefficients specific to voxel $v$. The corresponding matrix vector equation for all voxels is
\begin{equation}
    \label{eq:encmat}
    \mathbf{Y} = \mathbf{X}\mathbf{\beta}
\end{equation}
where $\mathbf{Y}$ and $\mathbf{X}$ are defined as above and $\mathbf{\beta}$ is a $[M \times V]$ matrix of regression coefficients per voxel.
To reduce over-fitting, the regression-coefficients were estimated using ridge regression \citep{HK70}. Independently for each voxel, the regularization parameter $\lambda$ with the lowest mean squared error in a generalized leave-one-out cross-validation \citep{GHW79} was chosen from a set of candidate values (ranging from $1e3$ to $1e6$). This set was chosen so that the highest and lowest values of $\lambda$ were only rarely selected in cross-validation. While this regularization parameter can exert a large influence on regression coefficients, and hence interpretation, we do not scrutinize its role here, since we are interested in choosing the amount of regularization that maximizes prediction accuracy.

\subsection*{Analysis software}
All analyses were implemented using custom code that heavily utilizes the SciPy ecosystem \citep{jones2014scipy}, Scikit-Learn \citep{PVG+11}, Nilearn \citep{abraham2014machine}, and PyMVPA \citep{HHS09b}.
Visualization was done using the Matplotlib \citep{hunter2007matplotlib} and Seaborn packages \citep{waskom2020seaborn}.
Statistical modeling was done using the Bambi \citep{yarkoni2016bambi} and PyMC \citep{patil2010pymc} packages.
All source code for analysis and visualization is available under \url{https://github.com/psychoinformatics-de/studyforrest-paper-pandoraencoding}.

\subsection*{Quality metrics}

Binary retrieval accuracy and the matching score quantify encoding model performance based on the difference between predicted and observed spatial patterns that stretch across multiple voxels. They assume that each stimulus is associated with one spatial pattern, and thus do not incorporate stimulus time courses with multiple BOLD samples. Since our stimuli encompass three BOLD samples each, we treat each BOLD sample as as an individual stimulus in both the binary retrieval accuracy and the matching score. However, we need to avoid misclassifications in the case in which predicted BOLD activity for one sample of the stimulus is closest to another sample in the same stimulus instead of the original sample, -- and thus would be counted as a misclassification in some quality metrices, even if predicted BOLD activity correctly classifies the stimulus.
To deal with this issue, when testing each predicted BOLD sample we temporarily remove all other samples of the same stimulus from the validation set.
Hence, for each BOLD sample of a stimulus we test against $N = 25\:\times 3 - 2$ other stimuli per run (25 stimuli with 3 BOLD samples each).
Since all quality metrics are computed on the validation set in a leave-one-run-out cross-validation scheme, this leads to different number of samples used for the binary retrieval accuracy and matching score (number of samples $N$ = 3 samples $\times$ 25 stimuli $\times$ 8 runs = 600), and decoding accuracy ($N$ = 25 stimuli $\times$ 8 runs = 200 or $N$ = 5 genres $\times$ 8 runs = 40).

\subsubsection*{Binary retrieval accuracy}

Binary retrieval accuracy \citep{ML08} tests if an encoding model's predictions can, on average, differentiate a stimulus' observed BOLD activity from the BOLD activity of a decoy stimulus. Specifically, a stimulus pair is counted as successfully classified, if the sum of the cosine similarities between predicted and observed f{MRI} responses is greater for the correctly matched predictions and observations than for the incorrectly matched ones.

%NEW
In other words, a pair of stimuli, A and B, is correctly matched if
$$sim(\mathbf{a},\mathbf{\hat{a}}) + sim(\mathbf{b},\mathbf{\hat{b}}) > sim(\mathbf{a}, \mathbf{\hat{b}}) + sim(\mathbf{b}, \mathbf{\hat{a}})$$
where $\mathbf{a}$ and $\mathbf{b}$ denote the vectors of fMRI activity in response to the stimuli, $\mathbf{\hat{a}}$ and $\mathbf{\hat{b}}$ denote the vectors of predicted fMRI activity, and $sim(\mathbf{a},\mathbf{b})=\frac{\mathbf{a}\cdot\mathbf{b}}{\norm{\mathbf{a}}\norm{\mathbf{b}}}$ denotes the cosine similarity of two vectors.

In our case, the binary retrieval measure for one predicted BOLD sample of a stimulus was computed by counting the correct matches for all (exhaustive) pair-wise combinations of all BOLD samples of all stimuli, independent of genre, while excluding other samples of the same stimulus, and then dividing by the number of combinations. This quantity was averaged across all samples of all stimuli.

\subsubsection*{Matching score}
%
An alternative measure of encoding performance is the correlation rank score or matching score \citep{SF14}. For each BOLD sample of a given stimulus $l_{n}$ in the validation set, its predicted BOLD activity $\widetilde{y}_{n}$ is correlated with the observed BOLD activity of every stimulus, $y_{n}$.
These correlations are then ordered, and the  matching score $m(l_{n})$ is \[m(l_{n}) = 1-\frac{rank(l_{n})-1}{N-1} \] Where $rank(l_{n})$ is the rank of the correlation between predicted $\widetilde{y}_{n}$ and observed $y_{n}$ BOLD activity of $l_{n}$, and $N$ = 25 stimuli $\times$ 3 samples - 2 = 73 is the number of BOLD samples in the validation set excluding the two other BOLD samples from the same stimulus.
Finally, the matching scores for all stimuli in the validation set are averaged.

\begin{figure}[!htb]
  \centering
  \includegraphics[width=.8\linewidth]{pics/decoding_scheme.pdf}

  \caption{A schematic overview of the decoding of stimulus identity and stimulus music category. The
    predicted f{MRI} time series of the validation run is reduced in
    dimensionality by principal component analysis, and a multivariate-normal
    likelihood function $p(y_{t}|x_{t})$ is constructed.  From there, the
  probability distribution over music stimuli and subsequently musical
categories is estimated.}

 \label{fig:decoding_scheme}
\end{figure}

\subsubsection*{Decoding}

Instead of testing the encoding performance, we can also test the performance of a decoder based on the individual encoding models \citep{NG11} (Figure \ref{fig:decoding_scheme}). Our goal is to obtain the probability of which music stimulus was presented from the predicted spatial activation pattern across voxels. To go from the probability of a voxel activation given stimulus features $p(y_{vt}|x_{t})$ to probability of stimulus features given voxel activation $p(x_{t}|y_{t})$ we follow \citet{NG09} and separate our decoding scheme into two parts.

\paragraph{Single- to Multi-Voxel Encoding}

First we condense the large number of voxel-specific encoding models into one multi-voxel encoding model.
%NEW
To do this we project the vectors of predicted and observed f{MRI} data of all voxels onto the first $k$ principal components of the $[N\times V]$ matrix of predicted BOLD activity resulting.
We denote the predicted and observed fMRI activity whitened by PCA as $\hat{y}^{pca}_{t}$ and $y^{pca}_{t}$ respectively.

We choose $k$ via cross-validation to estimate the number of principal components that maximize the decoding accuracy on the training set for each participant. As in \citet{NG09} we construct the $k$-dimensional multivariate normal probability density function $p(y_{t}|x_{t})$ to obtain a likelihood function across voxels.

%NEW
To construct $p(y_{t}|x_{t})$ we first compute the covariance matrix $\Sigma$ of the residuals between predicted and observed BOLD activity $y^{pca}_{t}-\hat{y}^{pca}_{t}$ in the training set.
For computing $p(y_{t}|x_{t})$ for a $y_{t}$ and $x_{t}$ in the test set, we use $p(y_{t}|x_{t}) = N(y^{pca}_{t};\hat{y}^{pca}_{t},\Sigma)$.

\paragraph{Multi-Voxel Encoding to Decoding}

We now express this likelihood in terms of the label of the music stimulus, instead of its LQ-MFS features. We use the simplifying assumption that the BOLD activity is influenced by the music stimuli only through their LQ-MFS coefficients $x$, and --- given that each music stimulus was associated with only three (lagged) LQ-MFS representations --- the likelihood to observe a given triple of consecutive $y$ for a specific music stimulus $l_{n}$ is $p(y|l_{n}) \propto p(y_{t-1}|x_{1})p(y_{t}|x_{2})p(y_{t+1}|x_{3})$ where $t$ is the sample \unit[6]{seconds} after the start of the music stimulus, $x$ are the three LQ-MFS feature vectors of this stimulus, and $n=1..25$ denotes the stimulus. For a given triple of consecutive BOLD activity $y$ from the same stimulus, we can now estimate the probability distribution over music stimuli $p(l_{n}|y)$  by using Bayes' rule: $p(l_{n}|y) \propto p(y|l_{n})p(l_{n})$. Since each stimulus was presented exactly once, it has an uniform prior distribution with $p(l_{n})=\frac{1}{25}$.
%NEW
To decode individual stimulus labels we use the mode of the probability distribution $p(l_{n}|y)$, since it will represent the most probable presented stimulus given the data.

Additionally we decode the musical genre of the presented stimulus given the observed BOLD activity as the mode of $p(c_{n}|y) = \sum\nolimits_{l \in  stim(c_{n})} p(l|y)$ where $stim(c_{n})$ are the labels of the stimuli belonging to the genre $c_{n}$.

%NEW
Genre decoding accuracy can be high, even if decoding individual stimulus labels is difficult.
This is possible because misclassified stimulus labels can nonetheless belong to the correct genre, i.e. the false positives for a stimulus can be true positives if only the genre is considered.

\subsection*{Voxel selection}

We varied the number of voxels used in the analysis, both for 3T and 7T f{MRI} data, and selected which voxels to keep by two different criteria. Both criteria were based on voxel characteristics in the training set only.

\paragraph{Selection by stability}

\citet{ML08} selected the 500 most stable voxels for their analysis. For a single voxel, each run can be represented as a vector of BOLD activity, where each entry is associated with one stimulus. A voxel's "stability score" is then the average (pair-wise) correlation between the vectors of the eight runs for all combinations of runs. Hence, this criterion selects voxels with consistent activation for each stimulus across runs.

\paragraph{Selection by $r^2$}

As we are interested in encoding performance, we can use the quality of predictions of each voxel's encoding model as a selection criterion. We compute the coefficient of determination $r^2$ for each voxel-specific encoding model in the training set. Using this criterion selects voxels whose activity can be explained best by an LQ-MFS-based encoding model.

\subsection*{Data availability}

The 7T f{MRI} data is available as part of the OpenNeuro \href{https://openneuro.org/datasets/ds000113/versions/1.3.0}{ds000113b} dataset \citep{ds113b}.
The 3T f{MRI} data is available upon request.

\section*{Results}

\noindent
\begin{table*}[!htbp]
\centering
%\resizebox{\linewidth}{!}{
\begin{adjustbox}{center}
\begin{tabular}{lcccccccccc}
\hline
Parameter &   \multicolumn{9}{c}{Options}  \tabularnewline
\hline
Number of Voxels & 25 & 50 & 100 & 250 & 500 & 1000 & 2500 & 5000 & 10000 \\
Field strength            & \multicolumn{4}{c}{3T} & & \multicolumn{4}{c}{7T}  &   \\
Selection strategy & \multicolumn{4}{c}{stability} & & \multicolumn{4}{c}{$r^{2}$} \\
Metrics & \multicolumn{3}{c}{Binary retrieval acc.} & \multicolumn{2}{c}{Matching score} & \multicolumn{2}{c}{Stimulus decoding} & \multicolumn{2}{c}{Genre decoding} \\
\hline
\end{tabular}
\end{adjustbox}
\caption{Different parameters and their candidate values for validating voxel-wise encoding models.}
\label{table:parameters}
\end{table*}

We trained voxel-wise encoding models to predict voxel activity using a LQ-MFS representation of music stimuli in 3T and 7T f{MRI} data.
First, we evaluate each voxel's encoding model individually. Figure \ref{fig:r2_plot} shows $r^{2}$ values from an eight fold leave-one-run-out cross-validation of the 10000 voxels with highest $r^{2}$ averaged across participants. While voxel-wise encoding models show higher peak $r^{2}$ in 3T than in 7T in the best performing voxels (left side of the figure), voxel-wise encoding models show higher $r^{2}$ in 7T than in 3T for all other voxels.

\begin{figure}[!htb]
  \centering
  \includegraphics[width=\linewidth]{pics/r2_plot.pdf}

    \caption{Voxel-wise $r^{2}$ between predicted and observed BOLD activity for each left-out run, averaged across participants and runs with 95\% confidence interval. Shown here are the 10000 voxels with highest $r^{2}$ for 3T and 7T.
  }

 \label{fig:r2_plot}\end{figure}

Next, we evaluate i\textbf{sets} of voxel-wise encoding models for 3T and 7T using several quality metrics. We varied the number of voxels used in the analysis and how they were selected, both for 3T and 7T f{MRI} data, and compared the resulting differences in three quality metrics of encoding models. To differentiate between effects of voxel number and overall volume of the voxels, which differs in 3T and 7T, we show the results as a function of the number of voxels, as well as overall volume of these voxels.
To illustrate the extent of 10000 selected voxels for different field strengths and selection criteria, we show for each voxel the proportion of validation sets across all participants in which it was selected in Figure \ref{fig:glassbrains}. In both 3T and 7T the voxels that are most often selected are situated in primary and secondary auditory areas.
%NEW
Table \ref{table:parameters} shows the different validation parameters and lists candidate values we used for these parameters.

\subsection*{Binary retrieval accuracy}

\begin{figure*}[!htb]
  \centering
  \includegraphics[width=\linewidth]{pics/binary.pdf}

  \caption{\textbf{A} Mean binary retrieval accuracy as a function of the
  included number of voxels for 3T and 7T, for stability- and $r^2$-based
  voxel selection. Error bars denote the bootstrapped 95\% confidence interval
  of the mean. The mean is taken over binary retrieval accuracies of eight runs
  for each of the 19 participants. \textbf{B} Mean binary retrieval accuracy as
a function of the overall volume of the included voxels for 3T and 7T, for
	stability- and $r^2$-based voxel selection. Chance level is 0.5. Stars mark for which number of voxels the two curves differ significantly.}

 \label{fig:binary_retrieval}\end{figure*}

Figure \ref{fig:binary_retrieval} shows differences in binary retrieval accuracy for different numbers of voxels, selection strategies, and field strength.
For both 3T and 7T data, and both selection strategies, the maximum binary retrieval accuracy is achieved with relatively low number of voxels (3T and $r^{2}$: \unit[250]{voxels}, \unit[6.75]{$cm^{3}$} volume, 7T and $r^{2}$: \unit[500]{voxels}, \unit[0.68]{$cm^{3}$} volume, 3T and stability selection: \unit[500]{voxels}, \unit[13.5]{$cm^{3}$} volume, 7T and stability selection: \unit[1000]{voxels}, \unit[2.7]{$cm^{3}$} volume) and decreases the more voxels are included. For both stability selection criteria, 3T outperforms or is equal to 7T across all numbers of voxels. With regard to the voxel selection strategies, selection by $r^{2}$ outperforms or is equal to selection by stability across all numbers of voxels (Figure \ref{fig:binary_retrieval_selection}).

%NEW
Furthermore, we use a hierarchical Bayesian model to test whether selection strategy and field strength influence the validation over and above the inter-individual variation and variation due to the number of voxels.
Table \ref{table:hierarchical_model} shows that both high field strength (7T) and selection by stability lead to a worse Binary retrieval accuracy.

\subsection*{Matching score}

\begin{figure*}[!htb]
  \centering
    \includegraphics[width=\linewidth]{pics/rank.pdf}

  \caption{\textbf{A} Mean matching rank score as a function of the included number
  of voxels for 3T and 7T, for stability- and $r^2$-based voxel selection.
  Error bars denote the bootstrapped 95\% confidence interval of the mean. The
  mean is taken over matching scores of eight runs for each of the
  19 participants. \textbf{B} Mean matching rank score as a function of the overall
volume of the included voxels for 3T and 7T, for stability- and
$r^2$-based voxel selection. Chance level is 0.5. Stars mark for which number of voxels the two curves differ significantly.}

 \label{fig:matching_score}
\end{figure*}

Figure \ref{fig:matching_score} shows differences in matching score for different numbers of voxels, selection strategies, and field strength. Matching scores in both 3T and 7T data peak at high numbers of voxels (3T and $r^{2}$: \unit[1000]{voxels}, \unit[27]{$cm^{3}$} volume, 7T and $r^{2}$: \unit[5000]{voxels}, \unit[13.5]{$cm^{3}$} volume, 3T and stability selection: \unit[500]{voxels}, \unit[13.5]{$cm^{3}$} volume, 7T and stability selection: \unit[5000]{voxels}, \unit[13.5]{$cm^{3}$} volume). Indexing by the overall volumes these voxels encompass reveals that the matching score peaks for 3T and 7T at (for stability selection) or close to (for selection by $r^{2}$) the same volume. For both stability selection and selection by $r^{2}$, 3T outperforms 7T for the same number of voxels, except for the largest two numbers of voxels, and --- in terms of the volume the voxels take --- 7T outperforms 3T for volumes up to \unit[2.7]{$cm^{3}$} and 3T outperforms 7T for volumes beyond \unit[27]{$cm^{3}$}. Furthermore, for 3T data, selection by stability and selection by $r^{2}$ perform similarly, while selection by $r^{2}$ outperforms selection by stability consistently in 7T data (Figure \ref{fig:matching_score_selection}).

%NEW
Furthermore, we use a hierarchical Bayesian model to test whether selection strategy and field strength influence the validation over and above the inter-individual variation and variation due to the number of voxels.
Table \ref{table:hierarchical_model} shows that both high field strength (7T) and selection by stability lead to a worse matching score.

\subsection*{Decoding accuracy}

\begin{figure*}[!htb]
  \centering
    \includegraphics[width=\linewidth]{pics/decoding.pdf}

	\caption{\textbf{A} Mean decoding accuracy of individual music stimuli as a function of
  the included number of voxels for 3T and 7T, for stability- and
  $r^2$-based voxel selection. Error bars denote the bootstrapped 95\%
  confidence interval of the mean. The mean is taken over decoding
  accuracies of eight runs for each of the 19 participants. Chance level is
    0.04. \textbf{B} Mean
decoding accuracy of individual music stimuli as a function of the overall volume of the
included voxels for 3T and 7T, for stability- and $r^2$-based voxel
selection. Chance level is 0.04. Stars mark for which number of voxels the two curves differ significantly.
}
 \label{fig:decoding_accuracy_stimulus}
\end{figure*}
Figure \ref{fig:decoding_accuracy_stimulus} shows the accuracy of decoding each individual stimulus (chance level 0.04) for different numbers of voxels, selection strategies, and field strength. 3T consistently outperforms 7T in decoding individual stimuli: if voxels are selected by stability, 3T outperforms 7T for all numbers of voxels, if voxels are selected by $r^{2}$, 3T outperforms 7T for all but the two highest numbers of voxels. For 7T data, the number of voxels matters little, however, for 3T data, optimal decoding accuracy is reached at a relatively low number of voxels (\unit[250]{voxels} for selection by $r^{2}$ and selection by stability, \unit[6.75]{$cm^{3}$} volume), and subsequently decreases if more voxels are included.

\begin{figure*}[!htb]
  \centering
  \includegraphics[width=\linewidth]{pics/decoding_genre.pdf}

  \caption{\textbf{A} Mean decoding accuracy of music category as a function of
  the included number of voxels for 3T and 7T, for stability- and
  $r^2$-based voxel selection. Error bars denote the bootstrapped 95\%
  confidence interval of the mean. The mean is taken over decoding
  accuracies of eight runs for each of the 19 participants. Chance level is 0.2. \textbf{B} Mean
decoding accuracy of music category as a function of the overall volume of the
included voxels for 3T and 7T, for stability- and $r^2$-based voxel
selection. Chance level is 0.2. Stars mark for which number of voxels the two curves differ significantly.
}

 \label{fig:decoding_accuracy}
\end{figure*}

Instead of decoding individual stimuli, we can also decode the musical genre of each individual stimulus. Figure \ref{fig:decoding_accuracy} shows the accuracy of decoding music genre (chance level 0.20) for different numbers of voxels, selection strategies, and field strength. Here a different pattern emerges: decoding accuracy is higher for 7T data than for 3T data, across all numbers of voxels and selection criteria. Potentially, this suggests that 7T BOLD activity contains different information than 3T.  For both selection strategies, 7T shows higher maximum genre decoding accuracy for high numbers of voxels, while 3T shows higher maximum genre decoding accuracy for lower numbers of voxels.

%NEW
Furthermore, we use a hierarchical Bayesian model to test whether selection strategy and field strength influence the validation over and above the inter-individual variation and variation due to the number of voxels.
Table \ref{table:hierarchical_model} shows that a high field strength leads to worse stimulus decoding accuracy and better genre decoding accuracy and that selection strategy does not clearly influence decoding accuracy.

\begin{figure*}[!htb]
  \centering
  \makebox[\textwidth][c]{\includegraphics[width=1.5\linewidth]{pics/conf_mats.pdf}}%
  \caption{Confusion matrices for genre decoding, separated by field-strength and voxel selection. Each confusion matrix is averaged across the different number of voxels, participants and runs.}

 \label{fig:confusion_matrices}
\end{figure*}

Next, we want to answer the question which musical genres can be best predicted, and if those differ between 3T and 7T data. For this we compute the confusion matrix from the out-of-sample predictions of all classifiers: Rows denote the stimulus' true genre, columns denote the predicted genre. Each cell contains the count -- or proportion -- of occurrences of this combination of true and predicted genres. Maximum classification accuracy leads to a diagonal confusion matrix.
Figure \ref{fig:confusion_matrices} shows the confusion matrices averaged across numbers of voxels for different field strength and selection criteria. The occurrences are normalized per row, and their mean proportion across numbers of voxels are indicated in each cell. All confusion matrices show a higher misclassification between genres that contain speech (country, rock'n'roll and heavy metal) and between instrumental genres (ambient and symphonic), while very few genres that contain speech are incorrectly classified as instrumental or vice versa. The increased genre decoding accuracy in 7T is thus due to a better discrimination between the two instrumental genres --- ambient and symphonic --- and between the three vocal genres --- country, rock'n'roll, and heavy metal.

%NEW
In other words, the rate of true negatives is high between the instrumental and vocal genres (i.e. vocal genres do not get classified as instrumental and vice versa) and the rate of false negatives and false positives is higher in between the vocal genres than in between the instrumental genres.
The effect of 7T can be seen insofar as that it reduces false negatives/positives between the genres of country and metal as well as between symphonic and ambient music.

\section*{Discussion}

Our results show that all three quality metrics vary with the number of voxels and voxel selection strategies in both low and high field strengths, even though individually each quality metric consistently exceeds chance level. We demonstrate that the patterns of this variation differ between quality metrics and that field strength, voxel selection, and voxel number all affect these metrics differently.

We believe these results emphasize the significance of an often under-appreciated aspect of computational models in neuroimaging: near arbitrary choices in the data analysis -- like the number of voxels used or how they are selected -- have a considerable impact on the performance evaluation of voxel-wise encoding models. Furthermore, the impact of these choices is inconsistent across the considered validation strategies. The effects of these parameters are rarely known a-priori, and each choice of a specific value can often be easily justified, yet this undisclosed flexibility leads to considerable researcher degrees-of-freedom \citep{SNS11,hong2019false} that will bias the comparison between different voxel-wise encoding models and hinder generalization of findings across individual studies.

In light of the substantial effects individual parameters exert on validation strategies, it does not surprise that one of our results -- a higher matching score in 7T than in 3T BOLD fMRI data -- contrasts a  previous study \citep{SF14}, which found higher matching scores for encoding natural sounds in 7T than in 3T, although using a different number of stimuli in the two conditions.

However, in spite of these variabilities, we can abstract several recommendations for applying and validating voxel-wise encoding models.

%r2 > stability selection
First, it is likely that selecting voxels by the quality of its encoding model, such as the $r^{2}$ score, will outperform a selection of voxels based on their stable response to the stimulus --- however, this difference diminishes for large numbers of voxels. One reason that this effect is especially prominent when only few voxels are included could be the quickly diminishing returns for higher number of voxels: voxels in which  an encoding model performs well are already included, and each additional voxel decreases the joint encoding performance. Interestingly, this pattern differs in the matching score, where performance in 7T is worse for the smallest number of voxels and increases when more voxels are included. In fact, this holds true for selection by both $r^{2}$ and a stability criterion, and thus differentiates the matching score from both binary retrieval accuracy, as well as decoding accuracy. This contrasts especially with binary retrieval accuracy, results from a previous study indicate that low numbers of voxels lead to high binary retrieval accuracy \citep{hoefle2018identifying}. Our results seem to corroborate this finding, suggesting that binary retrieval accuracy, in contrast to the matching score, is sensitive to the inclusion of worse performing voxels.

Second, the number of voxels has a large effect on all quality metrics, and to compare two sets of voxel-wise encoding models even on the same stimuli, one has to control for them. For most quality metrics, this control is most easily exercised by fixing the number of voxels directly, however, in case of the matching score, it can be more meaningful to fix the overall volume the two sets of encoding models encompass.
%NEW
For decoding accuracy and binary retrieval, our results suggest that fixing the number of voxels in the range of 250 to 1000 voxels provides a good starting point.
%NEW

Third, while higher field strength might lead to better individual encoding models, it might not improve their decoding accuracy. We show that voxel-wise encoding models in 3T f{MRI} data outperform voxel-wise encoding models in 7T f{MRI} data in all but one quality metric. This indicates that high field strength does not provide more information per se, since stimuli can't be differentiated better in general, in spite of the higher $r^{2}$ in most voxels in 7T. However, our key insight is, that higher field strength can decrease the decoding accuracy for some labels of the stimulus, i.e. the decoding of individual stimuli, yet increase the decoding accuracy for other labels, i.e the decoding of the musical genre of individual stimuli. Thus, one has to exercise caution, when choosing which label to decode from a set of encoding models. One reason for this, could be that training encoding models on stimulus features can make stimulus labels somewhat arbitrary. Two similar stimuli can lead to similar and accurate predictions of BOLD activity, and hence a higher $r^{2}$,  yet this similarity could decrease a quality measure that is based on the correct discrimination between individual stimuli.

While we believe that these three recommendations can guide neuroimaging researchers, to conclusively assess if these recommendations generalize, more data are needed, since the results in this study are based on only two f{MRI} datasets encompassing the same auditory stimuli recorded in 3- and 7T. This facilitates a comparison between different field strengths, yet, further work is needed to show that these conclusions hold beyond this set of stimuli, our chosen stimulus representation, and even beyond the auditory domain.

%NEW
Due to these limits in generalization, the main intent of our study is to serve as an example about the often hidden variability in validating voxel-wise encoding pipelines.
We hope this can be a starting point for testing the effect of different parameters on validation procedures more rigorously and a motivation for researchers to consider the effect of auxiliary validation parameters.

\section*{Author contributions}

MB performed the analysis and wrote the manuscript.
JSG contributed to the manuscript.
JWR contributed to the manuscript.
MH contributed to the analysis and the manuscript.

\section*{Competing Interests}

No competing interests were disclosed.

\section*{Grant Information}

This research was, in part, supported by the German Federal Ministry of
Education and Research (BMBF) as part of a US-German collaboration in
computational neuroscience (CRCNS; awarded to James Haxby, Peter Ramadge, and
Michael Hanke), co-funded by the BMBF and the US National Science Foundation
(BMBF 01GQ1112; NSF 1129855).  Work on the data-sharing technology employed for
this research was supported by US-German CRCNS project awarded to
Yaroslav~O.~Halchenko and Michael~Hanke, co-funded by the BMBF and the US
National Science Foundation (BMBF 01GQ1411; NSF 1429999).  Michael Hanke was
supported by funds from the German federal state of Saxony-Anhalt, Project:
Center for Behavioral Brain Sciences.
Moritz Boos and Jochem W. Rieger were supported by funds from the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) under Germany's Excellence Strategy – EXC 2177/1 - Project ID 390895286.

\section*{Acknowledgements}
%This section should acknowledge anyone who contributed to the research or the
%article but who does not qualify as an author based on the criteria provided
%earlier (e.g. someone or an organisation that provided writing assistance).
%Please state how they contributed; authors should obtain permission to
%acknowledge from all those mentioned in the Acknowledgements section.  Please
%do not list grant funding in this section (this should be included in the
%Grant information section - See above).

We are grateful to Michael Casey and the musicians.
We are also thankful for Cristiano Micheli's helpful feedback on an earlier version of this manuscript.

\bibliography{references}

\beginsupplement
\newpage
\section*{Supplementary information} \label{supplemental}

\begin{figure}[H]
  \centering
  \includegraphics[width=\linewidth]{pics/glassbrains.png}
  \caption{Glassbrains showing the proportion of how often a voxel was included across all participants and all validation sets for different field strengths and voxel selection criteria.}

 \label{fig:glassbrains}\end{figure}

\begin{figure}[H]
  \centering
  \includegraphics[width=\linewidth]{pics/binary_selection.pdf}

  \caption{\textbf{A} Mean binary retrieval accuracy as a function of the
  included number of voxels for 3T and 7T, for stability- and $r^2$-based
  voxel selection. Error bars denote the bootstrapped 95\% confidence interval
  of the mean. The mean is taken over binary retrieval accuracies of eight runs
  for each of the 19 participants. Chance level is 0.5. Stars mark for which number of voxels the two curves differ significantly.
}

 \label{fig:binary_retrieval_selection}\end{figure}

\begin{figure}[H]
  \centering
    \includegraphics[width=\linewidth]{pics/rank_selection.pdf}

  \caption{\textbf{A} Mean matching rank score as a function of the included number
  of voxels for 3T and 7T, for stability- and $r^2$-based voxel selection.
  Error bars denote the bootstrapped 95\% confidence interval of the mean. The
  mean is taken over matching scores accuracies of eight runs for each of the
  19 participants. Chance level is 0.5. Stars mark for which number of voxels the two curves differ significantly.}

 \label{fig:matching_score_selection}
\end{figure}

\begin{figure}[H]
  \centering
    \includegraphics[width=\linewidth]{pics/decoding_selection.pdf}

	\caption{\textbf{A} Mean decoding accuracy of individual music stimuli as a function of
  the included number of voxels for 3T and 7T, for stability- and
  $r^2$-based voxel selection. Error bars denote the bootstrapped 95\%
  confidence interval of the mean. The mean is taken over decoding
  accuracies of eight runs for each of the 19 participants. Chance level is
    0.04. Stars mark for which number of voxels the two curves differ significantly.
}
 \label{fig:decoding_accuracy_stimulus_selection}
\end{figure}

\begin{figure}[H]
  \centering
  \includegraphics[width=\linewidth]{pics/decoding_genre_selection.pdf}

  \caption{Mean decoding accuracy of music category as a function of
  the included number of voxels for 3T and 7T, for stability- and
  $r^2$-based voxel selection. Error bars denote the bootstrapped 95\%
  confidence interval of the mean. The mean is taken over decoding
  accuracies of eight runs for each of the 19 participants. Chance level is 0.2. Stars mark for which number of voxels the two curves differ significantly.
	}

 \label{fig:decoding_accuracy_selection}
\end{figure}

%NEW
So far we focused on demonstrating the effect of different analysis parameters visually.
In addition, we can use hierarchical Bayesian modeling \citep{gelman2013bayesian} to quantify the effect of individual parameters on different validation methods, while modeling inter-individual variation explicitly.
For each validation method we try to predict its score for each participant.
To do so, we model as main effects the analysis parameters field strength (as a binary indicator whether it field strength is 7T), selection strategy (as a binary indicator whether selection is done by stability), and the number of voxels (as categorical variable with an independent parameter for each number of voxels).
Additionally, we model inter-individual variation in these scores by allowing for a participant-specific parameter (i.e. model with "no pooling" across participants).
Table \ref{table:hierarchical_model} summarizes the estimated parameters for the main effects.
Categorical predictors with only two choices (Field strength and selection strategy) are dummy-coded.
This means that, for example, the coefficient of "Field strength (7T)" quantifies the influence of a field strength of 7T over the "baseline" of 3T.
One can see that selection by stability performs either worse than selection by $r^{2}$ (Binary retrieval accuracy and Matching score) or similarly to selection by $r^{2}$ (decoding accuracy and genre decoding accuracy).
Additionally, we find that high field strength (7T) outperforms 3T only for genre decoding and performs worse than 3T for all other validation methods.
These results confirm the pattern that could be seen throughout the earlier figures: 7T does not outperform across all validation strategies 3T and selection by $r^{2}$ performs better or similarly to selection by stability across all validation strategies.

\noindent
\begin{table*}[!htbp]
\centering
%\resizebox{\linewidth}{!}{
\begin{adjustbox}{center}
\begin{tabular}{lccccc}
\multicolumn{5}{c}{Binary retrieval accuracy} \tabularnewline
\hline
Parameter &   Mean &     SD &  hdi\_3\% &  hdi\_97\% \tabularnewline
\hline
Field strength (7T)            & -0.013 &  0.004 &  -0.020 &   -0.006 \tabularnewline
Selection strategy (stability) & -0.016 &  0.004 &  -0.022 &   -0.009 \\
\hline
\multicolumn{5}{c}{Matching score} \tabularnewline
\hline
Parameter &   Mean &     SD &  hdi\_3\% &  hdi\_97\% \\
\hline
Field strength (7T)             & -0.034 &  0.004 &  -0.041 &   -0.027 \\
Selection strategy (stability)  & -0.017 &  0.004 &  -0.024 &   -0.010 \\
\hline
\multicolumn{5}{c}{Decoding accuracy} \tabularnewline
\hline
Parameter &   Mean &     SD &  hdi\_3\% &  hdi\_97\% \\
\hline
Field strength (7T)            & -0.038 &  0.003 &  -0.043 &   -0.032 \\
Selection strategy (stability) &  0.003 &  0.003 &  -0.002 &    0.009 \\
\hline
\multicolumn{5}{c}{Genre decoding accuracy} \tabularnewline
\hline
Parameter &   Mean &     SD &  hdi\_3\% &  hdi\_97\% \\
\hline
Field strength (7T)            &  0.115 &  0.006 &   0.104 &    0.127 \\
Selection strategy (stability) &  0.002 &  0.006 &  -0.009 &    0.013 \\
\hline
\end{tabular}
\end{adjustbox}
\caption{Coefficients of the influence of different analysis parameters on different validation metrics. Mean and SD are the mean and standard deviation of the fitted regression coefficients. Hdi specifies the lower or upper end of the highest density interval that covers 95\% of the posterior distribution of the parameter.}
\label{table:hierarchical_model}
\end{table*}

\end{document}


\section{Results in \llll channel}
\label{sec:hmhzz_result_4l}

The statistical treatment in searching for heavy resonances in $ZZ \rightarrow \llll$ final state is described in this section.
Results are presented in both cut- and MVA- based analysis.

\subsection{Statistical procedure}

The upper limits on heavy resonances are obtained using the unbinned profile likelihood fits.
\mfl is the discriminant.
The likelihood function is a product of a Poisson term representing the probability of observing $n$ events
and a weighted sum of both signal and background probability distribution functions (pdfs) evaluated for all observed events.
\begingroup
\small
\begin{equation}
  {\cal L } (x_{1}..x_{n}| \sigma_{\textrm{ggF}}, \sigma_{\textrm{VBF}}) = \textrm{Pois}(n|S_{\textrm{ggF}} + S_{\textrm{VBF}} + B)  \left[ \prod\limits^{n}_{i=1}
  \frac{S_{\textrm{ggF}} f_{\textrm{ggF}}(x_{i}) + S_{\textrm{VBF}} f_{\textrm{VBF}}(x_{i}) + B f_{\textrm{B}}(x_{i})}{S_{\textrm{ggF}} + S_{\textrm{VBF}} + B}
  \right]
\end{equation}
\endgroup
where $f_X$s are the pdfs of signal and backgrounds modelled in section~\ref{sec:signal_model} and ~\ref{sec:hmhzz_bkg},
$S_X$ and $B$ are the normalizations of signal and sum of backgrounds.

The parameters of interest (POI) in the search is $\sigma_{\textrm{ggF}}$ (and $\sigma_{\textrm{VBF}}$ only for NWA signal), which is the cross section of signal model in ggF (and VBF) production mode.
In the case of two POIs, when testing one POI, the other one is profiled along with other nuisance parameters (except left unconstrained) during the minimization.
These POIs enter the likelihood inside the expected signal yields $S_{\mathrm{ggF}}$ and $S_{\mathrm{VBF}}$ as:
\begingroup
\small
\begin{equation}
S_{\mathrm{ggF(VBF)}} = \sigma_{\mathrm{ggF(VBF)}} \times B(S\rightarrow ZZ) \times A \times C \times \int\mathcal{L}
\end{equation}
\endgroup
where $A\times C$ is the signal acceptance as parameterized in~\ref{sec:hmhzz_signal_acc}, and $\int\mathcal{L} = \lumi $ is the integrated luminosity of the dataset.

The dependence of the expected number of signal and background events (normalizations) and the shape of the PDFs on the systematic uncertainties measured in section~\ref{sec:hmhzz_sys}
is described by a set of nuisance parameters (NPs) $\theta_{i}$, constrained by Gaussian distribution described in section~\ref{sec:NPs}.
The final likelihood function ${\cal L } ( \sigma_{\mathrm{ggF}}, \sigma_{\mathrm{VBF}},\mH,\theta_{i})$ is therefore a function of $\sigma_{\mathrm{ggF}}, \sigma_{\mathrm{VBF}} , \mH$, and $\theta_{i}$.

Furthermore, the normalization of SM background $pp\to ZZ$, including both \qqZZ\ and \ggZZ, is a free parameter ($\mu_{ZZ}$) and profiled during the minimization.
Floating $ZZ$ normalization in fit takes the advantage of reducing the dependence on theory predictions and their associated uncertainties,
especially given that the increased data luminosity would provide precise determination of the SM $ZZ$ background rate.

At the end, the upper limit on production cross-section $\sigma_{\mathrm{ggF(VBF)}}$ at a given heavy resonance model is obtained by setting the mass of signal $m_{H}$ parameter as constant at the desired value,
and maximising the likelihood function with respect to nuisance parameters.
%The test statistics $q_\mu$ is used to set upper limits following \cite{CowanTestStatistics}.
The $CLs$ method described in section~\ref{sec:CLs} is used to obtain exclusion limits.
%In order to measure the compatibility of the data with the background-only hypothesis ($\sigma_{\mathrm{ggF}}=\sigma_{\mathrm{VBF}}=0$), the test statistic $q_{0}$ is used as introduced in section~\ref{sec:teststat}.

\subsection{Fit to likelihood function under background-only hypothesis for MVA-based analysis}

Both MVA- and cut-based analysis are studied by performing likelihood fit to the (pseudo-) data under the background-only hypothesis and under different signal models.
First of all, table~\ref{tab:postfit_4lyields} (table~\ref{tab:postfit_4lyields_cut}) summaries the expected and observed number of events in MVA (Cut) -based analysis for region of $\mfl > 200~\gev~$ together with their systematic uncertainties after background-only fit.
The normalization of $ZZ$ background is taken from data for one category each, as shown in table~\ref{tab:muZZ_bonly_dnn} (table~\ref{tab:muZZ_bonly_cut}) in MVA (Cut) -based analysis.

\begin{table}[htbp]
  \centering
    \caption{$ZZ$ normalization factor in each category, obtained from a likelihood fit to the data under the background-only hypothesis in \textbf{MVA-based} analysis.}
  \label{tab:muZZ_bonly_dnn}
  \small
  \begin{spacing}{1.0}
  \begin{tabular}{ccc}
    \toprule
    Normalization factor  & Fitted value \\
    \midrule
    $\mu_{ZZ}^{ggF-MVA-high}$  & 1.07 $\pm$ 0.05 \\
    \hline
    $\mu_{ZZ}^{ggF-MVA-low}$   & 1.12 $\pm$ 0.03 \\
    \hline
    $\mu_{ZZ}^{VBF-MVA-enriched}$  & 0.9 $\pm$ 0.3 \\
    \bottomrule
  \end{tabular}
  \end{spacing}
\end{table}

\begin{table}[htbp]
  \centering
    \caption{$ZZ$ normalization factor in each category, obtained from a likelihood fit to the data under the background-only hypothesis in \textbf{Cut-based} analysis.}
  \label{tab:muZZ_bonly_cut}
  \small
  \begin{spacing}{1.0}
  \begin{tabular}{ccc}
    \toprule
    Normalization factor  & Fitted value \\
    \midrule
    $\mu_{ZZ}^{ggF-CBA-enriched}$  & 1.10 $\pm$ 0.02 \\
    \hline
    $\mu_{ZZ}^{VBF-CBA-enriched}$  & 1.1 $\pm$ 0.2 \\
    \bottomrule
  \end{tabular}
  \end{spacing}
\end{table}

\begin{table}[tbp]
    \caption{\label{tab:postfit_4lyields}
      Expected and observed numbers of events for $\mfl > 200~\gev{}$,
      together with their systematic uncertainties,
      for three \textbf{MVA-based} categories.
      The expected number of events, as well as their uncertainties, are obtained
      from a likelihood fit to the data under the background-only hypothesis.
      The uncertainties of the $ZZ$ normalisation factors, presented in table~\ref{tab:muZZ_bonly_dnn},
      are also taken into account.
    }
    \centering

    \footnotesize
    \resizebox{\textwidth}{!}{
      \begin{tabular}{l r@{ $\pm$ }l r@{ $\pm$ }l r@{ $\pm$ }l r@{ $\pm$ }l r@{ $\pm$ }l }
        \toprule
        \multirow{2}{*}{Process} & \multicolumn{2}{c}{VBF-MVA-enriched} & \multicolumn{6}{c}{ggF-MVA-high}  & \multicolumn{2}{c}{ggF-MVA-low} \\
        & \multicolumn{2}{c}{} & \multicolumn{2}{c}{$4\mu$ channel} & \multicolumn{2}{c}{$2e2\mu$ channel} & \multicolumn{2}{c}{$4e$ channel}  & \multicolumn{2}{c}{} \\
        \midrule

        \qqZZ	&	11	&	4	&	232	&	10	&	389	&	17	&	154	&	7	&	2008	&	47	\\
        \ggZZ	&	3	&	2	&	37	&	6	&	64	&	10	&	26	&	4	&	247	&	19	\\
        \ZZ (EW)	&	4.1	&	0.4	&	4.5	&	0.2	&	7.5	&	0.4	&	3	&	0.2	&	14.3	&	0.7	\\
        \Zjet,\ttbar	&	0.08	&	0.02	&	0.6	&	0.1	&	1.7	&	0.4	&	0.8	&	0.1	&	8.8	&	2.0	\\
        \ttv,\vvv	&	0.96	&	0.10	&	9.8	&	0.2	&	17.5	&	0.4	&	7.8	&	0.2	&	21.9	&	0.5	\\
        \midrule
        Total background	&	19	&	4	&	284	&	9	&	480	&	16	&	192	&	6	&	2300	&	48	\\

        \midrule
        Observed  & \multicolumn{2}{c}{19} & \multicolumn{2}{c}{271} & \multicolumn{2}{c}{493} & \multicolumn{2}{c}{191} & \multicolumn{2}{c}{2301} \\
        \bottomrule
      \end{tabular}
    }
\end{table}

\begin{table}[tbp]
    \caption{\label{tab:postfit_4lyields_cut}
      Expected and observed numbers of events for $\mfl > 200~\gev{}$,
      together with their systematic uncertainties,
      for two \textbf{Cut-based} categories.
      The expected number of events, as well as their uncertainties, are obtained
      from a likelihood fit to the data under the background-only hypothesis.
      The uncertainties of the $ZZ$ normalisation factors, presented in table~\ref{tab:muZZ_bonly_cut},
      are also taken into account.
    }
    \centering

    \footnotesize
    \resizebox{\textwidth}{!}{
      \begin{tabular}{l r@{ $\pm$ }l r@{ $\pm$ }l r@{ $\pm$ }l r@{ $\pm$ }l r@{ $\pm$ }l }
        \toprule
        \multirow{2}{*}{Process} & \multicolumn{2}{c}{VBF-CBA-enriched} & \multicolumn{6}{c}{ggF-CBA-enriched} \\
        & \multicolumn{2}{c}{} & \multicolumn{2}{c}{$4\mu$ channel} & \multicolumn{2}{c}{$2e2\mu$ channel} & \multicolumn{2}{c}{$4e$ channel} \\
        \midrule

        \qqZZ	        &	48	&	8	&	860	&	18	&1360	&	28	&	515	&	11	\\
        \ggZZ	        &	13	&	4	&	114	&	9	&	189	&	14	&	73	&	6	\\
        \ZZ (EW)    &	10.9	&	0.9	&	6.9	&	0.3	&11.1	&	0.4	&	4.4	&	0.2	\\
        \Zjet,\ttbar	&	0.3	&	0.1	&	2.1	&	0.4	&	6.7	&	1.6	&	3.1	&	0.4	\\
        \ttv,\vvv	    &	3.0	&	0.2	&16.3	&	0.4	&26.8	&	0.6	&11.8	&	0.3	\\
        \midrule
        Total background	&	75	&	9	&	1000	&	18	&	1593	&	28	&	607	&	11	\\

        \midrule
        Observed  & \multicolumn{2}{c}{75} & \multicolumn{2}{c}{932} & \multicolumn{2}{c}{1656} & \multicolumn{2}{c}{612} \\
        \bottomrule
      \end{tabular}
    }
\end{table}

Then due to the same background estimation and modelling procedures, as well as the same method of systematic measurements between MVA- and Cut- based analysis,
the following results of background-only fits in this selection are only shown in MVA-based analysis under the model of heavy Higgs resonance with narrow-width as an example.
The final results of interpretation in both MVA- and cut- based analysis in different signal models described in section~\ref{sec:signal_model} will be presented in next section.

The post-fit \mfl spectrum in each category is shown in figure~\ref{fig:m4l_postfit}.

\begin{figure}[!ht]
  \begin{center}
  \subfloat[]{\includegraphics[width=0.32\textwidth]{figures/HMHZZ/results/4l_postfit_m4l_ggF_4mu.pdf}}
  \subfloat[]{\includegraphics[width=0.32\textwidth]{figures/HMHZZ/results/4l_postfit_m4l_ggF_2mu2e.pdf}}
  \subfloat[]{\includegraphics[width=0.32\textwidth]{figures/HMHZZ/results/4l_postfit_m4l_ggF_4e.pdf}} \\
  \subfloat[]{\includegraphics[width=0.32\textwidth]{figures/HMHZZ/results/4l_postfit_m4l_rest_incl.pdf}}
  \subfloat[]{\includegraphics[width=0.32\textwidth]{figures/HMHZZ/results/4l_postfit_m4l_VBF_incl.pdf}}

    \caption{Distribution of the four-lepton invariant mass \mfl
        for (a), (b), (c) the ggF-MVA-high categories, (d) the ggF-MVA-low category
        and (e) the VBF-MVA-enriched category.
    The backgrounds are determined from a combined likelihood fit to the data under the background-only
    hypothesis. The simulated signal at 600~\gev~ is normalized to
    a cross section corresponding to 50 (5) times the observed upper limit given in section~\ref{sec:hmhzz_spin0nwa}
    for the ggF (VBF) production.
    The error bars on the data points indicate the statistical uncertainty,
    and the systematic uncertainty in MC prediction is shown by the hatched band.
    The lower panels show the ratio of data to prediction.
    The red arrows indicate data points that are outside the displayed range.}
  \label{fig:m4l_postfit}
  \end{center}
\end{figure}

To inspect the likelihood model, pulls and constraints as well as the correlation matrix of NPs are studied by performing a background only fit.
Figure~\ref{fig:NPpull_cb_asimov} shows the pulls and constraints when fitting to background only Asimov data (top) and observed data (bottom).
Figure~\ref{fig:NPcorr_cb_asimov} shows the correlation matrix, only for NPs with correlation between each others greater than 0.1 when fitting to background only Asimov data.

\begin{figure}[!ht]
\begin{center}
\subfloat[]{\includegraphics[width=0.8\figwidth]{figures/HMHZZ/results/Pulls_NWA_floatZZ_dnn_Asimov.pdf}}\\
\subfloat[]{\includegraphics[width=0.8\figwidth]{figures/HMHZZ/results/Pulls_NWA_floatZZ_dnn_Obs.pdf}}
\caption{Pulls and constraints of nuisance parameters after a background only fit to (a) Asimov data and (b) observed data in the \llll channel.
The Asimov data is generated with background data only, and the observed data includes datasets from 2015 to 2018.
}
\label{fig:NPpull_cb_asimov}
\end{center}
\end{figure}

\begin{figure}[!ht]
\begin{center}
\subfloat[]{\includegraphics[width=0.8\figwidth]{figures/HMHZZ/results/Correlation_NWA_floatZZ_dnn_Asimov.pdf}}
\caption{Correlation of nuisance parameters after a background only fit to Asimov data in the \llll channel.
The Asimov data is generated with background data only.
}
\label{fig:NPcorr_cb_asimov}
\end{center}
\end{figure}

The impact of a systematic uncertainty on the result depends on the production mode and the mass hypothesis.
To check the impact of systematic uncertainties on expected signal sensitivity, a NP ranking study is performed using
signal injected Asimov data with the injected cross section close to 95\% CLs upper limit in figure~\ref{fig:NWA201518_MVA} at the masses of 400~\gev~ and 1000~\gev.
%The results are shown in Fig.~\ref{fig:rank_cb_NWA_ggF} (\ref{fig:rank_cb_NWA_VBF}) for ggF (VBF) production mode.
The results are shown in table~\ref{tab:NPranking_NWA}.
For ggF production, at lower mass, the systematic uncertainties of parton showering variation for signal, the luminosity uncertainty,
and the parametrization of signal acceptance dominate,
while at higher mass, the shape uncertainties from PDF variation for ZZ (\qqZZ and \ggZZ) background become important, as also seen in VBF production mode.
In addition for VBF, jet related uncertainties become more important comparing to ggF production.
Moreover, the dominate uncertainties include the acceptance uncertainty from QCD scale variation for signal and the luminosity uncertainty.

\begin{table}[htbp!]
    \caption{\label{tab:NPranking_NWA}
    Impact of the leading systematic uncertainties, the data statistic uncertainties,
    as well as the total uncertainties on the
    predicted signal event yield with the cross section times branching ratio being
    set to the expected upper limit,
    expressed as a percentage of the signal yield
    for the ggF (left) and VBF (right) production modes at $\mH = 400$ and $1000~\gev$.
    }
    %\footnotesize
    \centering
    \begin{tabular}{l l | l l}
      \toprule
      \multicolumn{2}{c|}{ggF production} &  \multicolumn{2}{c}{VBF production} \\
      \multicolumn{2}{l|}{Systematic source    \hfill Impact [\%]}   & \multicolumn{2}{l}{Systematic source \hfill Impact [\%]}\\
      \midrule
      \multicolumn{4}{c}{$\bigstrut$  $\mH=400~\gev$}\\
      \midrule

       Parton showering of ggF       & 2.3 & QCD scale of VBF                     & 2.7 \\
       Luminosity                    & 1.8 & Jet flavor composition               & 2.5 \\
       PDF of \qqZZ                  & 1.6 & Luminosity                           & 1.8 \\
       Signal yield parameterization & 1.4 & Jet energy scale (in-su calibration) & 1.6 \\
      Data stat. uncertainty         &  48 & Data stat. uncertainty               & 57 \\
      Total Uncertainty              &  49 & Total Uncertainty                    & 58 \\
      \midrule
      \multicolumn{4}{c}{$\bigstrut$  $\mH=1000~\gev$}\\
      \midrule

       PDF of \qqZZ              & 2.5 & QCD scale of VBF       & 2.3 \\
       Parton showering of ggF   & 2.4 & PDF of \qqZZ           & 2.2 \\
       PDF of \ggZZ              & 1.9 & Luminosity             & 1.8 \\
       Luminosity                & 1.8 & PDF of \ggZZ           & 1.6 \\
      Data stat. uncertainty     &  84 & Data stat. uncertainty & 92 \\
      Total Uncertainty          &  86 & Total Uncertainty      & 93 \\
      \bottomrule

    \end{tabular}
  \end{table}

%\begin{figure}[!ht]
%\begin{center}
%\subfloat[]{\includegraphics[width=0.48\figwidth]{figures/HMHZZ/results/workspace_4l_NWA_dnn_400_ggF_newAsimov_0p226_pulls_paper.pdf}}
%\subfloat[]{\includegraphics[width=0.48\figwidth]{figures/HMHZZ/results/workspace_4l_NWA_dnn_1000_ggF_newAsimov_0p03_pulls_paper.pdf}}
%\caption{NP ranking plot for the \xsggF\ fit to Asimov data in the \llll channel.
%The Asimov data is injected with \xsggF\ = 0.226 fb for $m_H = 400$ GeV (left) and \xsggF\ = 0.030 fb for $m_H = 1000$ GeV (right).
%}
%\label{fig:rank_cb_NWA_ggF}
%\end{center}
%\end{figure}
%
%\begin{figure}[!ht]
%\begin{center}
%\subfloat[]{\includegraphics[width=0.48\figwidth]{figures/HMHZZ/results/workspace_4l_NWA_dnn_400_VBF_newAsimov_0p12_pulls_paper.pdf}}
%\subfloat[]{\includegraphics[width=0.48\figwidth]{figures/HMHZZ/results/workspace_4l_NWA_dnn_1000_VBF_newAsimov_0p0226_pulls_paper.pdf}}
%\caption{NP ranking plot for the \xsVBF\ fit to Asimov data in the \llll channel.
%The Asimov data is injected with \xsVBF\ = 0.120 fb for $m_H = 400$ GeV (left) and \xsVBF\ = 0.0226 fb for $m_H = 1000$ GeV (right).
%}
%\label{fig:rank_cb_NWA_VBF}
%\end{center}
%\end{figure}

%% =======================================================================================================
\subsection{Interpretations}
\label{sec:hmhzz_spin0nwa}

\subsubsection{Spin-0 resonance with NWA}

In the absence of a specific model, the ratio of ggF and VBF production mode is unknown for this additional heavy scalar.
For this reason, the fits for ggF and VBF processes are done separately, and in each case the cross section of the untested process is allowed to be a free parameter in the statistical fit.
The observed and expected upper limit at 95\% confidence level (CL) on the $\sigma \times B(H \rightarrow ZZ)$ of a narrow scalar resonance for both ggF and VBF production mode with the integrated luminosity of \lumi is shown in figure~\ref{fig:NWA201518_MVA} (\ref{fig:NWA201518_Cut}) for MVA- (cut-) based analysis.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/limits_DNN_ggF.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/limits_DNN_VBF.pdf}
    \caption{The expected and observed upper limits at 95\% CL on $\sigma \times B(H \rightarrow ZZ)$ using the MVA-based analysis for ggF (left) and VBF (right) production. The green and yellow bands represent the $\pm 1\sigma$ and $\pm 2\sigma$ uncertainties in the expected limits.
 }
    \label{fig:NWA201518_MVA}
\end{figure}

\begin{figure}[h]
    \centering
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/limits_Cut2020_ggF.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/limits_Cut2020_VBF.pdf}
    \caption{The expected and observed upper limits at 95\% CL on $\sigma \times B(H \rightarrow ZZ)$ using the cut-based analysis for ggF (left) and VBF (right) production. The green and yellow bands represent the $\pm 1\sigma$ and $\pm 2\sigma$ uncertainties in the expected limits.
 }
    \label{fig:NWA201518_Cut}
\end{figure}

In order to measure the compatibility of the data with the background-only hypothesis ($\sigma_{\mathrm{ggF}}=\sigma_{\mathrm{VBF}}=0$), the test statistic $q_{0}$, as described in equation~\ref{eq:q0}, is used.
Figure~\ref{fig:p0_MVA} shows its corresponding $p_0$ scan for ggF (left) and VBF (right) production as functions of \mH in MVA-based analysis.
\begin{figure}[h]
    \centering
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/p0_4ldnn_p0_ggF.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/p0_4ldnn_p0_VBF.pdf}
    \caption{The local p0 scan as functions of \mH on $\sigma \times B(H \rightarrow ZZ)$ using the MVA-based analysis for ggF (left) and VBF (right) production.
 }
    \label{fig:p0_MVA}
\end{figure}
No excess over 2$\sigma$ is found for ggF production, while for VBF production, the maximal local significance is found to be 2.46$\sigma$ at 620~\gev~
with corresponding global significance of 0.88$\sigma$ computed from Ref.~\cite{Gross:2010qma}.

\subsubsection{Spin-0 resonance with LWA}

In the case of LWA model, only ggF production mode is studied.
The interference between the heavy scalar and SM Higgs boson (H-h), as well as the heavy scalar and SM \ggZZ continuum background (H-B) as modelled in section~\ref{sec:int_model} are token into account.
The upper limit at 95\% confidence level (CL) on ggF cross section times branch ratio ($\sigma_{ggF} \times B(H \rightarrow ZZ)$) is shown in figure~\ref{fig:LWAlimits_ggF_201518_withInt} for a width of 1, 5, 10 and 15\% of \mH.

\begin{figure}[h]
    \begin{center}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/Limits_LWA_withInt_1.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/Limits_LWA_withInt_5.pdf} \\
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/Limits_LWA_withInt_10.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/Limits_LWA_withInt_15.pdf} \\
    \end{center}
    \caption{The upper limits at 95\% confidence level on $\sigma_{ggF} \times B(H\rightarrow ZZ)$
    as a function of the heavy resonance mass $m_{H}$ for the ggF production mode with an intrinsic width of 1\% (top left), 5\% (top right), 10\% (bottom left) and 15\% (bottom right) for both the case where interference with Standard Model processes is considered.
    The green and yellow bands represent the $\pm 1~\sigma$ and $\pm 2~\sigma$ uncertainties in the expected limits.
  }
    \label{fig:LWAlimits_ggF_201518_withInt}
\end{figure}

\subsubsection{Spin-2 RS Graviton resonance}

The observed and expected 95\% upper limit on the cross section times branching ratio for RS Graviton (RSG) scenario is shown in figure~\ref{fig:RSGlimits_ggF_201518}.
Similar to LWA, only 4$e$, 4$\mu$ and 2$e$2$\mu$ channel of ggF production mode are used.
On top of the expected and observed upper limits in this model, a predicted cross section as function of mass provided by theorist is also shown in the figure.
Comparing with the observed result provided by $ZZ \rightarrow \llll$ decay, this spin-2 graviton is excluded up to a mass of 1500~\gev.

\begin{figure}[h]
    \begin{center}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/Limits_RSG.pdf}
    \end{center}
    \caption{The upper limits at 95\% confidence level on $\sigma_{ggF} \times B(G_{KK}\rightarrow ZZ)$
    as a function of the heavy resonance mass $m(G_{KK})$ for the ggF production mode in RS Graviton model.
    The green and yellow bands represent the $\pm 1~\sigma$ and $\pm 2~\sigma$ uncertainties in the expected limits.
  }
    \label{fig:RSGlimits_ggF_201518}
\end{figure}

\subsubsection{Summary of interpretation}

As a summary, figure~\ref{fig:models_limits} shows the comparison of expected and observed 95\% CL upper limits between different models described above.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/4l_compareAll_exp.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/4l_compareAll_obs.pdf}
    \caption{The expected (left) and observed (right) upper limits at 95\% CL on $\sigma \times B(S\to ZZ)$ for ggF production mode at different hypothetical models including the MVA- and cut- based NWA, LWA with width assumptions of 1, 5, 10, 15\%, and the RSG. }
    \label{fig:models_limits}
\end{figure}

Figure~\ref{fig:comparison_limits} compares the expected 95\% CL upper limits as a function of the NWA resonance mass
in this analysis with full run-2 data and the one in previous publication~\cite{Aaboud:2017rel} with the integrated luminosity of 36.1~\ifb.
With a significant increase of integrated luminosity and an improved analysis strategy, comparing to the previous publication,
the expected sensitivities of searching for narrow-width heavy resonance reduced by up to 70\% in MVA-based analysis,
where 50\% of reduction is due to luminosity increase while other improvement mainly comes from inviting multivariate method.

\begin{figure}[h]
    \centering
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/4l_36vs139_ggF.pdf}
    \includegraphics[width=0.48\textwidth]{figures/HMHZZ/results/4l_36vs139_VBF.pdf}
    \caption{Comparisons of the expected upper limits at 95\% CL on the cross section times branching ratio as a
             function of the heavy resonance mass ~\mH~ for the ggF production mode (left) and for the VBF production mode
             (right) in the case of the NWA. The expected limits from the previous publication are shown in the green dashed
             line and are projected to the 139~\ifb~
             as shown in the blue dashed line. In addition, the current results based on either
             cut-based categorisation or the multivariate-based categorisation are shown in red and black lines.}
    \label{fig:comparison_limits}
\end{figure}

Figure~\ref{fig:4mu_eventdisplay} shows the display of one candidate event passing analysis selection in four-muon final state with four-muon invariant mass of 1.34~\tev.

\begin{figure}[h]
    \centering
    \includegraphics[width=1.0\textwidth]{figures/HMHZZ/results/display_run339037_evt365322232_2017-10-24T14-09-59.png}
    \caption{Display of one candidate event in 4$\mu$ final state with the mass of 1.35~\tev.}
    \label{fig:4mu_eventdisplay}
\end{figure}


\documentclass{article}

\usepackage{amsmath}

\begin{document}
\section{This is Sample Article}

sample content.

\begin{align*}
  1 = 2
\end{align*}

\end{document}


\section{Lid-driven cavity flow}

Lid-driven cavity flow is flow in a unit square containing a unit depth of water with horizontal topography. The top (north) boundary has a unit velocity and the other three boundaries are solid wall. The initial condition is water at rest. This is a standard test for numerical methods used to solve the two-dimensional Navier--Stokes equations. This is not a usual test for shallow water models~\cite{CCFNC2006}. However, it may benefit to check the behaviour of numerical solutions of \anuga{} to this problem.

The analytical solution to this problem is not available, but a large number of researchers have proposed numerical solutions. Some of the literatures amongst others are Cueto-Felgueroso et al.~\cite{CCFNC2006} and Erturk et al.~\cite{ECG2005}.

\subsection{Results}

The following figures show numerical solutions of \anuga{} to this lid-driven cavity flow problem. We focus on the velocity and velocity fields. Note that the current version of \anuga{} is set up for inviscid fluid (water). That is, the Reynolds number is infinity. An accurate result should show secondary vortices around the corners for high Reynolds numbers.

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{vel_t1_centroid.png}
\end{center}
\caption{Velocity at the centroids of computational elements at an instant of time.}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{vel_t1_vertex.png}
\end{center}
\caption{Velocity at the vertices of computational elements at an instant of time.}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{vel_t2_centroid.png}
\end{center}
\caption{Velocity at the centroids of computational elements at the final time step.}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{vel_t2_vertex.png}
\end{center}
\caption{Velocity at the vertices of computational elements at the final time step.}
\end{figure}

\begin{figure}
\begin{center}
\includegraphics[width=0.9\textwidth]{xvelocity_at_y05.png}
\end{center}
\caption{Xvelocity at $y=0.5$\,.}
\end{figure}

\endinput


\subsection{Diamond–Mirrlees}